Coverage Report

Created: 2026-10-31 07:15

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/Simd/src/Simd/SimdAvx2Winograd3.cpp
Line
Count
Source
1
/*
2
* Simd Library (http://ermig1979.github.io/Simd).
3
*
4
* Copyright (c) 2011-2024 Yermalayeu Ihar.
5
*
6
* Permission is hereby granted, free of charge, to any person obtaining a copy
7
* of this software and associated documentation files (the "Software"), to deal
8
* in the Software without restriction, including without limitation the rights
9
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
* copies of the Software, and to permit persons to whom the Software is
11
* furnished to do so, subject to the following conditions:
12
*
13
* The above copyright notice and this permission notice shall be included in
14
* all copies or substantial portions of the Software.
15
*
16
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
* SOFTWARE.
23
*/
24
#include "Simd/SimdMemory.h"
25
#include "Simd/SimdStore.h"
26
#include "Simd/SimdWinograd.h"
27
#include "Simd/SimdSse41.h"
28
#include "Simd/SimdSet.h"
29
#include "Simd/SimdBase.h"
30
31
namespace Simd
32
{
33
#if defined(SIMD_AVX2_ENABLE) && defined(SIMD_SYNET_ENABLE)    
34
    namespace Avx2
35
    {
36
        SIMD_INLINE void WinogradKernel3x3Block2x2SetFilter8t(const float* src, float* dst, size_t stride)
37
0
        {
38
0
            const __m256 r2 = _mm256_set1_ps(1.0f / 2.0f);
39
0
            const __m256 r4 = _mm256_set1_ps(1.0f / 4.0f);
40
41
0
            __m256 s[9];
42
0
            s[0] = _mm256_loadu_ps(src + 0 * stride);
43
0
            s[1] = _mm256_loadu_ps(src + 1 * stride);
44
0
            s[2] = _mm256_loadu_ps(src + 2 * stride);
45
0
            s[3] = _mm256_loadu_ps(src + 3 * stride);
46
0
            s[4] = _mm256_loadu_ps(src + 4 * stride);
47
0
            s[5] = _mm256_loadu_ps(src + 5 * stride);
48
0
            s[6] = _mm256_loadu_ps(src + 6 * stride);
49
0
            s[7] = _mm256_loadu_ps(src + 7 * stride);
50
0
            s[8] = _mm256_loadu_ps(src + 8 * stride);
51
52
0
            _mm256_storeu_ps(dst + 0 * stride, s[0]);
53
0
            __m256 _0a2 = _mm256_add_ps(s[0], s[2]);
54
0
            _mm256_storeu_ps(dst + 1 * stride, _mm256_mul_ps(_mm256_add_ps(_0a2, s[1]), r2));
55
0
            _mm256_storeu_ps(dst + 2 * stride, _mm256_mul_ps(_mm256_sub_ps(_0a2, s[1]), r2));
56
0
            _mm256_storeu_ps(dst + 3 * stride, s[2]);
57
58
0
            __m256 _0a6a3 = _mm256_add_ps(_mm256_add_ps(s[0], s[6]), s[3]);
59
0
            _mm256_storeu_ps(dst + 4 * stride, _mm256_mul_ps(_0a6a3, r2));
60
0
            __m256 _2a8a5 = _mm256_add_ps(_mm256_add_ps(s[2], s[8]), s[5]);
61
0
            __m256 _1a7a4 = _mm256_add_ps(_mm256_add_ps(s[1], s[7]), s[4]);
62
0
            _mm256_storeu_ps(dst + 5 * stride, _mm256_mul_ps(_mm256_add_ps(_mm256_add_ps(_0a6a3, _2a8a5), _1a7a4), r4));
63
0
            _mm256_storeu_ps(dst + 6 * stride, _mm256_mul_ps(_mm256_sub_ps(_mm256_add_ps(_0a6a3, _2a8a5), _1a7a4), r4));
64
0
            _mm256_storeu_ps(dst + 7 * stride, _mm256_mul_ps(_2a8a5, r2));
65
66
0
            __m256 _0a6s3 = _mm256_sub_ps(_mm256_add_ps(s[0], s[6]), s[3]);
67
0
            _mm256_storeu_ps(dst + 8 * stride, _mm256_mul_ps(_0a6s3, r2));
68
0
            __m256 _2a8s5 = _mm256_sub_ps(_mm256_add_ps(s[2], s[8]), s[5]);
69
0
            __m256 _1a7s4 = _mm256_sub_ps(_mm256_add_ps(s[1], s[7]), s[4]);
70
0
            _mm256_storeu_ps(dst + 9 * stride, _mm256_mul_ps(_mm256_add_ps(_mm256_add_ps(_0a6s3, _2a8s5), _1a7s4), r4));
71
0
            _mm256_storeu_ps(dst + 10 * stride, _mm256_mul_ps(_mm256_sub_ps(_mm256_add_ps(_0a6s3, _2a8s5), _1a7s4), r4));
72
0
            _mm256_storeu_ps(dst + 11 * stride, _mm256_mul_ps(_2a8s5, r2));
73
74
0
            _mm256_storeu_ps(dst + 12 * stride, s[6]);
75
0
            __m256 _6a8 = _mm256_add_ps(s[6], s[8]);
76
0
            _mm256_storeu_ps(dst + 13 * stride, _mm256_mul_ps(_mm256_add_ps(_6a8, s[7]), r2));
77
0
            _mm256_storeu_ps(dst + 14 * stride, _mm256_mul_ps(_mm256_sub_ps(_6a8, s[7]), r2));
78
0
            _mm256_storeu_ps(dst + 15 * stride, s[8]);
79
0
        }
80
81
        void WinogradKernel3x3Block2x2SetFilter(const float* src, size_t size, float* dst, SimdBool trans)
82
0
        {
83
0
            if (trans)
84
0
            {
85
0
                size_t size8 = AlignLo(size, 8), i = 0;
86
0
                for (; i < size8; i += 8)
87
0
                    WinogradKernel3x3Block2x2SetFilter8t(src + i, dst + i, size);
88
0
                for (; i < size; i += 1)
89
0
                    Base::WinogradKernel3x3Block2x2SetFilter1t(src + i, dst + i, size);
90
0
            }
91
0
            else
92
0
            {
93
0
                Sse41::WinogradKernel3x3Block2x2SetFilter(src, size, dst, trans);
94
0
            }
95
0
        }
96
97
        //-----------------------------------------------------------------------
98
99
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInputLoad8n(const float * src, __m256 * dst)
100
0
        {
101
0
            __m256 a0 = Load<false>(src + 0, src + 8);
102
0
            __m256 a1 = Load<false>(src + 2, src + 10);
103
0
            __m256 a2 = Load<false>(src + 4, src + 12);
104
0
            __m256 a3 = Load<false>(src + 6, src + 14);
105
0
            dst[0] = _mm256_shuffle_ps(a0, a2, 0x88);
106
0
            dst[1] = _mm256_shuffle_ps(a0, a2, 0xDD);
107
0
            dst[2] = _mm256_shuffle_ps(a1, a3, 0x88);
108
0
            dst[3] = _mm256_shuffle_ps(a1, a3, 0xDD);
109
0
        }
110
111
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInputLoad8n(const float * src, __m256 * dst, PadType pad)
112
0
        {
113
0
            __m256 a0 = Set(pad == PadNose1 ? Sse41::LoadPadZeroNose1(src + 0) : _mm_loadu_ps(src + 0), _mm_loadu_ps(src + 8));
114
0
            __m256 a1 = Load<false>(src + 2, src + 10);
115
0
            __m256 a2 = Load<false>(src + 4, src + 12);
116
0
            __m256 a3 = Set(_mm_loadu_ps(src + 6), pad == PadTail2 ? Sse41::LoadPadZeroTail2(src + 14) : (pad == PadTail1 ? Sse41::LoadPadZeroTail1(src + 14) : _mm_loadu_ps(src + 14)));
117
0
            dst[0] = _mm256_shuffle_ps(a0, a2, 0x88);
118
0
            dst[1] = _mm256_shuffle_ps(a0, a2, 0xDD);
119
0
            dst[2] = _mm256_shuffle_ps(a1, a3, 0x88);
120
0
            dst[3] = _mm256_shuffle_ps(a1, a3, 0xDD);
121
0
        }
122
123
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInputLoad8z(__m256 * dst)
124
0
        {
125
0
            dst[0] = _mm256_setzero_ps();
126
0
            dst[1] = _mm256_setzero_ps();
127
0
            dst[2] = _mm256_setzero_ps();
128
0
            dst[3] = _mm256_setzero_ps();
129
0
        }
130
131
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8Store(const __m256 * src, float * dst, size_t stride)
132
0
        {
133
0
            _mm256_storeu_ps(dst + 0 * stride, _mm256_sub_ps(_mm256_sub_ps(src[0], src[8]), _mm256_sub_ps(src[2], src[10])));
134
0
            _mm256_storeu_ps(dst + 1 * stride, _mm256_add_ps(_mm256_sub_ps(src[1], src[9]), _mm256_sub_ps(src[2], src[10])));
135
0
            _mm256_storeu_ps(dst + 2 * stride, _mm256_sub_ps(_mm256_sub_ps(src[2], src[10]), _mm256_sub_ps(src[1], src[9])));
136
0
            _mm256_storeu_ps(dst + 3 * stride, _mm256_sub_ps(_mm256_sub_ps(src[1], src[9]), _mm256_sub_ps(src[3], src[11])));
137
0
            _mm256_storeu_ps(dst + 4 * stride, _mm256_sub_ps(_mm256_add_ps(src[4], src[8]), _mm256_add_ps(src[6], src[10])));
138
0
            _mm256_storeu_ps(dst + 5 * stride, _mm256_add_ps(_mm256_add_ps(src[5], src[9]), _mm256_add_ps(src[6], src[10])));
139
0
            _mm256_storeu_ps(dst + 6 * stride, _mm256_sub_ps(_mm256_add_ps(src[6], src[10]), _mm256_add_ps(src[5], src[9])));
140
0
            _mm256_storeu_ps(dst + 7 * stride, _mm256_sub_ps(_mm256_add_ps(src[5], src[9]), _mm256_add_ps(src[7], src[11])));
141
0
            _mm256_storeu_ps(dst + 8 * stride, _mm256_sub_ps(_mm256_sub_ps(src[8], src[4]), _mm256_sub_ps(src[10], src[6])));
142
0
            _mm256_storeu_ps(dst + 9 * stride, _mm256_add_ps(_mm256_sub_ps(src[9], src[5]), _mm256_sub_ps(src[10], src[6])));
143
0
            _mm256_storeu_ps(dst + 10 * stride, _mm256_sub_ps(_mm256_sub_ps(src[10], src[6]), _mm256_sub_ps(src[9], src[5])));
144
0
            _mm256_storeu_ps(dst + 11 * stride, _mm256_sub_ps(_mm256_sub_ps(src[9], src[5]), _mm256_sub_ps(src[11], src[7])));
145
0
            _mm256_storeu_ps(dst + 12 * stride, _mm256_sub_ps(_mm256_sub_ps(src[4], src[12]), _mm256_sub_ps(src[6], src[14])));
146
0
            _mm256_storeu_ps(dst + 13 * stride, _mm256_add_ps(_mm256_sub_ps(src[5], src[13]), _mm256_sub_ps(src[6], src[14])));
147
0
            _mm256_storeu_ps(dst + 14 * stride, _mm256_sub_ps(_mm256_sub_ps(src[6], src[14]), _mm256_sub_ps(src[5], src[13])));
148
0
            _mm256_storeu_ps(dst + 15 * stride, _mm256_sub_ps(_mm256_sub_ps(src[5], src[13]), _mm256_sub_ps(src[7], src[15])));
149
0
        }
150
151
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8n(const float * src, size_t srcStride, float * dst, size_t dstStride)
152
0
        {
153
0
            __m256 t[16];
154
0
            WinogradKernel3x3Block2x2SetInputLoad8n(src + 0 * srcStride, t + 0);
155
0
            WinogradKernel3x3Block2x2SetInputLoad8n(src + 1 * srcStride, t + 4);
156
0
            WinogradKernel3x3Block2x2SetInputLoad8n(src + 2 * srcStride, t + 8);
157
0
            WinogradKernel3x3Block2x2SetInputLoad8n(src + 3 * srcStride, t + 12);
158
0
            WinogradKernel3x3Block2x2SetInput8Store(t, dst, dstStride);
159
0
        }
160
161
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8n(const float * src, size_t srcStride, PadType rowPad, PadType colPad, float * dst, size_t dstStride)
162
0
        {
163
0
            __m256 t[16];
164
0
            if (rowPad == PadNose1)
165
0
                WinogradKernel3x3Block2x2SetInputLoad8z(t + 0);
166
0
            else
167
0
                WinogradKernel3x3Block2x2SetInputLoad8n(src + 0 * srcStride, t + 0, colPad);
168
0
            WinogradKernel3x3Block2x2SetInputLoad8n(src + 1 * srcStride, t + 4, colPad);
169
0
            if (rowPad == PadTail2)
170
0
                WinogradKernel3x3Block2x2SetInputLoad8z(t + 8);
171
0
            else
172
0
                WinogradKernel3x3Block2x2SetInputLoad8n(src + 2 * srcStride, t + 8, colPad);
173
0
            if (rowPad >= PadTail1)
174
0
                WinogradKernel3x3Block2x2SetInputLoad8z(t + 12);
175
0
            else
176
0
                WinogradKernel3x3Block2x2SetInputLoad8n(src + 3 * srcStride, t + 12, colPad);
177
0
            WinogradKernel3x3Block2x2SetInput8Store(t, dst, dstStride);
178
0
        }
179
180
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcS, size_t srcC, __m256 dst[16])
181
0
        {
182
0
            dst[0] = _mm256_loadu_ps(src + 0 * srcS + 0 * srcC);
183
0
            dst[1] = _mm256_loadu_ps(src + 0 * srcS + 1 * srcC);
184
0
            dst[2] = _mm256_loadu_ps(src + 0 * srcS + 2 * srcC);
185
0
            dst[3] = _mm256_loadu_ps(src + 0 * srcS + 3 * srcC);
186
0
            dst[4] = _mm256_loadu_ps(src + 1 * srcS + 0 * srcC);
187
0
            dst[5] = _mm256_loadu_ps(src + 1 * srcS + 1 * srcC);
188
0
            dst[6] = _mm256_loadu_ps(src + 1 * srcS + 2 * srcC);
189
0
            dst[7] = _mm256_loadu_ps(src + 1 * srcS + 3 * srcC);
190
0
            dst[8] = _mm256_loadu_ps(src + 2 * srcS + 0 * srcC);
191
0
            dst[9] = _mm256_loadu_ps(src + 2 * srcS + 1 * srcC);
192
0
            dst[10] = _mm256_loadu_ps(src + 2 * srcS + 2 * srcC);
193
0
            dst[11] = _mm256_loadu_ps(src + 2 * srcS + 3 * srcC);
194
0
            dst[12] = _mm256_loadu_ps(src + 3 * srcS + 0 * srcC);
195
0
            dst[13] = _mm256_loadu_ps(src + 3 * srcS + 1 * srcC);
196
0
            dst[14] = _mm256_loadu_ps(src + 3 * srcS + 2 * srcC);
197
0
            dst[15] = _mm256_loadu_ps(src + 3 * srcS + 3 * srcC);
198
0
        }
199
200
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcW, size_t srcC, float * dst, size_t dstStride)
201
0
        {
202
0
            size_t srcS = srcW * srcC;
203
0
            size_t srcCF = AlignLo(srcC, F);
204
0
            for (size_t c = 0; c < srcCF; c += F)
205
0
            {
206
0
                __m256 tmp[16];
207
0
                WinogradKernel3x3Block2x2SetInput8t(src + c, srcS, srcC, tmp);
208
0
                WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + c, dstStride);
209
0
            }
210
0
            if (srcCF < srcC)
211
0
            {
212
0
                __m256 tmp[16];
213
0
                WinogradKernel3x3Block2x2SetInput8t(src + srcC - F, srcS, srcC, tmp);
214
0
                WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + srcC - F, dstStride);
215
0
            }
216
0
        }
217
218
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcS, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, __m256 dst[16])
219
0
        {
220
0
            for (size_t row = 0; row < rowB; ++row)
221
0
            {
222
0
                for (size_t col = 0; col < 4; ++col)
223
0
                    dst[col] = _mm256_setzero_ps();
224
0
                dst += 4;
225
0
            }
226
0
            for (size_t row = rowB; row < rowE; ++row)
227
0
            {
228
0
                for (size_t col = 0; col < colB; ++col)
229
0
                    dst[col] = _mm256_setzero_ps();
230
0
                for (size_t col = colB; col < colE; ++col)
231
0
                    dst[col] = _mm256_loadu_ps(src + row * srcS + col * srcC);
232
0
                for (size_t col = colE; col < 4; ++col)
233
0
                    dst[col] = _mm256_setzero_ps();
234
0
                dst += 4;
235
0
            }
236
0
            for (size_t row = rowE; row < 4; ++row)
237
0
            {
238
0
                for (size_t col = 0; col < 4; ++col)
239
0
                    dst[col] = _mm256_setzero_ps();
240
0
                dst += 4;
241
0
            }
242
0
        }
243
244
        SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcW, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, float * dst, size_t dstStride)
245
0
        {
246
0
            size_t srcS = srcW * srcC;
247
0
            size_t srcCF = AlignLo(srcC, F);
248
0
            for (size_t c = 0; c < srcCF; c += F)
249
0
            {
250
0
                __m256 tmp[16];
251
0
                WinogradKernel3x3Block2x2SetInput8t(src + c, srcS, srcC, rowB, rowE, colB, colE, tmp);
252
0
                WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + c, dstStride);
253
0
            }
254
0
            if (srcCF < srcC)
255
0
            {
256
0
                __m256 tmp[16];
257
0
                WinogradKernel3x3Block2x2SetInput8t(src + srcC - F, srcS, srcC, rowB, rowE, colB, colE, tmp);
258
0
                WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + srcC - F, dstStride);
259
0
            }
260
0
        }
261
262
        void WinogradKernel3x3Block2x2SetInput(const float* src, size_t srcChannels, size_t srcHeight, size_t srcWidth,
263
            size_t padY, size_t padX, size_t padH, size_t padW, float* dst, size_t dstStride, SimdBool trans)
264
0
        {
265
0
            assert(padY == padX && padY == padH && padY == padW && (padY == 0 || padY == 1));
266
0
            SimdBool pad = padY > 0 ? SimdTrue : SimdFalse;
267
0
            if (trans ? (srcChannels < 8) : (srcHeight < 4 || srcWidth < 18))
268
0
            {
269
0
                Sse41::WinogradKernel3x3Block2x2SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans);
270
0
                return;
271
0
            }
272
0
            size_t dstH = pad ? srcHeight : srcHeight - 2;
273
0
            size_t dstW = pad ? srcWidth : srcWidth - 2;
274
0
            size_t tileH = (dstH + 1) / 2;
275
0
            size_t tileW = (dstW + 1) / 2;
276
0
            size_t dstH2 = AlignLo(dstH, 2);
277
0
            size_t dstW2 = AlignLo(dstW, 2);
278
0
            if (trans)
279
0
            {
280
0
                size_t noseW = Simd::Min<size_t>(4, dstW + 1);
281
0
                size_t noseH = Simd::Min<size_t>(4, dstH + 1);
282
0
                size_t start = pad ? 2 : 0;
283
0
                if (pad)
284
0
                {
285
0
                    if (dstH == dstH2)
286
0
                        dstH2 -= 2;
287
0
                    if (dstW == dstW2)
288
0
                        dstW2 -= 2;
289
0
                    src -= (srcWidth + 1)*srcChannels;
290
0
                }
291
0
                size_t tailW = dstW - dstW2 + (pad ? 1 : 2);
292
0
                size_t tailH = dstH - dstH2 + (pad ? 1 : 2);
293
0
                size_t row = 0, col = 0;
294
0
                if (pad)
295
0
                {
296
0
                    if (pad)
297
0
                        WinogradKernel3x3Block2x2SetInput8t(src, srcWidth, srcChannels, 1, noseH, 1, noseW, dst, dstStride), dst += srcChannels;
298
0
                    for (col = start; col < dstW2; col += 2)
299
0
                        WinogradKernel3x3Block2x2SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, 4, dst, dstStride), dst += srcChannels;
300
0
                    if (col < dstW)
301
0
                        WinogradKernel3x3Block2x2SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, tailW, dst, dstStride), dst += srcChannels;
302
0
                }
303
0
                for (row = start; row < dstH2; row += 2)
304
0
                {
305
0
                    if (pad)
306
0
                        WinogradKernel3x3Block2x2SetInput8t(src + row * srcWidth * srcChannels, srcWidth, srcChannels, 0, 4, 1, noseW, dst, dstStride), dst += srcChannels;
307
0
                    for (col = start; col < dstW2; col += 2)
308
0
                        WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, dst, dstStride), dst += srcChannels;
309
0
                    if (col < dstW)
310
0
                        WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, 4, 0, tailW, dst, dstStride), dst += srcChannels;
311
0
                }
312
0
                if (row < dstH)
313
0
                {
314
0
                    if (pad)
315
0
                        WinogradKernel3x3Block2x2SetInput8t(src + row * srcWidth* srcChannels, srcWidth, srcChannels, 0, tailH, 1, noseW, dst, dstStride), dst += srcChannels;
316
0
                    for (col = start; col < dstW2; col += 2)
317
0
                        WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, 4, dst, dstStride), dst += srcChannels;
318
0
                    if (col < dstW)
319
0
                        WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, tailW, dst, dstStride), dst += srcChannels;
320
0
                }
321
0
            }
322
0
            else
323
0
            {
324
0
                size_t dstW16 = AlignLo(dstW, 16);
325
0
                if (pad && dstW16 == dstW)
326
0
                    dstW16 -= 16;
327
0
                PadType rowPad = dstH2 < dstH ? PadTail1 : PadNone;
328
0
                PadType colPad = dstW2 < dstW ? PadTail1 : PadNone;
329
0
                size_t tailCol = dstW2 < dstW ? dstW - 15 : dstW - 16;
330
0
                size_t tailRow = dstH2 < dstH ? dstH - 1 : dstH - 2;
331
0
                bool specialColTail = dstW16 < dstW || pad;
332
0
                bool specialRowTail = dstH2 < dstH || pad;
333
0
                if (pad)
334
0
                {
335
0
                    src -= srcWidth + 1;
336
0
                    rowPad = dstH2 < dstH ? PadTail2 : PadTail1;
337
0
                    colPad = dstW2 < dstW ? PadTail2 : PadTail1;
338
0
                    if (dstH2 == dstH)
339
0
                        dstH2 -= 2;
340
0
                }
341
0
                for (size_t c = 0; c < srcChannels; ++c)
342
0
                {
343
0
                    size_t row = 0, tileY = 0;
344
0
                    if (pad)
345
0
                    {
346
0
                        size_t col = 0, tileX = 0;
347
0
                        const float * s = src + row * srcWidth;
348
0
                        float * d = dst + tileY * tileW;
349
0
                        if (pad)
350
0
                            WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, PadNose1, PadNose1, d + tileX, dstStride), col += 16, tileX += 8;
351
0
                        for (; col < dstW16; col += 16, tileX += 8)
352
0
                            WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, PadNose1, PadNone, d + tileX, dstStride);
353
0
                        if (specialColTail)
354
0
                            WinogradKernel3x3Block2x2SetInput8n(s + tailCol, srcWidth, PadNose1, colPad, d + tileW - 8, dstStride);
355
0
                        row += 2, tileY += 1;
356
0
                    }
357
0
                    for (; row < dstH2; row += 2, tileY += 1)
358
0
                    {
359
0
                        size_t col = 0, tileX = 0;
360
0
                        const float * s = src + row * srcWidth;
361
0
                        float * d = dst + tileY * tileW;
362
0
                        if (pad)
363
0
                            WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, PadNone, PadNose1, d + tileX, dstStride), col += 16, tileX += 8;
364
0
                        for (; col < dstW16; col += 16, tileX += 8)
365
0
                            WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, d + tileX, dstStride);
366
0
                        if (specialColTail)
367
0
                            WinogradKernel3x3Block2x2SetInput8n(s + tailCol, srcWidth, PadNone, colPad, d + tileW - 8, dstStride);
368
0
                    }
369
0
                    if (specialRowTail)
370
0
                    {
371
0
                        size_t col = 0, tileX = 0;
372
0
                        const float * s = src + tailRow * srcWidth;
373
0
                        float * d = dst + (tileH - 1) * tileW;
374
0
                        if (pad)
375
0
                            WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, rowPad, PadNose1, d + tileX, dstStride), col += 16, tileX += 8;
376
0
                        for (; col < dstW16; col += 16, tileX += 8)
377
0
                            WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, rowPad, PadNone, d + tileX, dstStride);
378
0
                        if (specialColTail)
379
0
                            WinogradKernel3x3Block2x2SetInput8n(s + tailCol, srcWidth, rowPad, colPad, d + tileW - 8, dstStride);
380
0
                    }
381
0
                    src += srcWidth * srcHeight;
382
0
                    dst += tileW * tileH;
383
0
                }
384
0
            }
385
0
        }
386
387
        //-----------------------------------------------------------------------
388
389
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputLoad4(const float * src, size_t stride, __m256 * dst)
390
0
        {
391
0
            __m256 s0 = _mm256_loadu_ps(src + 0 * stride);
392
0
            __m256 s1 = _mm256_loadu_ps(src + 1 * stride);
393
0
            __m256 s2 = _mm256_loadu_ps(src + 2 * stride);
394
0
            __m256 s3 = _mm256_loadu_ps(src + 3 * stride);
395
0
            dst[0] = _mm256_add_ps(_mm256_add_ps(s0, s1), s2);
396
0
            dst[1] = _mm256_sub_ps(_mm256_sub_ps(s1, s2), s3);
397
0
        }
398
399
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputLoad16(const float * src, size_t stride, __m256 * dst)
400
0
        {
401
0
            __m256 tmp[8];
402
0
            WinogradKernel3x3Block2x2SetOutputLoad4(src + 0 * stride, stride, tmp + 0);
403
0
            WinogradKernel3x3Block2x2SetOutputLoad4(src + 4 * stride, stride, tmp + 2);
404
0
            WinogradKernel3x3Block2x2SetOutputLoad4(src + 8 * stride, stride, tmp + 4);
405
0
            WinogradKernel3x3Block2x2SetOutputLoad4(src + 12 * stride, stride, tmp + 6);
406
0
            dst[0] = _mm256_add_ps(_mm256_add_ps(tmp[0], tmp[2]), tmp[4]);
407
0
            dst[1] = _mm256_add_ps(_mm256_add_ps(tmp[1], tmp[3]), tmp[5]);
408
0
            dst[2] = _mm256_sub_ps(_mm256_sub_ps(tmp[2], tmp[4]), tmp[6]);
409
0
            dst[3] = _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[5]), tmp[7]);
410
0
        }
411
412
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8n(const float * src, size_t stride, __m256 * dst)
413
0
        {
414
0
            __m256 d[4], u[4];
415
0
            WinogradKernel3x3Block2x2SetOutputLoad16(src, stride, d);
416
0
            u[0] = _mm256_unpacklo_ps(d[0], d[1]);
417
0
            u[1] = _mm256_unpackhi_ps(d[0], d[1]);
418
0
            u[2] = _mm256_unpacklo_ps(d[2], d[3]);
419
0
            u[3] = _mm256_unpackhi_ps(d[2], d[3]);
420
0
            dst[0] = _mm256_permute2f128_ps(u[0], u[1], 0x20);
421
0
            dst[1] = _mm256_permute2f128_ps(u[0], u[1], 0x31);
422
0
            dst[2] = _mm256_permute2f128_ps(u[2], u[3], 0x20);
423
0
            dst[3] = _mm256_permute2f128_ps(u[2], u[3], 0x31);
424
0
        }
425
426
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8n(const float * src, size_t srcStride, float * dst, size_t dstStride)
427
0
        {
428
0
            __m256 d[4];
429
0
            WinogradKernel3x3Block2x2SetOutput8n(src, srcStride, d);
430
0
            _mm256_storeu_ps(dst + 0 * dstStride + 0, d[0]);
431
0
            _mm256_storeu_ps(dst + 0 * dstStride + 8, d[1]);
432
0
            _mm256_storeu_ps(dst + 1 * dstStride + 0, d[2]);
433
0
            _mm256_storeu_ps(dst + 1 * dstStride + 8, d[3]);
434
0
        }
435
436
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8n(const float * src, size_t srcStride, float * dst, size_t dstStride, bool lastRow, bool lastCol, const __m256 & mask)
437
0
        {
438
0
            __m256 d[4];
439
0
            WinogradKernel3x3Block2x2SetOutput8n(src, srcStride, d);
440
0
            _mm256_storeu_ps(dst + 0, d[0]);
441
0
            if (lastCol)
442
0
                _mm256_storeu_ps(dst + 8, d[1]);
443
0
            else
444
0
                StoreMasked<false>(dst + 8, d[1], mask);
445
0
            if (lastRow)
446
0
            {
447
0
                dst += dstStride;
448
0
                _mm256_storeu_ps(dst + 0, d[2]);
449
0
                if (lastCol)
450
0
                    _mm256_storeu_ps(dst + 8, d[3]);
451
0
                else
452
0
                    StoreMasked<false>(dst + 8, d[3], mask);
453
0
            }
454
0
        }
455
456
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputStore4(const __m256 src[4], float * dst, size_t dstS, size_t dstC)
457
0
        {
458
0
            _mm256_storeu_ps(dst + 0 * dstS + 0 * dstC, src[0]);
459
0
            _mm256_storeu_ps(dst + 0 * dstS + 1 * dstC, src[1]);
460
0
            _mm256_storeu_ps(dst + 1 * dstS + 0 * dstC, src[2]);
461
0
            _mm256_storeu_ps(dst + 1 * dstS + 1 * dstC, src[3]);
462
0
        }
463
464
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC)
465
0
        {
466
0
            size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F);
467
0
            for (size_t d = 0; d < dstCF; d += F)
468
0
            {
469
0
                __m256 tmp[4];
470
0
                WinogradKernel3x3Block2x2SetOutputLoad16(src + d, srcStride, tmp);
471
0
                WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + d, dstS, dstC);
472
0
            }
473
0
            if (dstCF < dstC)
474
0
            {
475
0
                __m256 tmp[4];
476
0
                WinogradKernel3x3Block2x2SetOutputLoad16(src + dstC - F, srcStride, tmp);
477
0
                WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + dstC - F, dstS, dstC);
478
0
            }
479
0
        }
480
481
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputStore4(const __m256 src[4], float * dst, size_t dstS, size_t dstC, size_t rowE, size_t colE)
482
0
        {
483
0
            for (size_t row = 0; row < rowE; ++row)
484
0
                for (size_t col = 0; col < colE; ++col)
485
0
                    _mm256_storeu_ps(dst + row * dstS + col * dstC, src[row * 2 + col]);
486
0
        }
487
488
        SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC, size_t rowE, size_t colE)
489
0
        {
490
0
            size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F);
491
0
            for (size_t d = 0; d < dstCF; d += F)
492
0
            {
493
0
                __m256 tmp[4];
494
0
                WinogradKernel3x3Block2x2SetOutputLoad16(src + d, srcStride, tmp);
495
0
                WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + d, dstS, dstC, rowE, colE);
496
0
            }
497
0
            if (dstCF < dstC)
498
0
            {
499
0
                __m256 tmp[4];
500
0
                WinogradKernel3x3Block2x2SetOutputLoad16(src + dstC - F, srcStride, tmp);
501
0
                WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + dstC - F, dstS, dstC, rowE, colE);
502
0
            }
503
0
        }
504
505
        void WinogradKernel3x3Block2x2SetOutput(const float * src, size_t srcStride, float * dst, size_t dstChannels, size_t dstHeight, size_t dstWidth, SimdBool trans)
506
0
        {
507
0
            if (trans ? (dstChannels < 8) : (dstHeight < 2 || dstWidth < 16))
508
0
            {
509
0
                Sse41::WinogradKernel3x3Block2x2SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans);
510
0
                return;
511
0
            }
512
0
            size_t tileH = (dstHeight + 1) / 2;
513
0
            size_t tileW = (dstWidth + 1) / 2;
514
0
            size_t dstH2 = AlignLo(dstHeight, 2);
515
0
            size_t dstW2 = AlignLo(dstWidth, 2);
516
0
            if (trans)
517
0
            {
518
0
                size_t row, col;
519
0
                for (row = 0; row < dstH2; row += 2)
520
0
                {
521
0
                    for (col = 0; col < dstW2; col += 2)
522
0
                        WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels), src += dstChannels;
523
0
                    if (col < dstWidth)
524
0
                        WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, 2, dstWidth - col), src += dstChannels;
525
0
                }
526
0
                if (row < dstHeight)
527
0
                {
528
0
                    for (col = 0; col < dstW2; col += 2)
529
0
                        WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, 2), src += dstChannels;
530
0
                    if (col < dstWidth)
531
0
                        WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, dstWidth - col), src += dstChannels;
532
0
                }
533
0
            }
534
0
            else
535
0
            {
536
0
                size_t dstW16 = AlignLo(dstWidth, 16);
537
0
                __m256 tailMask = LeftNotZero32f(8 + dstW2 - dstWidth);
538
0
                size_t tailCol = dstW2 < dstWidth ? dstWidth - 15 : dstWidth - 16;
539
0
                for (size_t c = 0; c < dstChannels; ++c)
540
0
                {
541
0
                    size_t row = 0, tileY = 0;
542
0
                    for (; row < dstH2; row += 2, tileY += 1)
543
0
                    {
544
0
                        size_t col = 0, tileX = 0;
545
0
                        const float * s = src + tileY * tileW;
546
0
                        float * d = dst + row * dstWidth;
547
0
                        for (; col < dstW16; col += 16, tileX += 8)
548
0
                            WinogradKernel3x3Block2x2SetOutput8n(s + tileX, srcStride, d + col, dstWidth);
549
0
                        if (col < dstWidth)
550
0
                            WinogradKernel3x3Block2x2SetOutput8n(s + tileW - 8, srcStride, d + tailCol, dstWidth, true, false, tailMask);
551
0
                    }
552
0
                    if (row < dstHeight)
553
0
                    {
554
0
                        size_t col = 0, tileX = 0;
555
0
                        const float * s = src + (tileH - 1) * tileW;
556
0
                        float * d = dst + (dstHeight - 1) * dstWidth;
557
0
                        for (; col < dstW16; col += 16, tileX += 8)
558
0
                            WinogradKernel3x3Block2x2SetOutput8n(s + tileX, srcStride, d + col, dstWidth, false, true, tailMask);
559
0
                        if (col < dstWidth)
560
0
                            WinogradKernel3x3Block2x2SetOutput8n(s + tileW - 8, srcStride, d + tailCol, dstWidth, false, false, tailMask);
561
0
                    }
562
0
                    src += tileW * tileH;
563
0
                    dst += dstHeight * dstWidth;
564
0
                }
565
0
            }
566
0
        }
567
568
        //-----------------------------------------------------------------------
569
570
        SIMD_INLINE void WinogradKernel3x3Block3x3SetFilter8Row(const __m256 * t, float * dst, size_t stride)
571
0
        {
572
0
            const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f);
573
0
            const __m256 r3 = _mm256_set1_ps(1.0f / 3.0f);
574
0
            const __m256 r2 = _mm256_set1_ps(1.0f / 2.0f);
575
0
            const __m256 f2_3 = _mm256_set1_ps(2.0f / 3.0f);
576
0
            const __m256 mr2 = _mm256_set1_ps(-1.0f / 2.0f);
577
578
0
            _mm256_storeu_ps(dst + 0 * stride, _mm256_mul_ps(r2, t[0]));
579
0
            __m256 t0 = _mm256_add_ps(t[0], t[2]);
580
0
            _mm256_storeu_ps(dst + 1 * stride, _mm256_mul_ps(mr2, _mm256_add_ps(t0, t[1])));
581
0
            _mm256_storeu_ps(dst + 2 * stride, _mm256_mul_ps(r6, _mm256_sub_ps(t[1], t0)));
582
0
            _mm256_storeu_ps(dst + 3 * stride, _mm256_add_ps(_mm256_mul_ps(r6, t[0]), _mm256_add_ps(_mm256_mul_ps(r3, t[1]), _mm256_mul_ps(f2_3, t[2]))));
583
0
            _mm256_storeu_ps(dst + 4 * stride, t[2]);
584
0
        }
585
586
        SIMD_INLINE void WinogradKernel3x3Block3x3SetFilter8All(const __m256 * s, float * dst, size_t stride)
587
0
        {
588
0
            const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f);
589
0
            const __m256 r3 = _mm256_set1_ps(1.0f / 3.0f);
590
0
            const __m256 r2 = _mm256_set1_ps(1.0f / 2.0f);
591
0
            const __m256 f2_3 = _mm256_set1_ps(2.0f / 3.0f);
592
0
            const __m256 mr2 = _mm256_set1_ps(-1.0f / 2.0f);
593
594
0
            __m256 t[3];
595
0
            t[0] = _mm256_mul_ps(r2, s[0]);
596
0
            t[1] = _mm256_mul_ps(r2, s[1]);
597
0
            t[2] = _mm256_mul_ps(r2, s[2]);
598
0
            WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 0 * stride, stride);
599
600
0
            t[0] = _mm256_mul_ps(mr2, _mm256_add_ps(_mm256_add_ps(s[0], s[6]), s[3]));
601
0
            t[1] = _mm256_mul_ps(mr2, _mm256_add_ps(_mm256_add_ps(s[1], s[7]), s[4]));
602
0
            t[2] = _mm256_mul_ps(mr2, _mm256_add_ps(_mm256_add_ps(s[2], s[8]), s[5]));
603
0
            WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 5 * stride, stride);
604
605
0
            t[0] = _mm256_mul_ps(r6, _mm256_sub_ps(s[3], _mm256_add_ps(s[0], s[6])));
606
0
            t[1] = _mm256_mul_ps(r6, _mm256_sub_ps(s[4], _mm256_add_ps(s[1], s[7])));
607
0
            t[2] = _mm256_mul_ps(r6, _mm256_sub_ps(s[5], _mm256_add_ps(s[2], s[8])));
608
0
            WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 10 * stride, stride);
609
610
0
            t[0] = _mm256_add_ps(_mm256_mul_ps(r6, s[0]), _mm256_add_ps(_mm256_mul_ps(r3, s[3]), _mm256_mul_ps(f2_3, s[6])));
611
0
            t[1] = _mm256_add_ps(_mm256_mul_ps(r6, s[1]), _mm256_add_ps(_mm256_mul_ps(r3, s[4]), _mm256_mul_ps(f2_3, s[7])));
612
0
            t[2] = _mm256_add_ps(_mm256_mul_ps(r6, s[2]), _mm256_add_ps(_mm256_mul_ps(r3, s[5]), _mm256_mul_ps(f2_3, s[8])));
613
0
            WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 15 * stride, stride);
614
615
0
            WinogradKernel3x3Block3x3SetFilter8Row(s + 6, dst + 20 * stride, stride);
616
0
        }
617
618
        SIMD_INLINE void WinogradKernel3x3Block3x3SetFilter8t(const float * src, float * dst, size_t stride)
619
0
        {
620
0
            __m256 s[9];
621
0
            s[0] = _mm256_loadu_ps(src + 0 * stride);
622
0
            s[1] = _mm256_loadu_ps(src + 1 * stride);
623
0
            s[2] = _mm256_loadu_ps(src + 2 * stride);
624
0
            s[3] = _mm256_loadu_ps(src + 3 * stride);
625
0
            s[4] = _mm256_loadu_ps(src + 4 * stride);
626
0
            s[5] = _mm256_loadu_ps(src + 5 * stride);
627
0
            s[6] = _mm256_loadu_ps(src + 6 * stride);
628
0
            s[7] = _mm256_loadu_ps(src + 7 * stride);
629
0
            s[8] = _mm256_loadu_ps(src + 8 * stride);
630
0
            WinogradKernel3x3Block3x3SetFilter8All(s, dst + 0 * stride, stride);
631
0
        }
632
633
        void WinogradKernel3x3Block3x3SetFilter(const float * src, size_t size, float * dst, SimdBool trans)
634
0
        {
635
0
            size_t size8 = AlignLo(size, 8), i = 0;
636
0
            if (trans)
637
0
            {
638
0
                for (; i < size8; i += 8)
639
0
                    WinogradKernel3x3Block3x3SetFilter8t(src + i, dst + i, size);
640
0
                for (; i < size; i += 1)
641
0
                    Base::WinogradKernel3x3Block3x3SetFilter1t(src + i, dst + i, size);
642
0
            }
643
0
            else
644
0
            {
645
0
                Sse41::WinogradKernel3x3Block3x3SetFilter(src, size, dst, trans);
646
0
            }
647
0
        }
648
649
        //-----------------------------------------------------------------------
650
651
        SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8Store(const __m256 src[25], float * dst, size_t stride)
652
0
        {
653
0
            __m256 _2 = _mm256_set1_ps(2.0f);
654
0
            __m256 _3 = _mm256_set1_ps(3.0f);
655
0
            __m256 tmp[5];
656
657
0
            tmp[0] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[0], src[10])), _mm256_sub_ps(src[15], src[5]));
658
0
            tmp[1] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[1], src[11])), _mm256_sub_ps(src[16], src[6]));
659
0
            tmp[2] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[2], src[12])), _mm256_sub_ps(src[17], src[7]));
660
0
            tmp[3] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[3], src[13])), _mm256_sub_ps(src[18], src[8]));
661
0
            tmp[4] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[4], src[14])), _mm256_sub_ps(src[19], src[9]));
662
0
            _mm256_storeu_ps(dst + 0 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1])));
663
0
            _mm256_storeu_ps(dst + 1 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1])));
664
0
            _mm256_storeu_ps(dst + 2 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2]))));
665
0
            _mm256_storeu_ps(dst + 3 * stride, _mm256_sub_ps(tmp[3], tmp[1]));
666
0
            _mm256_storeu_ps(dst + 4 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2])));
667
668
0
            tmp[0] = _mm256_sub_ps(_mm256_sub_ps(src[15], src[10]), _mm256_mul_ps(_2, src[5]));
669
0
            tmp[1] = _mm256_sub_ps(_mm256_sub_ps(src[16], src[11]), _mm256_mul_ps(_2, src[6]));
670
0
            tmp[2] = _mm256_sub_ps(_mm256_sub_ps(src[17], src[12]), _mm256_mul_ps(_2, src[7]));
671
0
            tmp[3] = _mm256_sub_ps(_mm256_sub_ps(src[18], src[13]), _mm256_mul_ps(_2, src[8]));
672
0
            tmp[4] = _mm256_sub_ps(_mm256_sub_ps(src[19], src[14]), _mm256_mul_ps(_2, src[9]));
673
0
            _mm256_storeu_ps(dst + 5 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1])));
674
0
            _mm256_storeu_ps(dst + 6 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1])));
675
0
            _mm256_storeu_ps(dst + 7 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2]))));
676
0
            _mm256_storeu_ps(dst + 8 * stride, _mm256_sub_ps(tmp[3], tmp[1]));
677
0
            _mm256_storeu_ps(dst + 9 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2])));
678
679
0
            tmp[0] = _mm256_add_ps(_mm256_mul_ps(_2, src[5]), _mm256_sub_ps(src[15], _mm256_mul_ps(_3, src[10])));
680
0
            tmp[1] = _mm256_add_ps(_mm256_mul_ps(_2, src[6]), _mm256_sub_ps(src[16], _mm256_mul_ps(_3, src[11])));
681
0
            tmp[2] = _mm256_add_ps(_mm256_mul_ps(_2, src[7]), _mm256_sub_ps(src[17], _mm256_mul_ps(_3, src[12])));
682
0
            tmp[3] = _mm256_add_ps(_mm256_mul_ps(_2, src[8]), _mm256_sub_ps(src[18], _mm256_mul_ps(_3, src[13])));
683
0
            tmp[4] = _mm256_add_ps(_mm256_mul_ps(_2, src[9]), _mm256_sub_ps(src[19], _mm256_mul_ps(_3, src[14])));
684
0
            _mm256_storeu_ps(dst + 10 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1])));
685
0
            _mm256_storeu_ps(dst + 11 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1])));
686
0
            _mm256_storeu_ps(dst + 12 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2]))));
687
0
            _mm256_storeu_ps(dst + 13 * stride, _mm256_sub_ps(tmp[3], tmp[1]));
688
0
            _mm256_storeu_ps(dst + 14 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2])));
689
690
0
            tmp[0] = _mm256_sub_ps(src[15], src[5]);
691
0
            tmp[1] = _mm256_sub_ps(src[16], src[6]);
692
0
            tmp[2] = _mm256_sub_ps(src[17], src[7]);
693
0
            tmp[3] = _mm256_sub_ps(src[18], src[8]);
694
0
            tmp[4] = _mm256_sub_ps(src[19], src[9]);
695
0
            _mm256_storeu_ps(dst + 15 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1])));
696
0
            _mm256_storeu_ps(dst + 16 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1])));
697
0
            _mm256_storeu_ps(dst + 17 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2]))));
698
0
            _mm256_storeu_ps(dst + 18 * stride, _mm256_sub_ps(tmp[3], tmp[1]));
699
0
            _mm256_storeu_ps(dst + 19 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2])));
700
701
0
            tmp[0] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[5], src[15])), _mm256_sub_ps(src[20], src[10]));
702
0
            tmp[1] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[6], src[16])), _mm256_sub_ps(src[21], src[11]));
703
0
            tmp[2] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[7], src[17])), _mm256_sub_ps(src[22], src[12]));
704
0
            tmp[3] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[8], src[18])), _mm256_sub_ps(src[23], src[13]));
705
0
            tmp[4] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[9], src[19])), _mm256_sub_ps(src[24], src[14]));
706
0
            _mm256_storeu_ps(dst + 20 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1])));
707
0
            _mm256_storeu_ps(dst + 21 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1])));
708
0
            _mm256_storeu_ps(dst + 22 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2]))));
709
0
            _mm256_storeu_ps(dst + 23 * stride, _mm256_sub_ps(tmp[3], tmp[1]));
710
0
            _mm256_storeu_ps(dst + 24 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2])));
711
0
        }
712
713
        SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcS, size_t srcC, __m256 dst[25])
714
0
        {
715
0
            dst[0] = _mm256_loadu_ps(src + 0 * srcS + 0 * srcC);
716
0
            dst[1] = _mm256_loadu_ps(src + 0 * srcS + 1 * srcC);
717
0
            dst[2] = _mm256_loadu_ps(src + 0 * srcS + 2 * srcC);
718
0
            dst[3] = _mm256_loadu_ps(src + 0 * srcS + 3 * srcC);
719
0
            dst[4] = _mm256_loadu_ps(src + 0 * srcS + 4 * srcC);
720
0
            dst[5] = _mm256_loadu_ps(src + 1 * srcS + 0 * srcC);
721
0
            dst[6] = _mm256_loadu_ps(src + 1 * srcS + 1 * srcC);
722
0
            dst[7] = _mm256_loadu_ps(src + 1 * srcS + 2 * srcC);
723
0
            dst[8] = _mm256_loadu_ps(src + 1 * srcS + 3 * srcC);
724
0
            dst[9] = _mm256_loadu_ps(src + 1 * srcS + 4 * srcC);
725
0
            dst[10] = _mm256_loadu_ps(src + 2 * srcS + 0 * srcC);
726
0
            dst[11] = _mm256_loadu_ps(src + 2 * srcS + 1 * srcC);
727
0
            dst[12] = _mm256_loadu_ps(src + 2 * srcS + 2 * srcC);
728
0
            dst[13] = _mm256_loadu_ps(src + 2 * srcS + 3 * srcC);
729
0
            dst[14] = _mm256_loadu_ps(src + 2 * srcS + 4 * srcC);
730
0
            dst[15] = _mm256_loadu_ps(src + 3 * srcS + 0 * srcC);
731
0
            dst[16] = _mm256_loadu_ps(src + 3 * srcS + 1 * srcC);
732
0
            dst[17] = _mm256_loadu_ps(src + 3 * srcS + 2 * srcC);
733
0
            dst[18] = _mm256_loadu_ps(src + 3 * srcS + 3 * srcC);
734
0
            dst[19] = _mm256_loadu_ps(src + 3 * srcS + 4 * srcC);
735
0
            dst[20] = _mm256_loadu_ps(src + 4 * srcS + 0 * srcC);
736
0
            dst[21] = _mm256_loadu_ps(src + 4 * srcS + 1 * srcC);
737
0
            dst[22] = _mm256_loadu_ps(src + 4 * srcS + 2 * srcC);
738
0
            dst[23] = _mm256_loadu_ps(src + 4 * srcS + 3 * srcC);
739
0
            dst[24] = _mm256_loadu_ps(src + 4 * srcS + 4 * srcC);
740
0
        }
741
742
        SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcW, size_t srcC, float * dst, size_t dstStride)
743
0
        {
744
0
            size_t srcS = srcW * srcC;
745
0
            size_t srcCF = AlignLo(srcC, F);
746
0
            for (size_t c = 0; c < srcCF; c += F)
747
0
            {
748
0
                __m256 tmp[25];
749
0
                WinogradKernel3x3Block3x3SetInput8t(src + c, srcS, srcC, tmp);
750
0
                WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + c, dstStride);
751
0
            }
752
0
            if (srcCF < srcC)
753
0
            {
754
0
                __m256 tmp[25];
755
0
                WinogradKernel3x3Block3x3SetInput8t(src + srcC - F, srcS, srcC, tmp);
756
0
                WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + srcC - F, dstStride);
757
0
            }
758
0
        }
759
760
        SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcS, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, __m256 dst[25])
761
0
        {
762
0
            for (size_t row = 0; row < rowB; ++row)
763
0
            {
764
0
                for (size_t col = 0; col < 5; ++col)
765
0
                    dst[col] = _mm256_setzero_ps();
766
0
                dst += 5;
767
0
            }
768
0
            for (size_t row = rowB; row < rowE; ++row)
769
0
            {
770
0
                for (size_t col = 0; col < colB; ++col)
771
0
                    dst[col] = _mm256_setzero_ps();
772
0
                for (size_t col = colB; col < colE; ++col)
773
0
                    dst[col] = _mm256_loadu_ps(src + row * srcS + col * srcC);
774
0
                for (size_t col = colE; col < 5; ++col)
775
0
                    dst[col] = _mm256_setzero_ps();
776
0
                dst += 5;
777
0
            }
778
0
            for (size_t row = rowE; row < 5; ++row)
779
0
            {
780
0
                for (size_t col = 0; col < 5; ++col)
781
0
                    dst[col] = _mm256_setzero_ps();
782
0
                dst += 5;
783
0
            }
784
0
        }
785
786
        SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcW, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, float * dst, size_t dstStride)
787
0
        {
788
0
            size_t srcS = srcW * srcC;
789
0
            size_t srcCF = AlignLo(srcC, F);
790
0
            for (size_t c = 0; c < srcCF; c += F)
791
0
            {
792
0
                __m256 tmp[25];
793
0
                WinogradKernel3x3Block3x3SetInput8t(src + c, srcS, srcC, rowB, rowE, colB, colE, tmp);
794
0
                WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + c, dstStride);
795
0
            }
796
0
            if (srcCF < srcC)
797
0
            {
798
0
                __m256 tmp[25];
799
0
                WinogradKernel3x3Block3x3SetInput8t(src + srcC - F, srcS, srcC, rowB, rowE, colB, colE, tmp);
800
0
                WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + srcC - F, dstStride);
801
0
            }
802
0
        }
803
804
        void WinogradKernel3x3Block3x3SetInput(const float* src, size_t srcChannels, size_t srcHeight, size_t srcWidth,
805
            size_t padY, size_t padX, size_t padH, size_t padW, float* dst, size_t dstStride, SimdBool trans)
806
0
        {
807
0
            assert(padY == padX && padY == padH && padY == padW && (padY == 0 || padY == 1));
808
0
            SimdBool pad = padY > 0 ? SimdTrue : SimdFalse;
809
0
            if (trans ? (srcChannels < 8) : (srcHeight < 5 || srcWidth < 15))
810
0
            {
811
0
                Sse41::WinogradKernel3x3Block3x3SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans);
812
0
                return;
813
0
            }
814
0
            size_t dstH = pad ? srcHeight : srcHeight - 2;
815
0
            size_t dstW = pad ? srcWidth : srcWidth - 2;
816
0
            size_t tileH = (dstH + 2) / 3;
817
0
            size_t tileW = (dstW + 2) / 3;
818
0
            size_t dstH3 = AlignLoAny(dstH, 3);
819
0
            size_t dstW3 = AlignLoAny(dstW, 3);
820
0
            if (trans)
821
0
            {
822
0
                size_t noseW = Simd::Min<size_t>(5, dstW + 1);
823
0
                size_t noseH = Simd::Min<size_t>(5, dstH + 1);
824
0
                size_t start = pad ? 3 : 0;
825
0
                if (pad)
826
0
                {
827
0
                    if (dstH == dstH3)
828
0
                        dstH3 -= 3;
829
0
                    if (dstW == dstW3)
830
0
                        dstW3 -= 3;
831
0
                    src -= (srcWidth + 1)*srcChannels;
832
0
                }
833
0
                size_t tailW = dstW - dstW3 + (pad ? 1 : 2);
834
0
                size_t tailH = dstH - dstH3 + (pad ? 1 : 2);
835
0
                size_t row = 0, col = 0;
836
0
                if (pad)
837
0
                {
838
0
                    if (pad)
839
0
                        WinogradKernel3x3Block3x3SetInput8t(src, srcWidth, srcChannels, 1, noseH, 1, noseW, dst, dstStride), dst += srcChannels;
840
0
                    for (col = start; col < dstW3; col += 3)
841
0
                        WinogradKernel3x3Block3x3SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, 5, dst, dstStride), dst += srcChannels;
842
0
                    if (col < dstW)
843
0
                        WinogradKernel3x3Block3x3SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, tailW, dst, dstStride), dst += srcChannels;
844
0
                }
845
0
                for (row = start; row < dstH3; row += 3)
846
0
                {
847
0
                    if (pad)
848
0
                        WinogradKernel3x3Block3x3SetInput8t(src + row * srcWidth * srcChannels, srcWidth, srcChannels, 0, 5, 1, noseW, dst, dstStride), dst += srcChannels;
849
0
                    for (col = start; col < dstW3; col += 3)
850
0
                        WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, dst, dstStride), dst += srcChannels;
851
0
                    if (col < dstW)
852
0
                        WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, 5, 0, tailW, dst, dstStride), dst += srcChannels;
853
0
                }
854
0
                if (row < dstH)
855
0
                {
856
0
                    if (pad)
857
0
                        WinogradKernel3x3Block3x3SetInput8t(src + row * srcWidth* srcChannels, srcWidth, srcChannels, 0, tailH, 1, noseW, dst, dstStride), dst += srcChannels;
858
0
                    for (col = start; col < dstW3; col += 3)
859
0
                        WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, 5, dst, dstStride), dst += srcChannels;
860
0
                    if (col < dstW)
861
0
                        WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, tailW, dst, dstStride), dst += srcChannels;
862
0
                }
863
0
            }
864
0
            else
865
0
            {
866
0
                Base::WinogradKernel3x3Block3x3SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans);
867
0
            }
868
0
        }
869
870
        //-----------------------------------------------------------------------
871
872
        SIMD_INLINE void WinogradKernel3x3Block3x3SetOutputLoad25(const float * src, size_t stride, __m256 dst[9])
873
0
        {
874
0
            __m256 s[25];
875
0
            s[0] = _mm256_loadu_ps(src + 0 * stride);
876
0
            s[1] = _mm256_loadu_ps(src + 1 * stride);
877
0
            s[2] = _mm256_loadu_ps(src + 2 * stride);
878
0
            s[3] = _mm256_loadu_ps(src + 3 * stride);
879
0
            s[4] = _mm256_loadu_ps(src + 4 * stride);
880
0
            s[5] = _mm256_loadu_ps(src + 5 * stride);
881
0
            s[6] = _mm256_loadu_ps(src + 6 * stride);
882
0
            s[7] = _mm256_loadu_ps(src + 7 * stride);
883
0
            s[8] = _mm256_loadu_ps(src + 8 * stride);
884
0
            s[9] = _mm256_loadu_ps(src + 9 * stride);
885
0
            s[10] = _mm256_loadu_ps(src + 10 * stride);
886
0
            s[11] = _mm256_loadu_ps(src + 11 * stride);
887
0
            s[12] = _mm256_loadu_ps(src + 12 * stride);
888
0
            s[13] = _mm256_loadu_ps(src + 13 * stride);
889
0
            s[14] = _mm256_loadu_ps(src + 14 * stride);
890
0
            s[15] = _mm256_loadu_ps(src + 15 * stride);
891
0
            s[16] = _mm256_loadu_ps(src + 16 * stride);
892
0
            s[17] = _mm256_loadu_ps(src + 17 * stride);
893
0
            s[18] = _mm256_loadu_ps(src + 18 * stride);
894
0
            s[19] = _mm256_loadu_ps(src + 19 * stride);
895
0
            s[20] = _mm256_loadu_ps(src + 20 * stride);
896
0
            s[21] = _mm256_loadu_ps(src + 21 * stride);
897
0
            s[22] = _mm256_loadu_ps(src + 22 * stride);
898
0
            s[23] = _mm256_loadu_ps(src + 23 * stride);
899
0
            s[24] = _mm256_loadu_ps(src + 24 * stride);
900
901
0
            __m256 _2 = _mm256_set1_ps(2.0f);
902
0
            __m256 _4 = _mm256_set1_ps(4.0f);
903
0
            __m256 t[5];
904
0
            t[0] = _mm256_add_ps(_mm256_add_ps(s[0], s[5]), _mm256_add_ps(s[10], s[15]));
905
0
            t[1] = _mm256_add_ps(_mm256_add_ps(s[1], s[6]), _mm256_add_ps(s[11], s[16]));
906
0
            t[2] = _mm256_add_ps(_mm256_add_ps(s[2], s[7]), _mm256_add_ps(s[12], s[17]));
907
0
            t[3] = _mm256_add_ps(_mm256_add_ps(s[3], s[8]), _mm256_add_ps(s[13], s[18]));
908
0
            t[4] = _mm256_add_ps(_mm256_add_ps(s[4], s[9]), _mm256_add_ps(s[14], s[19]));
909
0
            dst[0] = _mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3]));
910
0
            dst[1] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, t[3]));
911
0
            dst[2] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_add_ps(_mm256_mul_ps(_4, t[3]), t[4]));
912
913
0
            t[0] = _mm256_add_ps(_mm256_sub_ps(s[5], s[10]), _mm256_mul_ps(_2, s[15]));
914
0
            t[1] = _mm256_add_ps(_mm256_sub_ps(s[6], s[11]), _mm256_mul_ps(_2, s[16]));
915
0
            t[2] = _mm256_add_ps(_mm256_sub_ps(s[7], s[12]), _mm256_mul_ps(_2, s[17]));
916
0
            t[3] = _mm256_add_ps(_mm256_sub_ps(s[8], s[13]), _mm256_mul_ps(_2, s[18]));
917
0
            t[4] = _mm256_add_ps(_mm256_sub_ps(s[9], s[14]), _mm256_mul_ps(_2, s[19]));
918
0
            dst[3] = _mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3]));
919
0
            dst[4] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, t[3]));
920
0
            dst[5] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_add_ps(_mm256_mul_ps(_4, t[3]), t[4]));
921
922
0
            t[0] = _mm256_add_ps(_mm256_add_ps(s[5], s[10]), _mm256_add_ps(_mm256_mul_ps(_4, s[15]), s[20]));
923
0
            t[1] = _mm256_add_ps(_mm256_add_ps(s[6], s[11]), _mm256_add_ps(_mm256_mul_ps(_4, s[16]), s[21]));
924
0
            t[2] = _mm256_add_ps(_mm256_add_ps(s[7], s[12]), _mm256_add_ps(_mm256_mul_ps(_4, s[17]), s[22]));
925
0
            t[3] = _mm256_add_ps(_mm256_add_ps(s[8], s[13]), _mm256_add_ps(_mm256_mul_ps(_4, s[18]), s[23]));
926
0
            t[4] = _mm256_add_ps(_mm256_add_ps(s[9], s[14]), _mm256_add_ps(_mm256_mul_ps(_4, s[19]), s[24]));
927
0
            dst[6] = _mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3]));
928
0
            dst[7] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, t[3]));
929
0
            dst[8] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_add_ps(_mm256_mul_ps(_4, t[3]), t[4]));
930
0
        }
931
932
        SIMD_INLINE void WinogradKernel3x3Block3x3SetOutputStore9(const __m256 src[9], float * dst, size_t dstS, size_t dstC)
933
0
        {
934
0
            _mm256_storeu_ps(dst + 0 * dstS + 0 * dstC, src[0]);
935
0
            _mm256_storeu_ps(dst + 0 * dstS + 1 * dstC, src[1]);
936
0
            _mm256_storeu_ps(dst + 0 * dstS + 2 * dstC, src[2]);
937
0
            _mm256_storeu_ps(dst + 1 * dstS + 0 * dstC, src[3]);
938
0
            _mm256_storeu_ps(dst + 1 * dstS + 1 * dstC, src[4]);
939
0
            _mm256_storeu_ps(dst + 1 * dstS + 2 * dstC, src[5]);
940
0
            _mm256_storeu_ps(dst + 2 * dstS + 0 * dstC, src[6]);
941
0
            _mm256_storeu_ps(dst + 2 * dstS + 1 * dstC, src[7]);
942
0
            _mm256_storeu_ps(dst + 2 * dstS + 2 * dstC, src[8]);
943
0
        }
944
945
        SIMD_INLINE void WinogradKernel3x3Block3x3SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC)
946
0
        {
947
0
            size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F);
948
0
            for (size_t d = 0; d < dstCF; d += F)
949
0
            {
950
0
                __m256 tmp[9];
951
0
                WinogradKernel3x3Block3x3SetOutputLoad25(src + d, srcStride, tmp);
952
0
                WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + d, dstS, dstC);
953
0
            }
954
0
            if (dstCF < dstC)
955
0
            {
956
0
                __m256 tmp[9];
957
0
                WinogradKernel3x3Block3x3SetOutputLoad25(src + dstC - F, srcStride, tmp);
958
0
                WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + dstC - F, dstS, dstC);
959
0
            }
960
0
        }
961
962
        SIMD_INLINE void WinogradKernel3x3Block3x3SetOutputStore9(const __m256 src[16], float * dst, size_t dstS, size_t dstC, size_t rowE, size_t colE)
963
0
        {
964
0
            for (size_t row = 0; row < rowE; ++row)
965
0
                for (size_t col = 0; col < colE; ++col)
966
0
                    _mm256_storeu_ps(dst + row * dstS + col * dstC, src[row * 3 + col]);
967
0
        }
968
969
        SIMD_INLINE void WinogradKernel3x3Block3x3SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC, size_t rowE, size_t colE)
970
0
        {
971
0
            size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F);
972
0
            for (size_t d = 0; d < dstCF; d += F)
973
0
            {
974
0
                __m256 tmp[9];
975
0
                WinogradKernel3x3Block3x3SetOutputLoad25(src + d, srcStride, tmp);
976
0
                WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + d, dstS, dstC, rowE, colE);
977
0
            }
978
0
            if (dstCF < dstC)
979
0
            {
980
0
                __m256 tmp[9];
981
0
                WinogradKernel3x3Block3x3SetOutputLoad25(src + dstC - F, srcStride, tmp);
982
0
                WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + dstC - F, dstS, dstC, rowE, colE);
983
0
            }
984
0
        }
985
986
        void WinogradKernel3x3Block3x3SetOutput(const float * src, size_t srcStride, float * dst, size_t dstChannels, size_t dstHeight, size_t dstWidth, SimdBool trans)
987
0
        {
988
0
            if (trans ? (dstChannels < 8) : (dstHeight < 3 || dstWidth < 15))
989
0
            {
990
0
                Sse41::WinogradKernel3x3Block3x3SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans);
991
0
                return;
992
0
            }
993
0
            size_t tileH = (dstHeight + 2) / 3;
994
0
            size_t tileW = (dstWidth + 2) / 3;
995
0
            size_t dstH3 = AlignLoAny(dstHeight, 3);
996
0
            size_t dstW3 = AlignLoAny(dstWidth, 3);
997
0
            if (trans)
998
0
            {
999
0
                size_t row, col;
1000
0
                for (row = 0; row < dstH3; row += 3)
1001
0
                {
1002
0
                    for (col = 0; col < dstW3; col += 3)
1003
0
                        WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels), src += dstChannels;
1004
0
                    if (col < dstWidth)
1005
0
                        WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, 3, dstWidth - col), src += dstChannels;
1006
0
                }
1007
0
                if (row < dstHeight)
1008
0
                {
1009
0
                    for (col = 0; col < dstW3; col += 3)
1010
0
                        WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, 3), src += dstChannels;
1011
0
                    if (col < dstWidth)
1012
0
                        WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, dstWidth - col), src += dstChannels;
1013
0
                }
1014
0
            }
1015
0
            else
1016
0
            {
1017
0
                Base::WinogradKernel3x3Block3x3SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans);
1018
0
            }
1019
0
        }
1020
1021
        //-----------------------------------------------------------------------
1022
1023
        SIMD_INLINE void WinogradKernel3x3Block4x4SetFilter8Row(const __m256 * t, float * dst, size_t stride)
1024
0
        {
1025
0
            const __m256 r4 = _mm256_set1_ps(1.0f / 4.0f);
1026
0
            const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f);
1027
0
            const __m256 mr6 = _mm256_set1_ps(-1.0f / 6.0f);
1028
0
            const __m256 r12 = _mm256_set1_ps(1.0f / 12.0f);
1029
0
            const __m256 r24 = _mm256_set1_ps(1.0f / 24.0f);
1030
0
            _mm256_storeu_ps(dst + 0 * stride, _mm256_mul_ps(r4, t[0]));
1031
0
            __m256 t0 = _mm256_add_ps(t[0], t[2]);
1032
0
            _mm256_storeu_ps(dst + 1 * stride, _mm256_mul_ps(mr6, _mm256_add_ps(t0, t[1])));
1033
0
            _mm256_storeu_ps(dst + 2 * stride, _mm256_mul_ps(mr6, _mm256_sub_ps(t0, t[1])));
1034
0
            __m256 t1 = _mm256_add_ps(_mm256_mul_ps(r24, t[0]), _mm256_mul_ps(r6, t[2]));
1035
0
            __m256 t2 = _mm256_mul_ps(r12, t[1]);
1036
0
            _mm256_storeu_ps(dst + 3 * stride, _mm256_add_ps(t1, t2));
1037
0
            _mm256_storeu_ps(dst + 4 * stride, _mm256_sub_ps(t1, t2));
1038
0
            _mm256_storeu_ps(dst + 5 * stride, t[2]);
1039
0
        }
1040
1041
        SIMD_INLINE void WinogradKernel3x3Block4x4SetFilter8All(const __m256 * s, float * dst, size_t stride)
1042
0
        {
1043
0
            const __m256 r4 = _mm256_set1_ps(1.0f / 4.0f);
1044
0
            const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f);
1045
0
            const __m256 mr6 = _mm256_set1_ps(-1.0f / 6.0f);
1046
0
            const __m256 r12 = _mm256_set1_ps(1.0f / 12.0f);
1047
0
            const __m256 r24 = _mm256_set1_ps(1.0f / 24.0f);
1048
1049
0
            __m256 t[3];
1050
0
            t[0] = _mm256_mul_ps(r4, s[0]);
1051
0
            t[1] = _mm256_mul_ps(r4, s[1]);
1052
0
            t[2] = _mm256_mul_ps(r4, s[2]);
1053
0
            WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 0 * stride, stride);
1054
1055
0
            t[0] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_add_ps(s[0], s[3]), s[6]));
1056
0
            t[1] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_add_ps(s[1], s[4]), s[7]));
1057
0
            t[2] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_add_ps(s[2], s[5]), s[8]));
1058
0
            WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 6 * stride, stride);
1059
1060
0
            t[0] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_sub_ps(s[0], s[3]), s[6]));
1061
0
            t[1] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_sub_ps(s[1], s[4]), s[7]));
1062
0
            t[2] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_sub_ps(s[2], s[5]), s[8]));
1063
0
            WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 12 * stride, stride);
1064
1065
0
            t[0] = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(r24, s[0]), _mm256_mul_ps(r12, s[3])), _mm256_mul_ps(r6, s[6]));
1066
0
            t[1] = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(r24, s[1]), _mm256_mul_ps(r12, s[4])), _mm256_mul_ps(r6, s[7]));
1067
0
            t[2] = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(r24, s[2]), _mm256_mul_ps(r12, s[5])), _mm256_mul_ps(r6, s[8]));
1068
0
            WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 18 * stride, stride);
1069
1070
0
            t[0] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(r24, s[0]), _mm256_mul_ps(r12, s[3])), _mm256_mul_ps(r6, s[6]));
1071
0
            t[1] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(r24, s[1]), _mm256_mul_ps(r12, s[4])), _mm256_mul_ps(r6, s[7]));
1072
0
            t[2] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(r24, s[2]), _mm256_mul_ps(r12, s[5])), _mm256_mul_ps(r6, s[8]));
1073
0
            WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 24 * stride, stride);
1074
1075
0
            WinogradKernel3x3Block4x4SetFilter8Row(s + 6, dst + 30 * stride, stride);
1076
0
        }
1077
1078
        SIMD_INLINE void WinogradKernel3x3Block4x4SetFilter8t(const float * src, float * dst, size_t stride)
1079
0
        {
1080
0
            __m256 s[9];
1081
0
            s[0] = _mm256_loadu_ps(src + 0 * stride);
1082
0
            s[1] = _mm256_loadu_ps(src + 1 * stride);
1083
0
            s[2] = _mm256_loadu_ps(src + 2 * stride);
1084
0
            s[3] = _mm256_loadu_ps(src + 3 * stride);
1085
0
            s[4] = _mm256_loadu_ps(src + 4 * stride);
1086
0
            s[5] = _mm256_loadu_ps(src + 5 * stride);
1087
0
            s[6] = _mm256_loadu_ps(src + 6 * stride);
1088
0
            s[7] = _mm256_loadu_ps(src + 7 * stride);
1089
0
            s[8] = _mm256_loadu_ps(src + 8 * stride);
1090
0
            WinogradKernel3x3Block4x4SetFilter8All(s, dst + 0 * stride, stride);
1091
0
        }
1092
1093
        void WinogradKernel3x3Block4x4SetFilter(const float * src, size_t size, float * dst, SimdBool trans)
1094
0
        {
1095
0
            if (trans)
1096
0
            {
1097
0
                size_t size8 = AlignLo(size, 8), i = 0;
1098
0
                for (; i < size8; i += 8)
1099
0
                    WinogradKernel3x3Block4x4SetFilter8t(src + i, dst + i, size);
1100
0
                for (; i < size; i += 1)
1101
0
                    Base::WinogradKernel3x3Block4x4SetFilter1t(src + i, dst + i, size);
1102
0
            }
1103
0
            else
1104
0
            {
1105
0
                Sse41::WinogradKernel3x3Block4x4SetFilter(src, size, dst, trans);
1106
0
            }
1107
0
        }
1108
1109
        //-----------------------------------------------------------------------
1110
1111
        SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8Store(const __m256 src[36], float * dst, size_t stride)
1112
0
        {
1113
0
            __m256 _2 = _mm256_set1_ps(2.0f);
1114
0
            __m256 _4 = _mm256_set1_ps(4.0f);
1115
0
            __m256 _5 = _mm256_set1_ps(5.0f);
1116
0
            __m256 tmp[36];
1117
0
            tmp[0] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[0]), _mm256_mul_ps(_5, src[12])), src[24]);
1118
0
            tmp[1] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[1]), _mm256_mul_ps(_5, src[13])), src[25]);
1119
0
            tmp[2] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[2]), _mm256_mul_ps(_5, src[14])), src[26]);
1120
0
            tmp[3] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[3]), _mm256_mul_ps(_5, src[15])), src[27]);
1121
0
            tmp[4] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[4]), _mm256_mul_ps(_5, src[16])), src[28]);
1122
0
            tmp[5] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[5]), _mm256_mul_ps(_5, src[17])), src[29]);
1123
0
            tmp[6] = _mm256_sub_ps(_mm256_add_ps(src[18], src[24]), _mm256_mul_ps(_4, _mm256_add_ps(src[6], src[12])));
1124
0
            tmp[7] = _mm256_sub_ps(_mm256_add_ps(src[19], src[25]), _mm256_mul_ps(_4, _mm256_add_ps(src[7], src[13])));
1125
0
            tmp[8] = _mm256_sub_ps(_mm256_add_ps(src[20], src[26]), _mm256_mul_ps(_4, _mm256_add_ps(src[8], src[14])));
1126
0
            tmp[9] = _mm256_sub_ps(_mm256_add_ps(src[21], src[27]), _mm256_mul_ps(_4, _mm256_add_ps(src[9], src[15])));
1127
0
            tmp[10] = _mm256_sub_ps(_mm256_add_ps(src[22], src[28]), _mm256_mul_ps(_4, _mm256_add_ps(src[10], src[16])));
1128
0
            tmp[11] = _mm256_sub_ps(_mm256_add_ps(src[23], src[29]), _mm256_mul_ps(_4, _mm256_add_ps(src[11], src[17])));
1129
0
            tmp[12] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[6], src[12])), _mm256_sub_ps(src[24], src[18]));
1130
0
            tmp[13] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[7], src[13])), _mm256_sub_ps(src[25], src[19]));
1131
0
            tmp[14] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[8], src[14])), _mm256_sub_ps(src[26], src[20]));
1132
0
            tmp[15] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[9], src[15])), _mm256_sub_ps(src[27], src[21]));
1133
0
            tmp[16] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[10], src[16])), _mm256_sub_ps(src[28], src[22]));
1134
0
            tmp[17] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[11], src[17])), _mm256_sub_ps(src[29], src[23]));
1135
0
            tmp[18] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[18], src[6])), _mm256_sub_ps(src[24], src[12]));
1136
0
            tmp[19] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[19], src[7])), _mm256_sub_ps(src[25], src[13]));
1137
0
            tmp[20] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[20], src[8])), _mm256_sub_ps(src[26], src[14]));
1138
0
            tmp[21] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[21], src[9])), _mm256_sub_ps(src[27], src[15]));
1139
0
            tmp[22] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[22], src[10])), _mm256_sub_ps(src[28], src[16]));
1140
0
            tmp[23] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[23], src[11])), _mm256_sub_ps(src[29], src[17]));
1141
0
            tmp[24] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[6], src[18])), _mm256_sub_ps(src[24], src[12]));
1142
0
            tmp[25] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[7], src[19])), _mm256_sub_ps(src[25], src[13]));
1143
0
            tmp[26] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[8], src[20])), _mm256_sub_ps(src[26], src[14]));
1144
0
            tmp[27] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[9], src[21])), _mm256_sub_ps(src[27], src[15]));
1145
0
            tmp[28] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[10], src[22])), _mm256_sub_ps(src[28], src[16]));
1146
0
            tmp[29] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[11], src[23])), _mm256_sub_ps(src[29], src[17]));
1147
0
            tmp[30] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[6]), _mm256_mul_ps(_5, src[18])), src[30]);
1148
0
            tmp[31] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[7]), _mm256_mul_ps(_5, src[19])), src[31]);
1149
0
            tmp[32] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[8]), _mm256_mul_ps(_5, src[20])), src[32]);
1150
0
            tmp[33] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[9]), _mm256_mul_ps(_5, src[21])), src[33]);
1151
0
            tmp[34] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[10]), _mm256_mul_ps(_5, src[22])), src[34]);
1152
0
            tmp[35] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[11]), _mm256_mul_ps(_5, src[23])), src[35]);
1153
1154
0
            _mm256_storeu_ps(dst + 0 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[0]), _mm256_mul_ps(_5, tmp[2])), tmp[4]));
1155
0
            _mm256_storeu_ps(dst + 1 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[3], tmp[4]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[1], tmp[2]))));
1156
0
            _mm256_storeu_ps(dst + 2 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[1], tmp[2])), _mm256_sub_ps(tmp[4], tmp[3])));
1157
0
            _mm256_storeu_ps(dst + 3 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[3], tmp[1])), _mm256_sub_ps(tmp[4], tmp[2])));
1158
0
            _mm256_storeu_ps(dst + 4 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2])));
1159
0
            _mm256_storeu_ps(dst + 5 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[1]), _mm256_mul_ps(_5, tmp[3])), tmp[5]));
1160
0
            _mm256_storeu_ps(dst + 6 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[6]), _mm256_mul_ps(_5, tmp[8])), tmp[10]));
1161
0
            _mm256_storeu_ps(dst + 7 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[9], tmp[10]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[7], tmp[8]))));
1162
0
            _mm256_storeu_ps(dst + 8 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[7], tmp[8])), _mm256_sub_ps(tmp[10], tmp[9])));
1163
0
            _mm256_storeu_ps(dst + 9 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[9], tmp[7])), _mm256_sub_ps(tmp[10], tmp[8])));
1164
0
            _mm256_storeu_ps(dst + 10 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[7], tmp[9])), _mm256_sub_ps(tmp[10], tmp[8])));
1165
0
            _mm256_storeu_ps(dst + 11 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[7]), _mm256_mul_ps(_5, tmp[9])), tmp[11]));
1166
0
            _mm256_storeu_ps(dst + 12 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[12]), _mm256_mul_ps(_5, tmp[14])), tmp[16]));
1167
0
            _mm256_storeu_ps(dst + 13 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[15], tmp[16]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[13], tmp[14]))));
1168
0
            _mm256_storeu_ps(dst + 14 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[13], tmp[14])), _mm256_sub_ps(tmp[16], tmp[15])));
1169
0
            _mm256_storeu_ps(dst + 15 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[15], tmp[13])), _mm256_sub_ps(tmp[16], tmp[14])));
1170
0
            _mm256_storeu_ps(dst + 16 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[13], tmp[15])), _mm256_sub_ps(tmp[16], tmp[14])));
1171
0
            _mm256_storeu_ps(dst + 17 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[13]), _mm256_mul_ps(_5, tmp[15])), tmp[17]));
1172
0
            _mm256_storeu_ps(dst + 18 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[18]), _mm256_mul_ps(_5, tmp[20])), tmp[22]));
1173
0
            _mm256_storeu_ps(dst + 19 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[21], tmp[22]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[19], tmp[20]))));
1174
0
            _mm256_storeu_ps(dst + 20 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[19], tmp[20])), _mm256_sub_ps(tmp[22], tmp[21])));
1175
0
            _mm256_storeu_ps(dst + 21 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[21], tmp[19])), _mm256_sub_ps(tmp[22], tmp[20])));
1176
0
            _mm256_storeu_ps(dst + 22 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[19], tmp[21])), _mm256_sub_ps(tmp[22], tmp[20])));
1177
0
            _mm256_storeu_ps(dst + 23 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[19]), _mm256_mul_ps(_5, tmp[21])), tmp[23]));
1178
0
            _mm256_storeu_ps(dst + 24 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[24]), _mm256_mul_ps(_5, tmp[26])), tmp[28]));
1179
0
            _mm256_storeu_ps(dst + 25 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[27], tmp[28]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[25], tmp[26]))));
1180
0
            _mm256_storeu_ps(dst + 26 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[25], tmp[26])), _mm256_sub_ps(tmp[28], tmp[27])));
1181
0
            _mm256_storeu_ps(dst + 27 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[27], tmp[25])), _mm256_sub_ps(tmp[28], tmp[26])));
1182
0
            _mm256_storeu_ps(dst + 28 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[25], tmp[27])), _mm256_sub_ps(tmp[28], tmp[26])));
1183
0
            _mm256_storeu_ps(dst + 29 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[25]), _mm256_mul_ps(_5, tmp[27])), tmp[29]));
1184
0
            _mm256_storeu_ps(dst + 30 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[30]), _mm256_mul_ps(_5, tmp[32])), tmp[34]));
1185
0
            _mm256_storeu_ps(dst + 31 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[33], tmp[34]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[31], tmp[32]))));
1186
0
            _mm256_storeu_ps(dst + 32 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[31], tmp[32])), _mm256_sub_ps(tmp[34], tmp[33])));
1187
0
            _mm256_storeu_ps(dst + 33 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[33], tmp[31])), _mm256_sub_ps(tmp[34], tmp[32])));
1188
0
            _mm256_storeu_ps(dst + 34 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[31], tmp[33])), _mm256_sub_ps(tmp[34], tmp[32])));
1189
0
            _mm256_storeu_ps(dst + 35 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[31]), _mm256_mul_ps(_5, tmp[33])), tmp[35]));
1190
0
        }
1191
1192
        SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcS, size_t srcC, __m256 dst[36])
1193
0
        {
1194
0
            dst[0] = _mm256_loadu_ps(src + 0 * srcS + 0 * srcC);
1195
0
            dst[1] = _mm256_loadu_ps(src + 0 * srcS + 1 * srcC);
1196
0
            dst[2] = _mm256_loadu_ps(src + 0 * srcS + 2 * srcC);
1197
0
            dst[3] = _mm256_loadu_ps(src + 0 * srcS + 3 * srcC);
1198
0
            dst[4] = _mm256_loadu_ps(src + 0 * srcS + 4 * srcC);
1199
0
            dst[5] = _mm256_loadu_ps(src + 0 * srcS + 5 * srcC);
1200
0
            dst[6] = _mm256_loadu_ps(src + 1 * srcS + 0 * srcC);
1201
0
            dst[7] = _mm256_loadu_ps(src + 1 * srcS + 1 * srcC);
1202
0
            dst[8] = _mm256_loadu_ps(src + 1 * srcS + 2 * srcC);
1203
0
            dst[9] = _mm256_loadu_ps(src + 1 * srcS + 3 * srcC);
1204
0
            dst[10] = _mm256_loadu_ps(src + 1 * srcS + 4 * srcC);
1205
0
            dst[11] = _mm256_loadu_ps(src + 1 * srcS + 5 * srcC);
1206
0
            dst[12] = _mm256_loadu_ps(src + 2 * srcS + 0 * srcC);
1207
0
            dst[13] = _mm256_loadu_ps(src + 2 * srcS + 1 * srcC);
1208
0
            dst[14] = _mm256_loadu_ps(src + 2 * srcS + 2 * srcC);
1209
0
            dst[15] = _mm256_loadu_ps(src + 2 * srcS + 3 * srcC);
1210
0
            dst[16] = _mm256_loadu_ps(src + 2 * srcS + 4 * srcC);
1211
0
            dst[17] = _mm256_loadu_ps(src + 2 * srcS + 5 * srcC);
1212
0
            dst[18] = _mm256_loadu_ps(src + 3 * srcS + 0 * srcC);
1213
0
            dst[19] = _mm256_loadu_ps(src + 3 * srcS + 1 * srcC);
1214
0
            dst[20] = _mm256_loadu_ps(src + 3 * srcS + 2 * srcC);
1215
0
            dst[21] = _mm256_loadu_ps(src + 3 * srcS + 3 * srcC);
1216
0
            dst[22] = _mm256_loadu_ps(src + 3 * srcS + 4 * srcC);
1217
0
            dst[23] = _mm256_loadu_ps(src + 3 * srcS + 5 * srcC);
1218
0
            dst[24] = _mm256_loadu_ps(src + 4 * srcS + 0 * srcC);
1219
0
            dst[25] = _mm256_loadu_ps(src + 4 * srcS + 1 * srcC);
1220
0
            dst[26] = _mm256_loadu_ps(src + 4 * srcS + 2 * srcC);
1221
0
            dst[27] = _mm256_loadu_ps(src + 4 * srcS + 3 * srcC);
1222
0
            dst[28] = _mm256_loadu_ps(src + 4 * srcS + 4 * srcC);
1223
0
            dst[29] = _mm256_loadu_ps(src + 4 * srcS + 5 * srcC);
1224
0
            dst[30] = _mm256_loadu_ps(src + 5 * srcS + 0 * srcC);
1225
0
            dst[31] = _mm256_loadu_ps(src + 5 * srcS + 1 * srcC);
1226
0
            dst[32] = _mm256_loadu_ps(src + 5 * srcS + 2 * srcC);
1227
0
            dst[33] = _mm256_loadu_ps(src + 5 * srcS + 3 * srcC);
1228
0
            dst[34] = _mm256_loadu_ps(src + 5 * srcS + 4 * srcC);
1229
0
            dst[35] = _mm256_loadu_ps(src + 5 * srcS + 5 * srcC);
1230
0
        }
1231
1232
        SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcW, size_t srcC, float * dst, size_t dstStride)
1233
0
        {
1234
0
            size_t srcS = srcW * srcC;
1235
0
            size_t srcCF = AlignLo(srcC, F);
1236
0
            for (size_t c = 0; c < srcCF; c += F)
1237
0
            {
1238
0
                __m256 tmp[36];
1239
0
                WinogradKernel3x3Block4x4SetInput8t(src + c, srcS, srcC, tmp);
1240
0
                WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + c, dstStride);
1241
0
            }
1242
0
            if (srcCF < srcC)
1243
0
            {
1244
0
                __m256 tmp[36];
1245
0
                WinogradKernel3x3Block4x4SetInput8t(src + srcC - F, srcS, srcC, tmp);
1246
0
                WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + srcC - F, dstStride);
1247
0
            }
1248
0
        }
1249
1250
        SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcS, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, __m256 dst[36])
1251
0
        {
1252
0
            for (size_t row = 0; row < rowB; ++row)
1253
0
            {
1254
0
                dst[0] = _mm256_setzero_ps();
1255
0
                dst[1] = _mm256_setzero_ps();
1256
0
                dst[2] = _mm256_setzero_ps();
1257
0
                dst[3] = _mm256_setzero_ps();
1258
0
                dst[4] = _mm256_setzero_ps();
1259
0
                dst[5] = _mm256_setzero_ps();
1260
0
                dst += 6;
1261
0
            }
1262
0
            for (size_t row = rowB; row < rowE; ++row)
1263
0
            {
1264
0
                for (size_t col = 0; col < colB; ++col)
1265
0
                    dst[col] = _mm256_setzero_ps();
1266
0
                for (size_t col = colB; col < colE; ++col)
1267
0
                    dst[col] = _mm256_loadu_ps(src + row * srcS + col * srcC);
1268
0
                for (size_t col = colE; col < 6; ++col)
1269
0
                    dst[col] = _mm256_setzero_ps();
1270
0
                dst += 6;
1271
0
            }
1272
0
            for (size_t row = rowE; row < 6; ++row)
1273
0
            {
1274
0
                dst[0] = _mm256_setzero_ps();
1275
0
                dst[1] = _mm256_setzero_ps();
1276
0
                dst[2] = _mm256_setzero_ps();
1277
0
                dst[3] = _mm256_setzero_ps();
1278
0
                dst[4] = _mm256_setzero_ps();
1279
0
                dst[5] = _mm256_setzero_ps();
1280
0
                dst += 6;
1281
0
            }
1282
0
        }
1283
1284
        SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcW, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, float * dst, size_t dstStride)
1285
0
        {
1286
0
            size_t srcS = srcW * srcC;
1287
0
            size_t srcCF = AlignLo(srcC, F);
1288
0
            for (size_t c = 0; c < srcCF; c += F)
1289
0
            {
1290
0
                __m256 tmp[36];
1291
0
                WinogradKernel3x3Block4x4SetInput8t(src + c, srcS, srcC, rowB, rowE, colB, colE, tmp);
1292
0
                WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + c, dstStride);
1293
0
            }
1294
0
            if (srcCF < srcC)
1295
0
            {
1296
0
                __m256 tmp[36];
1297
0
                WinogradKernel3x3Block4x4SetInput8t(src + srcC - F, srcS, srcC, rowB, rowE, colB, colE, tmp);
1298
0
                WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + srcC - F, dstStride);
1299
0
            }
1300
0
        }
1301
1302
        void WinogradKernel3x3Block4x4SetInput(const float* src, size_t srcChannels, size_t srcHeight, size_t srcWidth,
1303
            size_t padY, size_t padX, size_t padH, size_t padW, float* dst, size_t dstStride, SimdBool trans)
1304
0
        {
1305
0
            if (trans ? (srcChannels < 8) : (srcHeight < 6 || srcWidth < 14))
1306
0
            {
1307
0
                Sse41::WinogradKernel3x3Block4x4SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans);
1308
0
                return;
1309
0
            }
1310
0
            if (trans)
1311
0
            {
1312
0
                assert(padY + padH <= 2 && padX + padW <= 2);
1313
0
                size_t dstH = srcHeight - 2 + padY + padH;
1314
0
                size_t dstW = srcWidth - 2 + padX + padW;
1315
0
                size_t dstH4 = dstH / 4 * 4;
1316
0
                size_t dstW4 = dstW / 4 * 4;
1317
0
                size_t noseW = Simd::Min<size_t>(6, srcWidth + padX);
1318
0
                size_t noseH = Simd::Min<size_t>(6, srcHeight + padY);
1319
0
                size_t startY = padY ? 4 : 0;
1320
0
                size_t startX = padX ? 4 : 0;
1321
0
                if (padH && dstH == dstH4)
1322
0
                    dstH4 -= 4;
1323
0
                if (padY)
1324
0
                    src -= srcWidth * srcChannels;
1325
0
                if (padW && dstW == dstW4)
1326
0
                    dstW4 -= 4;
1327
0
                if (padX)
1328
0
                    src -= srcChannels;
1329
0
                size_t tailW = dstW - dstW4 + (padW ? 1 : 2);
1330
0
                size_t tailH = dstH - dstH4 + (padH ? 1 : 2);
1331
0
                size_t row = 0, col = 0;
1332
0
                if (padY)
1333
0
                {
1334
0
                    if (padX)
1335
0
                        WinogradKernel3x3Block4x4SetInput8t(src, srcWidth, srcChannels, 1, noseH, 1, noseW, dst, dstStride), dst += srcChannels;
1336
0
                    for (col = startX; col < dstW4; col += 4)
1337
0
                        WinogradKernel3x3Block4x4SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, 6, dst, dstStride), dst += srcChannels;
1338
0
                    if (col < dstW)
1339
0
                        WinogradKernel3x3Block4x4SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, tailW, dst, dstStride), dst += srcChannels;
1340
0
                }
1341
0
                for (row = startY; row < dstH4; row += 4)
1342
0
                {
1343
0
                    if (padX)
1344
0
                        WinogradKernel3x3Block4x4SetInput8t(src + row * srcWidth * srcChannels, srcWidth, srcChannels, 0, 6, 1, noseW, dst, dstStride), dst += srcChannels;
1345
0
                    for (col = startX; col < dstW4; col += 4)
1346
0
                        WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, dst, dstStride), dst += srcChannels;
1347
0
                    if (col < dstW)
1348
0
                        WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, 6, 0, tailW, dst, dstStride), dst += srcChannels;
1349
0
                }
1350
0
                if (row < dstH)
1351
0
                {
1352
0
                    if (padX)
1353
0
                        WinogradKernel3x3Block4x4SetInput8t(src + row * srcWidth* srcChannels, srcWidth, srcChannels, 0, tailH, 1, noseW, dst, dstStride), dst += srcChannels;
1354
0
                    for (col = startX; col < dstW4; col += 4)
1355
0
                        WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, 6, dst, dstStride), dst += srcChannels;
1356
0
                    if (col < dstW)
1357
0
                        WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, tailW, dst, dstStride), dst += srcChannels;
1358
0
                }
1359
0
            }
1360
0
            else
1361
0
            {
1362
0
                Base::WinogradKernel3x3Block4x4SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans);
1363
0
            }
1364
0
        }
1365
1366
        //-----------------------------------------------------------------------
1367
1368
        SIMD_INLINE void WinogradKernel3x3Block4x4SetOutputLoad36(const float * src, size_t stride, __m256 dst[16])
1369
0
        {
1370
0
            __m256 s[36];
1371
0
            s[0] = _mm256_loadu_ps(src + 0 * stride);
1372
0
            s[1] = _mm256_loadu_ps(src + 1 * stride);
1373
0
            s[2] = _mm256_loadu_ps(src + 2 * stride);
1374
0
            s[3] = _mm256_loadu_ps(src + 3 * stride);
1375
0
            s[4] = _mm256_loadu_ps(src + 4 * stride);
1376
0
            s[5] = _mm256_loadu_ps(src + 5 * stride);
1377
0
            s[6] = _mm256_loadu_ps(src + 6 * stride);
1378
0
            s[7] = _mm256_loadu_ps(src + 7 * stride);
1379
0
            s[8] = _mm256_loadu_ps(src + 8 * stride);
1380
0
            s[9] = _mm256_loadu_ps(src + 9 * stride);
1381
0
            s[10] = _mm256_loadu_ps(src + 10 * stride);
1382
0
            s[11] = _mm256_loadu_ps(src + 11 * stride);
1383
0
            s[12] = _mm256_loadu_ps(src + 12 * stride);
1384
0
            s[13] = _mm256_loadu_ps(src + 13 * stride);
1385
0
            s[14] = _mm256_loadu_ps(src + 14 * stride);
1386
0
            s[15] = _mm256_loadu_ps(src + 15 * stride);
1387
0
            s[16] = _mm256_loadu_ps(src + 16 * stride);
1388
0
            s[17] = _mm256_loadu_ps(src + 17 * stride);
1389
0
            s[18] = _mm256_loadu_ps(src + 18 * stride);
1390
0
            s[19] = _mm256_loadu_ps(src + 19 * stride);
1391
0
            s[20] = _mm256_loadu_ps(src + 20 * stride);
1392
0
            s[21] = _mm256_loadu_ps(src + 21 * stride);
1393
0
            s[22] = _mm256_loadu_ps(src + 22 * stride);
1394
0
            s[23] = _mm256_loadu_ps(src + 23 * stride);
1395
0
            s[24] = _mm256_loadu_ps(src + 24 * stride);
1396
0
            s[25] = _mm256_loadu_ps(src + 25 * stride);
1397
0
            s[26] = _mm256_loadu_ps(src + 26 * stride);
1398
0
            s[27] = _mm256_loadu_ps(src + 27 * stride);
1399
0
            s[28] = _mm256_loadu_ps(src + 28 * stride);
1400
0
            s[29] = _mm256_loadu_ps(src + 29 * stride);
1401
0
            s[30] = _mm256_loadu_ps(src + 30 * stride);
1402
0
            s[31] = _mm256_loadu_ps(src + 31 * stride);
1403
0
            s[32] = _mm256_loadu_ps(src + 32 * stride);
1404
0
            s[33] = _mm256_loadu_ps(src + 33 * stride);
1405
0
            s[34] = _mm256_loadu_ps(src + 34 * stride);
1406
0
            s[35] = _mm256_loadu_ps(src + 35 * stride);
1407
1408
0
            __m256 _2 = _mm256_set1_ps(2.0f);
1409
0
            __m256 _4 = _mm256_set1_ps(4.0f);
1410
0
            __m256 _8 = _mm256_set1_ps(8.0f);
1411
0
            __m256 t[24];
1412
0
            t[0] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[0], s[6]), _mm256_add_ps(s[12], s[18])), s[24]);
1413
0
            t[1] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[1], s[7]), _mm256_add_ps(s[13], s[19])), s[25]);
1414
0
            t[2] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[2], s[8]), _mm256_add_ps(s[14], s[20])), s[26]);
1415
0
            t[3] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[3], s[9]), _mm256_add_ps(s[15], s[21])), s[27]);
1416
0
            t[4] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[4], s[10]), _mm256_add_ps(s[16], s[22])), s[28]);
1417
0
            t[5] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[5], s[11]), _mm256_add_ps(s[17], s[23])), s[29]);
1418
0
            t[6] = _mm256_add_ps(_mm256_sub_ps(s[6], s[12]), _mm256_mul_ps(_2, _mm256_sub_ps(s[18], s[24])));
1419
0
            t[7] = _mm256_add_ps(_mm256_sub_ps(s[7], s[13]), _mm256_mul_ps(_2, _mm256_sub_ps(s[19], s[25])));
1420
0
            t[8] = _mm256_add_ps(_mm256_sub_ps(s[8], s[14]), _mm256_mul_ps(_2, _mm256_sub_ps(s[20], s[26])));
1421
0
            t[9] = _mm256_add_ps(_mm256_sub_ps(s[9], s[15]), _mm256_mul_ps(_2, _mm256_sub_ps(s[21], s[27])));
1422
0
            t[10] = _mm256_add_ps(_mm256_sub_ps(s[10], s[16]), _mm256_mul_ps(_2, _mm256_sub_ps(s[22], s[28])));
1423
0
            t[11] = _mm256_add_ps(_mm256_sub_ps(s[11], s[17]), _mm256_mul_ps(_2, _mm256_sub_ps(s[23], s[29])));
1424
0
            t[12] = _mm256_add_ps(_mm256_add_ps(s[6], s[12]), _mm256_mul_ps(_4, _mm256_add_ps(s[18], s[24])));
1425
0
            t[13] = _mm256_add_ps(_mm256_add_ps(s[7], s[13]), _mm256_mul_ps(_4, _mm256_add_ps(s[19], s[25])));
1426
0
            t[14] = _mm256_add_ps(_mm256_add_ps(s[8], s[14]), _mm256_mul_ps(_4, _mm256_add_ps(s[20], s[26])));
1427
0
            t[15] = _mm256_add_ps(_mm256_add_ps(s[9], s[15]), _mm256_mul_ps(_4, _mm256_add_ps(s[21], s[27])));
1428
0
            t[16] = _mm256_add_ps(_mm256_add_ps(s[10], s[16]), _mm256_mul_ps(_4, _mm256_add_ps(s[22], s[28])));
1429
0
            t[17] = _mm256_add_ps(_mm256_add_ps(s[11], s[17]), _mm256_mul_ps(_4, _mm256_add_ps(s[23], s[29])));
1430
0
            t[18] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[6], s[12]), _mm256_mul_ps(_8, _mm256_sub_ps(s[18], s[24]))), s[30]);
1431
0
            t[19] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[7], s[13]), _mm256_mul_ps(_8, _mm256_sub_ps(s[19], s[25]))), s[31]);
1432
0
            t[20] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[8], s[14]), _mm256_mul_ps(_8, _mm256_sub_ps(s[20], s[26]))), s[32]);
1433
0
            t[21] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[9], s[15]), _mm256_mul_ps(_8, _mm256_sub_ps(s[21], s[27]))), s[33]);
1434
0
            t[22] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[10], s[16]), _mm256_mul_ps(_8, _mm256_sub_ps(s[22], s[28]))), s[34]);
1435
0
            t[23] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[11], s[17]), _mm256_mul_ps(_8, _mm256_sub_ps(s[23], s[29]))), s[35]);
1436
1437
0
            dst[0] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3])), t[4]);
1438
0
            dst[1] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, _mm256_sub_ps(t[3], t[4])));
1439
0
            dst[2] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_mul_ps(_4, _mm256_add_ps(t[3], t[4])));
1440
0
            dst[3] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_8, _mm256_sub_ps(t[3], t[4]))), t[5]);
1441
0
            dst[4] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[6], t[7]), _mm256_add_ps(t[8], t[9])), t[10]);
1442
0
            dst[5] = _mm256_add_ps(_mm256_sub_ps(t[7], t[8]), _mm256_mul_ps(_2, _mm256_sub_ps(t[9], t[10])));
1443
0
            dst[6] = _mm256_add_ps(_mm256_add_ps(t[7], t[8]), _mm256_mul_ps(_4, _mm256_add_ps(t[9], t[10])));
1444
0
            dst[7] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[7], t[8]), _mm256_mul_ps(_8, _mm256_sub_ps(t[9], t[10]))), t[11]);
1445
0
            dst[8] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[12], t[13]), _mm256_add_ps(t[14], t[15])), t[16]);
1446
0
            dst[9] = _mm256_add_ps(_mm256_sub_ps(t[13], t[14]), _mm256_mul_ps(_2, _mm256_sub_ps(t[15], t[16])));
1447
0
            dst[10] = _mm256_add_ps(_mm256_add_ps(t[13], t[14]), _mm256_mul_ps(_4, _mm256_add_ps(t[15], t[16])));
1448
0
            dst[11] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[13], t[14]), _mm256_mul_ps(_8, _mm256_sub_ps(t[15], t[16]))), t[17]);
1449
0
            dst[12] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[18], t[19]), _mm256_add_ps(t[20], t[21])), t[22]);
1450
0
            dst[13] = _mm256_add_ps(_mm256_sub_ps(t[19], t[20]), _mm256_mul_ps(_2, _mm256_sub_ps(t[21], t[22])));
1451
0
            dst[14] = _mm256_add_ps(_mm256_add_ps(t[19], t[20]), _mm256_mul_ps(_4, _mm256_add_ps(t[21], t[22])));
1452
0
            dst[15] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[19], t[20]), _mm256_mul_ps(_8, _mm256_sub_ps(t[21], t[22]))), t[23]);
1453
0
        }
1454
1455
        SIMD_INLINE void WinogradKernel3x3Block4x4SetOutputStore16(const __m256 src[16], float * dst, size_t dstS, size_t dstC)
1456
0
        {
1457
0
            _mm256_storeu_ps(dst + 0 * dstS + 0 * dstC, src[0]);
1458
0
            _mm256_storeu_ps(dst + 0 * dstS + 1 * dstC, src[1]);
1459
0
            _mm256_storeu_ps(dst + 0 * dstS + 2 * dstC, src[2]);
1460
0
            _mm256_storeu_ps(dst + 0 * dstS + 3 * dstC, src[3]);
1461
0
            _mm256_storeu_ps(dst + 1 * dstS + 0 * dstC, src[4]);
1462
0
            _mm256_storeu_ps(dst + 1 * dstS + 1 * dstC, src[5]);
1463
0
            _mm256_storeu_ps(dst + 1 * dstS + 2 * dstC, src[6]);
1464
0
            _mm256_storeu_ps(dst + 1 * dstS + 3 * dstC, src[7]);
1465
0
            _mm256_storeu_ps(dst + 2 * dstS + 0 * dstC, src[8]);
1466
0
            _mm256_storeu_ps(dst + 2 * dstS + 1 * dstC, src[9]);
1467
0
            _mm256_storeu_ps(dst + 2 * dstS + 2 * dstC, src[10]);
1468
0
            _mm256_storeu_ps(dst + 2 * dstS + 3 * dstC, src[11]);
1469
0
            _mm256_storeu_ps(dst + 3 * dstS + 0 * dstC, src[12]);
1470
0
            _mm256_storeu_ps(dst + 3 * dstS + 1 * dstC, src[13]);
1471
0
            _mm256_storeu_ps(dst + 3 * dstS + 2 * dstC, src[14]);
1472
0
            _mm256_storeu_ps(dst + 3 * dstS + 3 * dstC, src[15]);
1473
0
        }
1474
1475
        SIMD_INLINE void WinogradKernel3x3Block4x4SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC)
1476
0
        {
1477
0
            size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F);
1478
0
            for (size_t d = 0; d < dstCF; d += F)
1479
0
            {
1480
0
                __m256 tmp[16];
1481
0
                WinogradKernel3x3Block4x4SetOutputLoad36(src + d, srcStride, tmp);
1482
0
                WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + d, dstS, dstC);
1483
0
            }
1484
0
            if (dstCF < dstC)
1485
0
            {
1486
0
                __m256 tmp[16];
1487
0
                WinogradKernel3x3Block4x4SetOutputLoad36(src + dstC - F, srcStride, tmp);
1488
0
                WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + dstC - F, dstS, dstC);
1489
0
            }
1490
0
        }
1491
1492
        SIMD_INLINE void WinogradKernel3x3Block4x4SetOutputStore16(const __m256 src[16], float * dst, size_t dstS, size_t dstC, size_t rowE, size_t colE)
1493
0
        {
1494
0
            for (size_t row = 0; row < rowE; ++row)
1495
0
                for (size_t col = 0; col < colE; ++col)
1496
0
                    _mm256_storeu_ps(dst + row * dstS + col * dstC, src[row * 4 + col]);
1497
0
        }
1498
1499
        SIMD_INLINE void WinogradKernel3x3Block4x4SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC, size_t rowE, size_t colE)
1500
0
        {
1501
0
            size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F);
1502
0
            for (size_t d = 0; d < dstCF; d += F)
1503
0
            {
1504
0
                __m256 tmp[16];
1505
0
                WinogradKernel3x3Block4x4SetOutputLoad36(src + d, srcStride, tmp);
1506
0
                WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + d, dstS, dstC, rowE, colE);
1507
0
            }
1508
0
            if (dstCF < dstC)
1509
0
            {
1510
0
                __m256 tmp[16];
1511
0
                WinogradKernel3x3Block4x4SetOutputLoad36(src + dstC - F, srcStride, tmp);
1512
0
                WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + dstC - F, dstS, dstC, rowE, colE);
1513
0
            }
1514
0
        }
1515
1516
        void WinogradKernel3x3Block4x4SetOutput(const float * src, size_t srcStride, float * dst, size_t dstChannels, size_t dstHeight, size_t dstWidth, SimdBool trans)
1517
0
        {
1518
0
            if (trans ? (dstChannels < 8) : (dstHeight < 4 || dstWidth < 16))
1519
0
            {
1520
0
                Sse41::WinogradKernel3x3Block4x4SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans);
1521
0
                return;
1522
0
            }
1523
0
            size_t tileH = (dstHeight + 3) / 4;
1524
0
            size_t tileW = (dstWidth + 3) / 4;
1525
0
            size_t dstH4 = AlignLo(dstHeight, 4);
1526
0
            size_t dstW4 = AlignLo(dstWidth, 4);
1527
0
            if (trans)
1528
0
            {
1529
0
                size_t row, col;
1530
0
                for (row = 0; row < dstH4; row += 4)
1531
0
                {
1532
0
                    for (col = 0; col < dstW4; col += 4)
1533
0
                        WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels), src += dstChannels;
1534
0
                    if (col < dstWidth)
1535
0
                        WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, 4, dstWidth - col), src += dstChannels;
1536
0
                }
1537
0
                if (row < dstHeight)
1538
0
                {
1539
0
                    for (col = 0; col < dstW4; col += 4)
1540
0
                        WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, 4), src += dstChannels;
1541
0
                    if (col < dstWidth)
1542
0
                        WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, dstWidth - col), src += dstChannels;
1543
0
                }
1544
0
            }
1545
0
            else
1546
0
            {
1547
0
                Base::WinogradKernel3x3Block4x4SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans);
1548
0
            }
1549
0
        }
1550
    }
1551
#endif
1552
}