/src/Simd/src/Simd/SimdAvx2Winograd3.cpp
Line | Count | Source |
1 | | /* |
2 | | * Simd Library (http://ermig1979.github.io/Simd). |
3 | | * |
4 | | * Copyright (c) 2011-2024 Yermalayeu Ihar. |
5 | | * |
6 | | * Permission is hereby granted, free of charge, to any person obtaining a copy |
7 | | * of this software and associated documentation files (the "Software"), to deal |
8 | | * in the Software without restriction, including without limitation the rights |
9 | | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
10 | | * copies of the Software, and to permit persons to whom the Software is |
11 | | * furnished to do so, subject to the following conditions: |
12 | | * |
13 | | * The above copyright notice and this permission notice shall be included in |
14 | | * all copies or substantial portions of the Software. |
15 | | * |
16 | | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
19 | | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
20 | | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
21 | | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
22 | | * SOFTWARE. |
23 | | */ |
24 | | #include "Simd/SimdMemory.h" |
25 | | #include "Simd/SimdStore.h" |
26 | | #include "Simd/SimdWinograd.h" |
27 | | #include "Simd/SimdSse41.h" |
28 | | #include "Simd/SimdSet.h" |
29 | | #include "Simd/SimdBase.h" |
30 | | |
31 | | namespace Simd |
32 | | { |
33 | | #if defined(SIMD_AVX2_ENABLE) && defined(SIMD_SYNET_ENABLE) |
34 | | namespace Avx2 |
35 | | { |
36 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetFilter8t(const float* src, float* dst, size_t stride) |
37 | 0 | { |
38 | 0 | const __m256 r2 = _mm256_set1_ps(1.0f / 2.0f); |
39 | 0 | const __m256 r4 = _mm256_set1_ps(1.0f / 4.0f); |
40 | |
|
41 | 0 | __m256 s[9]; |
42 | 0 | s[0] = _mm256_loadu_ps(src + 0 * stride); |
43 | 0 | s[1] = _mm256_loadu_ps(src + 1 * stride); |
44 | 0 | s[2] = _mm256_loadu_ps(src + 2 * stride); |
45 | 0 | s[3] = _mm256_loadu_ps(src + 3 * stride); |
46 | 0 | s[4] = _mm256_loadu_ps(src + 4 * stride); |
47 | 0 | s[5] = _mm256_loadu_ps(src + 5 * stride); |
48 | 0 | s[6] = _mm256_loadu_ps(src + 6 * stride); |
49 | 0 | s[7] = _mm256_loadu_ps(src + 7 * stride); |
50 | 0 | s[8] = _mm256_loadu_ps(src + 8 * stride); |
51 | |
|
52 | 0 | _mm256_storeu_ps(dst + 0 * stride, s[0]); |
53 | 0 | __m256 _0a2 = _mm256_add_ps(s[0], s[2]); |
54 | 0 | _mm256_storeu_ps(dst + 1 * stride, _mm256_mul_ps(_mm256_add_ps(_0a2, s[1]), r2)); |
55 | 0 | _mm256_storeu_ps(dst + 2 * stride, _mm256_mul_ps(_mm256_sub_ps(_0a2, s[1]), r2)); |
56 | 0 | _mm256_storeu_ps(dst + 3 * stride, s[2]); |
57 | |
|
58 | 0 | __m256 _0a6a3 = _mm256_add_ps(_mm256_add_ps(s[0], s[6]), s[3]); |
59 | 0 | _mm256_storeu_ps(dst + 4 * stride, _mm256_mul_ps(_0a6a3, r2)); |
60 | 0 | __m256 _2a8a5 = _mm256_add_ps(_mm256_add_ps(s[2], s[8]), s[5]); |
61 | 0 | __m256 _1a7a4 = _mm256_add_ps(_mm256_add_ps(s[1], s[7]), s[4]); |
62 | 0 | _mm256_storeu_ps(dst + 5 * stride, _mm256_mul_ps(_mm256_add_ps(_mm256_add_ps(_0a6a3, _2a8a5), _1a7a4), r4)); |
63 | 0 | _mm256_storeu_ps(dst + 6 * stride, _mm256_mul_ps(_mm256_sub_ps(_mm256_add_ps(_0a6a3, _2a8a5), _1a7a4), r4)); |
64 | 0 | _mm256_storeu_ps(dst + 7 * stride, _mm256_mul_ps(_2a8a5, r2)); |
65 | |
|
66 | 0 | __m256 _0a6s3 = _mm256_sub_ps(_mm256_add_ps(s[0], s[6]), s[3]); |
67 | 0 | _mm256_storeu_ps(dst + 8 * stride, _mm256_mul_ps(_0a6s3, r2)); |
68 | 0 | __m256 _2a8s5 = _mm256_sub_ps(_mm256_add_ps(s[2], s[8]), s[5]); |
69 | 0 | __m256 _1a7s4 = _mm256_sub_ps(_mm256_add_ps(s[1], s[7]), s[4]); |
70 | 0 | _mm256_storeu_ps(dst + 9 * stride, _mm256_mul_ps(_mm256_add_ps(_mm256_add_ps(_0a6s3, _2a8s5), _1a7s4), r4)); |
71 | 0 | _mm256_storeu_ps(dst + 10 * stride, _mm256_mul_ps(_mm256_sub_ps(_mm256_add_ps(_0a6s3, _2a8s5), _1a7s4), r4)); |
72 | 0 | _mm256_storeu_ps(dst + 11 * stride, _mm256_mul_ps(_2a8s5, r2)); |
73 | |
|
74 | 0 | _mm256_storeu_ps(dst + 12 * stride, s[6]); |
75 | 0 | __m256 _6a8 = _mm256_add_ps(s[6], s[8]); |
76 | 0 | _mm256_storeu_ps(dst + 13 * stride, _mm256_mul_ps(_mm256_add_ps(_6a8, s[7]), r2)); |
77 | 0 | _mm256_storeu_ps(dst + 14 * stride, _mm256_mul_ps(_mm256_sub_ps(_6a8, s[7]), r2)); |
78 | 0 | _mm256_storeu_ps(dst + 15 * stride, s[8]); |
79 | 0 | } |
80 | | |
81 | | void WinogradKernel3x3Block2x2SetFilter(const float* src, size_t size, float* dst, SimdBool trans) |
82 | 0 | { |
83 | 0 | if (trans) |
84 | 0 | { |
85 | 0 | size_t size8 = AlignLo(size, 8), i = 0; |
86 | 0 | for (; i < size8; i += 8) |
87 | 0 | WinogradKernel3x3Block2x2SetFilter8t(src + i, dst + i, size); |
88 | 0 | for (; i < size; i += 1) |
89 | 0 | Base::WinogradKernel3x3Block2x2SetFilter1t(src + i, dst + i, size); |
90 | 0 | } |
91 | 0 | else |
92 | 0 | { |
93 | 0 | Sse41::WinogradKernel3x3Block2x2SetFilter(src, size, dst, trans); |
94 | 0 | } |
95 | 0 | } |
96 | | |
97 | | //----------------------------------------------------------------------- |
98 | | |
99 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInputLoad8n(const float * src, __m256 * dst) |
100 | 0 | { |
101 | 0 | __m256 a0 = Load<false>(src + 0, src + 8); |
102 | 0 | __m256 a1 = Load<false>(src + 2, src + 10); |
103 | 0 | __m256 a2 = Load<false>(src + 4, src + 12); |
104 | 0 | __m256 a3 = Load<false>(src + 6, src + 14); |
105 | 0 | dst[0] = _mm256_shuffle_ps(a0, a2, 0x88); |
106 | 0 | dst[1] = _mm256_shuffle_ps(a0, a2, 0xDD); |
107 | 0 | dst[2] = _mm256_shuffle_ps(a1, a3, 0x88); |
108 | 0 | dst[3] = _mm256_shuffle_ps(a1, a3, 0xDD); |
109 | 0 | } |
110 | | |
111 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInputLoad8n(const float * src, __m256 * dst, PadType pad) |
112 | 0 | { |
113 | 0 | __m256 a0 = Set(pad == PadNose1 ? Sse41::LoadPadZeroNose1(src + 0) : _mm_loadu_ps(src + 0), _mm_loadu_ps(src + 8)); |
114 | 0 | __m256 a1 = Load<false>(src + 2, src + 10); |
115 | 0 | __m256 a2 = Load<false>(src + 4, src + 12); |
116 | 0 | __m256 a3 = Set(_mm_loadu_ps(src + 6), pad == PadTail2 ? Sse41::LoadPadZeroTail2(src + 14) : (pad == PadTail1 ? Sse41::LoadPadZeroTail1(src + 14) : _mm_loadu_ps(src + 14))); |
117 | 0 | dst[0] = _mm256_shuffle_ps(a0, a2, 0x88); |
118 | 0 | dst[1] = _mm256_shuffle_ps(a0, a2, 0xDD); |
119 | 0 | dst[2] = _mm256_shuffle_ps(a1, a3, 0x88); |
120 | 0 | dst[3] = _mm256_shuffle_ps(a1, a3, 0xDD); |
121 | 0 | } |
122 | | |
123 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInputLoad8z(__m256 * dst) |
124 | 0 | { |
125 | 0 | dst[0] = _mm256_setzero_ps(); |
126 | 0 | dst[1] = _mm256_setzero_ps(); |
127 | 0 | dst[2] = _mm256_setzero_ps(); |
128 | 0 | dst[3] = _mm256_setzero_ps(); |
129 | 0 | } |
130 | | |
131 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8Store(const __m256 * src, float * dst, size_t stride) |
132 | 0 | { |
133 | 0 | _mm256_storeu_ps(dst + 0 * stride, _mm256_sub_ps(_mm256_sub_ps(src[0], src[8]), _mm256_sub_ps(src[2], src[10]))); |
134 | 0 | _mm256_storeu_ps(dst + 1 * stride, _mm256_add_ps(_mm256_sub_ps(src[1], src[9]), _mm256_sub_ps(src[2], src[10]))); |
135 | 0 | _mm256_storeu_ps(dst + 2 * stride, _mm256_sub_ps(_mm256_sub_ps(src[2], src[10]), _mm256_sub_ps(src[1], src[9]))); |
136 | 0 | _mm256_storeu_ps(dst + 3 * stride, _mm256_sub_ps(_mm256_sub_ps(src[1], src[9]), _mm256_sub_ps(src[3], src[11]))); |
137 | 0 | _mm256_storeu_ps(dst + 4 * stride, _mm256_sub_ps(_mm256_add_ps(src[4], src[8]), _mm256_add_ps(src[6], src[10]))); |
138 | 0 | _mm256_storeu_ps(dst + 5 * stride, _mm256_add_ps(_mm256_add_ps(src[5], src[9]), _mm256_add_ps(src[6], src[10]))); |
139 | 0 | _mm256_storeu_ps(dst + 6 * stride, _mm256_sub_ps(_mm256_add_ps(src[6], src[10]), _mm256_add_ps(src[5], src[9]))); |
140 | 0 | _mm256_storeu_ps(dst + 7 * stride, _mm256_sub_ps(_mm256_add_ps(src[5], src[9]), _mm256_add_ps(src[7], src[11]))); |
141 | 0 | _mm256_storeu_ps(dst + 8 * stride, _mm256_sub_ps(_mm256_sub_ps(src[8], src[4]), _mm256_sub_ps(src[10], src[6]))); |
142 | 0 | _mm256_storeu_ps(dst + 9 * stride, _mm256_add_ps(_mm256_sub_ps(src[9], src[5]), _mm256_sub_ps(src[10], src[6]))); |
143 | 0 | _mm256_storeu_ps(dst + 10 * stride, _mm256_sub_ps(_mm256_sub_ps(src[10], src[6]), _mm256_sub_ps(src[9], src[5]))); |
144 | 0 | _mm256_storeu_ps(dst + 11 * stride, _mm256_sub_ps(_mm256_sub_ps(src[9], src[5]), _mm256_sub_ps(src[11], src[7]))); |
145 | 0 | _mm256_storeu_ps(dst + 12 * stride, _mm256_sub_ps(_mm256_sub_ps(src[4], src[12]), _mm256_sub_ps(src[6], src[14]))); |
146 | 0 | _mm256_storeu_ps(dst + 13 * stride, _mm256_add_ps(_mm256_sub_ps(src[5], src[13]), _mm256_sub_ps(src[6], src[14]))); |
147 | 0 | _mm256_storeu_ps(dst + 14 * stride, _mm256_sub_ps(_mm256_sub_ps(src[6], src[14]), _mm256_sub_ps(src[5], src[13]))); |
148 | 0 | _mm256_storeu_ps(dst + 15 * stride, _mm256_sub_ps(_mm256_sub_ps(src[5], src[13]), _mm256_sub_ps(src[7], src[15]))); |
149 | 0 | } |
150 | | |
151 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8n(const float * src, size_t srcStride, float * dst, size_t dstStride) |
152 | 0 | { |
153 | 0 | __m256 t[16]; |
154 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 0 * srcStride, t + 0); |
155 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 1 * srcStride, t + 4); |
156 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 2 * srcStride, t + 8); |
157 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 3 * srcStride, t + 12); |
158 | 0 | WinogradKernel3x3Block2x2SetInput8Store(t, dst, dstStride); |
159 | 0 | } |
160 | | |
161 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8n(const float * src, size_t srcStride, PadType rowPad, PadType colPad, float * dst, size_t dstStride) |
162 | 0 | { |
163 | 0 | __m256 t[16]; |
164 | 0 | if (rowPad == PadNose1) |
165 | 0 | WinogradKernel3x3Block2x2SetInputLoad8z(t + 0); |
166 | 0 | else |
167 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 0 * srcStride, t + 0, colPad); |
168 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 1 * srcStride, t + 4, colPad); |
169 | 0 | if (rowPad == PadTail2) |
170 | 0 | WinogradKernel3x3Block2x2SetInputLoad8z(t + 8); |
171 | 0 | else |
172 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 2 * srcStride, t + 8, colPad); |
173 | 0 | if (rowPad >= PadTail1) |
174 | 0 | WinogradKernel3x3Block2x2SetInputLoad8z(t + 12); |
175 | 0 | else |
176 | 0 | WinogradKernel3x3Block2x2SetInputLoad8n(src + 3 * srcStride, t + 12, colPad); |
177 | 0 | WinogradKernel3x3Block2x2SetInput8Store(t, dst, dstStride); |
178 | 0 | } |
179 | | |
180 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcS, size_t srcC, __m256 dst[16]) |
181 | 0 | { |
182 | 0 | dst[0] = _mm256_loadu_ps(src + 0 * srcS + 0 * srcC); |
183 | 0 | dst[1] = _mm256_loadu_ps(src + 0 * srcS + 1 * srcC); |
184 | 0 | dst[2] = _mm256_loadu_ps(src + 0 * srcS + 2 * srcC); |
185 | 0 | dst[3] = _mm256_loadu_ps(src + 0 * srcS + 3 * srcC); |
186 | 0 | dst[4] = _mm256_loadu_ps(src + 1 * srcS + 0 * srcC); |
187 | 0 | dst[5] = _mm256_loadu_ps(src + 1 * srcS + 1 * srcC); |
188 | 0 | dst[6] = _mm256_loadu_ps(src + 1 * srcS + 2 * srcC); |
189 | 0 | dst[7] = _mm256_loadu_ps(src + 1 * srcS + 3 * srcC); |
190 | 0 | dst[8] = _mm256_loadu_ps(src + 2 * srcS + 0 * srcC); |
191 | 0 | dst[9] = _mm256_loadu_ps(src + 2 * srcS + 1 * srcC); |
192 | 0 | dst[10] = _mm256_loadu_ps(src + 2 * srcS + 2 * srcC); |
193 | 0 | dst[11] = _mm256_loadu_ps(src + 2 * srcS + 3 * srcC); |
194 | 0 | dst[12] = _mm256_loadu_ps(src + 3 * srcS + 0 * srcC); |
195 | 0 | dst[13] = _mm256_loadu_ps(src + 3 * srcS + 1 * srcC); |
196 | 0 | dst[14] = _mm256_loadu_ps(src + 3 * srcS + 2 * srcC); |
197 | 0 | dst[15] = _mm256_loadu_ps(src + 3 * srcS + 3 * srcC); |
198 | 0 | } |
199 | | |
200 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcW, size_t srcC, float * dst, size_t dstStride) |
201 | 0 | { |
202 | 0 | size_t srcS = srcW * srcC; |
203 | 0 | size_t srcCF = AlignLo(srcC, F); |
204 | 0 | for (size_t c = 0; c < srcCF; c += F) |
205 | 0 | { |
206 | 0 | __m256 tmp[16]; |
207 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + c, srcS, srcC, tmp); |
208 | 0 | WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + c, dstStride); |
209 | 0 | } |
210 | 0 | if (srcCF < srcC) |
211 | 0 | { |
212 | 0 | __m256 tmp[16]; |
213 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + srcC - F, srcS, srcC, tmp); |
214 | 0 | WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + srcC - F, dstStride); |
215 | 0 | } |
216 | 0 | } |
217 | | |
218 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcS, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, __m256 dst[16]) |
219 | 0 | { |
220 | 0 | for (size_t row = 0; row < rowB; ++row) |
221 | 0 | { |
222 | 0 | for (size_t col = 0; col < 4; ++col) |
223 | 0 | dst[col] = _mm256_setzero_ps(); |
224 | 0 | dst += 4; |
225 | 0 | } |
226 | 0 | for (size_t row = rowB; row < rowE; ++row) |
227 | 0 | { |
228 | 0 | for (size_t col = 0; col < colB; ++col) |
229 | 0 | dst[col] = _mm256_setzero_ps(); |
230 | 0 | for (size_t col = colB; col < colE; ++col) |
231 | 0 | dst[col] = _mm256_loadu_ps(src + row * srcS + col * srcC); |
232 | 0 | for (size_t col = colE; col < 4; ++col) |
233 | 0 | dst[col] = _mm256_setzero_ps(); |
234 | 0 | dst += 4; |
235 | 0 | } |
236 | 0 | for (size_t row = rowE; row < 4; ++row) |
237 | 0 | { |
238 | 0 | for (size_t col = 0; col < 4; ++col) |
239 | 0 | dst[col] = _mm256_setzero_ps(); |
240 | 0 | dst += 4; |
241 | 0 | } |
242 | 0 | } |
243 | | |
244 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetInput8t(const float * src, size_t srcW, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, float * dst, size_t dstStride) |
245 | 0 | { |
246 | 0 | size_t srcS = srcW * srcC; |
247 | 0 | size_t srcCF = AlignLo(srcC, F); |
248 | 0 | for (size_t c = 0; c < srcCF; c += F) |
249 | 0 | { |
250 | 0 | __m256 tmp[16]; |
251 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + c, srcS, srcC, rowB, rowE, colB, colE, tmp); |
252 | 0 | WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + c, dstStride); |
253 | 0 | } |
254 | 0 | if (srcCF < srcC) |
255 | 0 | { |
256 | 0 | __m256 tmp[16]; |
257 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + srcC - F, srcS, srcC, rowB, rowE, colB, colE, tmp); |
258 | 0 | WinogradKernel3x3Block2x2SetInput8Store(tmp, dst + srcC - F, dstStride); |
259 | 0 | } |
260 | 0 | } |
261 | | |
262 | | void WinogradKernel3x3Block2x2SetInput(const float* src, size_t srcChannels, size_t srcHeight, size_t srcWidth, |
263 | | size_t padY, size_t padX, size_t padH, size_t padW, float* dst, size_t dstStride, SimdBool trans) |
264 | 0 | { |
265 | 0 | assert(padY == padX && padY == padH && padY == padW && (padY == 0 || padY == 1)); |
266 | 0 | SimdBool pad = padY > 0 ? SimdTrue : SimdFalse; |
267 | 0 | if (trans ? (srcChannels < 8) : (srcHeight < 4 || srcWidth < 18)) |
268 | 0 | { |
269 | 0 | Sse41::WinogradKernel3x3Block2x2SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans); |
270 | 0 | return; |
271 | 0 | } |
272 | 0 | size_t dstH = pad ? srcHeight : srcHeight - 2; |
273 | 0 | size_t dstW = pad ? srcWidth : srcWidth - 2; |
274 | 0 | size_t tileH = (dstH + 1) / 2; |
275 | 0 | size_t tileW = (dstW + 1) / 2; |
276 | 0 | size_t dstH2 = AlignLo(dstH, 2); |
277 | 0 | size_t dstW2 = AlignLo(dstW, 2); |
278 | 0 | if (trans) |
279 | 0 | { |
280 | 0 | size_t noseW = Simd::Min<size_t>(4, dstW + 1); |
281 | 0 | size_t noseH = Simd::Min<size_t>(4, dstH + 1); |
282 | 0 | size_t start = pad ? 2 : 0; |
283 | 0 | if (pad) |
284 | 0 | { |
285 | 0 | if (dstH == dstH2) |
286 | 0 | dstH2 -= 2; |
287 | 0 | if (dstW == dstW2) |
288 | 0 | dstW2 -= 2; |
289 | 0 | src -= (srcWidth + 1)*srcChannels; |
290 | 0 | } |
291 | 0 | size_t tailW = dstW - dstW2 + (pad ? 1 : 2); |
292 | 0 | size_t tailH = dstH - dstH2 + (pad ? 1 : 2); |
293 | 0 | size_t row = 0, col = 0; |
294 | 0 | if (pad) |
295 | 0 | { |
296 | 0 | if (pad) |
297 | 0 | WinogradKernel3x3Block2x2SetInput8t(src, srcWidth, srcChannels, 1, noseH, 1, noseW, dst, dstStride), dst += srcChannels; |
298 | 0 | for (col = start; col < dstW2; col += 2) |
299 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, 4, dst, dstStride), dst += srcChannels; |
300 | 0 | if (col < dstW) |
301 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, tailW, dst, dstStride), dst += srcChannels; |
302 | 0 | } |
303 | 0 | for (row = start; row < dstH2; row += 2) |
304 | 0 | { |
305 | 0 | if (pad) |
306 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + row * srcWidth * srcChannels, srcWidth, srcChannels, 0, 4, 1, noseW, dst, dstStride), dst += srcChannels; |
307 | 0 | for (col = start; col < dstW2; col += 2) |
308 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, dst, dstStride), dst += srcChannels; |
309 | 0 | if (col < dstW) |
310 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, 4, 0, tailW, dst, dstStride), dst += srcChannels; |
311 | 0 | } |
312 | 0 | if (row < dstH) |
313 | 0 | { |
314 | 0 | if (pad) |
315 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + row * srcWidth* srcChannels, srcWidth, srcChannels, 0, tailH, 1, noseW, dst, dstStride), dst += srcChannels; |
316 | 0 | for (col = start; col < dstW2; col += 2) |
317 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, 4, dst, dstStride), dst += srcChannels; |
318 | 0 | if (col < dstW) |
319 | 0 | WinogradKernel3x3Block2x2SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, tailW, dst, dstStride), dst += srcChannels; |
320 | 0 | } |
321 | 0 | } |
322 | 0 | else |
323 | 0 | { |
324 | 0 | size_t dstW16 = AlignLo(dstW, 16); |
325 | 0 | if (pad && dstW16 == dstW) |
326 | 0 | dstW16 -= 16; |
327 | 0 | PadType rowPad = dstH2 < dstH ? PadTail1 : PadNone; |
328 | 0 | PadType colPad = dstW2 < dstW ? PadTail1 : PadNone; |
329 | 0 | size_t tailCol = dstW2 < dstW ? dstW - 15 : dstW - 16; |
330 | 0 | size_t tailRow = dstH2 < dstH ? dstH - 1 : dstH - 2; |
331 | 0 | bool specialColTail = dstW16 < dstW || pad; |
332 | 0 | bool specialRowTail = dstH2 < dstH || pad; |
333 | 0 | if (pad) |
334 | 0 | { |
335 | 0 | src -= srcWidth + 1; |
336 | 0 | rowPad = dstH2 < dstH ? PadTail2 : PadTail1; |
337 | 0 | colPad = dstW2 < dstW ? PadTail2 : PadTail1; |
338 | 0 | if (dstH2 == dstH) |
339 | 0 | dstH2 -= 2; |
340 | 0 | } |
341 | 0 | for (size_t c = 0; c < srcChannels; ++c) |
342 | 0 | { |
343 | 0 | size_t row = 0, tileY = 0; |
344 | 0 | if (pad) |
345 | 0 | { |
346 | 0 | size_t col = 0, tileX = 0; |
347 | 0 | const float * s = src + row * srcWidth; |
348 | 0 | float * d = dst + tileY * tileW; |
349 | 0 | if (pad) |
350 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, PadNose1, PadNose1, d + tileX, dstStride), col += 16, tileX += 8; |
351 | 0 | for (; col < dstW16; col += 16, tileX += 8) |
352 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, PadNose1, PadNone, d + tileX, dstStride); |
353 | 0 | if (specialColTail) |
354 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + tailCol, srcWidth, PadNose1, colPad, d + tileW - 8, dstStride); |
355 | 0 | row += 2, tileY += 1; |
356 | 0 | } |
357 | 0 | for (; row < dstH2; row += 2, tileY += 1) |
358 | 0 | { |
359 | 0 | size_t col = 0, tileX = 0; |
360 | 0 | const float * s = src + row * srcWidth; |
361 | 0 | float * d = dst + tileY * tileW; |
362 | 0 | if (pad) |
363 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, PadNone, PadNose1, d + tileX, dstStride), col += 16, tileX += 8; |
364 | 0 | for (; col < dstW16; col += 16, tileX += 8) |
365 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, d + tileX, dstStride); |
366 | 0 | if (specialColTail) |
367 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + tailCol, srcWidth, PadNone, colPad, d + tileW - 8, dstStride); |
368 | 0 | } |
369 | 0 | if (specialRowTail) |
370 | 0 | { |
371 | 0 | size_t col = 0, tileX = 0; |
372 | 0 | const float * s = src + tailRow * srcWidth; |
373 | 0 | float * d = dst + (tileH - 1) * tileW; |
374 | 0 | if (pad) |
375 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, rowPad, PadNose1, d + tileX, dstStride), col += 16, tileX += 8; |
376 | 0 | for (; col < dstW16; col += 16, tileX += 8) |
377 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + col, srcWidth, rowPad, PadNone, d + tileX, dstStride); |
378 | 0 | if (specialColTail) |
379 | 0 | WinogradKernel3x3Block2x2SetInput8n(s + tailCol, srcWidth, rowPad, colPad, d + tileW - 8, dstStride); |
380 | 0 | } |
381 | 0 | src += srcWidth * srcHeight; |
382 | 0 | dst += tileW * tileH; |
383 | 0 | } |
384 | 0 | } |
385 | 0 | } |
386 | | |
387 | | //----------------------------------------------------------------------- |
388 | | |
389 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputLoad4(const float * src, size_t stride, __m256 * dst) |
390 | 0 | { |
391 | 0 | __m256 s0 = _mm256_loadu_ps(src + 0 * stride); |
392 | 0 | __m256 s1 = _mm256_loadu_ps(src + 1 * stride); |
393 | 0 | __m256 s2 = _mm256_loadu_ps(src + 2 * stride); |
394 | 0 | __m256 s3 = _mm256_loadu_ps(src + 3 * stride); |
395 | 0 | dst[0] = _mm256_add_ps(_mm256_add_ps(s0, s1), s2); |
396 | 0 | dst[1] = _mm256_sub_ps(_mm256_sub_ps(s1, s2), s3); |
397 | 0 | } |
398 | | |
399 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputLoad16(const float * src, size_t stride, __m256 * dst) |
400 | 0 | { |
401 | 0 | __m256 tmp[8]; |
402 | 0 | WinogradKernel3x3Block2x2SetOutputLoad4(src + 0 * stride, stride, tmp + 0); |
403 | 0 | WinogradKernel3x3Block2x2SetOutputLoad4(src + 4 * stride, stride, tmp + 2); |
404 | 0 | WinogradKernel3x3Block2x2SetOutputLoad4(src + 8 * stride, stride, tmp + 4); |
405 | 0 | WinogradKernel3x3Block2x2SetOutputLoad4(src + 12 * stride, stride, tmp + 6); |
406 | 0 | dst[0] = _mm256_add_ps(_mm256_add_ps(tmp[0], tmp[2]), tmp[4]); |
407 | 0 | dst[1] = _mm256_add_ps(_mm256_add_ps(tmp[1], tmp[3]), tmp[5]); |
408 | 0 | dst[2] = _mm256_sub_ps(_mm256_sub_ps(tmp[2], tmp[4]), tmp[6]); |
409 | 0 | dst[3] = _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[5]), tmp[7]); |
410 | 0 | } |
411 | | |
412 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8n(const float * src, size_t stride, __m256 * dst) |
413 | 0 | { |
414 | 0 | __m256 d[4], u[4]; |
415 | 0 | WinogradKernel3x3Block2x2SetOutputLoad16(src, stride, d); |
416 | 0 | u[0] = _mm256_unpacklo_ps(d[0], d[1]); |
417 | 0 | u[1] = _mm256_unpackhi_ps(d[0], d[1]); |
418 | 0 | u[2] = _mm256_unpacklo_ps(d[2], d[3]); |
419 | 0 | u[3] = _mm256_unpackhi_ps(d[2], d[3]); |
420 | 0 | dst[0] = _mm256_permute2f128_ps(u[0], u[1], 0x20); |
421 | 0 | dst[1] = _mm256_permute2f128_ps(u[0], u[1], 0x31); |
422 | 0 | dst[2] = _mm256_permute2f128_ps(u[2], u[3], 0x20); |
423 | 0 | dst[3] = _mm256_permute2f128_ps(u[2], u[3], 0x31); |
424 | 0 | } |
425 | | |
426 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8n(const float * src, size_t srcStride, float * dst, size_t dstStride) |
427 | 0 | { |
428 | 0 | __m256 d[4]; |
429 | 0 | WinogradKernel3x3Block2x2SetOutput8n(src, srcStride, d); |
430 | 0 | _mm256_storeu_ps(dst + 0 * dstStride + 0, d[0]); |
431 | 0 | _mm256_storeu_ps(dst + 0 * dstStride + 8, d[1]); |
432 | 0 | _mm256_storeu_ps(dst + 1 * dstStride + 0, d[2]); |
433 | 0 | _mm256_storeu_ps(dst + 1 * dstStride + 8, d[3]); |
434 | 0 | } |
435 | | |
436 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8n(const float * src, size_t srcStride, float * dst, size_t dstStride, bool lastRow, bool lastCol, const __m256 & mask) |
437 | 0 | { |
438 | 0 | __m256 d[4]; |
439 | 0 | WinogradKernel3x3Block2x2SetOutput8n(src, srcStride, d); |
440 | 0 | _mm256_storeu_ps(dst + 0, d[0]); |
441 | 0 | if (lastCol) |
442 | 0 | _mm256_storeu_ps(dst + 8, d[1]); |
443 | 0 | else |
444 | 0 | StoreMasked<false>(dst + 8, d[1], mask); |
445 | 0 | if (lastRow) |
446 | 0 | { |
447 | 0 | dst += dstStride; |
448 | 0 | _mm256_storeu_ps(dst + 0, d[2]); |
449 | 0 | if (lastCol) |
450 | 0 | _mm256_storeu_ps(dst + 8, d[3]); |
451 | 0 | else |
452 | 0 | StoreMasked<false>(dst + 8, d[3], mask); |
453 | 0 | } |
454 | 0 | } |
455 | | |
456 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputStore4(const __m256 src[4], float * dst, size_t dstS, size_t dstC) |
457 | 0 | { |
458 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 0 * dstC, src[0]); |
459 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 1 * dstC, src[1]); |
460 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 0 * dstC, src[2]); |
461 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 1 * dstC, src[3]); |
462 | 0 | } |
463 | | |
464 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC) |
465 | 0 | { |
466 | 0 | size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F); |
467 | 0 | for (size_t d = 0; d < dstCF; d += F) |
468 | 0 | { |
469 | 0 | __m256 tmp[4]; |
470 | 0 | WinogradKernel3x3Block2x2SetOutputLoad16(src + d, srcStride, tmp); |
471 | 0 | WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + d, dstS, dstC); |
472 | 0 | } |
473 | 0 | if (dstCF < dstC) |
474 | 0 | { |
475 | 0 | __m256 tmp[4]; |
476 | 0 | WinogradKernel3x3Block2x2SetOutputLoad16(src + dstC - F, srcStride, tmp); |
477 | 0 | WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + dstC - F, dstS, dstC); |
478 | 0 | } |
479 | 0 | } |
480 | | |
481 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutputStore4(const __m256 src[4], float * dst, size_t dstS, size_t dstC, size_t rowE, size_t colE) |
482 | 0 | { |
483 | 0 | for (size_t row = 0; row < rowE; ++row) |
484 | 0 | for (size_t col = 0; col < colE; ++col) |
485 | 0 | _mm256_storeu_ps(dst + row * dstS + col * dstC, src[row * 2 + col]); |
486 | 0 | } |
487 | | |
488 | | SIMD_INLINE void WinogradKernel3x3Block2x2SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC, size_t rowE, size_t colE) |
489 | 0 | { |
490 | 0 | size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F); |
491 | 0 | for (size_t d = 0; d < dstCF; d += F) |
492 | 0 | { |
493 | 0 | __m256 tmp[4]; |
494 | 0 | WinogradKernel3x3Block2x2SetOutputLoad16(src + d, srcStride, tmp); |
495 | 0 | WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + d, dstS, dstC, rowE, colE); |
496 | 0 | } |
497 | 0 | if (dstCF < dstC) |
498 | 0 | { |
499 | 0 | __m256 tmp[4]; |
500 | 0 | WinogradKernel3x3Block2x2SetOutputLoad16(src + dstC - F, srcStride, tmp); |
501 | 0 | WinogradKernel3x3Block2x2SetOutputStore4(tmp, dst + dstC - F, dstS, dstC, rowE, colE); |
502 | 0 | } |
503 | 0 | } |
504 | | |
505 | | void WinogradKernel3x3Block2x2SetOutput(const float * src, size_t srcStride, float * dst, size_t dstChannels, size_t dstHeight, size_t dstWidth, SimdBool trans) |
506 | 0 | { |
507 | 0 | if (trans ? (dstChannels < 8) : (dstHeight < 2 || dstWidth < 16)) |
508 | 0 | { |
509 | 0 | Sse41::WinogradKernel3x3Block2x2SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans); |
510 | 0 | return; |
511 | 0 | } |
512 | 0 | size_t tileH = (dstHeight + 1) / 2; |
513 | 0 | size_t tileW = (dstWidth + 1) / 2; |
514 | 0 | size_t dstH2 = AlignLo(dstHeight, 2); |
515 | 0 | size_t dstW2 = AlignLo(dstWidth, 2); |
516 | 0 | if (trans) |
517 | 0 | { |
518 | 0 | size_t row, col; |
519 | 0 | for (row = 0; row < dstH2; row += 2) |
520 | 0 | { |
521 | 0 | for (col = 0; col < dstW2; col += 2) |
522 | 0 | WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels), src += dstChannels; |
523 | 0 | if (col < dstWidth) |
524 | 0 | WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, 2, dstWidth - col), src += dstChannels; |
525 | 0 | } |
526 | 0 | if (row < dstHeight) |
527 | 0 | { |
528 | 0 | for (col = 0; col < dstW2; col += 2) |
529 | 0 | WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, 2), src += dstChannels; |
530 | 0 | if (col < dstWidth) |
531 | 0 | WinogradKernel3x3Block2x2SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, dstWidth - col), src += dstChannels; |
532 | 0 | } |
533 | 0 | } |
534 | 0 | else |
535 | 0 | { |
536 | 0 | size_t dstW16 = AlignLo(dstWidth, 16); |
537 | 0 | __m256 tailMask = LeftNotZero32f(8 + dstW2 - dstWidth); |
538 | 0 | size_t tailCol = dstW2 < dstWidth ? dstWidth - 15 : dstWidth - 16; |
539 | 0 | for (size_t c = 0; c < dstChannels; ++c) |
540 | 0 | { |
541 | 0 | size_t row = 0, tileY = 0; |
542 | 0 | for (; row < dstH2; row += 2, tileY += 1) |
543 | 0 | { |
544 | 0 | size_t col = 0, tileX = 0; |
545 | 0 | const float * s = src + tileY * tileW; |
546 | 0 | float * d = dst + row * dstWidth; |
547 | 0 | for (; col < dstW16; col += 16, tileX += 8) |
548 | 0 | WinogradKernel3x3Block2x2SetOutput8n(s + tileX, srcStride, d + col, dstWidth); |
549 | 0 | if (col < dstWidth) |
550 | 0 | WinogradKernel3x3Block2x2SetOutput8n(s + tileW - 8, srcStride, d + tailCol, dstWidth, true, false, tailMask); |
551 | 0 | } |
552 | 0 | if (row < dstHeight) |
553 | 0 | { |
554 | 0 | size_t col = 0, tileX = 0; |
555 | 0 | const float * s = src + (tileH - 1) * tileW; |
556 | 0 | float * d = dst + (dstHeight - 1) * dstWidth; |
557 | 0 | for (; col < dstW16; col += 16, tileX += 8) |
558 | 0 | WinogradKernel3x3Block2x2SetOutput8n(s + tileX, srcStride, d + col, dstWidth, false, true, tailMask); |
559 | 0 | if (col < dstWidth) |
560 | 0 | WinogradKernel3x3Block2x2SetOutput8n(s + tileW - 8, srcStride, d + tailCol, dstWidth, false, false, tailMask); |
561 | 0 | } |
562 | 0 | src += tileW * tileH; |
563 | 0 | dst += dstHeight * dstWidth; |
564 | 0 | } |
565 | 0 | } |
566 | 0 | } |
567 | | |
568 | | //----------------------------------------------------------------------- |
569 | | |
570 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetFilter8Row(const __m256 * t, float * dst, size_t stride) |
571 | 0 | { |
572 | 0 | const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f); |
573 | 0 | const __m256 r3 = _mm256_set1_ps(1.0f / 3.0f); |
574 | 0 | const __m256 r2 = _mm256_set1_ps(1.0f / 2.0f); |
575 | 0 | const __m256 f2_3 = _mm256_set1_ps(2.0f / 3.0f); |
576 | 0 | const __m256 mr2 = _mm256_set1_ps(-1.0f / 2.0f); |
577 | |
|
578 | 0 | _mm256_storeu_ps(dst + 0 * stride, _mm256_mul_ps(r2, t[0])); |
579 | 0 | __m256 t0 = _mm256_add_ps(t[0], t[2]); |
580 | 0 | _mm256_storeu_ps(dst + 1 * stride, _mm256_mul_ps(mr2, _mm256_add_ps(t0, t[1]))); |
581 | 0 | _mm256_storeu_ps(dst + 2 * stride, _mm256_mul_ps(r6, _mm256_sub_ps(t[1], t0))); |
582 | 0 | _mm256_storeu_ps(dst + 3 * stride, _mm256_add_ps(_mm256_mul_ps(r6, t[0]), _mm256_add_ps(_mm256_mul_ps(r3, t[1]), _mm256_mul_ps(f2_3, t[2])))); |
583 | 0 | _mm256_storeu_ps(dst + 4 * stride, t[2]); |
584 | 0 | } |
585 | | |
586 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetFilter8All(const __m256 * s, float * dst, size_t stride) |
587 | 0 | { |
588 | 0 | const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f); |
589 | 0 | const __m256 r3 = _mm256_set1_ps(1.0f / 3.0f); |
590 | 0 | const __m256 r2 = _mm256_set1_ps(1.0f / 2.0f); |
591 | 0 | const __m256 f2_3 = _mm256_set1_ps(2.0f / 3.0f); |
592 | 0 | const __m256 mr2 = _mm256_set1_ps(-1.0f / 2.0f); |
593 | |
|
594 | 0 | __m256 t[3]; |
595 | 0 | t[0] = _mm256_mul_ps(r2, s[0]); |
596 | 0 | t[1] = _mm256_mul_ps(r2, s[1]); |
597 | 0 | t[2] = _mm256_mul_ps(r2, s[2]); |
598 | 0 | WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 0 * stride, stride); |
599 | |
|
600 | 0 | t[0] = _mm256_mul_ps(mr2, _mm256_add_ps(_mm256_add_ps(s[0], s[6]), s[3])); |
601 | 0 | t[1] = _mm256_mul_ps(mr2, _mm256_add_ps(_mm256_add_ps(s[1], s[7]), s[4])); |
602 | 0 | t[2] = _mm256_mul_ps(mr2, _mm256_add_ps(_mm256_add_ps(s[2], s[8]), s[5])); |
603 | 0 | WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 5 * stride, stride); |
604 | |
|
605 | 0 | t[0] = _mm256_mul_ps(r6, _mm256_sub_ps(s[3], _mm256_add_ps(s[0], s[6]))); |
606 | 0 | t[1] = _mm256_mul_ps(r6, _mm256_sub_ps(s[4], _mm256_add_ps(s[1], s[7]))); |
607 | 0 | t[2] = _mm256_mul_ps(r6, _mm256_sub_ps(s[5], _mm256_add_ps(s[2], s[8]))); |
608 | 0 | WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 10 * stride, stride); |
609 | |
|
610 | 0 | t[0] = _mm256_add_ps(_mm256_mul_ps(r6, s[0]), _mm256_add_ps(_mm256_mul_ps(r3, s[3]), _mm256_mul_ps(f2_3, s[6]))); |
611 | 0 | t[1] = _mm256_add_ps(_mm256_mul_ps(r6, s[1]), _mm256_add_ps(_mm256_mul_ps(r3, s[4]), _mm256_mul_ps(f2_3, s[7]))); |
612 | 0 | t[2] = _mm256_add_ps(_mm256_mul_ps(r6, s[2]), _mm256_add_ps(_mm256_mul_ps(r3, s[5]), _mm256_mul_ps(f2_3, s[8]))); |
613 | 0 | WinogradKernel3x3Block3x3SetFilter8Row(t, dst + 15 * stride, stride); |
614 | |
|
615 | 0 | WinogradKernel3x3Block3x3SetFilter8Row(s + 6, dst + 20 * stride, stride); |
616 | 0 | } |
617 | | |
618 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetFilter8t(const float * src, float * dst, size_t stride) |
619 | 0 | { |
620 | 0 | __m256 s[9]; |
621 | 0 | s[0] = _mm256_loadu_ps(src + 0 * stride); |
622 | 0 | s[1] = _mm256_loadu_ps(src + 1 * stride); |
623 | 0 | s[2] = _mm256_loadu_ps(src + 2 * stride); |
624 | 0 | s[3] = _mm256_loadu_ps(src + 3 * stride); |
625 | 0 | s[4] = _mm256_loadu_ps(src + 4 * stride); |
626 | 0 | s[5] = _mm256_loadu_ps(src + 5 * stride); |
627 | 0 | s[6] = _mm256_loadu_ps(src + 6 * stride); |
628 | 0 | s[7] = _mm256_loadu_ps(src + 7 * stride); |
629 | 0 | s[8] = _mm256_loadu_ps(src + 8 * stride); |
630 | 0 | WinogradKernel3x3Block3x3SetFilter8All(s, dst + 0 * stride, stride); |
631 | 0 | } |
632 | | |
633 | | void WinogradKernel3x3Block3x3SetFilter(const float * src, size_t size, float * dst, SimdBool trans) |
634 | 0 | { |
635 | 0 | size_t size8 = AlignLo(size, 8), i = 0; |
636 | 0 | if (trans) |
637 | 0 | { |
638 | 0 | for (; i < size8; i += 8) |
639 | 0 | WinogradKernel3x3Block3x3SetFilter8t(src + i, dst + i, size); |
640 | 0 | for (; i < size; i += 1) |
641 | 0 | Base::WinogradKernel3x3Block3x3SetFilter1t(src + i, dst + i, size); |
642 | 0 | } |
643 | 0 | else |
644 | 0 | { |
645 | 0 | Sse41::WinogradKernel3x3Block3x3SetFilter(src, size, dst, trans); |
646 | 0 | } |
647 | 0 | } |
648 | | |
649 | | //----------------------------------------------------------------------- |
650 | | |
651 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8Store(const __m256 src[25], float * dst, size_t stride) |
652 | 0 | { |
653 | 0 | __m256 _2 = _mm256_set1_ps(2.0f); |
654 | 0 | __m256 _3 = _mm256_set1_ps(3.0f); |
655 | 0 | __m256 tmp[5]; |
656 | |
|
657 | 0 | tmp[0] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[0], src[10])), _mm256_sub_ps(src[15], src[5])); |
658 | 0 | tmp[1] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[1], src[11])), _mm256_sub_ps(src[16], src[6])); |
659 | 0 | tmp[2] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[2], src[12])), _mm256_sub_ps(src[17], src[7])); |
660 | 0 | tmp[3] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[3], src[13])), _mm256_sub_ps(src[18], src[8])); |
661 | 0 | tmp[4] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[4], src[14])), _mm256_sub_ps(src[19], src[9])); |
662 | 0 | _mm256_storeu_ps(dst + 0 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1]))); |
663 | 0 | _mm256_storeu_ps(dst + 1 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1]))); |
664 | 0 | _mm256_storeu_ps(dst + 2 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2])))); |
665 | 0 | _mm256_storeu_ps(dst + 3 * stride, _mm256_sub_ps(tmp[3], tmp[1])); |
666 | 0 | _mm256_storeu_ps(dst + 4 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2]))); |
667 | |
|
668 | 0 | tmp[0] = _mm256_sub_ps(_mm256_sub_ps(src[15], src[10]), _mm256_mul_ps(_2, src[5])); |
669 | 0 | tmp[1] = _mm256_sub_ps(_mm256_sub_ps(src[16], src[11]), _mm256_mul_ps(_2, src[6])); |
670 | 0 | tmp[2] = _mm256_sub_ps(_mm256_sub_ps(src[17], src[12]), _mm256_mul_ps(_2, src[7])); |
671 | 0 | tmp[3] = _mm256_sub_ps(_mm256_sub_ps(src[18], src[13]), _mm256_mul_ps(_2, src[8])); |
672 | 0 | tmp[4] = _mm256_sub_ps(_mm256_sub_ps(src[19], src[14]), _mm256_mul_ps(_2, src[9])); |
673 | 0 | _mm256_storeu_ps(dst + 5 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1]))); |
674 | 0 | _mm256_storeu_ps(dst + 6 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1]))); |
675 | 0 | _mm256_storeu_ps(dst + 7 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2])))); |
676 | 0 | _mm256_storeu_ps(dst + 8 * stride, _mm256_sub_ps(tmp[3], tmp[1])); |
677 | 0 | _mm256_storeu_ps(dst + 9 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2]))); |
678 | |
|
679 | 0 | tmp[0] = _mm256_add_ps(_mm256_mul_ps(_2, src[5]), _mm256_sub_ps(src[15], _mm256_mul_ps(_3, src[10]))); |
680 | 0 | tmp[1] = _mm256_add_ps(_mm256_mul_ps(_2, src[6]), _mm256_sub_ps(src[16], _mm256_mul_ps(_3, src[11]))); |
681 | 0 | tmp[2] = _mm256_add_ps(_mm256_mul_ps(_2, src[7]), _mm256_sub_ps(src[17], _mm256_mul_ps(_3, src[12]))); |
682 | 0 | tmp[3] = _mm256_add_ps(_mm256_mul_ps(_2, src[8]), _mm256_sub_ps(src[18], _mm256_mul_ps(_3, src[13]))); |
683 | 0 | tmp[4] = _mm256_add_ps(_mm256_mul_ps(_2, src[9]), _mm256_sub_ps(src[19], _mm256_mul_ps(_3, src[14]))); |
684 | 0 | _mm256_storeu_ps(dst + 10 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1]))); |
685 | 0 | _mm256_storeu_ps(dst + 11 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1]))); |
686 | 0 | _mm256_storeu_ps(dst + 12 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2])))); |
687 | 0 | _mm256_storeu_ps(dst + 13 * stride, _mm256_sub_ps(tmp[3], tmp[1])); |
688 | 0 | _mm256_storeu_ps(dst + 14 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2]))); |
689 | |
|
690 | 0 | tmp[0] = _mm256_sub_ps(src[15], src[5]); |
691 | 0 | tmp[1] = _mm256_sub_ps(src[16], src[6]); |
692 | 0 | tmp[2] = _mm256_sub_ps(src[17], src[7]); |
693 | 0 | tmp[3] = _mm256_sub_ps(src[18], src[8]); |
694 | 0 | tmp[4] = _mm256_sub_ps(src[19], src[9]); |
695 | 0 | _mm256_storeu_ps(dst + 15 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1]))); |
696 | 0 | _mm256_storeu_ps(dst + 16 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1]))); |
697 | 0 | _mm256_storeu_ps(dst + 17 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2])))); |
698 | 0 | _mm256_storeu_ps(dst + 18 * stride, _mm256_sub_ps(tmp[3], tmp[1])); |
699 | 0 | _mm256_storeu_ps(dst + 19 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2]))); |
700 | |
|
701 | 0 | tmp[0] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[5], src[15])), _mm256_sub_ps(src[20], src[10])); |
702 | 0 | tmp[1] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[6], src[16])), _mm256_sub_ps(src[21], src[11])); |
703 | 0 | tmp[2] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[7], src[17])), _mm256_sub_ps(src[22], src[12])); |
704 | 0 | tmp[3] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[8], src[18])), _mm256_sub_ps(src[23], src[13])); |
705 | 0 | tmp[4] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[9], src[19])), _mm256_sub_ps(src[24], src[14])); |
706 | 0 | _mm256_storeu_ps(dst + 20 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[0], tmp[2])), _mm256_sub_ps(tmp[3], tmp[1]))); |
707 | 0 | _mm256_storeu_ps(dst + 21 * stride, _mm256_sub_ps(_mm256_sub_ps(tmp[3], tmp[2]), _mm256_mul_ps(_2, tmp[1]))); |
708 | 0 | _mm256_storeu_ps(dst + 22 * stride, _mm256_add_ps(_mm256_mul_ps(_2, tmp[1]), _mm256_sub_ps(tmp[3], _mm256_mul_ps(_3, tmp[2])))); |
709 | 0 | _mm256_storeu_ps(dst + 23 * stride, _mm256_sub_ps(tmp[3], tmp[1])); |
710 | 0 | _mm256_storeu_ps(dst + 24 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2]))); |
711 | 0 | } |
712 | | |
713 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcS, size_t srcC, __m256 dst[25]) |
714 | 0 | { |
715 | 0 | dst[0] = _mm256_loadu_ps(src + 0 * srcS + 0 * srcC); |
716 | 0 | dst[1] = _mm256_loadu_ps(src + 0 * srcS + 1 * srcC); |
717 | 0 | dst[2] = _mm256_loadu_ps(src + 0 * srcS + 2 * srcC); |
718 | 0 | dst[3] = _mm256_loadu_ps(src + 0 * srcS + 3 * srcC); |
719 | 0 | dst[4] = _mm256_loadu_ps(src + 0 * srcS + 4 * srcC); |
720 | 0 | dst[5] = _mm256_loadu_ps(src + 1 * srcS + 0 * srcC); |
721 | 0 | dst[6] = _mm256_loadu_ps(src + 1 * srcS + 1 * srcC); |
722 | 0 | dst[7] = _mm256_loadu_ps(src + 1 * srcS + 2 * srcC); |
723 | 0 | dst[8] = _mm256_loadu_ps(src + 1 * srcS + 3 * srcC); |
724 | 0 | dst[9] = _mm256_loadu_ps(src + 1 * srcS + 4 * srcC); |
725 | 0 | dst[10] = _mm256_loadu_ps(src + 2 * srcS + 0 * srcC); |
726 | 0 | dst[11] = _mm256_loadu_ps(src + 2 * srcS + 1 * srcC); |
727 | 0 | dst[12] = _mm256_loadu_ps(src + 2 * srcS + 2 * srcC); |
728 | 0 | dst[13] = _mm256_loadu_ps(src + 2 * srcS + 3 * srcC); |
729 | 0 | dst[14] = _mm256_loadu_ps(src + 2 * srcS + 4 * srcC); |
730 | 0 | dst[15] = _mm256_loadu_ps(src + 3 * srcS + 0 * srcC); |
731 | 0 | dst[16] = _mm256_loadu_ps(src + 3 * srcS + 1 * srcC); |
732 | 0 | dst[17] = _mm256_loadu_ps(src + 3 * srcS + 2 * srcC); |
733 | 0 | dst[18] = _mm256_loadu_ps(src + 3 * srcS + 3 * srcC); |
734 | 0 | dst[19] = _mm256_loadu_ps(src + 3 * srcS + 4 * srcC); |
735 | 0 | dst[20] = _mm256_loadu_ps(src + 4 * srcS + 0 * srcC); |
736 | 0 | dst[21] = _mm256_loadu_ps(src + 4 * srcS + 1 * srcC); |
737 | 0 | dst[22] = _mm256_loadu_ps(src + 4 * srcS + 2 * srcC); |
738 | 0 | dst[23] = _mm256_loadu_ps(src + 4 * srcS + 3 * srcC); |
739 | 0 | dst[24] = _mm256_loadu_ps(src + 4 * srcS + 4 * srcC); |
740 | 0 | } |
741 | | |
742 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcW, size_t srcC, float * dst, size_t dstStride) |
743 | 0 | { |
744 | 0 | size_t srcS = srcW * srcC; |
745 | 0 | size_t srcCF = AlignLo(srcC, F); |
746 | 0 | for (size_t c = 0; c < srcCF; c += F) |
747 | 0 | { |
748 | 0 | __m256 tmp[25]; |
749 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + c, srcS, srcC, tmp); |
750 | 0 | WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + c, dstStride); |
751 | 0 | } |
752 | 0 | if (srcCF < srcC) |
753 | 0 | { |
754 | 0 | __m256 tmp[25]; |
755 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + srcC - F, srcS, srcC, tmp); |
756 | 0 | WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + srcC - F, dstStride); |
757 | 0 | } |
758 | 0 | } |
759 | | |
760 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcS, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, __m256 dst[25]) |
761 | 0 | { |
762 | 0 | for (size_t row = 0; row < rowB; ++row) |
763 | 0 | { |
764 | 0 | for (size_t col = 0; col < 5; ++col) |
765 | 0 | dst[col] = _mm256_setzero_ps(); |
766 | 0 | dst += 5; |
767 | 0 | } |
768 | 0 | for (size_t row = rowB; row < rowE; ++row) |
769 | 0 | { |
770 | 0 | for (size_t col = 0; col < colB; ++col) |
771 | 0 | dst[col] = _mm256_setzero_ps(); |
772 | 0 | for (size_t col = colB; col < colE; ++col) |
773 | 0 | dst[col] = _mm256_loadu_ps(src + row * srcS + col * srcC); |
774 | 0 | for (size_t col = colE; col < 5; ++col) |
775 | 0 | dst[col] = _mm256_setzero_ps(); |
776 | 0 | dst += 5; |
777 | 0 | } |
778 | 0 | for (size_t row = rowE; row < 5; ++row) |
779 | 0 | { |
780 | 0 | for (size_t col = 0; col < 5; ++col) |
781 | 0 | dst[col] = _mm256_setzero_ps(); |
782 | 0 | dst += 5; |
783 | 0 | } |
784 | 0 | } |
785 | | |
786 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetInput8t(const float * src, size_t srcW, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, float * dst, size_t dstStride) |
787 | 0 | { |
788 | 0 | size_t srcS = srcW * srcC; |
789 | 0 | size_t srcCF = AlignLo(srcC, F); |
790 | 0 | for (size_t c = 0; c < srcCF; c += F) |
791 | 0 | { |
792 | 0 | __m256 tmp[25]; |
793 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + c, srcS, srcC, rowB, rowE, colB, colE, tmp); |
794 | 0 | WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + c, dstStride); |
795 | 0 | } |
796 | 0 | if (srcCF < srcC) |
797 | 0 | { |
798 | 0 | __m256 tmp[25]; |
799 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + srcC - F, srcS, srcC, rowB, rowE, colB, colE, tmp); |
800 | 0 | WinogradKernel3x3Block3x3SetInput8Store(tmp, dst + srcC - F, dstStride); |
801 | 0 | } |
802 | 0 | } |
803 | | |
804 | | void WinogradKernel3x3Block3x3SetInput(const float* src, size_t srcChannels, size_t srcHeight, size_t srcWidth, |
805 | | size_t padY, size_t padX, size_t padH, size_t padW, float* dst, size_t dstStride, SimdBool trans) |
806 | 0 | { |
807 | 0 | assert(padY == padX && padY == padH && padY == padW && (padY == 0 || padY == 1)); |
808 | 0 | SimdBool pad = padY > 0 ? SimdTrue : SimdFalse; |
809 | 0 | if (trans ? (srcChannels < 8) : (srcHeight < 5 || srcWidth < 15)) |
810 | 0 | { |
811 | 0 | Sse41::WinogradKernel3x3Block3x3SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans); |
812 | 0 | return; |
813 | 0 | } |
814 | 0 | size_t dstH = pad ? srcHeight : srcHeight - 2; |
815 | 0 | size_t dstW = pad ? srcWidth : srcWidth - 2; |
816 | 0 | size_t tileH = (dstH + 2) / 3; |
817 | 0 | size_t tileW = (dstW + 2) / 3; |
818 | 0 | size_t dstH3 = AlignLoAny(dstH, 3); |
819 | 0 | size_t dstW3 = AlignLoAny(dstW, 3); |
820 | 0 | if (trans) |
821 | 0 | { |
822 | 0 | size_t noseW = Simd::Min<size_t>(5, dstW + 1); |
823 | 0 | size_t noseH = Simd::Min<size_t>(5, dstH + 1); |
824 | 0 | size_t start = pad ? 3 : 0; |
825 | 0 | if (pad) |
826 | 0 | { |
827 | 0 | if (dstH == dstH3) |
828 | 0 | dstH3 -= 3; |
829 | 0 | if (dstW == dstW3) |
830 | 0 | dstW3 -= 3; |
831 | 0 | src -= (srcWidth + 1)*srcChannels; |
832 | 0 | } |
833 | 0 | size_t tailW = dstW - dstW3 + (pad ? 1 : 2); |
834 | 0 | size_t tailH = dstH - dstH3 + (pad ? 1 : 2); |
835 | 0 | size_t row = 0, col = 0; |
836 | 0 | if (pad) |
837 | 0 | { |
838 | 0 | if (pad) |
839 | 0 | WinogradKernel3x3Block3x3SetInput8t(src, srcWidth, srcChannels, 1, noseH, 1, noseW, dst, dstStride), dst += srcChannels; |
840 | 0 | for (col = start; col < dstW3; col += 3) |
841 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, 5, dst, dstStride), dst += srcChannels; |
842 | 0 | if (col < dstW) |
843 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, tailW, dst, dstStride), dst += srcChannels; |
844 | 0 | } |
845 | 0 | for (row = start; row < dstH3; row += 3) |
846 | 0 | { |
847 | 0 | if (pad) |
848 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + row * srcWidth * srcChannels, srcWidth, srcChannels, 0, 5, 1, noseW, dst, dstStride), dst += srcChannels; |
849 | 0 | for (col = start; col < dstW3; col += 3) |
850 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, dst, dstStride), dst += srcChannels; |
851 | 0 | if (col < dstW) |
852 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, 5, 0, tailW, dst, dstStride), dst += srcChannels; |
853 | 0 | } |
854 | 0 | if (row < dstH) |
855 | 0 | { |
856 | 0 | if (pad) |
857 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + row * srcWidth* srcChannels, srcWidth, srcChannels, 0, tailH, 1, noseW, dst, dstStride), dst += srcChannels; |
858 | 0 | for (col = start; col < dstW3; col += 3) |
859 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, 5, dst, dstStride), dst += srcChannels; |
860 | 0 | if (col < dstW) |
861 | 0 | WinogradKernel3x3Block3x3SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, tailW, dst, dstStride), dst += srcChannels; |
862 | 0 | } |
863 | 0 | } |
864 | 0 | else |
865 | 0 | { |
866 | 0 | Base::WinogradKernel3x3Block3x3SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans); |
867 | 0 | } |
868 | 0 | } |
869 | | |
870 | | //----------------------------------------------------------------------- |
871 | | |
872 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetOutputLoad25(const float * src, size_t stride, __m256 dst[9]) |
873 | 0 | { |
874 | 0 | __m256 s[25]; |
875 | 0 | s[0] = _mm256_loadu_ps(src + 0 * stride); |
876 | 0 | s[1] = _mm256_loadu_ps(src + 1 * stride); |
877 | 0 | s[2] = _mm256_loadu_ps(src + 2 * stride); |
878 | 0 | s[3] = _mm256_loadu_ps(src + 3 * stride); |
879 | 0 | s[4] = _mm256_loadu_ps(src + 4 * stride); |
880 | 0 | s[5] = _mm256_loadu_ps(src + 5 * stride); |
881 | 0 | s[6] = _mm256_loadu_ps(src + 6 * stride); |
882 | 0 | s[7] = _mm256_loadu_ps(src + 7 * stride); |
883 | 0 | s[8] = _mm256_loadu_ps(src + 8 * stride); |
884 | 0 | s[9] = _mm256_loadu_ps(src + 9 * stride); |
885 | 0 | s[10] = _mm256_loadu_ps(src + 10 * stride); |
886 | 0 | s[11] = _mm256_loadu_ps(src + 11 * stride); |
887 | 0 | s[12] = _mm256_loadu_ps(src + 12 * stride); |
888 | 0 | s[13] = _mm256_loadu_ps(src + 13 * stride); |
889 | 0 | s[14] = _mm256_loadu_ps(src + 14 * stride); |
890 | 0 | s[15] = _mm256_loadu_ps(src + 15 * stride); |
891 | 0 | s[16] = _mm256_loadu_ps(src + 16 * stride); |
892 | 0 | s[17] = _mm256_loadu_ps(src + 17 * stride); |
893 | 0 | s[18] = _mm256_loadu_ps(src + 18 * stride); |
894 | 0 | s[19] = _mm256_loadu_ps(src + 19 * stride); |
895 | 0 | s[20] = _mm256_loadu_ps(src + 20 * stride); |
896 | 0 | s[21] = _mm256_loadu_ps(src + 21 * stride); |
897 | 0 | s[22] = _mm256_loadu_ps(src + 22 * stride); |
898 | 0 | s[23] = _mm256_loadu_ps(src + 23 * stride); |
899 | 0 | s[24] = _mm256_loadu_ps(src + 24 * stride); |
900 | |
|
901 | 0 | __m256 _2 = _mm256_set1_ps(2.0f); |
902 | 0 | __m256 _4 = _mm256_set1_ps(4.0f); |
903 | 0 | __m256 t[5]; |
904 | 0 | t[0] = _mm256_add_ps(_mm256_add_ps(s[0], s[5]), _mm256_add_ps(s[10], s[15])); |
905 | 0 | t[1] = _mm256_add_ps(_mm256_add_ps(s[1], s[6]), _mm256_add_ps(s[11], s[16])); |
906 | 0 | t[2] = _mm256_add_ps(_mm256_add_ps(s[2], s[7]), _mm256_add_ps(s[12], s[17])); |
907 | 0 | t[3] = _mm256_add_ps(_mm256_add_ps(s[3], s[8]), _mm256_add_ps(s[13], s[18])); |
908 | 0 | t[4] = _mm256_add_ps(_mm256_add_ps(s[4], s[9]), _mm256_add_ps(s[14], s[19])); |
909 | 0 | dst[0] = _mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3])); |
910 | 0 | dst[1] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, t[3])); |
911 | 0 | dst[2] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_add_ps(_mm256_mul_ps(_4, t[3]), t[4])); |
912 | |
|
913 | 0 | t[0] = _mm256_add_ps(_mm256_sub_ps(s[5], s[10]), _mm256_mul_ps(_2, s[15])); |
914 | 0 | t[1] = _mm256_add_ps(_mm256_sub_ps(s[6], s[11]), _mm256_mul_ps(_2, s[16])); |
915 | 0 | t[2] = _mm256_add_ps(_mm256_sub_ps(s[7], s[12]), _mm256_mul_ps(_2, s[17])); |
916 | 0 | t[3] = _mm256_add_ps(_mm256_sub_ps(s[8], s[13]), _mm256_mul_ps(_2, s[18])); |
917 | 0 | t[4] = _mm256_add_ps(_mm256_sub_ps(s[9], s[14]), _mm256_mul_ps(_2, s[19])); |
918 | 0 | dst[3] = _mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3])); |
919 | 0 | dst[4] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, t[3])); |
920 | 0 | dst[5] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_add_ps(_mm256_mul_ps(_4, t[3]), t[4])); |
921 | |
|
922 | 0 | t[0] = _mm256_add_ps(_mm256_add_ps(s[5], s[10]), _mm256_add_ps(_mm256_mul_ps(_4, s[15]), s[20])); |
923 | 0 | t[1] = _mm256_add_ps(_mm256_add_ps(s[6], s[11]), _mm256_add_ps(_mm256_mul_ps(_4, s[16]), s[21])); |
924 | 0 | t[2] = _mm256_add_ps(_mm256_add_ps(s[7], s[12]), _mm256_add_ps(_mm256_mul_ps(_4, s[17]), s[22])); |
925 | 0 | t[3] = _mm256_add_ps(_mm256_add_ps(s[8], s[13]), _mm256_add_ps(_mm256_mul_ps(_4, s[18]), s[23])); |
926 | 0 | t[4] = _mm256_add_ps(_mm256_add_ps(s[9], s[14]), _mm256_add_ps(_mm256_mul_ps(_4, s[19]), s[24])); |
927 | 0 | dst[6] = _mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3])); |
928 | 0 | dst[7] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, t[3])); |
929 | 0 | dst[8] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_add_ps(_mm256_mul_ps(_4, t[3]), t[4])); |
930 | 0 | } |
931 | | |
932 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetOutputStore9(const __m256 src[9], float * dst, size_t dstS, size_t dstC) |
933 | 0 | { |
934 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 0 * dstC, src[0]); |
935 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 1 * dstC, src[1]); |
936 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 2 * dstC, src[2]); |
937 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 0 * dstC, src[3]); |
938 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 1 * dstC, src[4]); |
939 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 2 * dstC, src[5]); |
940 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 0 * dstC, src[6]); |
941 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 1 * dstC, src[7]); |
942 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 2 * dstC, src[8]); |
943 | 0 | } |
944 | | |
945 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC) |
946 | 0 | { |
947 | 0 | size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F); |
948 | 0 | for (size_t d = 0; d < dstCF; d += F) |
949 | 0 | { |
950 | 0 | __m256 tmp[9]; |
951 | 0 | WinogradKernel3x3Block3x3SetOutputLoad25(src + d, srcStride, tmp); |
952 | 0 | WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + d, dstS, dstC); |
953 | 0 | } |
954 | 0 | if (dstCF < dstC) |
955 | 0 | { |
956 | 0 | __m256 tmp[9]; |
957 | 0 | WinogradKernel3x3Block3x3SetOutputLoad25(src + dstC - F, srcStride, tmp); |
958 | 0 | WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + dstC - F, dstS, dstC); |
959 | 0 | } |
960 | 0 | } |
961 | | |
962 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetOutputStore9(const __m256 src[16], float * dst, size_t dstS, size_t dstC, size_t rowE, size_t colE) |
963 | 0 | { |
964 | 0 | for (size_t row = 0; row < rowE; ++row) |
965 | 0 | for (size_t col = 0; col < colE; ++col) |
966 | 0 | _mm256_storeu_ps(dst + row * dstS + col * dstC, src[row * 3 + col]); |
967 | 0 | } |
968 | | |
969 | | SIMD_INLINE void WinogradKernel3x3Block3x3SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC, size_t rowE, size_t colE) |
970 | 0 | { |
971 | 0 | size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F); |
972 | 0 | for (size_t d = 0; d < dstCF; d += F) |
973 | 0 | { |
974 | 0 | __m256 tmp[9]; |
975 | 0 | WinogradKernel3x3Block3x3SetOutputLoad25(src + d, srcStride, tmp); |
976 | 0 | WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + d, dstS, dstC, rowE, colE); |
977 | 0 | } |
978 | 0 | if (dstCF < dstC) |
979 | 0 | { |
980 | 0 | __m256 tmp[9]; |
981 | 0 | WinogradKernel3x3Block3x3SetOutputLoad25(src + dstC - F, srcStride, tmp); |
982 | 0 | WinogradKernel3x3Block3x3SetOutputStore9(tmp, dst + dstC - F, dstS, dstC, rowE, colE); |
983 | 0 | } |
984 | 0 | } |
985 | | |
986 | | void WinogradKernel3x3Block3x3SetOutput(const float * src, size_t srcStride, float * dst, size_t dstChannels, size_t dstHeight, size_t dstWidth, SimdBool trans) |
987 | 0 | { |
988 | 0 | if (trans ? (dstChannels < 8) : (dstHeight < 3 || dstWidth < 15)) |
989 | 0 | { |
990 | 0 | Sse41::WinogradKernel3x3Block3x3SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans); |
991 | 0 | return; |
992 | 0 | } |
993 | 0 | size_t tileH = (dstHeight + 2) / 3; |
994 | 0 | size_t tileW = (dstWidth + 2) / 3; |
995 | 0 | size_t dstH3 = AlignLoAny(dstHeight, 3); |
996 | 0 | size_t dstW3 = AlignLoAny(dstWidth, 3); |
997 | 0 | if (trans) |
998 | 0 | { |
999 | 0 | size_t row, col; |
1000 | 0 | for (row = 0; row < dstH3; row += 3) |
1001 | 0 | { |
1002 | 0 | for (col = 0; col < dstW3; col += 3) |
1003 | 0 | WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels), src += dstChannels; |
1004 | 0 | if (col < dstWidth) |
1005 | 0 | WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, 3, dstWidth - col), src += dstChannels; |
1006 | 0 | } |
1007 | 0 | if (row < dstHeight) |
1008 | 0 | { |
1009 | 0 | for (col = 0; col < dstW3; col += 3) |
1010 | 0 | WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, 3), src += dstChannels; |
1011 | 0 | if (col < dstWidth) |
1012 | 0 | WinogradKernel3x3Block3x3SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, dstWidth - col), src += dstChannels; |
1013 | 0 | } |
1014 | 0 | } |
1015 | 0 | else |
1016 | 0 | { |
1017 | 0 | Base::WinogradKernel3x3Block3x3SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans); |
1018 | 0 | } |
1019 | 0 | } |
1020 | | |
1021 | | //----------------------------------------------------------------------- |
1022 | | |
1023 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetFilter8Row(const __m256 * t, float * dst, size_t stride) |
1024 | 0 | { |
1025 | 0 | const __m256 r4 = _mm256_set1_ps(1.0f / 4.0f); |
1026 | 0 | const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f); |
1027 | 0 | const __m256 mr6 = _mm256_set1_ps(-1.0f / 6.0f); |
1028 | 0 | const __m256 r12 = _mm256_set1_ps(1.0f / 12.0f); |
1029 | 0 | const __m256 r24 = _mm256_set1_ps(1.0f / 24.0f); |
1030 | 0 | _mm256_storeu_ps(dst + 0 * stride, _mm256_mul_ps(r4, t[0])); |
1031 | 0 | __m256 t0 = _mm256_add_ps(t[0], t[2]); |
1032 | 0 | _mm256_storeu_ps(dst + 1 * stride, _mm256_mul_ps(mr6, _mm256_add_ps(t0, t[1]))); |
1033 | 0 | _mm256_storeu_ps(dst + 2 * stride, _mm256_mul_ps(mr6, _mm256_sub_ps(t0, t[1]))); |
1034 | 0 | __m256 t1 = _mm256_add_ps(_mm256_mul_ps(r24, t[0]), _mm256_mul_ps(r6, t[2])); |
1035 | 0 | __m256 t2 = _mm256_mul_ps(r12, t[1]); |
1036 | 0 | _mm256_storeu_ps(dst + 3 * stride, _mm256_add_ps(t1, t2)); |
1037 | 0 | _mm256_storeu_ps(dst + 4 * stride, _mm256_sub_ps(t1, t2)); |
1038 | 0 | _mm256_storeu_ps(dst + 5 * stride, t[2]); |
1039 | 0 | } |
1040 | | |
1041 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetFilter8All(const __m256 * s, float * dst, size_t stride) |
1042 | 0 | { |
1043 | 0 | const __m256 r4 = _mm256_set1_ps(1.0f / 4.0f); |
1044 | 0 | const __m256 r6 = _mm256_set1_ps(1.0f / 6.0f); |
1045 | 0 | const __m256 mr6 = _mm256_set1_ps(-1.0f / 6.0f); |
1046 | 0 | const __m256 r12 = _mm256_set1_ps(1.0f / 12.0f); |
1047 | 0 | const __m256 r24 = _mm256_set1_ps(1.0f / 24.0f); |
1048 | |
|
1049 | 0 | __m256 t[3]; |
1050 | 0 | t[0] = _mm256_mul_ps(r4, s[0]); |
1051 | 0 | t[1] = _mm256_mul_ps(r4, s[1]); |
1052 | 0 | t[2] = _mm256_mul_ps(r4, s[2]); |
1053 | 0 | WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 0 * stride, stride); |
1054 | |
|
1055 | 0 | t[0] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_add_ps(s[0], s[3]), s[6])); |
1056 | 0 | t[1] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_add_ps(s[1], s[4]), s[7])); |
1057 | 0 | t[2] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_add_ps(s[2], s[5]), s[8])); |
1058 | 0 | WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 6 * stride, stride); |
1059 | |
|
1060 | 0 | t[0] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_sub_ps(s[0], s[3]), s[6])); |
1061 | 0 | t[1] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_sub_ps(s[1], s[4]), s[7])); |
1062 | 0 | t[2] = _mm256_mul_ps(mr6, _mm256_add_ps(_mm256_sub_ps(s[2], s[5]), s[8])); |
1063 | 0 | WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 12 * stride, stride); |
1064 | |
|
1065 | 0 | t[0] = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(r24, s[0]), _mm256_mul_ps(r12, s[3])), _mm256_mul_ps(r6, s[6])); |
1066 | 0 | t[1] = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(r24, s[1]), _mm256_mul_ps(r12, s[4])), _mm256_mul_ps(r6, s[7])); |
1067 | 0 | t[2] = _mm256_add_ps(_mm256_add_ps(_mm256_mul_ps(r24, s[2]), _mm256_mul_ps(r12, s[5])), _mm256_mul_ps(r6, s[8])); |
1068 | 0 | WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 18 * stride, stride); |
1069 | |
|
1070 | 0 | t[0] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(r24, s[0]), _mm256_mul_ps(r12, s[3])), _mm256_mul_ps(r6, s[6])); |
1071 | 0 | t[1] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(r24, s[1]), _mm256_mul_ps(r12, s[4])), _mm256_mul_ps(r6, s[7])); |
1072 | 0 | t[2] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(r24, s[2]), _mm256_mul_ps(r12, s[5])), _mm256_mul_ps(r6, s[8])); |
1073 | 0 | WinogradKernel3x3Block4x4SetFilter8Row(t, dst + 24 * stride, stride); |
1074 | |
|
1075 | 0 | WinogradKernel3x3Block4x4SetFilter8Row(s + 6, dst + 30 * stride, stride); |
1076 | 0 | } |
1077 | | |
1078 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetFilter8t(const float * src, float * dst, size_t stride) |
1079 | 0 | { |
1080 | 0 | __m256 s[9]; |
1081 | 0 | s[0] = _mm256_loadu_ps(src + 0 * stride); |
1082 | 0 | s[1] = _mm256_loadu_ps(src + 1 * stride); |
1083 | 0 | s[2] = _mm256_loadu_ps(src + 2 * stride); |
1084 | 0 | s[3] = _mm256_loadu_ps(src + 3 * stride); |
1085 | 0 | s[4] = _mm256_loadu_ps(src + 4 * stride); |
1086 | 0 | s[5] = _mm256_loadu_ps(src + 5 * stride); |
1087 | 0 | s[6] = _mm256_loadu_ps(src + 6 * stride); |
1088 | 0 | s[7] = _mm256_loadu_ps(src + 7 * stride); |
1089 | 0 | s[8] = _mm256_loadu_ps(src + 8 * stride); |
1090 | 0 | WinogradKernel3x3Block4x4SetFilter8All(s, dst + 0 * stride, stride); |
1091 | 0 | } |
1092 | | |
1093 | | void WinogradKernel3x3Block4x4SetFilter(const float * src, size_t size, float * dst, SimdBool trans) |
1094 | 0 | { |
1095 | 0 | if (trans) |
1096 | 0 | { |
1097 | 0 | size_t size8 = AlignLo(size, 8), i = 0; |
1098 | 0 | for (; i < size8; i += 8) |
1099 | 0 | WinogradKernel3x3Block4x4SetFilter8t(src + i, dst + i, size); |
1100 | 0 | for (; i < size; i += 1) |
1101 | 0 | Base::WinogradKernel3x3Block4x4SetFilter1t(src + i, dst + i, size); |
1102 | 0 | } |
1103 | 0 | else |
1104 | 0 | { |
1105 | 0 | Sse41::WinogradKernel3x3Block4x4SetFilter(src, size, dst, trans); |
1106 | 0 | } |
1107 | 0 | } |
1108 | | |
1109 | | //----------------------------------------------------------------------- |
1110 | | |
1111 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8Store(const __m256 src[36], float * dst, size_t stride) |
1112 | 0 | { |
1113 | 0 | __m256 _2 = _mm256_set1_ps(2.0f); |
1114 | 0 | __m256 _4 = _mm256_set1_ps(4.0f); |
1115 | 0 | __m256 _5 = _mm256_set1_ps(5.0f); |
1116 | 0 | __m256 tmp[36]; |
1117 | 0 | tmp[0] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[0]), _mm256_mul_ps(_5, src[12])), src[24]); |
1118 | 0 | tmp[1] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[1]), _mm256_mul_ps(_5, src[13])), src[25]); |
1119 | 0 | tmp[2] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[2]), _mm256_mul_ps(_5, src[14])), src[26]); |
1120 | 0 | tmp[3] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[3]), _mm256_mul_ps(_5, src[15])), src[27]); |
1121 | 0 | tmp[4] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[4]), _mm256_mul_ps(_5, src[16])), src[28]); |
1122 | 0 | tmp[5] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[5]), _mm256_mul_ps(_5, src[17])), src[29]); |
1123 | 0 | tmp[6] = _mm256_sub_ps(_mm256_add_ps(src[18], src[24]), _mm256_mul_ps(_4, _mm256_add_ps(src[6], src[12]))); |
1124 | 0 | tmp[7] = _mm256_sub_ps(_mm256_add_ps(src[19], src[25]), _mm256_mul_ps(_4, _mm256_add_ps(src[7], src[13]))); |
1125 | 0 | tmp[8] = _mm256_sub_ps(_mm256_add_ps(src[20], src[26]), _mm256_mul_ps(_4, _mm256_add_ps(src[8], src[14]))); |
1126 | 0 | tmp[9] = _mm256_sub_ps(_mm256_add_ps(src[21], src[27]), _mm256_mul_ps(_4, _mm256_add_ps(src[9], src[15]))); |
1127 | 0 | tmp[10] = _mm256_sub_ps(_mm256_add_ps(src[22], src[28]), _mm256_mul_ps(_4, _mm256_add_ps(src[10], src[16]))); |
1128 | 0 | tmp[11] = _mm256_sub_ps(_mm256_add_ps(src[23], src[29]), _mm256_mul_ps(_4, _mm256_add_ps(src[11], src[17]))); |
1129 | 0 | tmp[12] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[6], src[12])), _mm256_sub_ps(src[24], src[18])); |
1130 | 0 | tmp[13] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[7], src[13])), _mm256_sub_ps(src[25], src[19])); |
1131 | 0 | tmp[14] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[8], src[14])), _mm256_sub_ps(src[26], src[20])); |
1132 | 0 | tmp[15] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[9], src[15])), _mm256_sub_ps(src[27], src[21])); |
1133 | 0 | tmp[16] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[10], src[16])), _mm256_sub_ps(src[28], src[22])); |
1134 | 0 | tmp[17] = _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(src[11], src[17])), _mm256_sub_ps(src[29], src[23])); |
1135 | 0 | tmp[18] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[18], src[6])), _mm256_sub_ps(src[24], src[12])); |
1136 | 0 | tmp[19] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[19], src[7])), _mm256_sub_ps(src[25], src[13])); |
1137 | 0 | tmp[20] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[20], src[8])), _mm256_sub_ps(src[26], src[14])); |
1138 | 0 | tmp[21] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[21], src[9])), _mm256_sub_ps(src[27], src[15])); |
1139 | 0 | tmp[22] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[22], src[10])), _mm256_sub_ps(src[28], src[16])); |
1140 | 0 | tmp[23] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[23], src[11])), _mm256_sub_ps(src[29], src[17])); |
1141 | 0 | tmp[24] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[6], src[18])), _mm256_sub_ps(src[24], src[12])); |
1142 | 0 | tmp[25] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[7], src[19])), _mm256_sub_ps(src[25], src[13])); |
1143 | 0 | tmp[26] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[8], src[20])), _mm256_sub_ps(src[26], src[14])); |
1144 | 0 | tmp[27] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[9], src[21])), _mm256_sub_ps(src[27], src[15])); |
1145 | 0 | tmp[28] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[10], src[22])), _mm256_sub_ps(src[28], src[16])); |
1146 | 0 | tmp[29] = _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(src[11], src[23])), _mm256_sub_ps(src[29], src[17])); |
1147 | 0 | tmp[30] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[6]), _mm256_mul_ps(_5, src[18])), src[30]); |
1148 | 0 | tmp[31] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[7]), _mm256_mul_ps(_5, src[19])), src[31]); |
1149 | 0 | tmp[32] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[8]), _mm256_mul_ps(_5, src[20])), src[32]); |
1150 | 0 | tmp[33] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[9]), _mm256_mul_ps(_5, src[21])), src[33]); |
1151 | 0 | tmp[34] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[10]), _mm256_mul_ps(_5, src[22])), src[34]); |
1152 | 0 | tmp[35] = _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, src[11]), _mm256_mul_ps(_5, src[23])), src[35]); |
1153 | |
|
1154 | 0 | _mm256_storeu_ps(dst + 0 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[0]), _mm256_mul_ps(_5, tmp[2])), tmp[4])); |
1155 | 0 | _mm256_storeu_ps(dst + 1 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[3], tmp[4]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[1], tmp[2])))); |
1156 | 0 | _mm256_storeu_ps(dst + 2 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[1], tmp[2])), _mm256_sub_ps(tmp[4], tmp[3]))); |
1157 | 0 | _mm256_storeu_ps(dst + 3 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[3], tmp[1])), _mm256_sub_ps(tmp[4], tmp[2]))); |
1158 | 0 | _mm256_storeu_ps(dst + 4 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[1], tmp[3])), _mm256_sub_ps(tmp[4], tmp[2]))); |
1159 | 0 | _mm256_storeu_ps(dst + 5 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[1]), _mm256_mul_ps(_5, tmp[3])), tmp[5])); |
1160 | 0 | _mm256_storeu_ps(dst + 6 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[6]), _mm256_mul_ps(_5, tmp[8])), tmp[10])); |
1161 | 0 | _mm256_storeu_ps(dst + 7 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[9], tmp[10]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[7], tmp[8])))); |
1162 | 0 | _mm256_storeu_ps(dst + 8 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[7], tmp[8])), _mm256_sub_ps(tmp[10], tmp[9]))); |
1163 | 0 | _mm256_storeu_ps(dst + 9 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[9], tmp[7])), _mm256_sub_ps(tmp[10], tmp[8]))); |
1164 | 0 | _mm256_storeu_ps(dst + 10 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[7], tmp[9])), _mm256_sub_ps(tmp[10], tmp[8]))); |
1165 | 0 | _mm256_storeu_ps(dst + 11 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[7]), _mm256_mul_ps(_5, tmp[9])), tmp[11])); |
1166 | 0 | _mm256_storeu_ps(dst + 12 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[12]), _mm256_mul_ps(_5, tmp[14])), tmp[16])); |
1167 | 0 | _mm256_storeu_ps(dst + 13 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[15], tmp[16]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[13], tmp[14])))); |
1168 | 0 | _mm256_storeu_ps(dst + 14 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[13], tmp[14])), _mm256_sub_ps(tmp[16], tmp[15]))); |
1169 | 0 | _mm256_storeu_ps(dst + 15 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[15], tmp[13])), _mm256_sub_ps(tmp[16], tmp[14]))); |
1170 | 0 | _mm256_storeu_ps(dst + 16 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[13], tmp[15])), _mm256_sub_ps(tmp[16], tmp[14]))); |
1171 | 0 | _mm256_storeu_ps(dst + 17 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[13]), _mm256_mul_ps(_5, tmp[15])), tmp[17])); |
1172 | 0 | _mm256_storeu_ps(dst + 18 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[18]), _mm256_mul_ps(_5, tmp[20])), tmp[22])); |
1173 | 0 | _mm256_storeu_ps(dst + 19 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[21], tmp[22]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[19], tmp[20])))); |
1174 | 0 | _mm256_storeu_ps(dst + 20 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[19], tmp[20])), _mm256_sub_ps(tmp[22], tmp[21]))); |
1175 | 0 | _mm256_storeu_ps(dst + 21 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[21], tmp[19])), _mm256_sub_ps(tmp[22], tmp[20]))); |
1176 | 0 | _mm256_storeu_ps(dst + 22 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[19], tmp[21])), _mm256_sub_ps(tmp[22], tmp[20]))); |
1177 | 0 | _mm256_storeu_ps(dst + 23 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[19]), _mm256_mul_ps(_5, tmp[21])), tmp[23])); |
1178 | 0 | _mm256_storeu_ps(dst + 24 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[24]), _mm256_mul_ps(_5, tmp[26])), tmp[28])); |
1179 | 0 | _mm256_storeu_ps(dst + 25 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[27], tmp[28]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[25], tmp[26])))); |
1180 | 0 | _mm256_storeu_ps(dst + 26 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[25], tmp[26])), _mm256_sub_ps(tmp[28], tmp[27]))); |
1181 | 0 | _mm256_storeu_ps(dst + 27 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[27], tmp[25])), _mm256_sub_ps(tmp[28], tmp[26]))); |
1182 | 0 | _mm256_storeu_ps(dst + 28 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[25], tmp[27])), _mm256_sub_ps(tmp[28], tmp[26]))); |
1183 | 0 | _mm256_storeu_ps(dst + 29 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[25]), _mm256_mul_ps(_5, tmp[27])), tmp[29])); |
1184 | 0 | _mm256_storeu_ps(dst + 30 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[30]), _mm256_mul_ps(_5, tmp[32])), tmp[34])); |
1185 | 0 | _mm256_storeu_ps(dst + 31 * stride, _mm256_sub_ps(_mm256_add_ps(tmp[33], tmp[34]), _mm256_mul_ps(_4, _mm256_add_ps(tmp[31], tmp[32])))); |
1186 | 0 | _mm256_storeu_ps(dst + 32 * stride, _mm256_add_ps(_mm256_mul_ps(_4, _mm256_sub_ps(tmp[31], tmp[32])), _mm256_sub_ps(tmp[34], tmp[33]))); |
1187 | 0 | _mm256_storeu_ps(dst + 33 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[33], tmp[31])), _mm256_sub_ps(tmp[34], tmp[32]))); |
1188 | 0 | _mm256_storeu_ps(dst + 34 * stride, _mm256_add_ps(_mm256_mul_ps(_2, _mm256_sub_ps(tmp[31], tmp[33])), _mm256_sub_ps(tmp[34], tmp[32]))); |
1189 | 0 | _mm256_storeu_ps(dst + 35 * stride, _mm256_add_ps(_mm256_sub_ps(_mm256_mul_ps(_4, tmp[31]), _mm256_mul_ps(_5, tmp[33])), tmp[35])); |
1190 | 0 | } |
1191 | | |
1192 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcS, size_t srcC, __m256 dst[36]) |
1193 | 0 | { |
1194 | 0 | dst[0] = _mm256_loadu_ps(src + 0 * srcS + 0 * srcC); |
1195 | 0 | dst[1] = _mm256_loadu_ps(src + 0 * srcS + 1 * srcC); |
1196 | 0 | dst[2] = _mm256_loadu_ps(src + 0 * srcS + 2 * srcC); |
1197 | 0 | dst[3] = _mm256_loadu_ps(src + 0 * srcS + 3 * srcC); |
1198 | 0 | dst[4] = _mm256_loadu_ps(src + 0 * srcS + 4 * srcC); |
1199 | 0 | dst[5] = _mm256_loadu_ps(src + 0 * srcS + 5 * srcC); |
1200 | 0 | dst[6] = _mm256_loadu_ps(src + 1 * srcS + 0 * srcC); |
1201 | 0 | dst[7] = _mm256_loadu_ps(src + 1 * srcS + 1 * srcC); |
1202 | 0 | dst[8] = _mm256_loadu_ps(src + 1 * srcS + 2 * srcC); |
1203 | 0 | dst[9] = _mm256_loadu_ps(src + 1 * srcS + 3 * srcC); |
1204 | 0 | dst[10] = _mm256_loadu_ps(src + 1 * srcS + 4 * srcC); |
1205 | 0 | dst[11] = _mm256_loadu_ps(src + 1 * srcS + 5 * srcC); |
1206 | 0 | dst[12] = _mm256_loadu_ps(src + 2 * srcS + 0 * srcC); |
1207 | 0 | dst[13] = _mm256_loadu_ps(src + 2 * srcS + 1 * srcC); |
1208 | 0 | dst[14] = _mm256_loadu_ps(src + 2 * srcS + 2 * srcC); |
1209 | 0 | dst[15] = _mm256_loadu_ps(src + 2 * srcS + 3 * srcC); |
1210 | 0 | dst[16] = _mm256_loadu_ps(src + 2 * srcS + 4 * srcC); |
1211 | 0 | dst[17] = _mm256_loadu_ps(src + 2 * srcS + 5 * srcC); |
1212 | 0 | dst[18] = _mm256_loadu_ps(src + 3 * srcS + 0 * srcC); |
1213 | 0 | dst[19] = _mm256_loadu_ps(src + 3 * srcS + 1 * srcC); |
1214 | 0 | dst[20] = _mm256_loadu_ps(src + 3 * srcS + 2 * srcC); |
1215 | 0 | dst[21] = _mm256_loadu_ps(src + 3 * srcS + 3 * srcC); |
1216 | 0 | dst[22] = _mm256_loadu_ps(src + 3 * srcS + 4 * srcC); |
1217 | 0 | dst[23] = _mm256_loadu_ps(src + 3 * srcS + 5 * srcC); |
1218 | 0 | dst[24] = _mm256_loadu_ps(src + 4 * srcS + 0 * srcC); |
1219 | 0 | dst[25] = _mm256_loadu_ps(src + 4 * srcS + 1 * srcC); |
1220 | 0 | dst[26] = _mm256_loadu_ps(src + 4 * srcS + 2 * srcC); |
1221 | 0 | dst[27] = _mm256_loadu_ps(src + 4 * srcS + 3 * srcC); |
1222 | 0 | dst[28] = _mm256_loadu_ps(src + 4 * srcS + 4 * srcC); |
1223 | 0 | dst[29] = _mm256_loadu_ps(src + 4 * srcS + 5 * srcC); |
1224 | 0 | dst[30] = _mm256_loadu_ps(src + 5 * srcS + 0 * srcC); |
1225 | 0 | dst[31] = _mm256_loadu_ps(src + 5 * srcS + 1 * srcC); |
1226 | 0 | dst[32] = _mm256_loadu_ps(src + 5 * srcS + 2 * srcC); |
1227 | 0 | dst[33] = _mm256_loadu_ps(src + 5 * srcS + 3 * srcC); |
1228 | 0 | dst[34] = _mm256_loadu_ps(src + 5 * srcS + 4 * srcC); |
1229 | 0 | dst[35] = _mm256_loadu_ps(src + 5 * srcS + 5 * srcC); |
1230 | 0 | } |
1231 | | |
1232 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcW, size_t srcC, float * dst, size_t dstStride) |
1233 | 0 | { |
1234 | 0 | size_t srcS = srcW * srcC; |
1235 | 0 | size_t srcCF = AlignLo(srcC, F); |
1236 | 0 | for (size_t c = 0; c < srcCF; c += F) |
1237 | 0 | { |
1238 | 0 | __m256 tmp[36]; |
1239 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + c, srcS, srcC, tmp); |
1240 | 0 | WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + c, dstStride); |
1241 | 0 | } |
1242 | 0 | if (srcCF < srcC) |
1243 | 0 | { |
1244 | 0 | __m256 tmp[36]; |
1245 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + srcC - F, srcS, srcC, tmp); |
1246 | 0 | WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + srcC - F, dstStride); |
1247 | 0 | } |
1248 | 0 | } |
1249 | | |
1250 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcS, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, __m256 dst[36]) |
1251 | 0 | { |
1252 | 0 | for (size_t row = 0; row < rowB; ++row) |
1253 | 0 | { |
1254 | 0 | dst[0] = _mm256_setzero_ps(); |
1255 | 0 | dst[1] = _mm256_setzero_ps(); |
1256 | 0 | dst[2] = _mm256_setzero_ps(); |
1257 | 0 | dst[3] = _mm256_setzero_ps(); |
1258 | 0 | dst[4] = _mm256_setzero_ps(); |
1259 | 0 | dst[5] = _mm256_setzero_ps(); |
1260 | 0 | dst += 6; |
1261 | 0 | } |
1262 | 0 | for (size_t row = rowB; row < rowE; ++row) |
1263 | 0 | { |
1264 | 0 | for (size_t col = 0; col < colB; ++col) |
1265 | 0 | dst[col] = _mm256_setzero_ps(); |
1266 | 0 | for (size_t col = colB; col < colE; ++col) |
1267 | 0 | dst[col] = _mm256_loadu_ps(src + row * srcS + col * srcC); |
1268 | 0 | for (size_t col = colE; col < 6; ++col) |
1269 | 0 | dst[col] = _mm256_setzero_ps(); |
1270 | 0 | dst += 6; |
1271 | 0 | } |
1272 | 0 | for (size_t row = rowE; row < 6; ++row) |
1273 | 0 | { |
1274 | 0 | dst[0] = _mm256_setzero_ps(); |
1275 | 0 | dst[1] = _mm256_setzero_ps(); |
1276 | 0 | dst[2] = _mm256_setzero_ps(); |
1277 | 0 | dst[3] = _mm256_setzero_ps(); |
1278 | 0 | dst[4] = _mm256_setzero_ps(); |
1279 | 0 | dst[5] = _mm256_setzero_ps(); |
1280 | 0 | dst += 6; |
1281 | 0 | } |
1282 | 0 | } |
1283 | | |
1284 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetInput8t(const float * src, size_t srcW, size_t srcC, size_t rowB, size_t rowE, size_t colB, size_t colE, float * dst, size_t dstStride) |
1285 | 0 | { |
1286 | 0 | size_t srcS = srcW * srcC; |
1287 | 0 | size_t srcCF = AlignLo(srcC, F); |
1288 | 0 | for (size_t c = 0; c < srcCF; c += F) |
1289 | 0 | { |
1290 | 0 | __m256 tmp[36]; |
1291 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + c, srcS, srcC, rowB, rowE, colB, colE, tmp); |
1292 | 0 | WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + c, dstStride); |
1293 | 0 | } |
1294 | 0 | if (srcCF < srcC) |
1295 | 0 | { |
1296 | 0 | __m256 tmp[36]; |
1297 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + srcC - F, srcS, srcC, rowB, rowE, colB, colE, tmp); |
1298 | 0 | WinogradKernel3x3Block4x4SetInput8Store(tmp, dst + srcC - F, dstStride); |
1299 | 0 | } |
1300 | 0 | } |
1301 | | |
1302 | | void WinogradKernel3x3Block4x4SetInput(const float* src, size_t srcChannels, size_t srcHeight, size_t srcWidth, |
1303 | | size_t padY, size_t padX, size_t padH, size_t padW, float* dst, size_t dstStride, SimdBool trans) |
1304 | 0 | { |
1305 | 0 | if (trans ? (srcChannels < 8) : (srcHeight < 6 || srcWidth < 14)) |
1306 | 0 | { |
1307 | 0 | Sse41::WinogradKernel3x3Block4x4SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans); |
1308 | 0 | return; |
1309 | 0 | } |
1310 | 0 | if (trans) |
1311 | 0 | { |
1312 | 0 | assert(padY + padH <= 2 && padX + padW <= 2); |
1313 | 0 | size_t dstH = srcHeight - 2 + padY + padH; |
1314 | 0 | size_t dstW = srcWidth - 2 + padX + padW; |
1315 | 0 | size_t dstH4 = dstH / 4 * 4; |
1316 | 0 | size_t dstW4 = dstW / 4 * 4; |
1317 | 0 | size_t noseW = Simd::Min<size_t>(6, srcWidth + padX); |
1318 | 0 | size_t noseH = Simd::Min<size_t>(6, srcHeight + padY); |
1319 | 0 | size_t startY = padY ? 4 : 0; |
1320 | 0 | size_t startX = padX ? 4 : 0; |
1321 | 0 | if (padH && dstH == dstH4) |
1322 | 0 | dstH4 -= 4; |
1323 | 0 | if (padY) |
1324 | 0 | src -= srcWidth * srcChannels; |
1325 | 0 | if (padW && dstW == dstW4) |
1326 | 0 | dstW4 -= 4; |
1327 | 0 | if (padX) |
1328 | 0 | src -= srcChannels; |
1329 | 0 | size_t tailW = dstW - dstW4 + (padW ? 1 : 2); |
1330 | 0 | size_t tailH = dstH - dstH4 + (padH ? 1 : 2); |
1331 | 0 | size_t row = 0, col = 0; |
1332 | 0 | if (padY) |
1333 | 0 | { |
1334 | 0 | if (padX) |
1335 | 0 | WinogradKernel3x3Block4x4SetInput8t(src, srcWidth, srcChannels, 1, noseH, 1, noseW, dst, dstStride), dst += srcChannels; |
1336 | 0 | for (col = startX; col < dstW4; col += 4) |
1337 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, 6, dst, dstStride), dst += srcChannels; |
1338 | 0 | if (col < dstW) |
1339 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + col * srcChannels, srcWidth, srcChannels, 1, noseH, 0, tailW, dst, dstStride), dst += srcChannels; |
1340 | 0 | } |
1341 | 0 | for (row = startY; row < dstH4; row += 4) |
1342 | 0 | { |
1343 | 0 | if (padX) |
1344 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + row * srcWidth * srcChannels, srcWidth, srcChannels, 0, 6, 1, noseW, dst, dstStride), dst += srcChannels; |
1345 | 0 | for (col = startX; col < dstW4; col += 4) |
1346 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, dst, dstStride), dst += srcChannels; |
1347 | 0 | if (col < dstW) |
1348 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, 6, 0, tailW, dst, dstStride), dst += srcChannels; |
1349 | 0 | } |
1350 | 0 | if (row < dstH) |
1351 | 0 | { |
1352 | 0 | if (padX) |
1353 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + row * srcWidth* srcChannels, srcWidth, srcChannels, 0, tailH, 1, noseW, dst, dstStride), dst += srcChannels; |
1354 | 0 | for (col = startX; col < dstW4; col += 4) |
1355 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, 6, dst, dstStride), dst += srcChannels; |
1356 | 0 | if (col < dstW) |
1357 | 0 | WinogradKernel3x3Block4x4SetInput8t(src + (row * srcWidth + col) * srcChannels, srcWidth, srcChannels, 0, tailH, 0, tailW, dst, dstStride), dst += srcChannels; |
1358 | 0 | } |
1359 | 0 | } |
1360 | 0 | else |
1361 | 0 | { |
1362 | 0 | Base::WinogradKernel3x3Block4x4SetInput(src, srcChannels, srcHeight, srcWidth, padY, padX, padH, padW, dst, dstStride, trans); |
1363 | 0 | } |
1364 | 0 | } |
1365 | | |
1366 | | //----------------------------------------------------------------------- |
1367 | | |
1368 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetOutputLoad36(const float * src, size_t stride, __m256 dst[16]) |
1369 | 0 | { |
1370 | 0 | __m256 s[36]; |
1371 | 0 | s[0] = _mm256_loadu_ps(src + 0 * stride); |
1372 | 0 | s[1] = _mm256_loadu_ps(src + 1 * stride); |
1373 | 0 | s[2] = _mm256_loadu_ps(src + 2 * stride); |
1374 | 0 | s[3] = _mm256_loadu_ps(src + 3 * stride); |
1375 | 0 | s[4] = _mm256_loadu_ps(src + 4 * stride); |
1376 | 0 | s[5] = _mm256_loadu_ps(src + 5 * stride); |
1377 | 0 | s[6] = _mm256_loadu_ps(src + 6 * stride); |
1378 | 0 | s[7] = _mm256_loadu_ps(src + 7 * stride); |
1379 | 0 | s[8] = _mm256_loadu_ps(src + 8 * stride); |
1380 | 0 | s[9] = _mm256_loadu_ps(src + 9 * stride); |
1381 | 0 | s[10] = _mm256_loadu_ps(src + 10 * stride); |
1382 | 0 | s[11] = _mm256_loadu_ps(src + 11 * stride); |
1383 | 0 | s[12] = _mm256_loadu_ps(src + 12 * stride); |
1384 | 0 | s[13] = _mm256_loadu_ps(src + 13 * stride); |
1385 | 0 | s[14] = _mm256_loadu_ps(src + 14 * stride); |
1386 | 0 | s[15] = _mm256_loadu_ps(src + 15 * stride); |
1387 | 0 | s[16] = _mm256_loadu_ps(src + 16 * stride); |
1388 | 0 | s[17] = _mm256_loadu_ps(src + 17 * stride); |
1389 | 0 | s[18] = _mm256_loadu_ps(src + 18 * stride); |
1390 | 0 | s[19] = _mm256_loadu_ps(src + 19 * stride); |
1391 | 0 | s[20] = _mm256_loadu_ps(src + 20 * stride); |
1392 | 0 | s[21] = _mm256_loadu_ps(src + 21 * stride); |
1393 | 0 | s[22] = _mm256_loadu_ps(src + 22 * stride); |
1394 | 0 | s[23] = _mm256_loadu_ps(src + 23 * stride); |
1395 | 0 | s[24] = _mm256_loadu_ps(src + 24 * stride); |
1396 | 0 | s[25] = _mm256_loadu_ps(src + 25 * stride); |
1397 | 0 | s[26] = _mm256_loadu_ps(src + 26 * stride); |
1398 | 0 | s[27] = _mm256_loadu_ps(src + 27 * stride); |
1399 | 0 | s[28] = _mm256_loadu_ps(src + 28 * stride); |
1400 | 0 | s[29] = _mm256_loadu_ps(src + 29 * stride); |
1401 | 0 | s[30] = _mm256_loadu_ps(src + 30 * stride); |
1402 | 0 | s[31] = _mm256_loadu_ps(src + 31 * stride); |
1403 | 0 | s[32] = _mm256_loadu_ps(src + 32 * stride); |
1404 | 0 | s[33] = _mm256_loadu_ps(src + 33 * stride); |
1405 | 0 | s[34] = _mm256_loadu_ps(src + 34 * stride); |
1406 | 0 | s[35] = _mm256_loadu_ps(src + 35 * stride); |
1407 | |
|
1408 | 0 | __m256 _2 = _mm256_set1_ps(2.0f); |
1409 | 0 | __m256 _4 = _mm256_set1_ps(4.0f); |
1410 | 0 | __m256 _8 = _mm256_set1_ps(8.0f); |
1411 | 0 | __m256 t[24]; |
1412 | 0 | t[0] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[0], s[6]), _mm256_add_ps(s[12], s[18])), s[24]); |
1413 | 0 | t[1] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[1], s[7]), _mm256_add_ps(s[13], s[19])), s[25]); |
1414 | 0 | t[2] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[2], s[8]), _mm256_add_ps(s[14], s[20])), s[26]); |
1415 | 0 | t[3] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[3], s[9]), _mm256_add_ps(s[15], s[21])), s[27]); |
1416 | 0 | t[4] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[4], s[10]), _mm256_add_ps(s[16], s[22])), s[28]); |
1417 | 0 | t[5] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(s[5], s[11]), _mm256_add_ps(s[17], s[23])), s[29]); |
1418 | 0 | t[6] = _mm256_add_ps(_mm256_sub_ps(s[6], s[12]), _mm256_mul_ps(_2, _mm256_sub_ps(s[18], s[24]))); |
1419 | 0 | t[7] = _mm256_add_ps(_mm256_sub_ps(s[7], s[13]), _mm256_mul_ps(_2, _mm256_sub_ps(s[19], s[25]))); |
1420 | 0 | t[8] = _mm256_add_ps(_mm256_sub_ps(s[8], s[14]), _mm256_mul_ps(_2, _mm256_sub_ps(s[20], s[26]))); |
1421 | 0 | t[9] = _mm256_add_ps(_mm256_sub_ps(s[9], s[15]), _mm256_mul_ps(_2, _mm256_sub_ps(s[21], s[27]))); |
1422 | 0 | t[10] = _mm256_add_ps(_mm256_sub_ps(s[10], s[16]), _mm256_mul_ps(_2, _mm256_sub_ps(s[22], s[28]))); |
1423 | 0 | t[11] = _mm256_add_ps(_mm256_sub_ps(s[11], s[17]), _mm256_mul_ps(_2, _mm256_sub_ps(s[23], s[29]))); |
1424 | 0 | t[12] = _mm256_add_ps(_mm256_add_ps(s[6], s[12]), _mm256_mul_ps(_4, _mm256_add_ps(s[18], s[24]))); |
1425 | 0 | t[13] = _mm256_add_ps(_mm256_add_ps(s[7], s[13]), _mm256_mul_ps(_4, _mm256_add_ps(s[19], s[25]))); |
1426 | 0 | t[14] = _mm256_add_ps(_mm256_add_ps(s[8], s[14]), _mm256_mul_ps(_4, _mm256_add_ps(s[20], s[26]))); |
1427 | 0 | t[15] = _mm256_add_ps(_mm256_add_ps(s[9], s[15]), _mm256_mul_ps(_4, _mm256_add_ps(s[21], s[27]))); |
1428 | 0 | t[16] = _mm256_add_ps(_mm256_add_ps(s[10], s[16]), _mm256_mul_ps(_4, _mm256_add_ps(s[22], s[28]))); |
1429 | 0 | t[17] = _mm256_add_ps(_mm256_add_ps(s[11], s[17]), _mm256_mul_ps(_4, _mm256_add_ps(s[23], s[29]))); |
1430 | 0 | t[18] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[6], s[12]), _mm256_mul_ps(_8, _mm256_sub_ps(s[18], s[24]))), s[30]); |
1431 | 0 | t[19] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[7], s[13]), _mm256_mul_ps(_8, _mm256_sub_ps(s[19], s[25]))), s[31]); |
1432 | 0 | t[20] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[8], s[14]), _mm256_mul_ps(_8, _mm256_sub_ps(s[20], s[26]))), s[32]); |
1433 | 0 | t[21] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[9], s[15]), _mm256_mul_ps(_8, _mm256_sub_ps(s[21], s[27]))), s[33]); |
1434 | 0 | t[22] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[10], s[16]), _mm256_mul_ps(_8, _mm256_sub_ps(s[22], s[28]))), s[34]); |
1435 | 0 | t[23] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(s[11], s[17]), _mm256_mul_ps(_8, _mm256_sub_ps(s[23], s[29]))), s[35]); |
1436 | |
|
1437 | 0 | dst[0] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[0], t[1]), _mm256_add_ps(t[2], t[3])), t[4]); |
1438 | 0 | dst[1] = _mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_2, _mm256_sub_ps(t[3], t[4]))); |
1439 | 0 | dst[2] = _mm256_add_ps(_mm256_add_ps(t[1], t[2]), _mm256_mul_ps(_4, _mm256_add_ps(t[3], t[4]))); |
1440 | 0 | dst[3] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[1], t[2]), _mm256_mul_ps(_8, _mm256_sub_ps(t[3], t[4]))), t[5]); |
1441 | 0 | dst[4] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[6], t[7]), _mm256_add_ps(t[8], t[9])), t[10]); |
1442 | 0 | dst[5] = _mm256_add_ps(_mm256_sub_ps(t[7], t[8]), _mm256_mul_ps(_2, _mm256_sub_ps(t[9], t[10]))); |
1443 | 0 | dst[6] = _mm256_add_ps(_mm256_add_ps(t[7], t[8]), _mm256_mul_ps(_4, _mm256_add_ps(t[9], t[10]))); |
1444 | 0 | dst[7] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[7], t[8]), _mm256_mul_ps(_8, _mm256_sub_ps(t[9], t[10]))), t[11]); |
1445 | 0 | dst[8] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[12], t[13]), _mm256_add_ps(t[14], t[15])), t[16]); |
1446 | 0 | dst[9] = _mm256_add_ps(_mm256_sub_ps(t[13], t[14]), _mm256_mul_ps(_2, _mm256_sub_ps(t[15], t[16]))); |
1447 | 0 | dst[10] = _mm256_add_ps(_mm256_add_ps(t[13], t[14]), _mm256_mul_ps(_4, _mm256_add_ps(t[15], t[16]))); |
1448 | 0 | dst[11] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[13], t[14]), _mm256_mul_ps(_8, _mm256_sub_ps(t[15], t[16]))), t[17]); |
1449 | 0 | dst[12] = _mm256_add_ps(_mm256_add_ps(_mm256_add_ps(t[18], t[19]), _mm256_add_ps(t[20], t[21])), t[22]); |
1450 | 0 | dst[13] = _mm256_add_ps(_mm256_sub_ps(t[19], t[20]), _mm256_mul_ps(_2, _mm256_sub_ps(t[21], t[22]))); |
1451 | 0 | dst[14] = _mm256_add_ps(_mm256_add_ps(t[19], t[20]), _mm256_mul_ps(_4, _mm256_add_ps(t[21], t[22]))); |
1452 | 0 | dst[15] = _mm256_add_ps(_mm256_add_ps(_mm256_sub_ps(t[19], t[20]), _mm256_mul_ps(_8, _mm256_sub_ps(t[21], t[22]))), t[23]); |
1453 | 0 | } |
1454 | | |
1455 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetOutputStore16(const __m256 src[16], float * dst, size_t dstS, size_t dstC) |
1456 | 0 | { |
1457 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 0 * dstC, src[0]); |
1458 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 1 * dstC, src[1]); |
1459 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 2 * dstC, src[2]); |
1460 | 0 | _mm256_storeu_ps(dst + 0 * dstS + 3 * dstC, src[3]); |
1461 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 0 * dstC, src[4]); |
1462 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 1 * dstC, src[5]); |
1463 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 2 * dstC, src[6]); |
1464 | 0 | _mm256_storeu_ps(dst + 1 * dstS + 3 * dstC, src[7]); |
1465 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 0 * dstC, src[8]); |
1466 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 1 * dstC, src[9]); |
1467 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 2 * dstC, src[10]); |
1468 | 0 | _mm256_storeu_ps(dst + 2 * dstS + 3 * dstC, src[11]); |
1469 | 0 | _mm256_storeu_ps(dst + 3 * dstS + 0 * dstC, src[12]); |
1470 | 0 | _mm256_storeu_ps(dst + 3 * dstS + 1 * dstC, src[13]); |
1471 | 0 | _mm256_storeu_ps(dst + 3 * dstS + 2 * dstC, src[14]); |
1472 | 0 | _mm256_storeu_ps(dst + 3 * dstS + 3 * dstC, src[15]); |
1473 | 0 | } |
1474 | | |
1475 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC) |
1476 | 0 | { |
1477 | 0 | size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F); |
1478 | 0 | for (size_t d = 0; d < dstCF; d += F) |
1479 | 0 | { |
1480 | 0 | __m256 tmp[16]; |
1481 | 0 | WinogradKernel3x3Block4x4SetOutputLoad36(src + d, srcStride, tmp); |
1482 | 0 | WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + d, dstS, dstC); |
1483 | 0 | } |
1484 | 0 | if (dstCF < dstC) |
1485 | 0 | { |
1486 | 0 | __m256 tmp[16]; |
1487 | 0 | WinogradKernel3x3Block4x4SetOutputLoad36(src + dstC - F, srcStride, tmp); |
1488 | 0 | WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + dstC - F, dstS, dstC); |
1489 | 0 | } |
1490 | 0 | } |
1491 | | |
1492 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetOutputStore16(const __m256 src[16], float * dst, size_t dstS, size_t dstC, size_t rowE, size_t colE) |
1493 | 0 | { |
1494 | 0 | for (size_t row = 0; row < rowE; ++row) |
1495 | 0 | for (size_t col = 0; col < colE; ++col) |
1496 | 0 | _mm256_storeu_ps(dst + row * dstS + col * dstC, src[row * 4 + col]); |
1497 | 0 | } |
1498 | | |
1499 | | SIMD_INLINE void WinogradKernel3x3Block4x4SetOutput8t(const float * src, size_t srcStride, float * dst, size_t dstW, size_t dstC, size_t rowE, size_t colE) |
1500 | 0 | { |
1501 | 0 | size_t dstS = dstW * dstC, dstCF = AlignLo(dstC, F); |
1502 | 0 | for (size_t d = 0; d < dstCF; d += F) |
1503 | 0 | { |
1504 | 0 | __m256 tmp[16]; |
1505 | 0 | WinogradKernel3x3Block4x4SetOutputLoad36(src + d, srcStride, tmp); |
1506 | 0 | WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + d, dstS, dstC, rowE, colE); |
1507 | 0 | } |
1508 | 0 | if (dstCF < dstC) |
1509 | 0 | { |
1510 | 0 | __m256 tmp[16]; |
1511 | 0 | WinogradKernel3x3Block4x4SetOutputLoad36(src + dstC - F, srcStride, tmp); |
1512 | 0 | WinogradKernel3x3Block4x4SetOutputStore16(tmp, dst + dstC - F, dstS, dstC, rowE, colE); |
1513 | 0 | } |
1514 | 0 | } |
1515 | | |
1516 | | void WinogradKernel3x3Block4x4SetOutput(const float * src, size_t srcStride, float * dst, size_t dstChannels, size_t dstHeight, size_t dstWidth, SimdBool trans) |
1517 | 0 | { |
1518 | 0 | if (trans ? (dstChannels < 8) : (dstHeight < 4 || dstWidth < 16)) |
1519 | 0 | { |
1520 | 0 | Sse41::WinogradKernel3x3Block4x4SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans); |
1521 | 0 | return; |
1522 | 0 | } |
1523 | 0 | size_t tileH = (dstHeight + 3) / 4; |
1524 | 0 | size_t tileW = (dstWidth + 3) / 4; |
1525 | 0 | size_t dstH4 = AlignLo(dstHeight, 4); |
1526 | 0 | size_t dstW4 = AlignLo(dstWidth, 4); |
1527 | 0 | if (trans) |
1528 | 0 | { |
1529 | 0 | size_t row, col; |
1530 | 0 | for (row = 0; row < dstH4; row += 4) |
1531 | 0 | { |
1532 | 0 | for (col = 0; col < dstW4; col += 4) |
1533 | 0 | WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels), src += dstChannels; |
1534 | 0 | if (col < dstWidth) |
1535 | 0 | WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, 4, dstWidth - col), src += dstChannels; |
1536 | 0 | } |
1537 | 0 | if (row < dstHeight) |
1538 | 0 | { |
1539 | 0 | for (col = 0; col < dstW4; col += 4) |
1540 | 0 | WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, 4), src += dstChannels; |
1541 | 0 | if (col < dstWidth) |
1542 | 0 | WinogradKernel3x3Block4x4SetOutput8t(src, srcStride, dst + (row * dstWidth + col)*dstChannels, dstWidth, dstChannels, dstHeight - row, dstWidth - col), src += dstChannels; |
1543 | 0 | } |
1544 | 0 | } |
1545 | 0 | else |
1546 | 0 | { |
1547 | 0 | Base::WinogradKernel3x3Block4x4SetOutput(src, srcStride, dst, dstChannels, dstHeight, dstWidth, trans); |
1548 | 0 | } |
1549 | 0 | } |
1550 | | } |
1551 | | #endif |
1552 | | } |