/src/Simd/src/Simd/SimdAvx2SynetQuantizedMergedConvolutionDepthwise.cpp
Line | Count | Source |
1 | | /* |
2 | | * Simd Library (http://ermig1979.github.io/Simd). |
3 | | * |
4 | | * Copyright (c) 2011-2025 Yermalayeu Ihar. |
5 | | * |
6 | | * Permission is hereby granted, free of charge, to any person obtaining a copy |
7 | | * of this software and associated documentation files (the "Software"), to deal |
8 | | * in the Software without restriction, including without limitation the rights |
9 | | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
10 | | * copies of the Software, and to permit persons to whom the Software is |
11 | | * furnished to do so, subject to the following conditions: |
12 | | * |
13 | | * The above copyright notice and this permission notice shall be included in |
14 | | * all copies or substantial portions of the Software. |
15 | | * |
16 | | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
19 | | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
20 | | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
21 | | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
22 | | * SOFTWARE. |
23 | | */ |
24 | | #include "Simd/SimdSynetQuantizedMergedConvolution.h" |
25 | | #include "Simd/SimdSynetQuantizedActivation.h" |
26 | | #include "Simd/SimdSynetQuantizeLinear.h" |
27 | | #include "Simd/SimdSynetConvolution8iCommon.h" |
28 | | #include "Simd/SimdSynet.h" |
29 | | #include "Simd/SimdMath.h" |
30 | | #include "Simd/SimdBase.h" |
31 | | #include "Simd/SimdCpu.h" |
32 | | #include "Simd/SimdLog.h" |
33 | | |
34 | | namespace Simd |
35 | | { |
36 | | #if defined(SIMD_AVX2_ENABLE) && defined(SIMD_SYNET_ENABLE) |
37 | | namespace Avx2 |
38 | | { |
39 | | typedef Base::SynetQuantizedMergedConvolution::AlgParam AlgParam; |
40 | | |
41 | | //------------------------------------------------------------------------------------------------- |
42 | | |
43 | | void QuantizedMergedConvolutionDepthwisePreprocess(const uint8_t* src, const uint8_t* zero, const ConvParam& p, const AlgParam& a, size_t maC, size_t dyBeg, size_t dyEnd, uint8_t* dst) |
44 | 0 | { |
45 | 0 | __m256i _zero = _mm256_set1_epi16(zero[0]); |
46 | 0 | size_t byMask = a.dbH - 1, byPad = p.kernelY - 1, byBeg = dyBeg ? dyBeg * p.strideY + byPad : 0, byEnd = dyEnd * p.strideY + byPad; |
47 | 0 | if (a.dsB) |
48 | 0 | { |
49 | 0 | size_t syMask = a.dsH - 1, sC = a.dsH * p.srcW, sR = p.srcW * F; |
50 | 0 | size_t bW = a.dbW * 2, bR = a.dbW * a.maC, xPad = p.padX * 2, wPad = p.padW * 2; |
51 | 0 | for (size_t c = 0; c < maC; c += F) |
52 | 0 | { |
53 | 0 | for (size_t by = byBeg; by < byEnd; by += 2) |
54 | 0 | { |
55 | 0 | int16_t* pd = (int16_t*)dst + (by & byMask) * bR; |
56 | 0 | size_t sy = by - p.padY; |
57 | 0 | const uint8_t* ps0 = (sy + 0) < p.srcH ? src + ((sy + 0) & syMask) * sR : zero; |
58 | 0 | const uint8_t* ps1 = (sy + 1) < p.srcH ? src + ((sy + 1) & syMask) * sR : zero; |
59 | 0 | if (xPad) |
60 | 0 | { |
61 | 0 | for (size_t x = 0; x < xPad; x += 2, pd += DF) |
62 | 0 | _mm256_storeu_si256((__m256i*)pd, _zero); |
63 | 0 | } |
64 | 0 | for (size_t sx = 0; sx < sR; sx += F, pd += DF) |
65 | 0 | { |
66 | 0 | __m256i s0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)(ps0 + sx))); |
67 | 0 | __m256i s1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)(ps1 + sx))); |
68 | 0 | _mm256_storeu_si256((__m256i*)(pd), _mm256_or_si256(s0, _mm256_slli_epi32(s1, 16))); |
69 | 0 | } |
70 | 0 | if (wPad) |
71 | 0 | { |
72 | 0 | for (size_t x = 0; x < wPad; x += 2, pd += DF) |
73 | 0 | _mm256_storeu_si256((__m256i*)pd, _zero); |
74 | 0 | } |
75 | 0 | } |
76 | 0 | src += sC * F; |
77 | 0 | dst += bW * DF; |
78 | 0 | } |
79 | 0 | } |
80 | 0 | else |
81 | 0 | { |
82 | 0 | size_t sR = p.srcW * p.srcC, sC = p.srcC; |
83 | 0 | size_t bW = a.dbW * 2, bC = a.maC, xPad = p.padX * 2, wPad = p.padW * 2, bR = a.dbW * a.maC; |
84 | 0 | for (size_t by = byBeg; by < byEnd; by += 2) |
85 | 0 | { |
86 | 0 | int16_t* pd = (int16_t*)dst + (by & byMask) * bR; |
87 | 0 | size_t sy = by - p.padY; |
88 | 0 | const uint8_t* ps0 = (sy + 0) < p.srcH ? src + (sy + 0) * sR : zero; |
89 | 0 | const uint8_t* ps1 = (sy + 1) < p.srcH ? src + (sy + 1) * sR : zero; |
90 | 0 | if (xPad) |
91 | 0 | { |
92 | 0 | for (size_t x = 0; x < xPad; x += 2, pd += DF) |
93 | 0 | for (size_t c = 0; c < bC; c += F) |
94 | 0 | _mm256_storeu_si256((__m256i*)(pd + c * bW), _zero); |
95 | 0 | } |
96 | 0 | for (size_t sx = 0; sx < p.srcW; sx++, pd += DF) |
97 | 0 | { |
98 | 0 | for (size_t sc = 0; sc < maC; sc += F) |
99 | 0 | { |
100 | 0 | __m256i s0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)(ps0 + sc))); |
101 | 0 | __m256i s1 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((__m128i*)(ps1 + sc))); |
102 | 0 | _mm256_storeu_si256((__m256i*)(pd + sc * bW), _mm256_or_si256(s0, _mm256_slli_epi32(s1, 16))); |
103 | 0 | } |
104 | 0 | ps0 += sC; |
105 | 0 | ps1 += sC; |
106 | 0 | } |
107 | 0 | if (wPad) |
108 | 0 | { |
109 | 0 | for (size_t x = 0; x < wPad; x += 2, pd += DF) |
110 | 0 | for (size_t c = 0; c < bC; c += F) |
111 | 0 | _mm256_storeu_si256((__m256i*)(pd + c * bW), _zero); |
112 | 0 | } |
113 | 0 | } |
114 | 0 | } |
115 | 0 | } |
116 | | |
117 | | //------------------------------------------------------------------------------------------------- |
118 | | |
119 | | SIMD_INLINE void Madd2(__m256i& i32, __m256i u8, __m256i i8) |
120 | | { |
121 | | i32 = _mm256_add_epi32(i32, _mm256_madd_epi16(u8, i8)); |
122 | | } |
123 | | |
124 | | SIMD_INLINE void Save1(uint8_t* dst, __m256i sum, const __m256i& bias, const __m256& norm, const __m256i& zero) |
125 | 0 | { |
126 | 0 | QuntizedTerm8i<Term8iLast8u>::template Save<0>(dst, (int32_t*)NULL, sum, &bias, &norm, zero); |
127 | 0 | } |
128 | | |
129 | | SIMD_INLINE void Save1(uint8_t* dst, __m256i sum, const __m256i& bias, const __m256& norm, const __m256i& zero, size_t tail) |
130 | 0 | { |
131 | 0 | QuntizedTerm8i<Term8iLast8u>::template Save<0>(dst, (int32_t*)NULL, sum, &bias, &norm, zero, tail); |
132 | 0 | } |
133 | | |
134 | | //------------------------------------------------------------------------------------------------ |
135 | | |
136 | | void QuantizedMergedConvolutionDepthwiseConvolutionAny(const uint8_t* src8, const ConvParam& p, const AlgParam& a, size_t maC, size_t dyBeg, size_t dyEnd, |
137 | | const int8_t* weight8, const int32_t* bias, const float* norm, int32_t zero, uint8_t* dst) |
138 | 0 | { |
139 | 0 | const int16_t* src = (int16_t*)src8, *weight = (int16_t*)weight8; |
140 | 0 | __m256 _norm; |
141 | 0 | __m256i _zero = _mm256_set1_epi32(zero), _bias; |
142 | 0 | __m256i d00, d10, d20, d30, d01, d11, d21, d31, w0, w1, s0; |
143 | 0 | size_t sC = maC, sCF = AlignLo(sC, F), kY = p.kernelY, kX = p.kernelX, sY = p.strideY, sX = p.strideX, dX = sX * DF, dW = a.dwStep; |
144 | 0 | size_t byMask = a.dbH - 1, bW = a.dbW * 2, bR = a.dbW * a.maC, dstW2 = AlignLo(p.dstW, 2), dstW4 = AlignLo(p.dstW, 4), dD = a.ddB ? a.maC : p.dstC; |
145 | 0 | size_t dyEnd2 = dyBeg + (sY == 1 ? AlignLo(dyEnd - dyBeg, 2) : 0), sizeW = a.dwSize, dyD = p.dstW * dD; |
146 | 0 | if(a.ddB) |
147 | 0 | dst += (dyBeg % a.ddStep) * p.dstW * dD; |
148 | 0 | else |
149 | 0 | dst += dyBeg * p.dstW * dD; |
150 | 0 | size_t dy = dyBeg; |
151 | 0 | for (; dy < dyEnd2; dy += 2) |
152 | 0 | { |
153 | 0 | size_t sc = 0, sy = dy * sY; |
154 | 0 | for (; sc < sCF; sc += F) |
155 | 0 | { |
156 | 0 | uint8_t* pd0 = dst + sc, * pd1 = pd0 + dyD; |
157 | 0 | const int16_t* ps0 = src + sc * bW; |
158 | 0 | _bias = _mm256_loadu_si256((__m256i*)(bias + sc)); |
159 | 0 | _norm = _mm256_loadu_ps(norm + sc); |
160 | 0 | size_t dx = 0; |
161 | 0 | for (; dx < dstW4; dx += 4, ps0 += 4 * dX) |
162 | 0 | { |
163 | 0 | d00 = _mm256_setzero_si256(); |
164 | 0 | d10 = _mm256_setzero_si256(); |
165 | 0 | d20 = _mm256_setzero_si256(); |
166 | 0 | d30 = _mm256_setzero_si256(); |
167 | 0 | d01 = _mm256_setzero_si256(); |
168 | 0 | d11 = _mm256_setzero_si256(); |
169 | 0 | d21 = _mm256_setzero_si256(); |
170 | 0 | d31 = _mm256_setzero_si256(); |
171 | 0 | const int16_t* pw0 = weight + sc * dW, * pw1 = pw0 + sizeW; |
172 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
173 | 0 | { |
174 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
175 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw0 += DF, pw1 += DF) |
176 | 0 | { |
177 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw0); |
178 | 0 | w1 = _mm256_loadu_si256((__m256i*)pw1); |
179 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 0 * dX)); |
180 | 0 | Madd2(d00, s0, w0); |
181 | 0 | Madd2(d01, s0, w1); |
182 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 1 * dX)); |
183 | 0 | Madd2(d10, s0, w0); |
184 | 0 | Madd2(d11, s0, w1); |
185 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 2 * dX)); |
186 | 0 | Madd2(d20, s0, w0); |
187 | 0 | Madd2(d21, s0, w1); |
188 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 3 * dX)); |
189 | 0 | Madd2(d30, s0, w0); |
190 | 0 | Madd2(d31, s0, w1); |
191 | 0 | } |
192 | 0 | } |
193 | 0 | Save1(pd0 + 0 * dD, d00, _bias, _norm, _zero); |
194 | 0 | Save1(pd0 + 1 * dD, d10, _bias, _norm, _zero); |
195 | 0 | Save1(pd0 + 2 * dD, d20, _bias, _norm, _zero); |
196 | 0 | Save1(pd0 + 3 * dD, d30, _bias, _norm, _zero); |
197 | 0 | Save1(pd1 + 0 * dD, d01, _bias, _norm, _zero); |
198 | 0 | Save1(pd1 + 1 * dD, d11, _bias, _norm, _zero); |
199 | 0 | Save1(pd1 + 2 * dD, d21, _bias, _norm, _zero); |
200 | 0 | Save1(pd1 + 3 * dD, d31, _bias, _norm, _zero); |
201 | 0 | pd0 += 4 * dD; |
202 | 0 | pd1 += 4 * dD; |
203 | 0 | } |
204 | 0 | for (; dx < dstW2; dx += 2, ps0 += 2 * dX) |
205 | 0 | { |
206 | 0 | d00 = _mm256_setzero_si256(); |
207 | 0 | d10 = _mm256_setzero_si256(); |
208 | 0 | d01 = _mm256_setzero_si256(); |
209 | 0 | d11 = _mm256_setzero_si256(); |
210 | 0 | const int16_t* pw0 = weight + sc * dW, * pw1 = pw0 + sizeW; |
211 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
212 | 0 | { |
213 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
214 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw0 += DF, pw1 += DF) |
215 | 0 | { |
216 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw0); |
217 | 0 | w1 = _mm256_loadu_si256((__m256i*)pw1); |
218 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 0 * dX)); |
219 | 0 | Madd2(d00, s0, w0); |
220 | 0 | Madd2(d01, s0, w1); |
221 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 1 * dX)); |
222 | 0 | Madd2(d10, s0, w0); |
223 | 0 | Madd2(d11, s0, w1); |
224 | 0 | } |
225 | 0 | } |
226 | 0 | Save1(pd0 + 0 * dD, d00, _bias, _norm, _zero); |
227 | 0 | Save1(pd0 + 1 * dD, d10, _bias, _norm, _zero); |
228 | 0 | Save1(pd1 + 0 * dD, d01, _bias, _norm, _zero); |
229 | 0 | Save1(pd1 + 1 * dD, d11, _bias, _norm, _zero); |
230 | 0 | pd0 += 2 * dD; |
231 | 0 | pd1 += 2 * dD; |
232 | 0 | } |
233 | 0 | for (; dx < p.dstW; ++dx, ps0 += dX) |
234 | 0 | { |
235 | 0 | d00 = _mm256_setzero_si256(); |
236 | 0 | d01 = _mm256_setzero_si256(); |
237 | 0 | const int16_t* pw0 = weight + sc * dW, * pw1 = pw0 + sizeW; |
238 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
239 | 0 | { |
240 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
241 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw0 += DF, pw1 += DF) |
242 | 0 | { |
243 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw0); |
244 | 0 | w1 = _mm256_loadu_si256((__m256i*)pw1); |
245 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 0 * dX)); |
246 | 0 | Madd2(d00, s0, w0); |
247 | 0 | Madd2(d01, s0, w1); |
248 | 0 | } |
249 | 0 | } |
250 | 0 | Save1(pd0 + 0 * dD, d00, _bias, _norm, _zero); |
251 | 0 | Save1(pd1 + 0 * dD, d01, _bias, _norm, _zero); |
252 | 0 | pd0 += dD; |
253 | 0 | pd1 += dD; |
254 | 0 | } |
255 | 0 | } |
256 | 0 | for (; sc < sC; sc += F) |
257 | 0 | { |
258 | 0 | uint8_t* pd0 = dst + sc, * pd1 = pd0 + dyD; |
259 | 0 | const int16_t* ps0 = src + sc * bW; |
260 | 0 | _bias = _mm256_loadu_si256((__m256i*)(bias + sc)); |
261 | 0 | _norm = _mm256_loadu_ps(norm + sc); |
262 | 0 | size_t dx = 0, tail = sC - sCF; |
263 | 0 | for (; dx < p.dstW; ++dx, ps0 += dX) |
264 | 0 | { |
265 | 0 | d00 = _mm256_setzero_si256(); |
266 | 0 | d01 = _mm256_setzero_si256(); |
267 | 0 | const int16_t* pw0 = weight + sc * dW, * pw1 = pw0 + sizeW; |
268 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
269 | 0 | { |
270 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
271 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw0 += DF, pw1 += DF) |
272 | 0 | { |
273 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw0); |
274 | 0 | w1 = _mm256_loadu_si256((__m256i*)pw1); |
275 | 0 | s0 = _mm256_loadu_si256((__m256i*)(ps + 0 * dX)); |
276 | 0 | Madd2(d00, s0, w0); |
277 | 0 | Madd2(d01, s0, w1); |
278 | 0 | } |
279 | 0 | } |
280 | 0 | Save1(pd0 + 0 * dD, d00, _bias, _norm, _zero, tail); |
281 | 0 | Save1(pd1 + 0 * dD, d01, _bias, _norm, _zero, tail); |
282 | 0 | pd0 += dD; |
283 | 0 | pd1 += dD; |
284 | 0 | } |
285 | 0 | } |
286 | 0 | dst += p.dstW * 2 * dD; |
287 | 0 | } |
288 | 0 | for (; dy < dyEnd; ++dy) |
289 | 0 | { |
290 | 0 | size_t sc = 0, sy = dy * sY; |
291 | 0 | for (; sc < sCF; sc += F) |
292 | 0 | { |
293 | 0 | uint8_t* pd = dst + sc; |
294 | 0 | const int16_t* ps0 = src + sc * bW; |
295 | 0 | _bias = _mm256_loadu_si256((__m256i*)(bias + sc)); |
296 | 0 | _norm = _mm256_loadu_ps(norm + sc); |
297 | 0 | size_t dx = 0; |
298 | 0 | for (; dx < dstW4; dx += 4, ps0 += 4 * dX) |
299 | 0 | { |
300 | 0 | d00 = _mm256_setzero_si256(); |
301 | 0 | d10 = _mm256_setzero_si256(); |
302 | 0 | d20 = _mm256_setzero_si256(); |
303 | 0 | d30 = _mm256_setzero_si256(); |
304 | 0 | const int16_t* pw = weight + sc * dW; |
305 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
306 | 0 | { |
307 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
308 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw += DF) |
309 | 0 | { |
310 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw); |
311 | 0 | Madd2(d00, _mm256_loadu_si256((__m256i*)(ps + 0 * dX)), w0); |
312 | 0 | Madd2(d10, _mm256_loadu_si256((__m256i*)(ps + 1 * dX)), w0); |
313 | 0 | Madd2(d20, _mm256_loadu_si256((__m256i*)(ps + 2 * dX)), w0); |
314 | 0 | Madd2(d30, _mm256_loadu_si256((__m256i*)(ps + 3 * dX)), w0); |
315 | 0 | } |
316 | 0 | } |
317 | 0 | Save1(pd + 0 * dD, d00, _bias, _norm, _zero); |
318 | 0 | Save1(pd + 1 * dD, d10, _bias, _norm, _zero); |
319 | 0 | Save1(pd + 2 * dD, d20, _bias, _norm, _zero); |
320 | 0 | Save1(pd + 3 * dD, d30, _bias, _norm, _zero); |
321 | 0 | pd += 4 * dD; |
322 | 0 | } |
323 | 0 | for (; dx < dstW2; dx += 2, ps0 += 2 * dX) |
324 | 0 | { |
325 | 0 | d00 = _mm256_setzero_si256(); |
326 | 0 | d10 = _mm256_setzero_si256(); |
327 | 0 | const int16_t* pw = weight + sc * dW; |
328 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
329 | 0 | { |
330 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
331 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw += DF) |
332 | 0 | { |
333 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw); |
334 | 0 | Madd2(d00, _mm256_loadu_si256((__m256i*)(ps + 0 * dX)), w0); |
335 | 0 | Madd2(d10, _mm256_loadu_si256((__m256i*)(ps + 1 * dX)), w0); |
336 | 0 | } |
337 | 0 | } |
338 | 0 | Save1(pd + 0 * dD, d00, _bias, _norm, _zero); |
339 | 0 | Save1(pd + 1 * dD, d10, _bias, _norm, _zero); |
340 | 0 | pd += 2 * dD; |
341 | 0 | } |
342 | 0 | for (; dx < p.dstW; ++dx, ps0 += dX) |
343 | 0 | { |
344 | 0 | d00 = _mm256_setzero_si256(); |
345 | 0 | const int16_t* pw = weight + sc * dW; |
346 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
347 | 0 | { |
348 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
349 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw += DF) |
350 | 0 | { |
351 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw); |
352 | 0 | Madd2(d00, _mm256_loadu_si256((__m256i*)ps), w0); |
353 | 0 | } |
354 | 0 | } |
355 | 0 | Save1(pd, d00, _bias, _norm, _zero); |
356 | 0 | pd += dD; |
357 | 0 | } |
358 | 0 | } |
359 | 0 | for (; sc < sC; sc += F) |
360 | 0 | { |
361 | 0 | uint8_t* pd = dst + sc; |
362 | 0 | const int16_t* ps0 = src + sc * bW; |
363 | 0 | _bias = _mm256_loadu_si256((__m256i*)(bias + sc)); |
364 | 0 | _norm = _mm256_loadu_ps(norm + sc); |
365 | 0 | size_t dx = 0, tail = sC - sCF; |
366 | 0 | for (; dx < p.dstW; ++dx, ps0 += dX) |
367 | 0 | { |
368 | 0 | d00 = _mm256_setzero_si256(); |
369 | 0 | const int16_t* pw = weight + sc * dW; |
370 | 0 | for (size_t ky = 0; ky < kY; ky += 2) |
371 | 0 | { |
372 | 0 | const int16_t* ps = ps0 + ((sy + ky) & byMask) * bR; |
373 | 0 | for (size_t kx = 0; kx < kX; ++kx, ps += DF, pw += DF) |
374 | 0 | { |
375 | 0 | w0 = _mm256_loadu_si256((__m256i*)pw); |
376 | 0 | Madd2(d00, _mm256_loadu_si256((__m256i*)ps), w0); |
377 | 0 | } |
378 | 0 | } |
379 | 0 | Save1(pd, d00, _bias, _norm, _zero, tail); |
380 | 0 | pd += dD; |
381 | 0 | } |
382 | 0 | } |
383 | 0 | dst += p.dstW * dD; |
384 | 0 | } |
385 | 0 | } |
386 | | |
387 | | //------------------------------------------------------------------------------------------------- |
388 | | |
389 | | void QuantizedMergedConvolutionDepthwiseConvolution3x3(const uint8_t* src8, const ConvParam& p, const AlgParam& a, size_t maC, size_t dyBeg, size_t dyEnd, |
390 | | const int8_t* weight8, const int32_t* bias, const float* norm, int32_t zero, uint8_t* dst) |
391 | 0 | { |
392 | 0 | const int16_t* src = (int16_t*)src8, * weight = (int16_t*)weight8; |
393 | 0 | __m256 _norm; |
394 | 0 | __m256i _zero = _mm256_set1_epi32(zero), _bias; |
395 | 0 | __m256i d00, d10, w03, w14, w25, s0; |
396 | 0 | size_t sC = maC, sCF = AlignLo(sC, F), kY = p.kernelY, kX = p.kernelX, sY = p.strideY, sX = p.strideX, dX = sX * DF, dW = a.dwStep; |
397 | 0 | size_t byMask = a.dbH - 1, bW = a.dbW * 2, bR = a.dbW * a.maC, dstW2 = (sX == 1 ? AlignLo(p.dstW, 2) : 0), dD = a.ddB ? a.maC : p.dstC; |
398 | 0 | size_t dyEnd2 = dyBeg + (sY == 1 ? AlignLo(dyEnd - dyBeg, 2) : 0), sizeW = a.dwSize, dyD = p.dstW * dD; |
399 | 0 | if (a.ddB) |
400 | 0 | dst += (dyBeg % a.ddStep) * p.dstW * dD; |
401 | 0 | else |
402 | 0 | dst += dyBeg * p.dstW * dD; |
403 | 0 | size_t dy = dyBeg; |
404 | 0 | for (; dy < dyEnd2; dy += 2) |
405 | 0 | { |
406 | 0 | __m256i d01, w36, w47, w58; |
407 | 0 | size_t sc = 0, sy = dy * sY; |
408 | 0 | for (; sc < sC; sc += F) |
409 | 0 | { |
410 | 0 | uint8_t* pd0 = dst + sc, * pd1 = pd0 + dyD; |
411 | 0 | const int16_t* ps0 = src + ((sy + 0) & byMask) * bR + sc * bW; |
412 | 0 | const int16_t* ps2 = src + ((sy + 2) & byMask) * bR + sc * bW; |
413 | 0 | const int16_t* pw0 = weight + sc * dW, * pw1 = pw0 + sizeW; |
414 | 0 | _bias = _mm256_loadu_si256((__m256i*)(bias + sc)); |
415 | 0 | _norm = _mm256_loadu_ps(norm + sc); |
416 | 0 | w03 = _mm256_loadu_si256((__m256i*)pw0 + 0); |
417 | 0 | w14 = _mm256_loadu_si256((__m256i*)pw0 + 1); |
418 | 0 | w25 = _mm256_loadu_si256((__m256i*)pw0 + 2); |
419 | 0 | w36 = _mm256_loadu_si256((__m256i*)pw1 + 3); |
420 | 0 | w47 = _mm256_loadu_si256((__m256i*)pw1 + 4); |
421 | 0 | w58 = _mm256_loadu_si256((__m256i*)pw1 + 5); |
422 | 0 | if (sc < sCF) |
423 | 0 | { |
424 | 0 | size_t dx = 0; |
425 | 0 | for (; dx < p.dstW; ++dx, ps0 += dX, ps2 += dX) |
426 | 0 | { |
427 | 0 | d00 = _mm256_setzero_si256(); |
428 | 0 | d01 = _mm256_setzero_si256(); |
429 | |
|
430 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 0); |
431 | 0 | Madd2(d00, s0, w03); |
432 | 0 | Madd2(d01, s0, _mm256_slli_epi32(w03, 16)); |
433 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 1); |
434 | 0 | Madd2(d00, s0, w14); |
435 | 0 | Madd2(d01, s0, _mm256_slli_epi32(w14, 16)); |
436 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 2); |
437 | 0 | Madd2(d00, s0, w25); |
438 | 0 | Madd2(d01, s0, _mm256_slli_epi32(w25, 16)); |
439 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 0); |
440 | 0 | Madd2(d00, s0, _mm256_srli_epi32(w36, 16)); |
441 | 0 | Madd2(d01, s0, w36); |
442 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 1); |
443 | 0 | Madd2(d00, s0, _mm256_srli_epi32(w47, 16)); |
444 | 0 | Madd2(d01, s0, w47); |
445 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 2); |
446 | 0 | Madd2(d00, s0, _mm256_srli_epi32(w58, 16)); |
447 | 0 | Madd2(d01, s0, w58); |
448 | |
|
449 | 0 | Save1(pd0, d00, _bias, _norm, _zero); |
450 | 0 | Save1(pd1, d01, _bias, _norm, _zero); |
451 | 0 | pd0 += dD; |
452 | 0 | pd1 += dD; |
453 | 0 | } |
454 | 0 | } |
455 | 0 | else |
456 | 0 | { |
457 | 0 | size_t tail = sC - sCF; |
458 | 0 | for (size_t dx = 0; dx < p.dstW; ++dx, ps0 += dX, ps2 += dX) |
459 | 0 | { |
460 | 0 | d00 = _mm256_setzero_si256(); |
461 | 0 | d01 = _mm256_setzero_si256(); |
462 | |
|
463 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 0); |
464 | 0 | Madd2(d00, s0, w03); |
465 | 0 | Madd2(d01, s0, _mm256_slli_epi32(w03, 16)); |
466 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 1); |
467 | 0 | Madd2(d00, s0, w14); |
468 | 0 | Madd2(d01, s0, _mm256_slli_epi32(w14, 16)); |
469 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 2); |
470 | 0 | Madd2(d00, s0, w25); |
471 | 0 | Madd2(d01, s0, _mm256_slli_epi32(w25, 16)); |
472 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 0); |
473 | 0 | Madd2(d00, s0, _mm256_srli_epi32(w36, 16)); |
474 | 0 | Madd2(d01, s0, w36); |
475 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 1); |
476 | 0 | Madd2(d00, s0, _mm256_srli_epi32(w47, 16)); |
477 | 0 | Madd2(d01, s0, w47); |
478 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 2); |
479 | 0 | Madd2(d00, s0, _mm256_srli_epi32(w58, 16)); |
480 | 0 | Madd2(d01, s0, w58); |
481 | |
|
482 | 0 | Save1(pd0, d00, _bias, _norm, _zero, tail); |
483 | 0 | Save1(pd1, d01, _bias, _norm, _zero, tail); |
484 | 0 | pd0 += dD; |
485 | 0 | pd1 += dD; |
486 | 0 | } |
487 | 0 | } |
488 | 0 | } |
489 | 0 | dst += p.dstW * dD * 2; |
490 | 0 | } |
491 | 0 | for (; dy < dyEnd; ++dy) |
492 | 0 | { |
493 | 0 | __m256i w6, w7, w8; |
494 | 0 | size_t sc = 0, sy = dy * sY; |
495 | 0 | for (; sc < sC; sc += F) |
496 | 0 | { |
497 | 0 | uint8_t* pd = dst + sc; |
498 | 0 | const int16_t* ps0 = src + ((sy + 0) & byMask) * bR + sc * bW; |
499 | 0 | const int16_t* ps2 = src + ((sy + 2) & byMask) * bR + sc * bW; |
500 | 0 | const int16_t* pw = weight + sc * dW; |
501 | 0 | _bias = _mm256_loadu_si256((__m256i*)(bias + sc)); |
502 | 0 | _norm = _mm256_loadu_ps(norm + sc); |
503 | 0 | w03 = _mm256_loadu_si256((__m256i*)pw + 0); |
504 | 0 | w14 = _mm256_loadu_si256((__m256i*)pw + 1); |
505 | 0 | w25 = _mm256_loadu_si256((__m256i*)pw + 2); |
506 | 0 | w6 = _mm256_loadu_si256((__m256i*)pw + 3); |
507 | 0 | w7 = _mm256_loadu_si256((__m256i*)pw + 4); |
508 | 0 | w8 = _mm256_loadu_si256((__m256i*)pw + 5); |
509 | 0 | if (sc < sCF) |
510 | 0 | { |
511 | 0 | size_t dx = 0; |
512 | 0 | for (; dx < dstW2; dx += 2, ps0 += QF, ps2 += QF) |
513 | 0 | { |
514 | 0 | d00 = _mm256_setzero_si256(); |
515 | 0 | d10 = _mm256_setzero_si256(); |
516 | |
|
517 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 0); |
518 | 0 | Madd2(d00, s0, w03); |
519 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 1); |
520 | 0 | Madd2(d00, s0, w14); |
521 | 0 | Madd2(d10, s0, w03); |
522 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 2); |
523 | 0 | Madd2(d00, s0, w25); |
524 | 0 | Madd2(d10, s0, w14); |
525 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 3); |
526 | 0 | Madd2(d10, s0, w25); |
527 | |
|
528 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 0); |
529 | 0 | Madd2(d00, s0, w6); |
530 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 1); |
531 | 0 | Madd2(d00, s0, w7); |
532 | 0 | Madd2(d10, s0, w6); |
533 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 2); |
534 | 0 | Madd2(d00, s0, w8); |
535 | 0 | Madd2(d10, s0, w7); |
536 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 3); |
537 | 0 | Madd2(d10, s0, w8); |
538 | |
|
539 | 0 | Save1(pd + 0 * dD, d00, _bias, _norm, _zero); |
540 | 0 | Save1(pd + 1 * dD, d10, _bias, _norm, _zero); |
541 | 0 | pd += 2 * dD; |
542 | 0 | } |
543 | 0 | for (; dx < p.dstW; ++dx, ps0 += dX, ps2 += dX) |
544 | 0 | { |
545 | 0 | d00 = _mm256_setzero_si256(); |
546 | |
|
547 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 0); |
548 | 0 | Madd2(d00, s0, w03); |
549 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 1); |
550 | 0 | Madd2(d00, s0, w14); |
551 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 2); |
552 | 0 | Madd2(d00, s0, w25); |
553 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 0); |
554 | 0 | Madd2(d00, s0, w6); |
555 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 1); |
556 | 0 | Madd2(d00, s0, w7); |
557 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 2); |
558 | 0 | Madd2(d00, s0, w8); |
559 | |
|
560 | 0 | Save1(pd, d00, _bias, _norm, _zero); |
561 | 0 | pd += dD; |
562 | 0 | } |
563 | 0 | } |
564 | 0 | else |
565 | 0 | { |
566 | 0 | size_t tail = sC - sCF; |
567 | 0 | for (size_t dx = 0; dx < p.dstW; ++dx, ps0 += dX, ps2 += dX) |
568 | 0 | { |
569 | 0 | d00 = _mm256_setzero_si256(); |
570 | |
|
571 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 0); |
572 | 0 | Madd2(d00, s0, w03); |
573 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 1); |
574 | 0 | Madd2(d00, s0, w14); |
575 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps0 + 2); |
576 | 0 | Madd2(d00, s0, w25); |
577 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 0); |
578 | 0 | Madd2(d00, s0, w6); |
579 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 1); |
580 | 0 | Madd2(d00, s0, w7); |
581 | 0 | s0 = _mm256_loadu_si256((__m256i*)ps2 + 2); |
582 | 0 | Madd2(d00, s0, w8); |
583 | |
|
584 | 0 | Save1(pd, d00, _bias, _norm, _zero, tail); |
585 | 0 | pd += dD; |
586 | 0 | } |
587 | 0 | } |
588 | 0 | } |
589 | 0 | dst += p.dstW * dD; |
590 | 0 | } |
591 | 0 | } |
592 | | |
593 | | //------------------------------------------------------------------------------------------------- |
594 | | |
595 | | void SetDepthwisePreprocess(const ConvParam& p, const Base::SynetQuantizedMergedConvolution::AlgParam& a, Base::SynetQuantizedMergedConvolution::DepthwisePreprocessPtr& func) |
596 | 0 | { |
597 | 0 | func = QuantizedMergedConvolutionDepthwisePreprocess; |
598 | 0 | } |
599 | | |
600 | | void SetDepthwiseConvolution(const ConvParam& p, const Base::SynetQuantizedMergedConvolution::AlgParam& a, Base::SynetQuantizedMergedConvolution::DepthwiseConvolutionPtr& func) |
601 | 0 | { |
602 | 0 | if(p.IsKernel(3)) |
603 | 0 | func = QuantizedMergedConvolutionDepthwiseConvolution3x3; |
604 | 0 | else |
605 | 0 | func = QuantizedMergedConvolutionDepthwiseConvolutionAny; |
606 | 0 | } |
607 | | } |
608 | | #endif |
609 | | } |