/src/Simd/src/Simd/SimdSse41SynetConvolution32fGemm.cpp
Line | Count | Source |
1 | | /* |
2 | | * Simd Library (http://ermig1979.github.io/Simd). |
3 | | * |
4 | | * Copyright (c) 2011-2024 Yermalayeu Ihar. |
5 | | * |
6 | | * Permission is hereby granted, free of charge, to any person obtaining a copy |
7 | | * of this software and associated documentation files (the "Software"), to deal |
8 | | * in the Software without restriction, including without limitation the rights |
9 | | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
10 | | * copies of the Software, and to permit persons to whom the Software is |
11 | | * furnished to do so, subject to the following conditions: |
12 | | * |
13 | | * The above copyright notice and this permission notice shall be included in |
14 | | * all copies or substantial portions of the Software. |
15 | | * |
16 | | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
19 | | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
20 | | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
21 | | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
22 | | * SOFTWARE. |
23 | | */ |
24 | | #include "Simd/SimdSynetConvolution32f.h" |
25 | | #include "Simd/SimdSynet.h" |
26 | | #include "Simd/SimdGemm.h" |
27 | | #include "Simd/SimdExp.h" |
28 | | #include "Simd/SimdErf.h" |
29 | | #include "Simd/SimdSse41.h" |
30 | | #include "Simd/SimdExtract.h" |
31 | | #include "Simd/SimdStore.h" |
32 | | |
33 | | namespace Simd |
34 | | { |
35 | | #if defined(SIMD_SSE41_ENABLE) && defined(SIMD_SYNET_ENABLE) |
36 | | namespace Sse41 |
37 | | { |
38 | | void ConvolutionBiasAndActivation(const float* bias, size_t count, size_t size, ::SimdConvolutionActivationType activation, const float* params, ::SimdBool trans, float* dst) |
39 | 0 | { |
40 | 0 | size_t aligned = trans ? AlignLo(count, F) : AlignLo(size, F); |
41 | 0 | if (activation == ::SimdConvolutionActivationIdentity) |
42 | 0 | { |
43 | 0 | if (bias) |
44 | 0 | Sse41::SynetAddBias(bias, count, size, dst, (SimdTensorFormatType)trans); |
45 | 0 | } |
46 | 0 | else if (activation == ::SimdConvolutionActivationRelu) |
47 | 0 | { |
48 | 0 | if (bias) |
49 | 0 | { |
50 | 0 | __m128 _0 = _mm_set1_ps(0.0f); |
51 | 0 | if (trans) |
52 | 0 | { |
53 | 0 | for (size_t j = 0; j < size; ++j) |
54 | 0 | { |
55 | 0 | size_t i = 0; |
56 | 0 | for (; i < aligned; i += F) |
57 | 0 | { |
58 | 0 | __m128 _dst = _mm_loadu_ps(dst + i); |
59 | 0 | __m128 _bias = _mm_loadu_ps(bias + i); |
60 | 0 | _mm_storeu_ps(dst + i, _mm_max_ps(_0, _mm_add_ps(_dst, _bias))); |
61 | 0 | } |
62 | 0 | for (; i < count; ++i) |
63 | 0 | dst[i] = Simd::Max(0.0f, dst[i] + bias[i]); |
64 | 0 | dst += count; |
65 | 0 | } |
66 | 0 | } |
67 | 0 | else |
68 | 0 | { |
69 | 0 | for (size_t i = 0; i < count; ++i) |
70 | 0 | { |
71 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
72 | 0 | size_t j = 0; |
73 | 0 | for (; j < aligned; j += F) |
74 | 0 | { |
75 | 0 | __m128 _dst = _mm_loadu_ps(dst + j); |
76 | 0 | _mm_storeu_ps(dst + j, _mm_max_ps(_0, _mm_add_ps(_dst, _bias))); |
77 | 0 | } |
78 | 0 | for (; j < size; ++j) |
79 | 0 | dst[j] = Simd::Max(0.0f, dst[j] + bias[i]); |
80 | 0 | dst += size; |
81 | 0 | } |
82 | 0 | } |
83 | 0 | } |
84 | 0 | else |
85 | 0 | { |
86 | 0 | float slope = 0; |
87 | 0 | SynetRelu32f(dst, size * count, &slope, dst); |
88 | 0 | } |
89 | 0 | } |
90 | 0 | else if (activation == ::SimdConvolutionActivationLeakyRelu) |
91 | 0 | { |
92 | 0 | float slope = params[0]; |
93 | 0 | if (bias) |
94 | 0 | { |
95 | 0 | __m128 _slope = _mm_set1_ps(slope); |
96 | 0 | if (trans) |
97 | 0 | { |
98 | 0 | for (size_t j = 0; j < size; ++j) |
99 | 0 | { |
100 | 0 | size_t i = 0; |
101 | 0 | for (; i < aligned; i += F) |
102 | 0 | { |
103 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i)); |
104 | 0 | _mm_storeu_ps(dst + i, SynetRelu32f(value, _slope)); |
105 | 0 | } |
106 | 0 | for (; i < count; ++i) |
107 | 0 | dst[i] = Base::SynetRelu32f(dst[i] + bias[i], slope); |
108 | 0 | dst += count; |
109 | 0 | } |
110 | 0 | } |
111 | 0 | else |
112 | 0 | { |
113 | 0 | for (size_t i = 0; i < count; ++i) |
114 | 0 | { |
115 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
116 | 0 | size_t j = 0; |
117 | 0 | for (; j < aligned; j += F) |
118 | 0 | { |
119 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias); |
120 | 0 | _mm_storeu_ps(dst + j, SynetRelu32f(value, _slope)); |
121 | 0 | } |
122 | 0 | for (; j < size; ++j) |
123 | 0 | dst[j] = Base::SynetRelu32f(dst[j] + bias[i], slope); |
124 | 0 | dst += size; |
125 | 0 | } |
126 | 0 | } |
127 | 0 | } |
128 | 0 | else |
129 | 0 | SynetRelu32f(dst, size * count, &slope, dst); |
130 | 0 | } |
131 | 0 | else if (activation == ::SimdConvolutionActivationRestrictRange) |
132 | 0 | { |
133 | 0 | float lower = params[0]; |
134 | 0 | float upper = params[1]; |
135 | 0 | if (bias) |
136 | 0 | { |
137 | 0 | __m128 _lower = _mm_set1_ps(lower); |
138 | 0 | __m128 _upper = _mm_set1_ps(upper); |
139 | 0 | if (trans) |
140 | 0 | { |
141 | 0 | for (size_t j = 0; j < size; ++j) |
142 | 0 | { |
143 | 0 | size_t i = 0; |
144 | 0 | for (; i < aligned; i += F) |
145 | 0 | { |
146 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i)); |
147 | 0 | _mm_storeu_ps(dst + i, _mm_min_ps(_mm_max_ps(_lower, value), _upper)); |
148 | 0 | } |
149 | 0 | for (; i < count; ++i) |
150 | 0 | dst[i] = Simd::RestrictRange(dst[i] + bias[i], lower, upper); |
151 | 0 | dst += count; |
152 | 0 | } |
153 | 0 | } |
154 | 0 | else |
155 | 0 | { |
156 | 0 | for (size_t i = 0; i < count; ++i) |
157 | 0 | { |
158 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
159 | 0 | size_t j = 0; |
160 | 0 | for (; j < aligned; j += F) |
161 | 0 | { |
162 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias); |
163 | 0 | _mm_storeu_ps(dst + j, _mm_min_ps(_mm_max_ps(_lower, value), _upper)); |
164 | 0 | } |
165 | 0 | for (; j < size; ++j) |
166 | 0 | dst[j] = Simd::RestrictRange(dst[j] + bias[i], lower, upper); |
167 | 0 | dst += size; |
168 | 0 | } |
169 | 0 | } |
170 | 0 | } |
171 | 0 | else |
172 | 0 | SynetRestrictRange32f(dst, size * count, &lower, &upper, dst); |
173 | 0 | } |
174 | 0 | else if (activation == ::SimdConvolutionActivationPrelu) |
175 | 0 | { |
176 | 0 | if (bias) |
177 | 0 | { |
178 | 0 | if (trans) |
179 | 0 | { |
180 | 0 | for (size_t j = 0; j < size; ++j) |
181 | 0 | { |
182 | 0 | size_t i = 0; |
183 | 0 | for (; i < aligned; i += F) |
184 | 0 | { |
185 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i)); |
186 | 0 | _mm_storeu_ps(dst + i, SynetRelu32f(value, _mm_loadu_ps(params + i))); |
187 | 0 | } |
188 | 0 | for (; i < count; ++i) |
189 | 0 | dst[i] = Base::SynetRelu32f(dst[i] + bias[i], params[i]); |
190 | 0 | dst += count; |
191 | 0 | } |
192 | 0 | } |
193 | 0 | else |
194 | 0 | { |
195 | 0 | for (size_t i = 0; i < count; ++i) |
196 | 0 | { |
197 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
198 | 0 | __m128 _slope = _mm_set1_ps(params[i]); |
199 | 0 | size_t j = 0; |
200 | 0 | for (; j < aligned; j += F) |
201 | 0 | { |
202 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias); |
203 | 0 | _mm_storeu_ps(dst + j, SynetRelu32f(value, _slope)); |
204 | 0 | } |
205 | 0 | for (; j < size; ++j) |
206 | 0 | dst[j] = Base::SynetRelu32f(dst[j] + bias[i], params[i]); |
207 | 0 | dst += size; |
208 | 0 | } |
209 | 0 | } |
210 | 0 | } |
211 | 0 | else |
212 | 0 | SynetPreluLayerForward(dst, params, count, size, dst, (SimdTensorFormatType)trans); |
213 | 0 | } |
214 | 0 | else if (activation == ::SimdConvolutionActivationElu) |
215 | 0 | { |
216 | 0 | float alpha = params[0]; |
217 | 0 | if (bias) |
218 | 0 | { |
219 | 0 | __m128 _alpha = _mm_set1_ps(alpha); |
220 | 0 | if (trans) |
221 | 0 | { |
222 | 0 | for (size_t j = 0; j < size; ++j) |
223 | 0 | { |
224 | 0 | size_t i = 0; |
225 | 0 | for (; i < aligned; i += F) |
226 | 0 | { |
227 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i)); |
228 | 0 | _mm_storeu_ps(dst + i, Elu(value, _alpha)); |
229 | 0 | } |
230 | 0 | for (; i < count; ++i) |
231 | 0 | dst[i] = Base::SynetElu32f(dst[i] + bias[i], alpha); |
232 | 0 | dst += count; |
233 | 0 | } |
234 | 0 | } |
235 | 0 | else |
236 | 0 | { |
237 | 0 | for (size_t i = 0; i < count; ++i) |
238 | 0 | { |
239 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
240 | 0 | size_t j = 0; |
241 | 0 | for (; j < aligned; j += F) |
242 | 0 | { |
243 | 0 | __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias); |
244 | 0 | _mm_storeu_ps(dst + j, Elu(value, _alpha)); |
245 | 0 | } |
246 | 0 | for (; j < size; ++j) |
247 | 0 | dst[j] = Base::SynetElu32f(dst[j] + bias[i], alpha); |
248 | 0 | dst += size; |
249 | 0 | } |
250 | 0 | } |
251 | 0 | } |
252 | 0 | else |
253 | 0 | SynetElu32f(dst, size * count, &alpha, dst); |
254 | 0 | } |
255 | 0 | else if (activation == ::SimdConvolutionActivationHswish) |
256 | 0 | { |
257 | 0 | float shift = params[0]; |
258 | 0 | float scale = params[1]; |
259 | 0 | if (bias) |
260 | 0 | { |
261 | 0 | __m128 _shift = _mm_set1_ps(shift); |
262 | 0 | __m128 _scale = _mm_set1_ps(scale); |
263 | 0 | if (trans) |
264 | 0 | { |
265 | 0 | for (size_t j = 0; j < size; ++j) |
266 | 0 | { |
267 | 0 | size_t i = 0; |
268 | 0 | for (; i < aligned; i += F) |
269 | 0 | { |
270 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i)); |
271 | 0 | Store<false>(dst + i, SynetHswish32f(value, _shift, _scale)); |
272 | 0 | } |
273 | 0 | for (; i < count; ++i) |
274 | 0 | dst[i] = Base::SynetHswish32f(dst[i] + bias[i], shift, scale); |
275 | 0 | dst += count; |
276 | 0 | } |
277 | 0 | } |
278 | 0 | else |
279 | 0 | { |
280 | 0 | for (size_t i = 0; i < count; ++i) |
281 | 0 | { |
282 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
283 | 0 | size_t j = 0; |
284 | 0 | for (; j < aligned; j += F) |
285 | 0 | { |
286 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + j), _bias); |
287 | 0 | Store<false>(dst + j, SynetHswish32f(value, _shift, _scale)); |
288 | 0 | } |
289 | 0 | for (; j < size; ++j) |
290 | 0 | dst[j] = Base::SynetHswish32f(dst[j] + bias[i], shift, scale); |
291 | 0 | dst += size; |
292 | 0 | } |
293 | 0 | } |
294 | 0 | } |
295 | 0 | else |
296 | 0 | SynetHswish32f(dst, count * size, &shift, &scale, dst); |
297 | 0 | } |
298 | 0 | else if (activation == ::SimdConvolutionActivationMish) |
299 | 0 | { |
300 | 0 | float threshold = params[0]; |
301 | 0 | if (bias) |
302 | 0 | { |
303 | 0 | __m128 _threshold = _mm_set1_ps(threshold); |
304 | 0 | if (trans) |
305 | 0 | { |
306 | 0 | for (size_t j = 0; j < size; ++j) |
307 | 0 | { |
308 | 0 | size_t i = 0; |
309 | 0 | for (; i < aligned; i += F) |
310 | 0 | { |
311 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i)); |
312 | 0 | Store<false>(dst + i, Mish(value, _threshold)); |
313 | 0 | } |
314 | 0 | for (; i < count; ++i) |
315 | 0 | dst[i] = Base::SynetMish32f(dst[i] + bias[i], threshold); |
316 | 0 | dst += count; |
317 | 0 | } |
318 | 0 | } |
319 | 0 | else |
320 | 0 | { |
321 | 0 | for (size_t i = 0; i < count; ++i) |
322 | 0 | { |
323 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
324 | 0 | size_t j = 0; |
325 | 0 | for (; j < aligned; j += F) |
326 | 0 | { |
327 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + j), _bias); |
328 | 0 | Store<false>(dst + j, Mish(value, _threshold)); |
329 | 0 | } |
330 | 0 | for (; j < size; ++j) |
331 | 0 | dst[j] = Base::SynetMish32f(dst[j] + bias[i], threshold); |
332 | 0 | dst += size; |
333 | 0 | } |
334 | 0 | } |
335 | 0 | } |
336 | 0 | else |
337 | 0 | SynetMish32f(dst, size * count, &threshold, dst); |
338 | 0 | } |
339 | 0 | else if (activation == ::SimdConvolutionActivationHardSigmoid) |
340 | 0 | { |
341 | 0 | float scale = params[0]; |
342 | 0 | float shift = params[1]; |
343 | 0 | if (bias) |
344 | 0 | { |
345 | 0 | __m128 _scale = _mm_set1_ps(scale); |
346 | 0 | __m128 _shift = _mm_set1_ps(shift); |
347 | 0 | if (trans) |
348 | 0 | { |
349 | 0 | for (size_t j = 0; j < size; ++j) |
350 | 0 | { |
351 | 0 | size_t i = 0; |
352 | 0 | for (; i < aligned; i += F) |
353 | 0 | { |
354 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i)); |
355 | 0 | Store<false>(dst + i, SynetHardSigmoid32f(value, _scale, _shift)); |
356 | 0 | } |
357 | 0 | for (; i < count; ++i) |
358 | 0 | dst[i] = Base::SynetHardSigmoid32f(dst[i] + bias[i], scale, shift); |
359 | 0 | dst += count; |
360 | 0 | } |
361 | 0 | } |
362 | 0 | else |
363 | 0 | { |
364 | 0 | for (size_t i = 0; i < count; ++i) |
365 | 0 | { |
366 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
367 | 0 | size_t j = 0; |
368 | 0 | for (; j < aligned; j += F) |
369 | 0 | { |
370 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + j), _bias); |
371 | 0 | Store<false>(dst + j, SynetHardSigmoid32f(value, _scale, _shift)); |
372 | 0 | } |
373 | 0 | for (; j < size; ++j) |
374 | 0 | dst[j] = Base::SynetHardSigmoid32f(dst[j] + bias[i], scale, shift); |
375 | 0 | dst += size; |
376 | 0 | } |
377 | 0 | } |
378 | 0 | } |
379 | 0 | else |
380 | 0 | SynetHardSigmoid32f(dst, count * size, &scale, &shift, dst); |
381 | 0 | } |
382 | 0 | else if (activation == ::SimdConvolutionActivationSwish) |
383 | 0 | { |
384 | 0 | float slope = params[0]; |
385 | 0 | if (bias) |
386 | 0 | { |
387 | 0 | __m128 _slope = _mm_set1_ps(slope); |
388 | 0 | if (trans) |
389 | 0 | { |
390 | 0 | for (size_t j = 0; j < size; ++j) |
391 | 0 | { |
392 | 0 | size_t i = 0; |
393 | 0 | for (; i < aligned; i += F) |
394 | 0 | { |
395 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i)); |
396 | 0 | Store<false>(dst + i, Swish(value, _slope)); |
397 | 0 | } |
398 | 0 | for (; i < count; ++i) |
399 | 0 | dst[i] = Base::SynetSwish32f(dst[i] + bias[i], slope); |
400 | 0 | dst += count; |
401 | 0 | } |
402 | 0 | } |
403 | 0 | else |
404 | 0 | { |
405 | 0 | for (size_t i = 0; i < count; ++i) |
406 | 0 | { |
407 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
408 | 0 | size_t j = 0; |
409 | 0 | for (; j < aligned; j += F) |
410 | 0 | { |
411 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + j), _bias); |
412 | 0 | Store<false>(dst + j, Swish(value, _slope)); |
413 | 0 | } |
414 | 0 | for (; j < size; ++j) |
415 | 0 | dst[j] = Base::SynetSwish32f(dst[j] + bias[i], slope); |
416 | 0 | dst += size; |
417 | 0 | } |
418 | 0 | } |
419 | 0 | } |
420 | 0 | else |
421 | 0 | SynetSwish32f(dst, count * size, &slope, dst); |
422 | 0 | } |
423 | 0 | else if (activation == ::SimdConvolutionActivationGelu) |
424 | 0 | { |
425 | 0 | if (bias) |
426 | 0 | { |
427 | 0 | if (trans) |
428 | 0 | { |
429 | 0 | for (size_t j = 0; j < size; ++j) |
430 | 0 | { |
431 | 0 | size_t i = 0; |
432 | 0 | for (; i < aligned; i += F) |
433 | 0 | { |
434 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i)); |
435 | 0 | Store<false>(dst + i, Gelu(value)); |
436 | 0 | } |
437 | 0 | for (; i < count; ++i) |
438 | 0 | dst[i] = Base::Gelu(dst[i] + bias[i]); |
439 | 0 | dst += count; |
440 | 0 | } |
441 | 0 | } |
442 | 0 | else |
443 | 0 | { |
444 | 0 | for (size_t i = 0; i < count; ++i) |
445 | 0 | { |
446 | 0 | __m128 _bias = _mm_set1_ps(bias[i]); |
447 | 0 | size_t j = 0; |
448 | 0 | for (; j < aligned; j += F) |
449 | 0 | { |
450 | 0 | __m128 value = _mm_add_ps(Load<false>(dst + j), _bias); |
451 | 0 | Store<false>(dst + j, Gelu(value)); |
452 | 0 | } |
453 | 0 | for (; j < size; ++j) |
454 | 0 | dst[j] = Base::Gelu(dst[j] + bias[i]); |
455 | 0 | dst += size; |
456 | 0 | } |
457 | 0 | } |
458 | 0 | } |
459 | 0 | else |
460 | 0 | SynetGelu32f(dst, count * size, dst); |
461 | 0 | } |
462 | 0 | else |
463 | 0 | { |
464 | 0 | Base::ConvolutionBiasAndActivation(bias, count, size, activation, params, trans, dst); |
465 | 0 | } |
466 | 0 | } |
467 | | |
468 | | //------------------------------------------------------------------------------------------------- |
469 | | |
470 | | SynetConvolution32fGemmNN::SynetConvolution32fGemmNN(const ConvParam& p) |
471 | 0 | : Base::SynetConvolution32fGemmNN(p) |
472 | 0 | { |
473 | 0 | _gemm.Init(InitGemmFuncs(Sse41::Gemm32fNN, "Sse41")); |
474 | 0 | if (_param.trans && _param.group == 1) |
475 | 0 | { |
476 | 0 | if (GemmRuntime()) |
477 | 0 | { |
478 | 0 | _gemmCb.Init(InitGemmCbFuncs(Sse41::Gemm32fNNcbBufferSize, Sse41::Gemm32fNNcbReorderB, Sse41::Gemm32fNNcbRun, "Sse41", GemmKernelF2, GemmKernelF3)); |
479 | 0 | _nhwcWeight.Resize(_gemmCb.At(0).BufferSize(_M * _merge, _N, _K)); |
480 | 0 | } |
481 | 0 | else |
482 | 0 | _nhwcWeight.Resize(Sse41::Gemm32fNNcbBufferSize(_M * _merge, _N, _K, GemmKernelAny, NHWC_GEMM_COMPATIBLE)); |
483 | 0 | _nhwcRun = Sse41::Gemm32fNNcbRun; |
484 | 0 | _nhwcReorderB = Sse41::Gemm32fNNcbReorderB; |
485 | 0 | } |
486 | 0 | _biasAndActivation = Sse41::ConvolutionBiasAndActivation; |
487 | 0 | } |
488 | | |
489 | | //------------------------------------------------------------------------------------------------- |
490 | | |
491 | | SynetConvolution32fGemmNT::SynetConvolution32fGemmNT(const ConvParam& p) |
492 | 0 | : Base::SynetConvolution32fGemmNT(p) |
493 | 0 | { |
494 | 0 | _gemm.Init(InitGemmFuncs(Sse41::Gemm32fNT, "Sse41")); |
495 | 0 | _biasAndActivation = Sse41::ConvolutionBiasAndActivation; |
496 | 0 | } |
497 | | |
498 | | //------------------------------------------------------------------------------------------------- |
499 | | |
500 | | SynetConvolution32fWinograd::SynetConvolution32fWinograd(const ConvParam& p) |
501 | 0 | : Base::SynetConvolution32fWinograd(p) |
502 | 0 | { |
503 | 0 | if (p.kernelY == 1 && p.kernelX == 3) |
504 | 0 | { |
505 | 0 | { |
506 | 0 | SetBlock(1, 4); |
507 | 0 | _setFilter = Sse41::WinogradKernel1x3Block1x4SetFilter; |
508 | 0 | _setInput = Sse41::WinogradKernel1x3Block1x4SetInput; |
509 | 0 | _setOutput = Sse41::WinogradKernel1x3Block1x4SetOutput; |
510 | 0 | } |
511 | 0 | } |
512 | 0 | else if (p.kernelY == 1 && p.kernelX == 5) |
513 | 0 | { |
514 | 0 | { |
515 | 0 | SetBlock(1, 4); |
516 | 0 | _setFilter = Sse41::WinogradKernel1x5Block1x4SetFilter; |
517 | 0 | _setInput = Sse41::WinogradKernel1x5Block1x4SetInput; |
518 | 0 | _setOutput = Sse41::WinogradKernel1x5Block1x4SetOutput; |
519 | 0 | } |
520 | 0 | } |
521 | 0 | else if (p.kernelY == 2 && p.kernelX == 2) |
522 | 0 | { |
523 | 0 | if (_blockY == 4 && _blockX == 4) |
524 | 0 | { |
525 | 0 | SetBlock(4, 4); |
526 | 0 | _setFilter = Sse41::WinogradKernel2x2Block4x4SetFilter; |
527 | 0 | _setInput = Sse41::WinogradKernel2x2Block4x4SetInput; |
528 | 0 | _setOutput = Sse41::WinogradKernel2x2Block4x4SetOutput; |
529 | 0 | } |
530 | 0 | else if (_blockY == 2 && _blockX == 2) |
531 | 0 | { |
532 | 0 | SetBlock(2, 2); |
533 | 0 | _setFilter = Sse41::WinogradKernel2x2Block2x2SetFilter; |
534 | 0 | _setInput = Sse41::WinogradKernel2x2Block2x2SetInput; |
535 | 0 | _setOutput = Sse41::WinogradKernel2x2Block2x2SetOutput; |
536 | 0 | } |
537 | 0 | else |
538 | 0 | assert(0); |
539 | 0 | } |
540 | 0 | else if (p.kernelY == 3 && p.kernelX == 3) |
541 | 0 | { |
542 | 0 | if (_blockY == 4 && _blockX == 4) |
543 | 0 | { |
544 | 0 | _setFilter = Sse41::WinogradKernel3x3Block4x4SetFilter; |
545 | 0 | _setInput = Sse41::WinogradKernel3x3Block4x4SetInput; |
546 | 0 | _setOutput = Sse41::WinogradKernel3x3Block4x4SetOutput; |
547 | 0 | } |
548 | 0 | else if (_blockY == 3 && _blockX == 3) |
549 | 0 | { |
550 | 0 | _setFilter = Sse41::WinogradKernel3x3Block3x3SetFilter; |
551 | 0 | _setInput = Sse41::WinogradKernel3x3Block3x3SetInput; |
552 | 0 | _setOutput = Sse41::WinogradKernel3x3Block3x3SetOutput; |
553 | 0 | } |
554 | 0 | else if (_blockY == 2 && _blockX == 2) |
555 | 0 | { |
556 | 0 | _setFilter = Sse41::WinogradKernel3x3Block2x2SetFilter; |
557 | 0 | _setInput = Sse41::WinogradKernel3x3Block2x2SetInput; |
558 | 0 | _setOutput = Sse41::WinogradKernel3x3Block2x2SetOutput; |
559 | 0 | } |
560 | 0 | else |
561 | 0 | assert(0); |
562 | 0 | } |
563 | 0 | else |
564 | 0 | assert(0); |
565 | 0 | _gemm.Init(InitGemmFuncs(Sse41::Gemm32fNN, "Sse41")); |
566 | 0 | if (_param.trans) |
567 | 0 | { |
568 | 0 | if (NHWC_GEMM_RUNTIME) |
569 | 0 | { |
570 | 0 | _gemmCb.Init(InitGemmCbFuncs(Sse41::Gemm32fNNcbBufferSize, Sse41::Gemm32fNNcbReorderB, Sse41::Gemm32fNNcbRun, "Sse41", GemmKernelF2, GemmKernelF3)); |
571 | 0 | _nhwcStrideW = _gemmCb.At(0).BufferSize(_M * _merge, _N, _K); |
572 | 0 | } |
573 | 0 | else |
574 | 0 | _nhwcStrideW = Sse41::Gemm32fNNcbBufferSize(_M * _merge, _N, _K, GemmKernelAny, NHWC_GEMM_COMPATIBLE); |
575 | 0 | _nhwcWeight.Resize(_nhwcStrideW * _count); |
576 | 0 | _nhwcRun = Sse41::Gemm32fNNcbRun; |
577 | 0 | _nhwcReorderB = Sse41::Gemm32fNNcbReorderB; |
578 | 0 | } |
579 | 0 | _biasAndActivation = Sse41::ConvolutionBiasAndActivation; |
580 | 0 | } |
581 | | } |
582 | | #endif |
583 | | } |