Coverage Report

Created: 2026-10-31 07:15

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/Simd/src/Simd/SimdSse41SynetConvolution32fGemm.cpp
Line
Count
Source
1
/*
2
* Simd Library (http://ermig1979.github.io/Simd).
3
*
4
* Copyright (c) 2011-2024 Yermalayeu Ihar.
5
*
6
* Permission is hereby granted, free of charge, to any person obtaining a copy
7
* of this software and associated documentation files (the "Software"), to deal
8
* in the Software without restriction, including without limitation the rights
9
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
* copies of the Software, and to permit persons to whom the Software is
11
* furnished to do so, subject to the following conditions:
12
*
13
* The above copyright notice and this permission notice shall be included in
14
* all copies or substantial portions of the Software.
15
*
16
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
* SOFTWARE.
23
*/
24
#include "Simd/SimdSynetConvolution32f.h"
25
#include "Simd/SimdSynet.h"
26
#include "Simd/SimdGemm.h"
27
#include "Simd/SimdExp.h"
28
#include "Simd/SimdErf.h"
29
#include "Simd/SimdSse41.h"
30
#include "Simd/SimdExtract.h"
31
#include "Simd/SimdStore.h"
32
33
namespace Simd
34
{
35
#if defined(SIMD_SSE41_ENABLE) && defined(SIMD_SYNET_ENABLE) 
36
    namespace Sse41
37
    {
38
        void ConvolutionBiasAndActivation(const float* bias, size_t count, size_t size, ::SimdConvolutionActivationType activation, const float* params, ::SimdBool trans, float* dst)
39
0
        {
40
0
            size_t aligned = trans ? AlignLo(count, F) : AlignLo(size, F);
41
0
            if (activation == ::SimdConvolutionActivationIdentity)
42
0
            {
43
0
                if (bias)
44
0
                    Sse41::SynetAddBias(bias, count, size, dst, (SimdTensorFormatType)trans);
45
0
            }
46
0
            else if (activation == ::SimdConvolutionActivationRelu)
47
0
            {
48
0
                if (bias)
49
0
                {
50
0
                    __m128 _0 = _mm_set1_ps(0.0f);
51
0
                    if (trans)
52
0
                    {
53
0
                        for (size_t j = 0; j < size; ++j)
54
0
                        {
55
0
                            size_t i = 0;
56
0
                            for (; i < aligned; i += F)
57
0
                            {
58
0
                                __m128 _dst = _mm_loadu_ps(dst + i);
59
0
                                __m128 _bias = _mm_loadu_ps(bias + i);
60
0
                                _mm_storeu_ps(dst + i, _mm_max_ps(_0, _mm_add_ps(_dst, _bias)));
61
0
                            }
62
0
                            for (; i < count; ++i)
63
0
                                dst[i] = Simd::Max(0.0f, dst[i] + bias[i]);
64
0
                            dst += count;
65
0
                        }
66
0
                    }
67
0
                    else
68
0
                    {
69
0
                        for (size_t i = 0; i < count; ++i)
70
0
                        {
71
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
72
0
                            size_t j = 0;
73
0
                            for (; j < aligned; j += F)
74
0
                            {
75
0
                                __m128 _dst = _mm_loadu_ps(dst + j);
76
0
                                _mm_storeu_ps(dst + j, _mm_max_ps(_0, _mm_add_ps(_dst, _bias)));
77
0
                            }
78
0
                            for (; j < size; ++j)
79
0
                                dst[j] = Simd::Max(0.0f, dst[j] + bias[i]);
80
0
                            dst += size;
81
0
                        }
82
0
                    }
83
0
                }
84
0
                else
85
0
                {
86
0
                    float slope = 0;
87
0
                    SynetRelu32f(dst, size * count, &slope, dst);
88
0
                }
89
0
            }
90
0
            else if (activation == ::SimdConvolutionActivationLeakyRelu)
91
0
            {
92
0
                float slope = params[0];
93
0
                if (bias)
94
0
                {
95
0
                    __m128 _slope = _mm_set1_ps(slope);
96
0
                    if (trans)
97
0
                    {
98
0
                        for (size_t j = 0; j < size; ++j)
99
0
                        {
100
0
                            size_t i = 0;
101
0
                            for (; i < aligned; i += F)
102
0
                            {
103
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i));
104
0
                                _mm_storeu_ps(dst + i, SynetRelu32f(value, _slope));
105
0
                            }
106
0
                            for (; i < count; ++i)
107
0
                                dst[i] = Base::SynetRelu32f(dst[i] + bias[i], slope);
108
0
                            dst += count;
109
0
                        }
110
0
                    }
111
0
                    else
112
0
                    {
113
0
                        for (size_t i = 0; i < count; ++i)
114
0
                        {
115
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
116
0
                            size_t j = 0;
117
0
                            for (; j < aligned; j += F)
118
0
                            {
119
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias);
120
0
                                _mm_storeu_ps(dst + j, SynetRelu32f(value, _slope));
121
0
                            }
122
0
                            for (; j < size; ++j)
123
0
                                dst[j] = Base::SynetRelu32f(dst[j] + bias[i], slope);
124
0
                            dst += size;
125
0
                        }
126
0
                    }
127
0
                }
128
0
                else
129
0
                    SynetRelu32f(dst, size * count, &slope, dst);
130
0
            }
131
0
            else if (activation == ::SimdConvolutionActivationRestrictRange)
132
0
            {
133
0
                float lower = params[0];
134
0
                float upper = params[1];
135
0
                if (bias)
136
0
                {
137
0
                    __m128 _lower = _mm_set1_ps(lower);
138
0
                    __m128 _upper = _mm_set1_ps(upper);
139
0
                    if (trans)
140
0
                    {
141
0
                        for (size_t j = 0; j < size; ++j)
142
0
                        {
143
0
                            size_t i = 0;
144
0
                            for (; i < aligned; i += F)
145
0
                            {
146
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i));
147
0
                                _mm_storeu_ps(dst + i, _mm_min_ps(_mm_max_ps(_lower, value), _upper));
148
0
                            }
149
0
                            for (; i < count; ++i)
150
0
                                dst[i] = Simd::RestrictRange(dst[i] + bias[i], lower, upper);
151
0
                            dst += count;
152
0
                        }
153
0
                    }
154
0
                    else
155
0
                    {
156
0
                        for (size_t i = 0; i < count; ++i)
157
0
                        {
158
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
159
0
                            size_t j = 0;
160
0
                            for (; j < aligned; j += F)
161
0
                            {
162
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias);
163
0
                                _mm_storeu_ps(dst + j, _mm_min_ps(_mm_max_ps(_lower, value), _upper));
164
0
                            }
165
0
                            for (; j < size; ++j)
166
0
                                dst[j] = Simd::RestrictRange(dst[j] + bias[i], lower, upper);
167
0
                            dst += size;
168
0
                        }
169
0
                    }
170
0
                }
171
0
                else
172
0
                    SynetRestrictRange32f(dst, size * count, &lower, &upper, dst);
173
0
            }
174
0
            else if (activation == ::SimdConvolutionActivationPrelu)
175
0
            {
176
0
                if (bias)
177
0
                {
178
0
                    if (trans)
179
0
                    {
180
0
                        for (size_t j = 0; j < size; ++j)
181
0
                        {
182
0
                            size_t i = 0;
183
0
                            for (; i < aligned; i += F)
184
0
                            {
185
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i));
186
0
                                _mm_storeu_ps(dst + i, SynetRelu32f(value, _mm_loadu_ps(params + i)));
187
0
                            }
188
0
                            for (; i < count; ++i)
189
0
                                dst[i] = Base::SynetRelu32f(dst[i] + bias[i], params[i]);
190
0
                            dst += count;
191
0
                        }
192
0
                    }
193
0
                    else
194
0
                    {
195
0
                        for (size_t i = 0; i < count; ++i)
196
0
                        {
197
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
198
0
                            __m128 _slope = _mm_set1_ps(params[i]);
199
0
                            size_t j = 0;
200
0
                            for (; j < aligned; j += F)
201
0
                            {
202
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias);
203
0
                                _mm_storeu_ps(dst + j, SynetRelu32f(value, _slope));
204
0
                            }
205
0
                            for (; j < size; ++j)
206
0
                                dst[j] = Base::SynetRelu32f(dst[j] + bias[i], params[i]);
207
0
                            dst += size;
208
0
                        }
209
0
                    }
210
0
                }
211
0
                else
212
0
                    SynetPreluLayerForward(dst, params, count, size, dst, (SimdTensorFormatType)trans);
213
0
            }
214
0
            else if (activation == ::SimdConvolutionActivationElu)
215
0
            {
216
0
                float alpha = params[0];
217
0
                if (bias)
218
0
                {
219
0
                    __m128 _alpha = _mm_set1_ps(alpha);
220
0
                    if (trans)
221
0
                    {
222
0
                        for (size_t j = 0; j < size; ++j)
223
0
                        {
224
0
                            size_t i = 0;
225
0
                            for (; i < aligned; i += F)
226
0
                            {
227
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + i), _mm_loadu_ps(bias + i));
228
0
                                _mm_storeu_ps(dst + i, Elu(value, _alpha));
229
0
                            }
230
0
                            for (; i < count; ++i)
231
0
                                dst[i] = Base::SynetElu32f(dst[i] + bias[i], alpha);
232
0
                            dst += count;
233
0
                        }
234
0
                    }
235
0
                    else
236
0
                    {
237
0
                        for (size_t i = 0; i < count; ++i)
238
0
                        {
239
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
240
0
                            size_t j = 0;
241
0
                            for (; j < aligned; j += F)
242
0
                            {
243
0
                                __m128 value = _mm_add_ps(_mm_loadu_ps(dst + j), _bias);
244
0
                                _mm_storeu_ps(dst + j, Elu(value, _alpha));
245
0
                            }
246
0
                            for (; j < size; ++j)
247
0
                                dst[j] = Base::SynetElu32f(dst[j] + bias[i], alpha);
248
0
                            dst += size;
249
0
                        }
250
0
                    }
251
0
                }
252
0
                else
253
0
                    SynetElu32f(dst, size * count, &alpha, dst);
254
0
            }
255
0
            else if (activation == ::SimdConvolutionActivationHswish)
256
0
            {
257
0
                float shift = params[0];
258
0
                float scale = params[1];
259
0
                if (bias)
260
0
                {
261
0
                    __m128 _shift = _mm_set1_ps(shift);
262
0
                    __m128 _scale = _mm_set1_ps(scale);
263
0
                    if (trans)
264
0
                    {
265
0
                        for (size_t j = 0; j < size; ++j)
266
0
                        {
267
0
                            size_t i = 0;
268
0
                            for (; i < aligned; i += F)
269
0
                            {
270
0
                                __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i));
271
0
                                Store<false>(dst + i, SynetHswish32f(value, _shift, _scale));
272
0
                            }
273
0
                            for (; i < count; ++i)
274
0
                                dst[i] = Base::SynetHswish32f(dst[i] + bias[i], shift, scale);
275
0
                            dst += count;
276
0
                        }
277
0
                    }
278
0
                    else
279
0
                    {
280
0
                        for (size_t i = 0; i < count; ++i)
281
0
                        {
282
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
283
0
                            size_t j = 0;
284
0
                            for (; j < aligned; j += F)
285
0
                            {
286
0
                                __m128 value = _mm_add_ps(Load<false>(dst + j), _bias);
287
0
                                Store<false>(dst + j, SynetHswish32f(value, _shift, _scale));
288
0
                            }
289
0
                            for (; j < size; ++j)
290
0
                                dst[j] = Base::SynetHswish32f(dst[j] + bias[i], shift, scale);
291
0
                            dst += size;
292
0
                        }
293
0
                    }
294
0
                }
295
0
                else
296
0
                    SynetHswish32f(dst, count * size, &shift, &scale, dst);
297
0
            }
298
0
            else if (activation == ::SimdConvolutionActivationMish)
299
0
            {
300
0
                float threshold = params[0];
301
0
                if (bias)
302
0
                {
303
0
                    __m128 _threshold = _mm_set1_ps(threshold);
304
0
                    if (trans)
305
0
                    {
306
0
                        for (size_t j = 0; j < size; ++j)
307
0
                        {
308
0
                            size_t i = 0;
309
0
                            for (; i < aligned; i += F)
310
0
                            {
311
0
                                __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i));
312
0
                                Store<false>(dst + i, Mish(value, _threshold));
313
0
                            }
314
0
                            for (; i < count; ++i)
315
0
                                dst[i] = Base::SynetMish32f(dst[i] + bias[i], threshold);
316
0
                            dst += count;
317
0
                        }
318
0
                    }
319
0
                    else
320
0
                    {
321
0
                        for (size_t i = 0; i < count; ++i)
322
0
                        {
323
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
324
0
                            size_t j = 0;
325
0
                            for (; j < aligned; j += F)
326
0
                            {
327
0
                                __m128 value = _mm_add_ps(Load<false>(dst + j), _bias);
328
0
                                Store<false>(dst + j, Mish(value, _threshold));
329
0
                            }
330
0
                            for (; j < size; ++j)
331
0
                                dst[j] = Base::SynetMish32f(dst[j] + bias[i], threshold);
332
0
                            dst += size;
333
0
                        }
334
0
                    }
335
0
                }
336
0
                else
337
0
                    SynetMish32f(dst, size * count, &threshold, dst);
338
0
            }
339
0
            else if (activation == ::SimdConvolutionActivationHardSigmoid)
340
0
            {
341
0
                float scale = params[0];
342
0
                float shift = params[1];
343
0
                if (bias)
344
0
                {
345
0
                    __m128 _scale = _mm_set1_ps(scale);
346
0
                    __m128 _shift = _mm_set1_ps(shift);
347
0
                    if (trans)
348
0
                    {
349
0
                        for (size_t j = 0; j < size; ++j)
350
0
                        {
351
0
                            size_t i = 0;
352
0
                            for (; i < aligned; i += F)
353
0
                            {
354
0
                                __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i));
355
0
                                Store<false>(dst + i, SynetHardSigmoid32f(value, _scale, _shift));
356
0
                            }
357
0
                            for (; i < count; ++i)
358
0
                                dst[i] = Base::SynetHardSigmoid32f(dst[i] + bias[i], scale, shift);
359
0
                            dst += count;
360
0
                        }
361
0
                    }
362
0
                    else
363
0
                    {
364
0
                        for (size_t i = 0; i < count; ++i)
365
0
                        {
366
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
367
0
                            size_t j = 0;
368
0
                            for (; j < aligned; j += F)
369
0
                            {
370
0
                                __m128 value = _mm_add_ps(Load<false>(dst + j), _bias);
371
0
                                Store<false>(dst + j, SynetHardSigmoid32f(value, _scale, _shift));
372
0
                            }
373
0
                            for (; j < size; ++j)
374
0
                                dst[j] = Base::SynetHardSigmoid32f(dst[j] + bias[i], scale, shift);
375
0
                            dst += size;
376
0
                        }
377
0
                    }
378
0
                }
379
0
                else
380
0
                    SynetHardSigmoid32f(dst, count * size, &scale, &shift, dst);
381
0
            }
382
0
            else if (activation == ::SimdConvolutionActivationSwish)
383
0
            {
384
0
                float slope = params[0];
385
0
                if (bias)
386
0
                {
387
0
                    __m128 _slope = _mm_set1_ps(slope);
388
0
                    if (trans)
389
0
                    {
390
0
                        for (size_t j = 0; j < size; ++j)
391
0
                        {
392
0
                            size_t i = 0;
393
0
                            for (; i < aligned; i += F)
394
0
                            {
395
0
                                __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i));
396
0
                                Store<false>(dst + i, Swish(value, _slope));
397
0
                            }
398
0
                            for (; i < count; ++i)
399
0
                                dst[i] = Base::SynetSwish32f(dst[i] + bias[i], slope);
400
0
                            dst += count;
401
0
                        }
402
0
                    }
403
0
                    else
404
0
                    {
405
0
                        for (size_t i = 0; i < count; ++i)
406
0
                        {
407
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
408
0
                            size_t j = 0;
409
0
                            for (; j < aligned; j += F)
410
0
                            {
411
0
                                __m128 value = _mm_add_ps(Load<false>(dst + j), _bias);
412
0
                                Store<false>(dst + j, Swish(value, _slope));
413
0
                            }
414
0
                            for (; j < size; ++j)
415
0
                                dst[j] = Base::SynetSwish32f(dst[j] + bias[i], slope);
416
0
                            dst += size;
417
0
                        }
418
0
                    }
419
0
                }
420
0
                else
421
0
                    SynetSwish32f(dst, count * size, &slope, dst);
422
0
            }
423
0
            else if (activation == ::SimdConvolutionActivationGelu)
424
0
            {
425
0
                if (bias)
426
0
                {
427
0
                    if (trans)
428
0
                    {
429
0
                        for (size_t j = 0; j < size; ++j)
430
0
                        {
431
0
                            size_t i = 0;
432
0
                            for (; i < aligned; i += F)
433
0
                            {
434
0
                                __m128 value = _mm_add_ps(Load<false>(dst + i), Load<false>(bias + i));
435
0
                                Store<false>(dst + i, Gelu(value));
436
0
                            }
437
0
                            for (; i < count; ++i)
438
0
                                dst[i] = Base::Gelu(dst[i] + bias[i]);
439
0
                            dst += count;
440
0
                        }
441
0
                    }
442
0
                    else
443
0
                    {
444
0
                        for (size_t i = 0; i < count; ++i)
445
0
                        {
446
0
                            __m128 _bias = _mm_set1_ps(bias[i]);
447
0
                            size_t j = 0;
448
0
                            for (; j < aligned; j += F)
449
0
                            {
450
0
                                __m128 value = _mm_add_ps(Load<false>(dst + j), _bias);
451
0
                                Store<false>(dst + j, Gelu(value));
452
0
                            }
453
0
                            for (; j < size; ++j)
454
0
                                dst[j] = Base::Gelu(dst[j] + bias[i]);
455
0
                            dst += size;
456
0
                        }
457
0
                    }
458
0
                }
459
0
                else
460
0
                    SynetGelu32f(dst, count * size, dst);
461
0
            }
462
0
            else
463
0
            {
464
0
                Base::ConvolutionBiasAndActivation(bias, count, size, activation, params, trans, dst);
465
0
            }
466
0
        }
467
468
        //-------------------------------------------------------------------------------------------------
469
470
        SynetConvolution32fGemmNN::SynetConvolution32fGemmNN(const ConvParam& p)
471
0
            : Base::SynetConvolution32fGemmNN(p)
472
0
        {
473
0
            _gemm.Init(InitGemmFuncs(Sse41::Gemm32fNN, "Sse41"));
474
0
            if (_param.trans && _param.group == 1)
475
0
            {
476
0
                if (GemmRuntime())
477
0
                {
478
0
                    _gemmCb.Init(InitGemmCbFuncs(Sse41::Gemm32fNNcbBufferSize, Sse41::Gemm32fNNcbReorderB, Sse41::Gemm32fNNcbRun, "Sse41", GemmKernelF2, GemmKernelF3));
479
0
                    _nhwcWeight.Resize(_gemmCb.At(0).BufferSize(_M * _merge, _N, _K));
480
0
                }
481
0
                else
482
0
                    _nhwcWeight.Resize(Sse41::Gemm32fNNcbBufferSize(_M * _merge, _N, _K, GemmKernelAny, NHWC_GEMM_COMPATIBLE));
483
0
                _nhwcRun = Sse41::Gemm32fNNcbRun;
484
0
                _nhwcReorderB = Sse41::Gemm32fNNcbReorderB;
485
0
            }
486
0
            _biasAndActivation = Sse41::ConvolutionBiasAndActivation;
487
0
        }
488
489
        //-------------------------------------------------------------------------------------------------
490
491
        SynetConvolution32fGemmNT::SynetConvolution32fGemmNT(const ConvParam& p)
492
0
            : Base::SynetConvolution32fGemmNT(p)
493
0
        {
494
0
            _gemm.Init(InitGemmFuncs(Sse41::Gemm32fNT, "Sse41"));
495
0
            _biasAndActivation = Sse41::ConvolutionBiasAndActivation;
496
0
        }
497
498
        //-------------------------------------------------------------------------------------------------
499
500
        SynetConvolution32fWinograd::SynetConvolution32fWinograd(const ConvParam& p)
501
0
            : Base::SynetConvolution32fWinograd(p)
502
0
        {
503
0
            if (p.kernelY == 1 && p.kernelX == 3)
504
0
            {
505
0
                {
506
0
                    SetBlock(1, 4);
507
0
                    _setFilter = Sse41::WinogradKernel1x3Block1x4SetFilter;
508
0
                    _setInput = Sse41::WinogradKernel1x3Block1x4SetInput;
509
0
                    _setOutput = Sse41::WinogradKernel1x3Block1x4SetOutput;
510
0
                }
511
0
            }
512
0
            else if (p.kernelY == 1 && p.kernelX == 5)
513
0
            {
514
0
                {
515
0
                    SetBlock(1, 4);
516
0
                    _setFilter = Sse41::WinogradKernel1x5Block1x4SetFilter;
517
0
                    _setInput = Sse41::WinogradKernel1x5Block1x4SetInput;
518
0
                    _setOutput = Sse41::WinogradKernel1x5Block1x4SetOutput;
519
0
                }
520
0
            }
521
0
            else if (p.kernelY == 2 && p.kernelX == 2)
522
0
            {
523
0
                if (_blockY == 4 && _blockX == 4)
524
0
                {
525
0
                    SetBlock(4, 4);
526
0
                    _setFilter = Sse41::WinogradKernel2x2Block4x4SetFilter;
527
0
                    _setInput = Sse41::WinogradKernel2x2Block4x4SetInput;
528
0
                    _setOutput = Sse41::WinogradKernel2x2Block4x4SetOutput;
529
0
                }
530
0
                else if (_blockY == 2 && _blockX == 2)
531
0
                {
532
0
                    SetBlock(2, 2);
533
0
                    _setFilter = Sse41::WinogradKernel2x2Block2x2SetFilter;
534
0
                    _setInput = Sse41::WinogradKernel2x2Block2x2SetInput;
535
0
                    _setOutput = Sse41::WinogradKernel2x2Block2x2SetOutput;
536
0
                }
537
0
                else
538
0
                    assert(0);
539
0
            }
540
0
            else if (p.kernelY == 3 && p.kernelX == 3)
541
0
            {
542
0
                if (_blockY == 4 && _blockX == 4)
543
0
                {
544
0
                    _setFilter = Sse41::WinogradKernel3x3Block4x4SetFilter;
545
0
                    _setInput = Sse41::WinogradKernel3x3Block4x4SetInput;
546
0
                    _setOutput = Sse41::WinogradKernel3x3Block4x4SetOutput;
547
0
                }
548
0
                else if (_blockY == 3 && _blockX == 3)
549
0
                {
550
0
                    _setFilter = Sse41::WinogradKernel3x3Block3x3SetFilter;
551
0
                    _setInput = Sse41::WinogradKernel3x3Block3x3SetInput;
552
0
                    _setOutput = Sse41::WinogradKernel3x3Block3x3SetOutput;
553
0
                }
554
0
                else if (_blockY == 2 && _blockX == 2)
555
0
                {
556
0
                    _setFilter = Sse41::WinogradKernel3x3Block2x2SetFilter;
557
0
                    _setInput = Sse41::WinogradKernel3x3Block2x2SetInput;
558
0
                    _setOutput = Sse41::WinogradKernel3x3Block2x2SetOutput;
559
0
                }
560
0
                else
561
0
                    assert(0);
562
0
            }
563
0
            else
564
0
                assert(0);
565
0
            _gemm.Init(InitGemmFuncs(Sse41::Gemm32fNN, "Sse41"));
566
0
            if (_param.trans)
567
0
            {
568
0
                if (NHWC_GEMM_RUNTIME)
569
0
                {
570
0
                    _gemmCb.Init(InitGemmCbFuncs(Sse41::Gemm32fNNcbBufferSize, Sse41::Gemm32fNNcbReorderB, Sse41::Gemm32fNNcbRun, "Sse41", GemmKernelF2, GemmKernelF3));
571
0
                    _nhwcStrideW = _gemmCb.At(0).BufferSize(_M * _merge, _N, _K);
572
0
                }
573
0
                else
574
0
                    _nhwcStrideW = Sse41::Gemm32fNNcbBufferSize(_M * _merge, _N, _K, GemmKernelAny, NHWC_GEMM_COMPATIBLE);
575
0
                _nhwcWeight.Resize(_nhwcStrideW * _count);
576
0
                _nhwcRun = Sse41::Gemm32fNNcbRun;
577
0
                _nhwcReorderB = Sse41::Gemm32fNNcbReorderB;
578
0
            }
579
0
            _biasAndActivation = Sse41::ConvolutionBiasAndActivation;
580
0
        }
581
    }
582
#endif
583
}