/src/Simd/src/Simd/SimdSynetInnerProduct16b.h

Source
/*
* Simd Library (http://ermig1979.github.io/Simd).
*
* Copyright (c) 2011-2026 Yermalayeu Ihar.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef __SimdSynetInnerProduct16b_h__
#define __SimdSynetInnerProduct16b_h__

#include "Simd/SimdArray.h"
#include "Simd/SimdPerformance.h"
#include "Simd/SimdSynetConvParam.h"

namespace Simd
{
    struct InnerProductParam16b
    {
        size_t M, N, K;
        SimdTensorDataType typeA, typeB, typeC;
        SimdBool transB, constB, bias;
        SimdConvolutionActivationType activation;

        InnerProductParam16b(size_t m, size_t n, size_t k,
            SimdTensorDataType ta, SimdTensorDataType tb, SimdTensorDataType tc,
            SimdBool t, SimdBool c, SimdBool b, SimdConvolutionActivationType a)
            : M(m), N(n), K(k)
            , typeA(ta), typeB(tb), typeC(tc)
            , transB(t), constB(c), bias(b)
            , activation(a)
        {
        }

        bool Valid()
        {
            return
                (typeA == SimdTensorData32f || typeA == SimdTensorData16b) &&
                (typeB == SimdTensorData32f || typeB == SimdTensorData16b) &&
                (typeC == SimdTensorData32f || typeC == SimdTensorData16b);
        }

        String Info(bool detail = true) const
        {
            std::stringstream ss;
            ss << M << "x" << N << "x" << K << "-";
            ss << ToChar(typeA) << ToChar(typeB) << ToChar(typeC) << "-";
            ss << (transB ? "t" : "n") << (constB ? "1" : "2") << (bias ? "b" : "o");
            if (detail)
                ss << "-" << ToStr(activation);
            return ss.str();
        }

        int64_t Flop() const
        {
            return int64_t(M) * N * K * 2;
        }
    };

    //-------------------------------------------------------------------------------------------------

    namespace Base
    {
        class SynetInnerProduct16b : public Deletable
        {
        public:
            SynetInnerProduct16b(const InnerProductParam16b& p);

            const InnerProductParam16b& Param() const
            {
                return _param;
            }

            virtual size_t InternalBufferSize() const;
            virtual size_t ExternalBufferSize() const;

            virtual String Ext() const = 0;
            virtual String Desc() const = 0;

            virtual void SetParams(const float* weight, const float* bias, const float* params) = 0;
            virtual void Forward(const uint8_t* A, const uint8_t* B, uint8_t* buf, uint8_t* C) = 0;

    #if defined(SIMD_PERFORMANCE_STATISTIC) && (defined(NDEBUG) || defined(SIMD_PERF_STAT_IN_DEBUG))
            Base::PerformanceMeasurer* Perf(const char* func);
    #endif

            const char* Info() const
            {
                _info = Desc();
                return _info.c_str();
            }

        protected:
            InnerProductParam16b _param;
    #if defined(SIMD_PERFORMANCE_STATISTIC) && (defined(NDEBUG) || defined(SIMD_PERF_STAT_IN_DEBUG))
            Base::PerformanceMeasurer* _perf;
    #endif
            Array8u _buffer;
            Array16u _weight;
            Array32f _bias, _params;
            mutable String _info;
            size_t _sizeA, _sizeB, _sizeC, _sizeS;

            uint8_t* Buffer(uint8_t* buffer);

            void SetBias(const float* bias, size_t align);
            void SetParams(const float* params, size_t align);
        };

        //------------------------------------------------------------------------------------------------- 
            
        class SynetInnerProduct16bRef : public SynetInnerProduct16b
        {
        public:
            SynetInnerProduct16bRef(const InnerProductParam16b& p);
            virtual String Ext() const { return "Base"; }
            virtual String Desc() const;
            virtual void SetParams(const float* weight, const float* bias, const float* params);
            virtual void Forward(const uint8_t* A, const uint8_t* B, uint8_t* buf, uint8_t* C);

        protected:
            void GemmAndBias(const uint16_t* A, const uint16_t* B, float* C);
        };

        class SynetInnerProduct16bGemmNN : public SynetInnerProduct16b
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
            virtual String Ext() const { return "Base"; }
            virtual String Desc() const;
            virtual void SetParams(const float* weight, const float* bias, const float* params);
            virtual void Forward(const uint8_t* A, const uint8_t* B, uint8_t* buf, uint8_t* C);

            static bool Preferable(const InnerProductParam16b& p);

            struct AlgParam
            {
                size_t F, microM, microN, microK;
                size_t macroM, macroN, macroK;
                size_t aM, aN, aK, eA, eB, eC, bK, cN;
            };

            typedef void(*PrepPtr)(const uint8_t* src, const InnerProductParam16b& p, const AlgParam& a, size_t size, size_t K, uint16_t* dst);
            typedef void(*GemmPtr)(const uint16_t* A, const InnerProductParam16b& p, const AlgParam& a, size_t M, size_t N, size_t K, int update, 
                const uint16_t* B, float* C, int post, const float* bias, const float* params, float * sum, uint8_t* dst);

        protected:
            void SetAlgParam(size_t F, size_t microM, size_t microN, size_t microK, size_t L1, size_t L2, size_t L3);

            AlgParam _alg;
            PrepPtr _prepA, _prepB;
            GemmPtr _gemm;
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }

#ifdef SIMD_SSE41_ENABLE    
    namespace Sse41
    {
        class SynetInnerProduct16bGemmNN : public Base::SynetInnerProduct16bGemmNN
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);

            virtual String Ext() const { return "Sse41"; }
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }
#endif

#ifdef SIMD_AVX2_ENABLE    
    namespace Avx2
    {
        class SynetInnerProduct16bGemmNN : public Sse41::SynetInnerProduct16bGemmNN
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);

            virtual String Ext() const { return "Avx2"; }
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }
#endif

#ifdef SIMD_AVX512BW_ENABLE    
    namespace Avx512bw
    {
        void InnerProduct16bGemmNN_ReorderA(const uint8_t* src8, const InnerProductParam16b& p, const Base::SynetInnerProduct16bGemmNN::AlgParam& a, size_t M, size_t K, uint16_t* dst);

        //-------------------------------------------------------------------------------------------------

        class SynetInnerProduct16bGemmNN : public Avx2::SynetInnerProduct16bGemmNN
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);

            virtual String Ext() const { return "Avx512bw"; }
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }
#endif

#if (defined(SIMD_AMXBF16_ENABLE) || (defined(SIMD_AVX512BW_ENABLE) && defined(SIMD_AMX_EMULATE)))   
    namespace AmxBf16
    {
        class SynetInnerProduct16bGemmNN : public Avx512bw::SynetInnerProduct16bGemmNN
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);

            virtual String Ext() const { return "AmxBf16"; }
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }
#endif

#ifdef SIMD_SVE2_ENABLE
    namespace Sve2
    {
        class SynetInnerProduct16bGemmNN : public Base::SynetInnerProduct16bGemmNN
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);

            virtual String Ext() const { return "Sve2"; }
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }
#endif

#ifdef SIMD_NEON_ENABLE
    namespace Neon
    {
        class SynetInnerProduct16bGemmNN : public Base::SynetInnerProduct16bGemmNN
        {
        public:
            SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);

            virtual String Ext() const { return "Neon"; }
        };

        //-------------------------------------------------------------------------------------------------

        void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
    }
#endif
}

#endif

Coverage Report

Created: 2026-10-28 07:09

Line	Count	Source
1		/*
2		* Simd Library (http://ermig1979.github.io/Simd).
3		*
4		* Copyright (c) 2011-2026 Yermalayeu Ihar.
5		*
6		* Permission is hereby granted, free of charge, to any person obtaining a copy
7		* of this software and associated documentation files (the "Software"), to deal
8		* in the Software without restriction, including without limitation the rights
9		* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10		* copies of the Software, and to permit persons to whom the Software is
11		* furnished to do so, subject to the following conditions:
12		*
13		* The above copyright notice and this permission notice shall be included in
14		* all copies or substantial portions of the Software.
15		*
16		* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17		* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18		* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19		* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20		* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21		* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22		* SOFTWARE.
23		*/
24		#ifndef __SimdSynetInnerProduct16b_h__
25		#define __SimdSynetInnerProduct16b_h__
26
27		#include "Simd/SimdArray.h"
28		#include "Simd/SimdPerformance.h"
29		#include "Simd/SimdSynetConvParam.h"
30
31		namespace Simd
32		{
33		struct InnerProductParam16b
34		{
35		size_t M, N, K;
36		SimdTensorDataType typeA, typeB, typeC;
37		SimdBool transB, constB, bias;
38		SimdConvolutionActivationType activation;
39
40		InnerProductParam16b(size_t m, size_t n, size_t k,
41		SimdTensorDataType ta, SimdTensorDataType tb, SimdTensorDataType tc,
42		SimdBool t, SimdBool c, SimdBool b, SimdConvolutionActivationType a)
43	0	: M(m), N(n), K(k)
44	0	, typeA(ta), typeB(tb), typeC(tc)
45	0	, transB(t), constB(c), bias(b)
46	0	, activation(a)
47	0	{
48	0	}
49
50		bool Valid()
51	0	{
52	0	return
53	0	(typeA == SimdTensorData32f \|\| typeA == SimdTensorData16b) &&
54	0	(typeB == SimdTensorData32f \|\| typeB == SimdTensorData16b) &&
55	0	(typeC == SimdTensorData32f \|\| typeC == SimdTensorData16b);
56	0	}
57
58		String Info(bool detail = true) const
59	0	{
60	0	std::stringstream ss;
61	0	ss << M << "x" << N << "x" << K << "-";
62	0	ss << ToChar(typeA) << ToChar(typeB) << ToChar(typeC) << "-";
63	0	ss << (transB ? "t" : "n") << (constB ? "1" : "2") << (bias ? "b" : "o");
64	0	if (detail)
65	0	ss << "-" << ToStr(activation);
66	0	return ss.str();
67	0	}
68
69		int64_t Flop() const
70	0	{
71	0	return int64_t(M) * N * K * 2;
72	0	}
73		};
74
75		//-------------------------------------------------------------------------------------------------
76
77		namespace Base
78		{
79		class SynetInnerProduct16b : public Deletable
80		{
81		public:
82		SynetInnerProduct16b(const InnerProductParam16b& p);
83
84		const InnerProductParam16b& Param() const
85	0	{
86	0	return _param;
87	0	}
88
89		virtual size_t InternalBufferSize() const;
90		virtual size_t ExternalBufferSize() const;
91
92		virtual String Ext() const = 0;
93		virtual String Desc() const = 0;
94
95		virtual void SetParams(const float* weight, const float* bias, const float* params) = 0;
96		virtual void Forward(const uint8_t* A, const uint8_t* B, uint8_t* buf, uint8_t* C) = 0;
97
98		#if defined(SIMD_PERFORMANCE_STATISTIC) && (defined(NDEBUG) \|\| defined(SIMD_PERF_STAT_IN_DEBUG))
99		Base::PerformanceMeasurer* Perf(const char* func);
100		#endif
101
102		const char* Info() const
103	0	{
104	0	_info = Desc();
105	0	return _info.c_str();
106	0	}
107
108		protected:
109		InnerProductParam16b _param;
110		#if defined(SIMD_PERFORMANCE_STATISTIC) && (defined(NDEBUG) \|\| defined(SIMD_PERF_STAT_IN_DEBUG))
111		Base::PerformanceMeasurer* _perf;
112		#endif
113		Array8u _buffer;
114		Array16u _weight;
115		Array32f _bias, _params;
116		mutable String _info;
117		size_t _sizeA, _sizeB, _sizeC, _sizeS;
118
119		uint8_t* Buffer(uint8_t* buffer);
120
121		void SetBias(const float* bias, size_t align);
122		void SetParams(const float* params, size_t align);
123		};
124
125		//-------------------------------------------------------------------------------------------------
126
127		class SynetInnerProduct16bRef : public SynetInnerProduct16b
128		{
129		public:
130		SynetInnerProduct16bRef(const InnerProductParam16b& p);
131	0	virtual String Ext() const { return "Base"; }
132		virtual String Desc() const;
133		virtual void SetParams(const float* weight, const float* bias, const float* params);
134		virtual void Forward(const uint8_t* A, const uint8_t* B, uint8_t* buf, uint8_t* C);
135
136		protected:
137		void GemmAndBias(const uint16_t* A, const uint16_t* B, float* C);
138		};
139
140		class SynetInnerProduct16bGemmNN : public SynetInnerProduct16b
141		{
142		public:
143		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
144	0	virtual String Ext() const { return "Base"; }
145		virtual String Desc() const;
146		virtual void SetParams(const float* weight, const float* bias, const float* params);
147		virtual void Forward(const uint8_t* A, const uint8_t* B, uint8_t* buf, uint8_t* C);
148
149		static bool Preferable(const InnerProductParam16b& p);
150
151		struct AlgParam
152		{
153		size_t F, microM, microN, microK;
154		size_t macroM, macroN, macroK;
155		size_t aM, aN, aK, eA, eB, eC, bK, cN;
156		};
157
158		typedef void(PrepPtr)(const uint8_t src, const InnerProductParam16b& p, const AlgParam& a, size_t size, size_t K, uint16_t* dst);
159		typedef void(GemmPtr)(const uint16_t A, const InnerProductParam16b& p, const AlgParam& a, size_t M, size_t N, size_t K, int update,
160		const uint16_t* B, float* C, int post, const float* bias, const float* params, float * sum, uint8_t* dst);
161
162		protected:
163		void SetAlgParam(size_t F, size_t microM, size_t microN, size_t microK, size_t L1, size_t L2, size_t L3);
164
165		AlgParam _alg;
166		PrepPtr _prepA, _prepB;
167		GemmPtr _gemm;
168		};
169
170		//-------------------------------------------------------------------------------------------------
171
172		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
173		}
174
175		#ifdef SIMD_SSE41_ENABLE
176		namespace Sse41
177		{
178		class SynetInnerProduct16bGemmNN : public Base::SynetInnerProduct16bGemmNN
179		{
180		public:
181		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
182
183	0	virtual String Ext() const { return "Sse41"; }
184		};
185
186		//-------------------------------------------------------------------------------------------------
187
188		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
189		}
190		#endif
191
192		#ifdef SIMD_AVX2_ENABLE
193		namespace Avx2
194		{
195		class SynetInnerProduct16bGemmNN : public Sse41::SynetInnerProduct16bGemmNN
196		{
197		public:
198		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
199
200	0	virtual String Ext() const { return "Avx2"; }
201		};
202
203		//-------------------------------------------------------------------------------------------------
204
205		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
206		}
207		#endif
208
209		#ifdef SIMD_AVX512BW_ENABLE
210		namespace Avx512bw
211		{
212		void InnerProduct16bGemmNN_ReorderA(const uint8_t* src8, const InnerProductParam16b& p, const Base::SynetInnerProduct16bGemmNN::AlgParam& a, size_t M, size_t K, uint16_t* dst);
213
214		//-------------------------------------------------------------------------------------------------
215
216		class SynetInnerProduct16bGemmNN : public Avx2::SynetInnerProduct16bGemmNN
217		{
218		public:
219		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
220
221	0	virtual String Ext() const { return "Avx512bw"; }
222		};
223
224		//-------------------------------------------------------------------------------------------------
225
226		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
227		}
228		#endif
229
230		#if (defined(SIMD_AMXBF16_ENABLE) \|\| (defined(SIMD_AVX512BW_ENABLE) && defined(SIMD_AMX_EMULATE)))
231		namespace AmxBf16
232		{
233		class SynetInnerProduct16bGemmNN : public Avx512bw::SynetInnerProduct16bGemmNN
234		{
235		public:
236		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
237
238		virtual String Ext() const { return "AmxBf16"; }
239		};
240
241		//-------------------------------------------------------------------------------------------------
242
243		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
244		}
245		#endif
246
247		#ifdef SIMD_SVE2_ENABLE
248		namespace Sve2
249		{
250		class SynetInnerProduct16bGemmNN : public Base::SynetInnerProduct16bGemmNN
251		{
252		public:
253		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
254
255		virtual String Ext() const { return "Sve2"; }
256		};
257
258		//-------------------------------------------------------------------------------------------------
259
260		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
261		}
262		#endif
263
264		#ifdef SIMD_NEON_ENABLE
265		namespace Neon
266		{
267		class SynetInnerProduct16bGemmNN : public Base::SynetInnerProduct16bGemmNN
268		{
269		public:
270		SynetInnerProduct16bGemmNN(const InnerProductParam16b& p);
271
272		virtual String Ext() const { return "Neon"; }
273		};
274
275		//-------------------------------------------------------------------------------------------------
276
277		void* SynetInnerProduct16bInit(size_t M, size_t N, size_t K, SimdTensorDataType typeA, SimdTensorDataType typeB, SimdTensorDataType typeC, SimdBool transB, SimdBool constB, SimdBool bias, SimdConvolutionActivationType activation);
278		}
279		#endif
280		}
281
282		#endif