/src/llama.cpp/ggml/src/ggml-cpu/simd-mappings.h
Line | Count | Source |
1 | | #pragma once |
2 | | |
3 | | #include "ggml-cpu-impl.h" |
4 | | |
5 | | #ifdef __ARM_FEATURE_SVE |
6 | | #include <arm_sve.h> |
7 | | #endif // __ARM_FEATURE_SVE |
8 | | |
9 | | #if defined(__ARM_NEON) && !defined(__CUDACC__) && !defined(__MUSACC__) |
10 | | // if YCM cannot find <arm_neon.h>, make a symbolic link to it, for example: |
11 | | // |
12 | | // $ ln -sfn /Library/Developer/CommandLineTools/usr/lib/clang/13.1.6/include/arm_neon.h ./src/ |
13 | | // |
14 | | #include <arm_neon.h> |
15 | | #endif |
16 | | |
17 | | #if defined(__riscv_v_intrinsic) |
18 | | #include <riscv_vector.h> |
19 | | #endif |
20 | | |
21 | | #ifdef __cplusplus |
22 | | extern "C" { |
23 | | #endif |
24 | | |
25 | | // |
26 | | // simd mappings |
27 | | // |
28 | | |
29 | | // FP16 to FP32 conversion |
30 | | |
31 | | // 16-bit float |
32 | | // on Arm, we use __fp16 |
33 | | // on x86, we use uint16_t |
34 | | // |
35 | | // for old CUDA compilers (<= 11), we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/10616 |
36 | | // for MUSA compilers , we use uint16_t: ref https://github.com/ggml-org/llama.cpp/pull/11843 |
37 | | // |
38 | | #if defined(__ARM_NEON) && !(defined(__CUDACC__) && __CUDACC_VER_MAJOR__ <= 11) && !defined(__MUSACC__) |
39 | | #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) neon_compute_fp16_to_fp32(x) |
40 | | #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) neon_compute_fp32_to_fp16(x) |
41 | | |
42 | | #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) |
43 | | |
44 | | static inline float neon_compute_fp16_to_fp32(ggml_fp16_t h) { |
45 | | __fp16 tmp; |
46 | | memcpy(&tmp, &h, sizeof(ggml_fp16_t)); |
47 | | return (float)tmp; |
48 | | } |
49 | | |
50 | | static inline ggml_fp16_t neon_compute_fp32_to_fp16(float f) { |
51 | | ggml_fp16_t res; |
52 | | __fp16 tmp = f; |
53 | | memcpy(&res, &tmp, sizeof(ggml_fp16_t)); |
54 | | return res; |
55 | | } |
56 | | #elif defined(__F16C__) |
57 | | #ifdef _MSC_VER |
58 | | #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(x))) |
59 | | #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _mm_extract_epi16(_mm_cvtps_ph(_mm_set_ss(x), 0), 0) |
60 | | #else |
61 | | #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) _cvtsh_ss(x) |
62 | | #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) _cvtss_sh(x, 0) |
63 | | #endif |
64 | | #elif defined(__POWER9_VECTOR__) |
65 | | #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) power_compute_fp16_to_fp32(x) |
66 | | #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) power_compute_fp32_to_fp16(x) |
67 | | /* the inline asm below is about 12% faster than the lookup method */ |
68 | | #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) |
69 | | #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x) |
70 | | |
71 | | static inline float power_compute_fp16_to_fp32(ggml_fp16_t h) { |
72 | | float f; |
73 | | double d; |
74 | | __asm__( |
75 | | "mtfprd %0,%2\n" |
76 | | "xscvhpdp %0,%0\n" |
77 | | "frsp %1,%0\n" : |
78 | | /* temp */ "=d"(d), |
79 | | /* out */ "=f"(f): |
80 | | /* in */ "r"(h)); |
81 | | return f; |
82 | | } |
83 | | |
84 | | static inline ggml_fp16_t power_compute_fp32_to_fp16(float f) { |
85 | | double d; |
86 | | ggml_fp16_t r; |
87 | | __asm__( /* xscvdphp can work on double or single precision */ |
88 | | "xscvdphp %0,%2\n" |
89 | | "mffprd %1,%0\n" : |
90 | | /* temp */ "=d"(d), |
91 | | /* out */ "=r"(r): |
92 | | /* in */ "f"(f)); |
93 | | return r; |
94 | | } |
95 | | #elif defined(__riscv) && defined(__riscv_zfhmin) |
96 | | static inline float riscv_compute_fp16_to_fp32(ggml_fp16_t h) { |
97 | | _Float16 hf; |
98 | | memcpy(&hf, &h, sizeof(ggml_fp16_t)); |
99 | | return hf; |
100 | | } |
101 | | |
102 | | static inline ggml_fp16_t riscv_compute_fp32_to_fp16(float f) { |
103 | | ggml_fp16_t res; |
104 | | _Float16 hf = (_Float16)f; |
105 | | memcpy(&res, &hf, sizeof(ggml_fp16_t)); |
106 | | return res; |
107 | | } |
108 | | |
109 | | #define GGML_CPU_COMPUTE_FP16_TO_FP32(x) riscv_compute_fp16_to_fp32(x) |
110 | | #define GGML_CPU_COMPUTE_FP32_TO_FP16(x) riscv_compute_fp32_to_fp16(x) |
111 | | #define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x) |
112 | | #define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x) |
113 | | #endif |
114 | | |
115 | | // precomputed f32 table for f16 (256 KB) |
116 | | // defined in ggml-cpu.c, initialized in ggml_cpu_init() |
117 | | extern float ggml_table_f32_f16[1 << 16]; |
118 | | |
119 | | // precomputed f32 table for e8m0 half (1 KB) |
120 | | // defined in ggml-cpu.c, initialized in ggml_cpu_init() |
121 | | extern float ggml_table_f32_e8m0_half[1 << 8]; |
122 | | |
123 | | // precomputed f32 table for ue4m3 (1 KB) |
124 | | // defined in ggml-cpu.c, initialized in ggml_cpu_init() |
125 | | extern float ggml_table_f32_ue4m3[1 << 8]; |
126 | | |
127 | | // Use lookup table for E8M0 on x86 (faster than bit manipulation) |
128 | | #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) |
129 | 0 | #define GGML_CPU_E8M0_TO_FP32_HALF(x) ggml_table_f32_e8m0_half[(uint8_t)(x)] |
130 | | #else |
131 | | #define GGML_CPU_E8M0_TO_FP32_HALF(x) GGML_E8M0_TO_FP32_HALF(x) |
132 | | #endif |
133 | | |
134 | | // Use lookup table for UE4M3 on x86 and ARM (faster than bit manipulation) |
135 | | #if defined(__AVX__) || defined(__AVX2__) || defined(__AVX512F__) || defined(__ARM_NEON) |
136 | 0 | #define GGML_CPU_UE4M3_TO_FP32(x) ggml_table_f32_ue4m3[(uint8_t)(x)] |
137 | | #else |
138 | | #define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x) |
139 | | #endif |
140 | | |
141 | | // On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32, |
142 | | // so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON. |
143 | | // This is also true for POWER9. |
144 | | #if !defined(GGML_CPU_FP16_TO_FP32) |
145 | 0 | inline static float ggml_lookup_fp16_to_fp32(ggml_fp16_t f) { |
146 | 0 | uint16_t s; |
147 | 0 | memcpy(&s, &f, sizeof(uint16_t)); |
148 | 0 | return ggml_table_f32_f16[s]; |
149 | 0 | } Unexecuted instantiation: repack.cpp:ggml_lookup_fp16_to_fp32(unsigned short) Unexecuted instantiation: ggml-cpu.c:ggml_lookup_fp16_to_fp32 Unexecuted instantiation: quants.c:ggml_lookup_fp16_to_fp32 Unexecuted instantiation: binary-ops.cpp:ggml_lookup_fp16_to_fp32(unsigned short) Unexecuted instantiation: unary-ops.cpp:ggml_lookup_fp16_to_fp32(unsigned short) Unexecuted instantiation: vec.cpp:ggml_lookup_fp16_to_fp32(unsigned short) Unexecuted instantiation: ops.cpp:ggml_lookup_fp16_to_fp32(unsigned short) Unexecuted instantiation: sgemm.cpp:ggml_lookup_fp16_to_fp32(unsigned short) |
150 | | |
151 | 0 | #define GGML_CPU_FP16_TO_FP32(x) ggml_lookup_fp16_to_fp32(x) |
152 | | #endif |
153 | | |
154 | | #if !defined(GGML_CPU_FP32_TO_FP16) |
155 | 131k | #define GGML_CPU_FP32_TO_FP16(x) GGML_COMPUTE_FP32_TO_FP16(x) |
156 | | #endif |
157 | | |
158 | | |
159 | | // we define a common set of C macros which map to specific intrinsics based on the current architecture |
160 | | // we then implement the fundamental computation operations below using only these macros |
161 | | // adding support for new architectures requires to define the corresponding SIMD macros |
162 | | // |
163 | | // GGML_F32_STEP / GGML_F16_STEP |
164 | | // number of elements to process in a single step |
165 | | // |
166 | | // GGML_F32_EPR / GGML_F16_EPR |
167 | | // number of elements to fit in a single register |
168 | | // |
169 | | |
170 | | #if defined(__ARM_FEATURE_SVE) && defined(__ARM_FEATURE_FMA) |
171 | | |
172 | | #define GGML_SIMD |
173 | | |
174 | | // F32 SVE |
175 | | #define GGML_F32_EPR 8 |
176 | | #define DEFAULT_PG svptrue_b32() |
177 | | |
178 | | #define GGML_F32xt svfloat32_t |
179 | | #define GGML_F32xt_ZERO svdup_n_f32(0.0f) |
180 | | #define GGML_F32xt_SET1(x) svdup_n_f32(x) |
181 | | #define GGML_F32xt_LOAD_IMPL(pg, a) svld1_f32(pg, a) |
182 | | #define GGML_F32xt_LOAD(a) GGML_F32xt_LOAD_IMPL(DEFAULT_PG, a) |
183 | | #define GGML_F32xt_STORE_IMPL(pg, a, b) svst1_f32(pg, a, b) |
184 | | #define GGML_F32xt_STORE(a, b) GGML_F32xt_STORE_IMPL(DEFAULT_PG, a, b) |
185 | | #define GGML_F32xt_FMA_IMPL(pg, a, b, c) svmad_f32_m(pg, b, c, a) |
186 | | #define GGML_F32xt_FMA(a, b, c) GGML_F32xt_FMA_IMPL(DEFAULT_PG, a, b, c) |
187 | | #define GGML_F32xt_ADD_IMPL(pg, a, b) svadd_f32_m(pg, a, b) |
188 | | #define GGML_F32xt_ADD(a, b) GGML_F32xt_ADD_IMPL(DEFAULT_PG, a, b) |
189 | | #define GGML_F32xt_MUL_IMPL(pg, a, b) svmul_f32_m(pg, a, b) |
190 | | #define GGML_F32xt_MUL(a, b) GGML_F32xt_MUL_IMPL(DEFAULT_PG, a, b) |
191 | | #define GGML_F32xt_REDUCE_ONE_IMPL(pg, a) svaddv(pg, a) |
192 | | #define GGML_F32xt_REDUCE_ONE(a) GGML_F32xt_REDUCE_ONE_IMPL(DEFAULT_PG, a) |
193 | | #define GGML_F32xt_REDUCE_IMPL(pg, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) \ |
194 | | { \ |
195 | | sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum2); \ |
196 | | sum3 = svadd_f32_m(DEFAULT_PG, sum3, sum4); \ |
197 | | sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum6); \ |
198 | | sum7 = svadd_f32_m(DEFAULT_PG, sum7, sum8); \ |
199 | | sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum3); \ |
200 | | sum5 = svadd_f32_m(DEFAULT_PG, sum5, sum7); \ |
201 | | sum1 = svadd_f32_m(DEFAULT_PG, sum1, sum5); \ |
202 | | (res) = (ggml_float) GGML_F32xt_REDUCE_ONE(sum1); \ |
203 | | } |
204 | | #define GGML_F32xt_REDUCE(res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) \ |
205 | | GGML_F32xt_REDUCE_IMPL(DEFAULT_PG, res, sum1, sum2, sum3, sum4, sum5, sum6, sum7, sum8) |
206 | | |
207 | | #define GGML_F32_VEC GGML_F32xt |
208 | | #define GGML_F32_VEC_ZERO GGML_F32xt_ZERO |
209 | | #define GGML_F32_VEC_SET1 GGML_F32xt_SET1 |
210 | | #define GGML_F32_VEC_LOAD GGML_F32xt_LOAD |
211 | | #define GGML_F32_VEC_STORE GGML_F32xt_STORE |
212 | | #define GGML_F32_VEC_FMA GGML_F32xt_FMA |
213 | | #define GGML_F32_VEC_ADD GGML_F32xt_ADD |
214 | | #define GGML_F32_VEC_MUL GGML_F32xt_MUL |
215 | | #define GGML_F32_VEC_REDUCE GGML_F32xt_REDUCE |
216 | | |
217 | | // F16 SVE |
218 | | #define DEFAULT_PG32 svptrue_b32() |
219 | | #define DEFAULT_PG16 svptrue_b16() |
220 | | |
221 | | #define GGML_F32Cxt svfloat16_t |
222 | | #define GGML_F32Cxt_ZERO svdup_n_f16(0.0f) |
223 | | #define GGML_F32Cxt_SET1(x) svdup_n_f16(x) |
224 | | #define GGML_F32Cxt_LOAD(p) svld1_f16(DEFAULT_PG16, (const __fp16 *)(p)) |
225 | | #define GGML_F32Cxt_STORE(dst_ptr, src_vec) svst1_f16(DEFAULT_PG16, (__fp16 *)(dst_ptr), (src_vec)) |
226 | | |
227 | | #define GGML_F32Cxt_FMA_IMPL(pg, a, b, c) svmad_f16_x(pg, b, c, a) |
228 | | #define GGML_F32Cxt_FMA(a, b, c) GGML_F32Cxt_FMA_IMPL(DEFAULT_PG16, a, b, c) |
229 | | #define GGML_F32Cxt_ADD_IMPL(pg, a, b) svadd_f16_x(pg, a, b) |
230 | | #define GGML_F32Cxt_ADD(a, b) GGML_F32Cxt_ADD_IMPL(DEFAULT_PG16, a, b) |
231 | | #define GGML_F32Cxt_MUL_IMPL(pg, a, b) svmul_f16_x(pg, a, b) |
232 | | #define GGML_F32Cxt_MUL(a, b) GGML_F32Cxt_MUL_IMPL(DEFAULT_PG16, a, b) |
233 | | #define GGML_F32Cxt_REDUCE GGML_F16xt_REDUCE_MIXED |
234 | | |
235 | | #define GGML_F16x_VEC GGML_F32Cxt |
236 | | #define GGML_F16x_VEC_ZERO GGML_F32Cxt_ZERO |
237 | | #define GGML_F16x_VEC_SET1 GGML_F32Cxt_SET1 |
238 | | #define GGML_F16x_VEC_LOAD(p, i) GGML_F32Cxt_LOAD(p) |
239 | | #define GGML_F16x_VEC_STORE(p, r, i) GGML_F32Cxt_STORE((__fp16 *)(p), r) |
240 | | #define GGML_F16x_VEC_FMA GGML_F32Cxt_FMA |
241 | | #define GGML_F16x_VEC_ADD GGML_F32Cxt_ADD |
242 | | #define GGML_F16x_VEC_MUL GGML_F32Cxt_MUL |
243 | | #define GGML_F16x_VEC_REDUCE GGML_F32Cxt_REDUCE |
244 | | |
245 | | #define GGML_F16xt_REDUCE_ONE_IMPL(pg, a) svaddv_f16(pg, a) |
246 | | #define GGML_F16xt_REDUCE_ONE(a) GGML_F16xt_REDUCE_ONE_IMPL(DEFAULT_PG16, a) |
247 | | |
248 | | #define GGML_F16xt_REDUCE_MIXED_IMPL(pg16, res, sum1, sum2, sum3, sum4) \ |
249 | | { \ |
250 | | sum1 = svadd_f16_x(pg16, sum1, sum2); \ |
251 | | sum3 = svadd_f16_x(pg16, sum3, sum4); \ |
252 | | sum1 = svadd_f16_x(pg16, sum1, sum3); \ |
253 | | __fp16 sum_f16 = svaddv_f16(pg16, sum1); \ |
254 | | (res) = (ggml_float) sum_f16; \ |
255 | | } |
256 | | #define GGML_F16xt_REDUCE_MIXED(res, sum1, sum2, sum3, sum4) \ |
257 | | GGML_F16xt_REDUCE_MIXED_IMPL(DEFAULT_PG16, res, sum1, sum2, sum3, sum4) |
258 | | |
259 | | // F16 NEON |
260 | | |
261 | | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) |
262 | | #define GGML_F16_STEP 32 |
263 | | #define GGML_F16_EPR 8 |
264 | | |
265 | | #define GGML_F16x8 float16x8_t |
266 | | #define GGML_F16x8_ZERO vdupq_n_f16(0.0f) |
267 | | #define GGML_F16x8_SET1(x) vdupq_n_f16(x) |
268 | | #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x)) |
269 | | #define GGML_F16x8_STORE vst1q_f16 |
270 | | #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c) |
271 | | #define GGML_F16x8_ADD vaddq_f16 |
272 | | #define GGML_F16x8_MUL vmulq_f16 |
273 | | #define GGML_F16x8_REDUCE(res, x) \ |
274 | | do { \ |
275 | | int offset = GGML_F16_ARR >> 1; \ |
276 | | for (int i = 0; i < offset; ++i) { \ |
277 | | (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ |
278 | | } \ |
279 | | offset >>= 1; \ |
280 | | for (int i = 0; i < offset; ++i) { \ |
281 | | (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ |
282 | | } \ |
283 | | offset >>= 1; \ |
284 | | for (int i = 0; i < offset; ++i) { \ |
285 | | (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ |
286 | | } \ |
287 | | const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \ |
288 | | const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \ |
289 | | (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \ |
290 | | } while (0) |
291 | | |
292 | | #define GGML_F16_VEC GGML_F16x8 |
293 | | #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO |
294 | | #define GGML_F16_VEC_SET1 GGML_F16x8_SET1 |
295 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p) |
296 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i]) |
297 | | #define GGML_F16_VEC_FMA GGML_F16x8_FMA |
298 | | #define GGML_F16_VEC_ADD GGML_F16x8_ADD |
299 | | #define GGML_F16_VEC_MUL GGML_F16x8_MUL |
300 | | #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE |
301 | | #else |
302 | | // if FP16 vector arithmetic is not supported, we use FP32 instead |
303 | | // and take advantage of the vcvt_ functions to convert to/from FP16 |
304 | | |
305 | | #define GGML_F16_STEP 16 |
306 | | #define GGML_F16_EPR 4 |
307 | | |
308 | | #define GGML_F32Cx4 float32x4_t |
309 | | #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f) |
310 | | #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x) |
311 | | #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x))) |
312 | | #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y)) |
313 | | #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c) |
314 | | #define GGML_F32Cx4_ADD vaddq_f32 |
315 | | #define GGML_F32Cx4_MUL vmulq_f32 |
316 | | #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE |
317 | | |
318 | | #define GGML_F16_VEC GGML_F32Cx4 |
319 | | #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO |
320 | | #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 |
321 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p) |
322 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i]) |
323 | | #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA |
324 | | #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD |
325 | | #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL |
326 | | #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE |
327 | | #endif |
328 | | |
329 | | #elif defined(__ARM_NEON) && defined(__ARM_FEATURE_FMA) |
330 | | |
331 | | #define GGML_SIMD |
332 | | |
333 | | // F32 NEON |
334 | | |
335 | | #define GGML_F32_STEP 16 |
336 | | #define GGML_F32_EPR 4 |
337 | | |
338 | | #define GGML_F32x4 float32x4_t |
339 | | #define GGML_F32x4_ZERO vdupq_n_f32(0.0f) |
340 | | #define GGML_F32x4_SET1(x) vdupq_n_f32(x) |
341 | | #define GGML_F32x4_LOAD vld1q_f32 |
342 | | #define GGML_F32x4_STORE vst1q_f32 |
343 | | #define GGML_F32x4_FMA(a, b, c) vfmaq_f32(a, b, c) |
344 | | #define GGML_F32x4_ADD vaddq_f32 |
345 | | #define GGML_F32x4_MUL vmulq_f32 |
346 | | #define GGML_F32x4_REDUCE_ONE(x) vaddvq_f32(x) |
347 | | #define GGML_F32x4_REDUCE(res, x) \ |
348 | | { \ |
349 | | int offset = GGML_F32_ARR >> 1; \ |
350 | | for (int i = 0; i < offset; ++i) { \ |
351 | | (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \ |
352 | | } \ |
353 | | offset >>= 1; \ |
354 | | for (int i = 0; i < offset; ++i) { \ |
355 | | (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \ |
356 | | } \ |
357 | | offset >>= 1; \ |
358 | | for (int i = 0; i < offset; ++i) { \ |
359 | | (x)[i] = vaddq_f32((x)[i], (x)[offset+i]); \ |
360 | | } \ |
361 | | (res) = (ggml_float) GGML_F32x4_REDUCE_ONE((x)[0]); \ |
362 | | } |
363 | | |
364 | | #define GGML_F32_VEC GGML_F32x4 |
365 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
366 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
367 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
368 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
369 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
370 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
371 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
372 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
373 | | |
374 | | // F16 NEON |
375 | | |
376 | | #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) |
377 | | #define GGML_F16_STEP 32 |
378 | | #define GGML_F16_EPR 8 |
379 | | |
380 | | #define GGML_F16x8 float16x8_t |
381 | | #define GGML_F16x8_ZERO vdupq_n_f16(0.0f) |
382 | | #define GGML_F16x8_SET1(x) vdupq_n_f16(x) |
383 | | #define GGML_F16x8_LOAD(x) vld1q_f16((const __fp16 *)(x)) |
384 | | #define GGML_F16x8_STORE vst1q_f16 |
385 | | #define GGML_F16x8_FMA(a, b, c) vfmaq_f16(a, b, c) |
386 | | #define GGML_F16x8_ADD vaddq_f16 |
387 | | #define GGML_F16x8_MUL vmulq_f16 |
388 | | #define GGML_F16x8_REDUCE(res, x) \ |
389 | | do { \ |
390 | | int offset = GGML_F16_ARR >> 1; \ |
391 | | for (int i = 0; i < offset; ++i) { \ |
392 | | (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ |
393 | | } \ |
394 | | offset >>= 1; \ |
395 | | for (int i = 0; i < offset; ++i) { \ |
396 | | (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ |
397 | | } \ |
398 | | offset >>= 1; \ |
399 | | for (int i = 0; i < offset; ++i) { \ |
400 | | (x)[i] = vaddq_f16((x)[i], (x)[offset+i]); \ |
401 | | } \ |
402 | | const float32x4_t t0 = vcvt_f32_f16(vget_low_f16 ((x)[0])); \ |
403 | | const float32x4_t t1 = vcvt_f32_f16(vget_high_f16((x)[0])); \ |
404 | | (res) = (ggml_float) vaddvq_f32(vaddq_f32(t0, t1)); \ |
405 | | } while (0) |
406 | | |
407 | | #define GGML_F16_VEC GGML_F16x8 |
408 | | #define GGML_F16_VEC_ZERO GGML_F16x8_ZERO |
409 | | #define GGML_F16_VEC_SET1 GGML_F16x8_SET1 |
410 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F16x8_LOAD(p) |
411 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x8_STORE((__fp16 *)(p), (r)[i]) |
412 | | #define GGML_F16_VEC_FMA GGML_F16x8_FMA |
413 | | #define GGML_F16_VEC_ADD GGML_F16x8_ADD |
414 | | #define GGML_F16_VEC_MUL GGML_F16x8_MUL |
415 | | #define GGML_F16_VEC_REDUCE GGML_F16x8_REDUCE |
416 | | #else |
417 | | // if FP16 vector arithmetic is not supported, we use FP32 instead |
418 | | // and take advantage of the vcvt_ functions to convert to/from FP16 |
419 | | |
420 | | #define GGML_F16_STEP 16 |
421 | | #define GGML_F16_EPR 4 |
422 | | |
423 | | #define GGML_F32Cx4 float32x4_t |
424 | | #define GGML_F32Cx4_ZERO vdupq_n_f32(0.0f) |
425 | | #define GGML_F32Cx4_SET1(x) vdupq_n_f32(x) |
426 | | #define GGML_F32Cx4_LOAD(x) vcvt_f32_f16(vld1_f16((const __fp16 *)(x))) |
427 | | #define GGML_F32Cx4_STORE(x, y) vst1_f16(x, vcvt_f16_f32(y)) |
428 | | #define GGML_F32Cx4_FMA(a, b, c) vfmaq_f32(a, b, c) |
429 | | #define GGML_F32Cx4_ADD vaddq_f32 |
430 | | #define GGML_F32Cx4_MUL vmulq_f32 |
431 | | #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE |
432 | | |
433 | | #define GGML_F16_VEC GGML_F32Cx4 |
434 | | #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO |
435 | | #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 |
436 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p) |
437 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE((__fp16 *)(p), r[i]) |
438 | | #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA |
439 | | #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD |
440 | | #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL |
441 | | #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE |
442 | | #endif |
443 | | |
444 | | #elif defined(__AVX512F__) |
445 | | |
446 | | #define GGML_SIMD |
447 | | |
448 | | // F32 AVX512 |
449 | | |
450 | | #define GGML_F32_STEP 64 |
451 | | #define GGML_F32_EPR 16 |
452 | | |
453 | | #define GGML_F32x16 __m512 |
454 | | #define GGML_F32x16_ZERO _mm512_setzero_ps() |
455 | | #define GGML_F32x16_SET1(x) _mm512_set1_ps(x) |
456 | | #define GGML_F32x16_LOAD _mm512_loadu_ps |
457 | | #define GGML_F32x16_STORE _mm512_storeu_ps |
458 | | // _mm512_fmadd_ps is defined in AVX512F so no guard is required |
459 | | #define GGML_F32x16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a) |
460 | | #define GGML_F32x16_ADD _mm512_add_ps |
461 | | #define GGML_F32x16_MUL _mm512_mul_ps |
462 | | #define GGML_F32x16_REDUCE(res, x) \ |
463 | | do { \ |
464 | | int offset = GGML_F32_ARR >> 1; \ |
465 | | for (int i = 0; i < offset; ++i) { \ |
466 | | x[i] = _mm512_add_ps(x[i], x[offset+i]); \ |
467 | | } \ |
468 | | offset >>= 1; \ |
469 | | for (int i = 0; i < offset; ++i) { \ |
470 | | x[i] = _mm512_add_ps(x[i], x[offset+i]); \ |
471 | | } \ |
472 | | offset >>= 1; \ |
473 | | for (int i = 0; i < offset; ++i) { \ |
474 | | x[i] = _mm512_add_ps(x[i], x[offset+i]); \ |
475 | | } \ |
476 | | res = (ggml_float) _mm512_reduce_add_ps(x[0]); \ |
477 | | } while (0) |
478 | | |
479 | | // TODO: is this optimal ? |
480 | | |
481 | | #define GGML_F32_VEC GGML_F32x16 |
482 | | #define GGML_F32_VEC_ZERO GGML_F32x16_ZERO |
483 | | #define GGML_F32_VEC_SET1 GGML_F32x16_SET1 |
484 | | #define GGML_F32_VEC_LOAD GGML_F32x16_LOAD |
485 | | #define GGML_F32_VEC_STORE GGML_F32x16_STORE |
486 | | #define GGML_F32_VEC_FMA GGML_F32x16_FMA |
487 | | #define GGML_F32_VEC_ADD GGML_F32x16_ADD |
488 | | #define GGML_F32_VEC_MUL GGML_F32x16_MUL |
489 | | #define GGML_F32_VEC_REDUCE GGML_F32x16_REDUCE |
490 | | |
491 | | // F16 AVX512 |
492 | | |
493 | | #if defined(__AVX512FP16__) |
494 | | |
495 | | #define GGML_F16_STEP 128 |
496 | | #define GGML_F16_EPR 32 |
497 | | |
498 | | #define GGML_F16x32 __m512h |
499 | | #define GGML_F16x32_ZERO _mm512_setzero_ph() |
500 | | #define GGML_F16x32_SET1(x) _mm512_set1_ph(__extension__(_Float16)(x)) |
501 | | #define GGML_F16x32_LOAD(x) _mm512_loadu_ph(x) |
502 | | #define GGML_F16x32_STORE(x, y) _mm512_storeu_ph(x, y) |
503 | | #define GGML_F16x32_FMA(a, b, c) _mm512_fmadd_ph(b, c, a) |
504 | | #define GGML_F16x32_ADD _mm512_add_ph |
505 | | #define GGML_F16x32_MUL _mm512_mul_ph |
506 | | #define GGML_F16x32_REDUCE(res, x) \ |
507 | | do { \ |
508 | | int offset = GGML_F16_ARR >> 1; \ |
509 | | for (int i = 0; i < offset; ++i) { \ |
510 | | x[i] = _mm512_add_ph(x[i], x[offset+i]); \ |
511 | | } \ |
512 | | offset >>= 1; \ |
513 | | for (int i = 0; i < offset; ++i) { \ |
514 | | x[i] = _mm512_add_ph(x[i], x[offset+i]); \ |
515 | | } \ |
516 | | offset >>= 1; \ |
517 | | for (int i = 0; i < offset; ++i) { \ |
518 | | x[i] = _mm512_add_ph(x[i], x[offset+i]); \ |
519 | | } \ |
520 | | res = (ggml_float) _mm512_reduce_add_ph(x[0]); \ |
521 | | } while (0) |
522 | | |
523 | | #define GGML_F16_VEC GGML_F16x32 |
524 | | #define GGML_F16_VEC_ZERO GGML_F16x32_ZERO |
525 | | #define GGML_F16_VEC_SET1 GGML_F16x32_SET1 |
526 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F16x32_LOAD(p) |
527 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x32_STORE(p, r[i]) |
528 | | #define GGML_F16_VEC_FMA GGML_F16x32_FMA |
529 | | #define GGML_F16_VEC_ADD GGML_F16x32_ADD |
530 | | #define GGML_F16_VEC_MUL GGML_F16x32_MUL |
531 | | #define GGML_F16_VEC_REDUCE GGML_F16x32_REDUCE |
532 | | |
533 | | #else // Fallback FP16 <-> FP32 |
534 | | |
535 | | #define GGML_F16_STEP 64 |
536 | | #define GGML_F16_EPR 16 |
537 | | |
538 | | #define GGML_F32Cx16 __m512 |
539 | | #define GGML_F32Cx16_ZERO _mm512_setzero_ps() |
540 | | #define GGML_F32Cx16_SET1(x) _mm512_set1_ps(x) |
541 | | |
542 | | // unlike _mm256_cvt intrinsics that require F16C, _mm512_cvt is defined in AVX512F |
543 | | // so F16C guard isn't required |
544 | | #define GGML_F32Cx16_LOAD(x) _mm512_cvtph_ps(_mm256_loadu_si256((const __m256i *)(x))) |
545 | | #define GGML_F32Cx16_STORE(x, y) _mm256_storeu_si256((__m256i *)(x), _mm512_cvtps_ph(y, 0)) |
546 | | |
547 | | #define GGML_F32Cx16_FMA(a, b, c) _mm512_fmadd_ps(b, c, a) |
548 | | #define GGML_F32Cx16_ADD _mm512_add_ps |
549 | | #define GGML_F32Cx16_MUL _mm512_mul_ps |
550 | | #define GGML_F32Cx16_REDUCE(res, x) \ |
551 | | do { \ |
552 | | int offset = GGML_F32_ARR >> 1; \ |
553 | | for (int i = 0; i < offset; ++i) { \ |
554 | | x[i] = _mm512_add_ps(x[i], x[offset+i]); \ |
555 | | } \ |
556 | | offset >>= 1; \ |
557 | | for (int i = 0; i < offset; ++i) { \ |
558 | | x[i] = _mm512_add_ps(x[i], x[offset+i]); \ |
559 | | } \ |
560 | | offset >>= 1; \ |
561 | | for (int i = 0; i < offset; ++i) { \ |
562 | | x[i] = _mm512_add_ps(x[i], x[offset+i]); \ |
563 | | } \ |
564 | | res = (ggml_float) _mm512_reduce_add_ps(x[0]); \ |
565 | | } while (0) |
566 | | |
567 | | #define GGML_F16_VEC GGML_F32Cx16 |
568 | | #define GGML_F16_VEC_ZERO GGML_F32Cx16_ZERO |
569 | | #define GGML_F16_VEC_SET1 GGML_F32Cx16_SET1 |
570 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx16_LOAD(p) |
571 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx16_STORE(p, r[i]) |
572 | | #define GGML_F16_VEC_FMA GGML_F32Cx16_FMA |
573 | | #define GGML_F16_VEC_ADD GGML_F32Cx16_ADD |
574 | | #define GGML_F16_VEC_MUL GGML_F32Cx16_MUL |
575 | | |
576 | | #define GGML_F16_VEC_REDUCE GGML_F32Cx16_REDUCE |
577 | | |
578 | | #endif // __AVX512FP16__ |
579 | | #elif defined(__AVX__) |
580 | | |
581 | | #define GGML_SIMD |
582 | | |
583 | | // F32 AVX |
584 | | |
585 | 0 | #define GGML_F32_STEP 32 |
586 | 0 | #define GGML_F32_EPR 8 |
587 | | |
588 | 0 | #define GGML_F32x8 __m256 |
589 | 0 | #define GGML_F32x8_ZERO _mm256_setzero_ps() |
590 | 0 | #define GGML_F32x8_SET1(x) _mm256_set1_ps(x) |
591 | 0 | #define GGML_F32x8_LOAD _mm256_loadu_ps |
592 | 0 | #define GGML_F32x8_STORE _mm256_storeu_ps |
593 | | #if defined(__FMA__) |
594 | 0 | #define GGML_F32x8_FMA(a, b, c) _mm256_fmadd_ps(b, c, a) |
595 | | #else |
596 | | #define GGML_F32x8_FMA(a, b, c) _mm256_add_ps(_mm256_mul_ps(b, c), a) |
597 | | #endif |
598 | 0 | #define GGML_F32x8_ADD _mm256_add_ps |
599 | 0 | #define GGML_F32x8_MUL _mm256_mul_ps |
600 | 0 | #define GGML_F32x8_REDUCE(res, x) \ |
601 | 0 | do { \ |
602 | 0 | int offset = GGML_F32_ARR >> 1; \ |
603 | 0 | for (int i = 0; i < offset; ++i) { \ |
604 | 0 | x[i] = _mm256_add_ps(x[i], x[offset+i]); \ |
605 | 0 | } \ |
606 | 0 | offset >>= 1; \ |
607 | 0 | for (int i = 0; i < offset; ++i) { \ |
608 | 0 | x[i] = _mm256_add_ps(x[i], x[offset+i]); \ |
609 | 0 | } \ |
610 | 0 | offset >>= 1; \ |
611 | 0 | for (int i = 0; i < offset; ++i) { \ |
612 | 0 | x[i] = _mm256_add_ps(x[i], x[offset+i]); \ |
613 | 0 | } \ |
614 | 0 | const __m128 t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), \ |
615 | 0 | _mm256_extractf128_ps(x[0], 1)); \ |
616 | 0 | const __m128 t1 = _mm_hadd_ps(t0, t0); \ |
617 | 0 | res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); \ |
618 | 0 | } while (0) |
619 | | // TODO: is this optimal ? |
620 | | |
621 | 0 | #define GGML_F32_VEC GGML_F32x8 |
622 | 0 | #define GGML_F32_VEC_ZERO GGML_F32x8_ZERO |
623 | 0 | #define GGML_F32_VEC_SET1 GGML_F32x8_SET1 |
624 | 0 | #define GGML_F32_VEC_LOAD GGML_F32x8_LOAD |
625 | 0 | #define GGML_F32_VEC_STORE GGML_F32x8_STORE |
626 | 0 | #define GGML_F32_VEC_FMA GGML_F32x8_FMA |
627 | 0 | #define GGML_F32_VEC_ADD GGML_F32x8_ADD |
628 | 0 | #define GGML_F32_VEC_MUL GGML_F32x8_MUL |
629 | 0 | #define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE |
630 | | |
631 | | // F16 AVX |
632 | | |
633 | 0 | #define GGML_F16_STEP 32 |
634 | 0 | #define GGML_F16_EPR 8 |
635 | | |
636 | | // F16 arithmetic is not supported by AVX, so we use F32 instead |
637 | | |
638 | 0 | #define GGML_F32Cx8 __m256 |
639 | 0 | #define GGML_F32Cx8_ZERO _mm256_setzero_ps() |
640 | 0 | #define GGML_F32Cx8_SET1(x) _mm256_set1_ps(x) |
641 | | |
642 | | #if defined(__F16C__) |
643 | | // the _mm256_cvt intrinsics require F16C |
644 | 0 | #define GGML_F32Cx8_LOAD(x) _mm256_cvtph_ps(_mm_loadu_si128((const __m128i *)(x))) |
645 | 0 | #define GGML_F32Cx8_STORE(x, y) _mm_storeu_si128((__m128i *)(x), _mm256_cvtps_ph(y, 0)) |
646 | | #else |
647 | | static inline __m256 __avx_f32cx8_load(const ggml_fp16_t * x) { |
648 | | float tmp[8]; |
649 | | |
650 | | for (int i = 0; i < 8; i++) { |
651 | | tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]); |
652 | | } |
653 | | |
654 | | return _mm256_loadu_ps(tmp); |
655 | | } |
656 | | static inline void __avx_f32cx8_store(ggml_fp16_t *x, __m256 y) { |
657 | | float arr[8]; |
658 | | |
659 | | _mm256_storeu_ps(arr, y); |
660 | | |
661 | | for (int i = 0; i < 8; i++) |
662 | | x[i] = GGML_CPU_FP32_TO_FP16(arr[i]); |
663 | | } |
664 | | #define GGML_F32Cx8_LOAD(x) __avx_f32cx8_load(x) |
665 | | #define GGML_F32Cx8_STORE(x, y) __avx_f32cx8_store(x, y) |
666 | | #endif |
667 | | |
668 | 0 | #define GGML_F32Cx8_FMA GGML_F32x8_FMA |
669 | | #define GGML_F32Cx8_ADD _mm256_add_ps |
670 | 0 | #define GGML_F32Cx8_MUL _mm256_mul_ps |
671 | 0 | #define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE |
672 | | |
673 | 0 | #define GGML_F16_VEC GGML_F32Cx8 |
674 | 0 | #define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO |
675 | 0 | #define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1 |
676 | 0 | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p) |
677 | 0 | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i]) |
678 | 0 | #define GGML_F16_VEC_FMA GGML_F32Cx8_FMA |
679 | | #define GGML_F16_VEC_ADD GGML_F32Cx8_ADD |
680 | 0 | #define GGML_F16_VEC_MUL GGML_F32Cx8_MUL |
681 | 0 | #define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE |
682 | | |
683 | | #elif defined(__POWER9_VECTOR__) |
684 | | |
685 | | #define GGML_SIMD |
686 | | |
687 | | // F32 POWER9 |
688 | | |
689 | | #define GGML_F32_STEP 32 |
690 | | #define GGML_F32_EPR 4 |
691 | | |
692 | | #define GGML_F32x4 vector float |
693 | | #define GGML_F32x4_ZERO {0.0f} |
694 | | #define GGML_F32x4_SET1 vec_splats |
695 | | #define GGML_F32x4_LOAD(p) vec_xl(0, p) |
696 | | #define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p) |
697 | | #define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a) |
698 | | #define GGML_F32x4_ADD vec_add |
699 | | #define GGML_F32x4_MUL vec_mul |
700 | | #define GGML_F32x4_REDUCE(res, x) \ |
701 | | { \ |
702 | | int offset = GGML_F32_ARR >> 1; \ |
703 | | for (int i = 0; i < offset; ++i) { \ |
704 | | x[i] = vec_add(x[i], x[offset+i]); \ |
705 | | } \ |
706 | | offset >>= 1; \ |
707 | | for (int i = 0; i < offset; ++i) { \ |
708 | | x[i] = vec_add(x[i], x[offset+i]); \ |
709 | | } \ |
710 | | offset >>= 1; \ |
711 | | for (int i = 0; i < offset; ++i) { \ |
712 | | x[i] = vec_add(x[i], x[offset+i]); \ |
713 | | } \ |
714 | | res = vec_extract(x[0], 0) + \ |
715 | | vec_extract(x[0], 1) + \ |
716 | | vec_extract(x[0], 2) + \ |
717 | | vec_extract(x[0], 3); \ |
718 | | } |
719 | | #define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \ |
720 | | { \ |
721 | | vector float v = vec_add(vec_add(s0, s1), \ |
722 | | vec_add(s2, s3)); \ |
723 | | v = vec_add(v, vec_sld(v, v, 8)); \ |
724 | | v = vec_add(v, vec_sld(v, v, 4)); \ |
725 | | res += (ggml_float) vec_extract(v, 0); \ |
726 | | } |
727 | | |
728 | | #define GGML_F32_VEC GGML_F32x4 |
729 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
730 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
731 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
732 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
733 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
734 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
735 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
736 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
737 | | |
738 | | // F16 POWER9 |
739 | | #define GGML_F16_STEP GGML_F32_STEP |
740 | | #define GGML_F16_EPR GGML_F32_EPR |
741 | | #define GGML_F16_VEC GGML_F32x4 |
742 | | #define GGML_F16_VEC_ZERO GGML_F32x4_ZERO |
743 | | #define GGML_F16_VEC_SET1 GGML_F32x4_SET1 |
744 | | #define GGML_F16_VEC_FMA GGML_F32x4_FMA |
745 | | #define GGML_F16_VEC_ADD GGML_F32x4_ADD |
746 | | #define GGML_F16_VEC_MUL GGML_F32x4_MUL |
747 | | #define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE |
748 | | // Use vec_xl, not vec_ld, in case the load address is not aligned. |
749 | | #define GGML_F16_VEC_LOAD(p, i) (i & 0x1) ? \ |
750 | | vec_extract_fp32_from_shorth(vec_xl(0, p - GGML_F16_EPR)) : \ |
751 | | vec_extract_fp32_from_shortl(vec_xl(0, p)) |
752 | | static inline unsigned char ggml_endian_byte(int i) { |
753 | | uint16_t tmp_val = 1; |
754 | | return ((unsigned char *)&tmp_val)[i]; |
755 | | } |
756 | | #define GGML_ENDIAN_BYTE(i) ggml_endian_byte(i) |
757 | | #define GGML_F16_VEC_STORE(p, r, i) \ |
758 | | if (i & 0x1) \ |
759 | | vec_xst(vec_pack_to_short_fp32(r[i - GGML_ENDIAN_BYTE(1)], \ |
760 | | r[i - GGML_ENDIAN_BYTE(0)]), \ |
761 | | 0, p - GGML_F16_EPR) |
762 | | |
763 | | //BF16 POWER9 |
764 | | #define GGML_BF16_STEP 16 |
765 | | #define GGML_BF16_EPR 8 |
766 | | |
767 | | #define GGML_BF16x8 vector unsigned short |
768 | | #define GGML_BF16x8_ZERO vec_splats((unsigned short)0) |
769 | | #define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p)) |
770 | | |
771 | | #define GGML_BF16_VEC GGML_BF16x8 |
772 | | #define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO |
773 | | #define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD |
774 | | #if defined(__LITTLE_ENDIAN__) |
775 | | #define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel(GGML_BF16_VEC_ZERO, (v))) |
776 | | #define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh(GGML_BF16_VEC_ZERO, (v))) |
777 | | #else |
778 | | #define GGML_BF16_TO_F32_LO(v) ((vector float) vec_mergel((v), GGML_BF16_VEC_ZERO)) |
779 | | #define GGML_BF16_TO_F32_HI(v) ((vector float) vec_mergeh((v), GGML_BF16_VEC_ZERO)) |
780 | | #endif |
781 | | #define GGML_BF16_FMA_LO(acc, x, y) \ |
782 | | (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y)) |
783 | | #define GGML_BF16_FMA_HI(acc, x, y) \ |
784 | | (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) |
785 | | |
786 | | #elif defined(__wasm_simd128__) |
787 | | |
788 | | #define GGML_SIMD |
789 | | |
790 | | // F32 WASM |
791 | | |
792 | | #define GGML_F32_STEP 16 |
793 | | #define GGML_F32_EPR 4 |
794 | | |
795 | | #define GGML_F32x4 v128_t |
796 | | #define GGML_F32x4_ZERO wasm_f32x4_splat(0.0f) |
797 | | #define GGML_F32x4_SET1(x) wasm_f32x4_splat(x) |
798 | | #define GGML_F32x4_LOAD wasm_v128_load |
799 | | #define GGML_F32x4_STORE wasm_v128_store |
800 | | #define GGML_F32x4_FMA(a, b, c) wasm_f32x4_add(wasm_f32x4_mul(b, c), a) |
801 | | #define GGML_F32x4_ADD wasm_f32x4_add |
802 | | #define GGML_F32x4_MUL wasm_f32x4_mul |
803 | | #define GGML_F32x4_REDUCE(res, x) \ |
804 | | { \ |
805 | | int offset = GGML_F32_ARR >> 1; \ |
806 | | for (int i = 0; i < offset; ++i) { \ |
807 | | x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ |
808 | | } \ |
809 | | offset >>= 1; \ |
810 | | for (int i = 0; i < offset; ++i) { \ |
811 | | x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ |
812 | | } \ |
813 | | offset >>= 1; \ |
814 | | for (int i = 0; i < offset; ++i) { \ |
815 | | x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ |
816 | | } \ |
817 | | res = wasm_f32x4_extract_lane(x[0], 0) + \ |
818 | | wasm_f32x4_extract_lane(x[0], 1) + \ |
819 | | wasm_f32x4_extract_lane(x[0], 2) + \ |
820 | | wasm_f32x4_extract_lane(x[0], 3); \ |
821 | | } |
822 | | |
823 | | #define GGML_F32_VEC GGML_F32x4 |
824 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
825 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
826 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
827 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
828 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
829 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
830 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
831 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
832 | | |
833 | | // F16 WASM |
834 | | |
835 | | #define GGML_F16_STEP 16 |
836 | | #define GGML_F16_EPR 4 |
837 | | |
838 | | inline static v128_t __wasm_f16x4_load(const ggml_fp16_t * p) { |
839 | | float tmp[4]; |
840 | | |
841 | | tmp[0] = GGML_CPU_FP16_TO_FP32(p[0]); |
842 | | tmp[1] = GGML_CPU_FP16_TO_FP32(p[1]); |
843 | | tmp[2] = GGML_CPU_FP16_TO_FP32(p[2]); |
844 | | tmp[3] = GGML_CPU_FP16_TO_FP32(p[3]); |
845 | | |
846 | | return wasm_v128_load(tmp); |
847 | | } |
848 | | |
849 | | inline static void __wasm_f16x4_store(ggml_fp16_t * p, v128_t x) { |
850 | | float tmp[4]; |
851 | | |
852 | | wasm_v128_store(tmp, x); |
853 | | |
854 | | p[0] = GGML_CPU_FP32_TO_FP16(tmp[0]); |
855 | | p[1] = GGML_CPU_FP32_TO_FP16(tmp[1]); |
856 | | p[2] = GGML_CPU_FP32_TO_FP16(tmp[2]); |
857 | | p[3] = GGML_CPU_FP32_TO_FP16(tmp[3]); |
858 | | } |
859 | | |
860 | | #define GGML_F16x4 v128_t |
861 | | #define GGML_F16x4_ZERO wasm_f32x4_splat(0.0f) |
862 | | #define GGML_F16x4_SET1(x) wasm_f32x4_splat(x) |
863 | | #define GGML_F16x4_LOAD(x) __wasm_f16x4_load(x) |
864 | | #define GGML_F16x4_STORE(x, y) __wasm_f16x4_store(x, y) |
865 | | #define GGML_F16x4_FMA GGML_F32x4_FMA |
866 | | #define GGML_F16x4_ADD wasm_f32x4_add |
867 | | #define GGML_F16x4_MUL wasm_f32x4_mul |
868 | | #define GGML_F16x4_REDUCE(res, x) \ |
869 | | { \ |
870 | | int offset = GGML_F16_ARR >> 1; \ |
871 | | for (int i = 0; i < offset; ++i) { \ |
872 | | x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ |
873 | | } \ |
874 | | offset >>= 1; \ |
875 | | for (int i = 0; i < offset; ++i) { \ |
876 | | x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ |
877 | | } \ |
878 | | offset >>= 1; \ |
879 | | for (int i = 0; i < offset; ++i) { \ |
880 | | x[i] = wasm_f32x4_add(x[i], x[offset+i]); \ |
881 | | } \ |
882 | | res = (ggml_float) (wasm_f32x4_extract_lane(x[0], 0) + \ |
883 | | wasm_f32x4_extract_lane(x[0], 1) + \ |
884 | | wasm_f32x4_extract_lane(x[0], 2) + \ |
885 | | wasm_f32x4_extract_lane(x[0], 3)); \ |
886 | | } |
887 | | |
888 | | #define GGML_F16_VEC GGML_F16x4 |
889 | | #define GGML_F16_VEC_ZERO GGML_F16x4_ZERO |
890 | | #define GGML_F16_VEC_SET1 GGML_F16x4_SET1 |
891 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F16x4_LOAD(p) |
892 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F16x4_STORE(p, r[i]) |
893 | | #define GGML_F16_VEC_FMA GGML_F16x4_FMA |
894 | | #define GGML_F16_VEC_ADD GGML_F16x4_ADD |
895 | | #define GGML_F16_VEC_MUL GGML_F16x4_MUL |
896 | | #define GGML_F16_VEC_REDUCE GGML_F16x4_REDUCE |
897 | | |
898 | | #elif defined(__SSE3__) |
899 | | |
900 | | #define GGML_SIMD |
901 | | |
902 | | // F32 SSE |
903 | | |
904 | | #define GGML_F32_STEP 32 |
905 | | #define GGML_F32_EPR 4 |
906 | | |
907 | | #define GGML_F32x4 __m128 |
908 | | #define GGML_F32x4_ZERO _mm_setzero_ps() |
909 | | #define GGML_F32x4_SET1(x) _mm_set1_ps(x) |
910 | | #define GGML_F32x4_LOAD _mm_loadu_ps |
911 | | #define GGML_F32x4_STORE _mm_storeu_ps |
912 | | #if defined(__FMA__) |
913 | | // TODO: Does this work? |
914 | | #define GGML_F32x4_FMA(a, b, c) _mm_fmadd_ps(b, c, a) |
915 | | #else |
916 | | #define GGML_F32x4_FMA(a, b, c) _mm_add_ps(_mm_mul_ps(b, c), a) |
917 | | #endif |
918 | | #define GGML_F32x4_ADD _mm_add_ps |
919 | | #define GGML_F32x4_MUL _mm_mul_ps |
920 | | #define GGML_F32x4_REDUCE(res, x) \ |
921 | | { \ |
922 | | int offset = GGML_F32_ARR >> 1; \ |
923 | | for (int i = 0; i < offset; ++i) { \ |
924 | | x[i] = _mm_add_ps(x[i], x[offset+i]); \ |
925 | | } \ |
926 | | offset >>= 1; \ |
927 | | for (int i = 0; i < offset; ++i) { \ |
928 | | x[i] = _mm_add_ps(x[i], x[offset+i]); \ |
929 | | } \ |
930 | | offset >>= 1; \ |
931 | | for (int i = 0; i < offset; ++i) { \ |
932 | | x[i] = _mm_add_ps(x[i], x[offset+i]); \ |
933 | | } \ |
934 | | const __m128 t0 = _mm_hadd_ps(x[0], x[0]); \ |
935 | | res = (ggml_float) _mm_cvtss_f32(_mm_hadd_ps(t0, t0)); \ |
936 | | } |
937 | | // TODO: is this optimal ? |
938 | | |
939 | | #define GGML_F32_VEC GGML_F32x4 |
940 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
941 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
942 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
943 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
944 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
945 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
946 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
947 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
948 | | |
949 | | // F16 SSE |
950 | | |
951 | | #define GGML_F16_STEP 32 |
952 | | #define GGML_F16_EPR 4 |
953 | | |
954 | | static inline __m128 __sse_f16x4_load(const ggml_fp16_t * x) { |
955 | | float tmp[4]; |
956 | | |
957 | | tmp[0] = GGML_CPU_FP16_TO_FP32(x[0]); |
958 | | tmp[1] = GGML_CPU_FP16_TO_FP32(x[1]); |
959 | | tmp[2] = GGML_CPU_FP16_TO_FP32(x[2]); |
960 | | tmp[3] = GGML_CPU_FP16_TO_FP32(x[3]); |
961 | | |
962 | | return _mm_loadu_ps(tmp); |
963 | | } |
964 | | |
965 | | static inline void __sse_f16x4_store(ggml_fp16_t * x, __m128 y) { |
966 | | float arr[4]; |
967 | | |
968 | | _mm_storeu_ps(arr, y); |
969 | | |
970 | | x[0] = GGML_CPU_FP32_TO_FP16(arr[0]); |
971 | | x[1] = GGML_CPU_FP32_TO_FP16(arr[1]); |
972 | | x[2] = GGML_CPU_FP32_TO_FP16(arr[2]); |
973 | | x[3] = GGML_CPU_FP32_TO_FP16(arr[3]); |
974 | | } |
975 | | |
976 | | #define GGML_F32Cx4 __m128 |
977 | | #define GGML_F32Cx4_ZERO _mm_setzero_ps() |
978 | | #define GGML_F32Cx4_SET1(x) _mm_set1_ps(x) |
979 | | #define GGML_F32Cx4_LOAD(x) __sse_f16x4_load(x) |
980 | | #define GGML_F32Cx4_STORE(x, y) __sse_f16x4_store(x, y) |
981 | | #define GGML_F32Cx4_FMA GGML_F32x4_FMA |
982 | | #define GGML_F32Cx4_ADD _mm_add_ps |
983 | | #define GGML_F32Cx4_MUL _mm_mul_ps |
984 | | #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE |
985 | | |
986 | | #define GGML_F16_VEC GGML_F32Cx4 |
987 | | #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO |
988 | | #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 |
989 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p) |
990 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i]) |
991 | | #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA |
992 | | #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD |
993 | | #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL |
994 | | #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE |
995 | | |
996 | | #elif defined(__loongarch_asx) |
997 | | |
998 | | #define GGML_SIMD |
999 | | |
1000 | | // F32 LASX |
1001 | | #define GGML_F32_STEP 32 |
1002 | | #define GGML_F32_EPR 8 |
1003 | | |
1004 | | #define GGML_F32x8 __m256 |
1005 | | #define GGML_F32x8_ZERO (__m256)__lasx_xvldi(0) |
1006 | | #define GGML_F32x8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x)) |
1007 | | #define GGML_F32x8_LOAD(x) (__m256)__lasx_xvld((x), 0) |
1008 | | #define GGML_F32x8_STORE(x,y) __lasx_xvst((y), (x), 0) |
1009 | | #define GGML_F32x8_FMA(a, b, c) __lasx_xvfmadd_s(b, c, a) |
1010 | | #define GGML_F32x8_ADD __lasx_xvfadd_s |
1011 | | #define GGML_F32x8_MUL __lasx_xvfmul_s |
1012 | | #define GGML_F32x8_REDUCE(res, x) \ |
1013 | | do { \ |
1014 | | int offset = GGML_F32_ARR >> 1; \ |
1015 | | for (int i = 0; i < offset; ++i) { \ |
1016 | | x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ |
1017 | | } \ |
1018 | | offset >>= 1; \ |
1019 | | for (int i = 0; i < offset; ++i) { \ |
1020 | | x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ |
1021 | | } \ |
1022 | | offset >>= 1; \ |
1023 | | for (int i = 0; i < offset; ++i) { \ |
1024 | | x[i] = __lasx_xvfadd_s(x[i], x[offset+i]); \ |
1025 | | } \ |
1026 | | float *tmp_p = (float *)&x[0]; \ |
1027 | | res = tmp_p[0] + tmp_p[1] + tmp_p[2] + tmp_p[3] + tmp_p[4] + tmp_p[5] + tmp_p[6] + tmp_p[7]; \ |
1028 | | } while (0) |
1029 | | // TODO: is this optimal ? |
1030 | | |
1031 | | #define GGML_F32_VEC GGML_F32x8 |
1032 | | #define GGML_F32_VEC_ZERO GGML_F32x8_ZERO |
1033 | | #define GGML_F32_VEC_SET1 GGML_F32x8_SET1 |
1034 | | #define GGML_F32_VEC_LOAD GGML_F32x8_LOAD |
1035 | | #define GGML_F32_VEC_STORE GGML_F32x8_STORE |
1036 | | #define GGML_F32_VEC_FMA GGML_F32x8_FMA |
1037 | | #define GGML_F32_VEC_ADD GGML_F32x8_ADD |
1038 | | #define GGML_F32_VEC_MUL GGML_F32x8_MUL |
1039 | | #define GGML_F32_VEC_REDUCE GGML_F32x8_REDUCE |
1040 | | |
1041 | | // F16 LASX |
1042 | | |
1043 | | #define GGML_F16_STEP 32 |
1044 | | #define GGML_F16_EPR 8 |
1045 | | |
1046 | | // F16 arithmetic is not supported by LASX, so we use F32 instead |
1047 | | |
1048 | | #define GGML_F32Cx8 __m256 |
1049 | | #define GGML_F32Cx8_ZERO (__m256)__lasx_xvldi(0) |
1050 | | #define GGML_F32Cx8_SET1(x) (__m256)__lasx_xvreplfr2vr_s((x)) |
1051 | | |
1052 | | static inline __m256 __lasx_f32cx8_load(const ggml_fp16_t * x) { |
1053 | | __m256i a; |
1054 | | memcpy(&a, x, sizeof(ggml_fp16_t) * 8); |
1055 | | a = __lasx_xvpermi_d(a, 0 | (1 << 4)); |
1056 | | return __lasx_xvfcvtl_s_h(a); |
1057 | | } |
1058 | | |
1059 | | static inline void __lasx_f32cx8_store(ggml_fp16_t * x, __m256 y) { |
1060 | | __m256i a = __lasx_xvfcvt_h_s(y, y); |
1061 | | a = __lasx_xvpermi_d(a, 0 | (2 << 2)); |
1062 | | memcpy(x, &a, sizeof(ggml_fp16_t) * 8); |
1063 | | } |
1064 | | #define GGML_F32Cx8_LOAD(x) __lasx_f32cx8_load(x) |
1065 | | #define GGML_F32Cx8_STORE(x, y) __lasx_f32cx8_store(x, y) |
1066 | | |
1067 | | #define GGML_F32Cx8_FMA GGML_F32x8_FMA |
1068 | | #define GGML_F32Cx8_ADD __lasx_xvfadd_s |
1069 | | #define GGML_F32Cx8_MUL __lasx_xvfmul_s |
1070 | | #define GGML_F32Cx8_REDUCE GGML_F32x8_REDUCE |
1071 | | |
1072 | | #define GGML_F16_VEC GGML_F32Cx8 |
1073 | | #define GGML_F16_VEC_ZERO GGML_F32Cx8_ZERO |
1074 | | #define GGML_F16_VEC_SET1 GGML_F32Cx8_SET1 |
1075 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx8_LOAD(p) |
1076 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx8_STORE(p, r[i]) |
1077 | | #define GGML_F16_VEC_FMA GGML_F32Cx8_FMA |
1078 | | #define GGML_F16_VEC_ADD GGML_F32Cx8_ADD |
1079 | | #define GGML_F16_VEC_MUL GGML_F32Cx8_MUL |
1080 | | #define GGML_F16_VEC_REDUCE GGML_F32Cx8_REDUCE |
1081 | | |
1082 | | #elif defined(__loongarch_sx) |
1083 | | |
1084 | | #define GGML_SIMD |
1085 | | |
1086 | | // F32 LSX |
1087 | | |
1088 | | #define GGML_F32_STEP 32 |
1089 | | #define GGML_F32_EPR 4 |
1090 | | |
1091 | | #define GGML_F32x4 __m128 |
1092 | | #define GGML_F32x4_ZERO (__m128)__lsx_vldi(0) |
1093 | | #define GGML_F32x4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x)) |
1094 | | #define GGML_F32x4_LOAD(x) (__m128)__lsx_vld((x), 0) |
1095 | | #define GGML_F32x4_STORE(x, y) __lsx_vst(y, x, 0) |
1096 | | #define GGML_F32x4_FMA(a, b, c) __lsx_vfmadd_s(b, c, a) |
1097 | | #define GGML_F32x4_ADD __lsx_vfadd_s |
1098 | | #define GGML_F32x4_MUL __lsx_vfmul_s |
1099 | | |
1100 | | #define GGML_F32x4_REDUCE(res, x) \ |
1101 | | { \ |
1102 | | int offset = GGML_F32_ARR >> 1; \ |
1103 | | for (int i = 0; i < offset; ++i) { \ |
1104 | | x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ |
1105 | | } \ |
1106 | | offset >>= 1; \ |
1107 | | for (int i = 0; i < offset; ++i) { \ |
1108 | | x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ |
1109 | | } \ |
1110 | | offset >>= 1; \ |
1111 | | for (int i = 0; i < offset; ++i) { \ |
1112 | | x[i] = __lsx_vfadd_s(x[i], x[offset+i]); \ |
1113 | | } \ |
1114 | | __m128i t0 = __lsx_vpickev_w((__m128i)x[0], (__m128i)x[0]); \ |
1115 | | __m128i t1 = __lsx_vpickod_w((__m128i)x[0], (__m128i)x[0]); \ |
1116 | | __m128 t2 = __lsx_vfadd_s((__m128)t0, (__m128)t1); \ |
1117 | | __m128i t3 = __lsx_vpickev_w((__m128i)t2, (__m128i)t2); \ |
1118 | | __m128i t4 = __lsx_vpickod_w((__m128i)t2, (__m128i)t2); \ |
1119 | | __m128 t5 = __lsx_vfadd_s((__m128)t3, (__m128)t4); \ |
1120 | | res = (ggml_float) ((v4f32)t5)[0]; \ |
1121 | | } |
1122 | | |
1123 | | #define GGML_F32_VEC GGML_F32x4 |
1124 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
1125 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
1126 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
1127 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
1128 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
1129 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
1130 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
1131 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
1132 | | |
1133 | | // F16 LSX |
1134 | | |
1135 | | #define GGML_F16_STEP 32 |
1136 | | #define GGML_F16_EPR 4 |
1137 | | |
1138 | | static inline __m128 __lsx_f16x4_load(const ggml_fp16_t * x) { |
1139 | | return __lsx_vfcvtl_s_h(__lsx_vld((const void *)x, 0)); |
1140 | | } |
1141 | | |
1142 | | static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) { |
1143 | | __m128i a = __lsx_vfcvt_h_s(y, y); |
1144 | | memcpy(x, &a, sizeof(ggml_fp16_t) * 4); |
1145 | | } |
1146 | | |
1147 | | #define GGML_F32Cx4 __m128 |
1148 | | #define GGML_F32Cx4_ZERO (__m128)__lsx_vldi(0) |
1149 | | #define GGML_F32Cx4_SET1(x) (__m128)__lsx_vreplfr2vr_s((x)) |
1150 | | #define GGML_F32Cx4_LOAD(x) (__m128)__lsx_f16x4_load(x) |
1151 | | #define GGML_F32Cx4_STORE(x, y) __lsx_f16x4_store(x, y) |
1152 | | #define GGML_F32Cx4_FMA GGML_F32x4_FMA |
1153 | | #define GGML_F32Cx4_ADD __lsx_vfadd_s |
1154 | | #define GGML_F32Cx4_MUL __lsx_vfmul_s |
1155 | | #define GGML_F32Cx4_REDUCE GGML_F32x4_REDUCE |
1156 | | |
1157 | | #define GGML_F16_VEC GGML_F32Cx4 |
1158 | | #define GGML_F16_VEC_ZERO GGML_F32Cx4_ZERO |
1159 | | #define GGML_F16_VEC_SET1 GGML_F32Cx4_SET1 |
1160 | | #define GGML_F16_VEC_LOAD(p, i) GGML_F32Cx4_LOAD(p) |
1161 | | #define GGML_F16_VEC_STORE(p, r, i) GGML_F32Cx4_STORE(p, r[i]) |
1162 | | #define GGML_F16_VEC_FMA GGML_F32Cx4_FMA |
1163 | | #define GGML_F16_VEC_ADD GGML_F32Cx4_ADD |
1164 | | #define GGML_F16_VEC_MUL GGML_F32Cx4_MUL |
1165 | | #define GGML_F16_VEC_REDUCE GGML_F32Cx4_REDUCE |
1166 | | |
1167 | | #elif defined(__VXE__) || defined(__VXE2__) |
1168 | | |
1169 | | #define GGML_SIMD |
1170 | | |
1171 | | // F32 s390x |
1172 | | |
1173 | | #define GGML_F32_STEP 32 |
1174 | | #define GGML_F32_EPR 4 |
1175 | | |
1176 | | #define GGML_F32x4 float32x4_t |
1177 | | #define GGML_F32x4_ZERO vec_splats(0.0f) |
1178 | | #define GGML_F32x4_SET1 vec_splats |
1179 | | #define GGML_F32x4_LOAD(p) vec_xl(0, p) |
1180 | | #define GGML_F32x4_STORE(p, r) vec_xst(r, 0, p) |
1181 | | #define GGML_F32x4_FMA(a, b, c) vec_madd(b, c, a) |
1182 | | #define GGML_F32x4_ADD vec_add |
1183 | | #define GGML_F32x4_MUL vec_mul |
1184 | | #define GGML_F32x4_REDUCE(res, x) \ |
1185 | | { \ |
1186 | | int offset = GGML_F32_ARR >> 1; \ |
1187 | | for (int i = 0; i < offset; ++i) { \ |
1188 | | x[i] = vec_add(x[i], x[offset + i]); \ |
1189 | | } \ |
1190 | | offset >>= 1; \ |
1191 | | for (int i = 0; i < offset; ++i) { \ |
1192 | | x[i] = vec_add(x[i], x[offset + i]); \ |
1193 | | } \ |
1194 | | offset >>= 1; \ |
1195 | | for (int i = 0; i < offset; ++i) { \ |
1196 | | x[i] = vec_add(x[i], x[offset + i]); \ |
1197 | | } \ |
1198 | | float32x4_t tmp = x[0] + vec_reve(x[0]); \ |
1199 | | res = tmp[0] + tmp[1]; \ |
1200 | | } |
1201 | | #define GGML_F32x4_REDUCE_4(res, s0, s1, s2, s3) \ |
1202 | | { \ |
1203 | | float32x4_t v = vec_add(vec_add(s0, s1), \ |
1204 | | vec_add(s2, s3)); \ |
1205 | | v = vec_add(v, vec_sld(v, v, 8)); \ |
1206 | | v = vec_add(v, vec_sld(v, v, 4)); \ |
1207 | | res += (ggml_float)vec_extract(v, 0); \ |
1208 | | } |
1209 | | |
1210 | | #define GGML_F32_VEC GGML_F32x4 |
1211 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
1212 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
1213 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
1214 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
1215 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
1216 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
1217 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
1218 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
1219 | | |
1220 | | // F16 s390x |
1221 | | #define GGML_F16_STEP GGML_F32_STEP |
1222 | | #define GGML_F16_EPR GGML_F32_EPR |
1223 | | |
1224 | | static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) { |
1225 | | float tmp[4]; |
1226 | | |
1227 | | for (int i = 0; i < 4; i++) { |
1228 | | tmp[i] = GGML_CPU_FP16_TO_FP32(x[i]); |
1229 | | } |
1230 | | |
1231 | | // note: keep type-cast here to prevent compiler bugs |
1232 | | // see: https://github.com/ggml-org/llama.cpp/issues/12846 |
1233 | | return vec_xl(0, (const float *)(tmp)); |
1234 | | } |
1235 | | |
1236 | | static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) { |
1237 | | float arr[4]; |
1238 | | |
1239 | | // note: keep type-cast here to prevent compiler bugs |
1240 | | // see: https://github.com/ggml-org/llama.cpp/issues/12846 |
1241 | | vec_xst(v_y, 0, (float *)(arr)); |
1242 | | |
1243 | | for (int i = 0; i < 4; i++) { |
1244 | | x[i] = GGML_CPU_FP32_TO_FP16(arr[i]); |
1245 | | } |
1246 | | } |
1247 | | |
1248 | | #define GGML_F16_VEC GGML_F32x4 |
1249 | | #define GGML_F16_VEC_ZERO GGML_F32x4_ZERO |
1250 | | #define GGML_F16_VEC_SET1 GGML_F32x4_SET1 |
1251 | | #define GGML_F16_VEC_LOAD(p, i) __lzs_f16cx4_load(p) |
1252 | | #define GGML_F16_VEC_STORE(p, r, i) __lzs_f16cx4_store(p, r[i]) |
1253 | | #define GGML_F16_VEC_FMA GGML_F32x4_FMA |
1254 | | #define GGML_F16_VEC_ADD GGML_F32x4_ADD |
1255 | | #define GGML_F16_VEC_MUL GGML_F32x4_MUL |
1256 | | #define GGML_F16_VEC_REDUCE GGML_F32x4_REDUCE |
1257 | | |
1258 | | // BF16 s390x |
1259 | | #define GGML_BF16_STEP 16 |
1260 | | #define GGML_BF16_EPR 8 |
1261 | | |
1262 | | #define GGML_BF16x8 __vector unsigned short |
1263 | | #define GGML_BF16x8_ZERO vec_splats((unsigned short)0) |
1264 | | #define GGML_BF16x8_LOAD(p) vec_xl(0, (const unsigned short *)(p)) |
1265 | | |
1266 | | #define GGML_BF16_VEC GGML_BF16x8 |
1267 | | #define GGML_BF16_VEC_ZERO GGML_BF16x8_ZERO |
1268 | | #define GGML_BF16_VEC_LOAD GGML_BF16x8_LOAD |
1269 | | #define GGML_BF16_TO_F32_LO(v) ((float32x4_t) vec_mergel((v), GGML_BF16_VEC_ZERO)) |
1270 | | #define GGML_BF16_TO_F32_HI(v) ((float32x4_t) vec_mergeh((v), GGML_BF16_VEC_ZERO)) |
1271 | | #define GGML_BF16_FMA_LO(acc, x, y) \ |
1272 | | (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_LO(x), GGML_BF16_TO_F32_LO(y)) |
1273 | | #define GGML_BF16_FMA_HI(acc, x, y) \ |
1274 | | (acc) = GGML_F32x4_FMA((acc), GGML_BF16_TO_F32_HI(x), GGML_BF16_TO_F32_HI(y)) |
1275 | | |
1276 | | #elif defined(__riscv_v_intrinsic) |
1277 | | |
1278 | | // compatible with vlen >= 128 |
1279 | | |
1280 | | #define GGML_SIMD |
1281 | | |
1282 | | // F32 |
1283 | | |
1284 | | #define GGML_F32_STEP 16 |
1285 | | #define GGML_F32_EPR 4 |
1286 | | |
1287 | | #define GGML_F32x4 vfloat32m1_t |
1288 | | #define GGML_F32x4_ZERO __riscv_vfmv_v_f_f32m1(0.0f, GGML_F32_EPR) |
1289 | | #define GGML_F32x4_SET1(x) __riscv_vfmv_v_f_f32m1(x, GGML_F32_EPR) |
1290 | | #define GGML_F32x4_LOAD(x) __riscv_vle32_v_f32m1(x, GGML_F32_EPR) |
1291 | | #define GGML_F32x4_STORE(b, v) __riscv_vse32_v_f32m1(b, v, GGML_F32_EPR) |
1292 | | #define GGML_F32x4_FMA(a, b, c) __riscv_vfmacc_vv_f32m1(a, b, c, GGML_F32_EPR) |
1293 | | #define GGML_F32x4_ADD(a, b) __riscv_vfadd_vv_f32m1(a, b, GGML_F32_EPR) |
1294 | | #define GGML_F32x4_MUL(a, b) __riscv_vfmul_vv_f32m1(a, b, GGML_F32_EPR) |
1295 | | |
1296 | | #define GGML_F32_VEC GGML_F32x4 |
1297 | | #define GGML_F32_VEC_ZERO GGML_F32x4_ZERO |
1298 | | #define GGML_F32_VEC_SET1 GGML_F32x4_SET1 |
1299 | | #define GGML_F32_VEC_LOAD GGML_F32x4_LOAD |
1300 | | #define GGML_F32_VEC_STORE GGML_F32x4_STORE |
1301 | | #define GGML_F32_VEC_FMA GGML_F32x4_FMA |
1302 | | #define GGML_F32_VEC_ADD GGML_F32x4_ADD |
1303 | | #define GGML_F32_VEC_MUL GGML_F32x4_MUL |
1304 | | #define GGML_F32_VEC_REDUCE GGML_F32x4_REDUCE |
1305 | | |
1306 | | #endif |
1307 | | |
1308 | | // GGML_F32_ARR / GGML_F16_ARR |
1309 | | // number of registers to use per step |
1310 | | #ifdef GGML_SIMD |
1311 | 0 | #define GGML_F32_ARR (GGML_F32_STEP/GGML_F32_EPR) |
1312 | 0 | #define GGML_F16_ARR (GGML_F16_STEP/GGML_F16_EPR) |
1313 | | #endif |
1314 | | |
1315 | | #ifdef __cplusplus |
1316 | | } |
1317 | | #endif |