/src/llama.cpp/ggml/src/ggml-cpu/binary-ops.cpp

Source
#include "binary-ops.h"

#if defined(GGML_USE_ACCELERATE)
#include <Accelerate/Accelerate.h>

using vDSP_fn_t = void (*)(const float *, vDSP_Stride, const float *, vDSP_Stride, float *, vDSP_Stride, vDSP_Length);
#endif

static inline float op_add(float a, float b) {
    return a + b;
}

static inline float op_sub(float a, float b) {
    return a - b;
}

static inline float op_mul(float a, float b) {
    return a * b;
}

static inline float op_div(float a, float b) {
    return a / b;
}

template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
static inline void vec_binary_op_contiguous(const int64_t n, dst_t * z, const src0_t * x, const src1_t * y) {
    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
    constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;
    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;

    for (int i = 0; i < n; i++) {
        z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(y[i])));
    }
}

template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
static inline void vec_binary_op_non_contiguous(const int64_t n, const int64_t ne10, const int64_t nb10, dst_t * z, const src0_t * x, const src1_t * y) {
    constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
    constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;
    constexpr auto f32_to_dst  = type_conversion_table<dst_t >::from_f32;

    for (int i = 0; i < n; i++) {
        int i10 = i % ne10;
        const src1_t * y_ptr = (const src1_t *)((const char *)y + i10*nb10);
        z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(*y_ptr)));
    }
}

template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
static void apply_binary_op(const ggml_compute_params * params, ggml_tensor * dst) {
    const ggml_tensor * src0 = dst->src[0];
    const ggml_tensor * src1 = dst->src[1];

    GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));

    GGML_TENSOR_BINARY_OP_LOCALS

    GGML_ASSERT( nb0 == sizeof(dst_t));
    GGML_ASSERT(nb00 == sizeof(src0_t));

    const auto [ir0, ir1] = get_thread_range(params, src0);
    const bool is_src1_contiguous = (nb10 == sizeof(src1_t));

    if (!is_src1_contiguous) { // broadcast not implemented yet for non-contiguous
        GGML_ASSERT(ggml_are_same_shape(src0, src1));
    }

#ifdef GGML_USE_ACCELERATE
    vDSP_fn_t vDSP_op = nullptr;
    // TODO - avoid the f32-only check using type 'trait' lookup tables and row-based src-to-float conversion functions
    if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
        if (op == op_add) {
            vDSP_op = vDSP_vadd;
        } else if (op == op_sub) {
            vDSP_op = vDSP_vsub;
        } else if (op == op_mul) {
            vDSP_op = vDSP_vmul;
        } else if (op == op_div) {
            vDSP_op = vDSP_vdiv;
        }
    }
#endif

    for (int64_t ir = ir0; ir < ir1; ++ir) {
        const int64_t i03 = ir/(ne02*ne01);
        const int64_t i02 = (ir - i03*ne02*ne01)/ne01;
        const int64_t i01 = (ir - i03*ne02*ne01 - i02*ne01);

        const int64_t i13 = i03 % ne13;
        const int64_t i12 = i02 % ne12;
        const int64_t i11 = i01 % ne11;

        dst_t        * dst_ptr  = (dst_t  *)       ((char *)       dst->data  + i03*nb3  + i02*nb2  + i01*nb1 );
        const src0_t * src0_ptr = (const src0_t *) ((const char *) src0->data + i03*nb03 + i02*nb02 + i01*nb01);
        const src1_t * src1_ptr = (const src1_t *) ((const char *) src1->data + i13*nb13 + i12*nb12 + i11*nb11);

        if (is_src1_contiguous) {
            // src1 is broadcastable across src0 and dst in i1, i2, i3
            const int64_t nr0 = ne00 / ne10;

            for (int64_t r = 0; r < nr0; ++r) {
#ifdef GGML_USE_ACCELERATE
                if constexpr (std::is_same_v<src0_t, float> && std::is_same_v<src1_t, float> && std::is_same_v<dst_t, float>) {
                    if (vDSP_op != nullptr) {
                        vDSP_op(src1_ptr, 1, src0_ptr + r*ne10, 1, dst_ptr + r*ne10, 1, ne10);
                        continue;
                    }
                }
#endif
                vec_binary_op_contiguous<op>(ne10, dst_ptr + r*ne10, src0_ptr + r*ne10, src1_ptr);
            }
        } else {
            vec_binary_op_non_contiguous<op>(ne0, ne10, nb10, dst_ptr, src0_ptr, src1_ptr);
        }
    }
}

// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates
template <float (*op)(float, float)>
static void binary_op(const ggml_compute_params * params, ggml_tensor * dst) {
    const ggml_tensor * src0 = dst->src[0];
    const ggml_tensor * src1 = dst->src[1];

    /*  */ if (src0->type == GGML_TYPE_F32  && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) { // all f32
        apply_binary_op<op, float, float, float>(params, dst);
    } else if (src0->type == GGML_TYPE_F16  && src1->type == GGML_TYPE_F16  && dst->type == GGML_TYPE_F16) { // all f16
        apply_binary_op<op, ggml_fp16_t, ggml_fp16_t, ggml_fp16_t>(params, dst);
    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16
        apply_binary_op<op, ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(params, dst);
    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_BF16) {
        apply_binary_op<op, ggml_bf16_t, float, ggml_bf16_t>(params, dst);
    } else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) {
        apply_binary_op<op, ggml_bf16_t, float, float>(params, dst);
    } else if (src0->type == GGML_TYPE_F16  && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F16) {
        apply_binary_op<op, ggml_fp16_t, float, ggml_fp16_t>(params, dst);
    } else if (src0->type == GGML_TYPE_F16  && src1->type == GGML_TYPE_F32  && dst->type == GGML_TYPE_F32) {
        apply_binary_op<op, ggml_fp16_t, float, float>(params, dst);
    } else {
        GGML_ABORT("%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,
            ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));
    }
}

void ggml_compute_forward_add_non_quantized(const ggml_compute_params * params, ggml_tensor * dst) {
    binary_op<op_add>(params, dst);
}

void ggml_compute_forward_sub(const ggml_compute_params * params, ggml_tensor * dst) {
    binary_op<op_sub>(params, dst);
}

void ggml_compute_forward_mul(const ggml_compute_params * params, ggml_tensor * dst) {
    binary_op<op_mul>(params, dst);
}

void ggml_compute_forward_div(const ggml_compute_params * params, ggml_tensor * dst) {
    binary_op<op_div>(params, dst);
}

Line	Count	Source
1		#include "binary-ops.h"
2
3		#if defined(GGML_USE_ACCELERATE)
4		#include <Accelerate/Accelerate.h>
5
6		using vDSP_fn_t = void ()(const float , vDSP_Stride, const float , vDSP_Stride, float , vDSP_Stride, vDSP_Length);
7		#endif
8
9	0	static inline float op_add(float a, float b) {
10	0	return a + b;
11	0	}
12
13	0	static inline float op_sub(float a, float b) {
14	0	return a - b;
15	0	}
16
17	0	static inline float op_mul(float a, float b) {
18	0	return a * b;
19	0	}
20
21	0	static inline float op_div(float a, float b) {
22	0	return a / b;
23	0	}
24
25		template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
26	0	static inline void vec_binary_op_contiguous(const int64_t n, dst_t * z, const src0_t * x, const src1_t * y) {
27	0	constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
28	0	constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;
29	0	constexpr auto f32_to_dst = type_conversion_table<dst_t >::from_f32;
30
31	0	for (int i = 0; i < n; i++) {
32	0	z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(y[i])));
33	0	}
34	0	} Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), float, float, float>(long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), unsigned short, unsigned short, unsigned short>(long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), ggml_bf16_t, float, float>(long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), unsigned short, float, unsigned short>(long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_add(float, float)), unsigned short, float, float>(long, float, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), float, float, float>(long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), unsigned short, unsigned short, unsigned short>(long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), ggml_bf16_t, float, float>(long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), unsigned short, float, unsigned short>(long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_sub(float, float)), unsigned short, float, float>(long, float, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), float, float, float>(long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), unsigned short, unsigned short, unsigned short>(long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), ggml_bf16_t, float, float>(long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), unsigned short, float, unsigned short>(long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_mul(float, float)), unsigned short, float, float>(long, float, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), float, float, float>(long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), unsigned short, unsigned short, unsigned short>(long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), ggml_bf16_t, float, float>(long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), unsigned short, float, unsigned short>(long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_contiguous<&(op_div(float, float)), unsigned short, float, float>(long, float, unsigned short const, float const)
35
36		template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
37	0	static inline void vec_binary_op_non_contiguous(const int64_t n, const int64_t ne10, const int64_t nb10, dst_t * z, const src0_t * x, const src1_t * y) {
38	0	constexpr auto src0_to_f32 = type_conversion_table<src0_t>::to_f32;
39	0	constexpr auto src1_to_f32 = type_conversion_table<src1_t>::to_f32;
40	0	constexpr auto f32_to_dst = type_conversion_table<dst_t >::from_f32;
41
42	0	for (int i = 0; i < n; i++) {
43	0	int i10 = i % ne10;
44	0	const src1_t * y_ptr = (const src1_t )((const char )y + i10*nb10);
45	0	z[i] = f32_to_dst(op(src0_to_f32(x[i]), src1_to_f32(*y_ptr)));
46	0	}
47	0	} Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), float, float, float>(long, long, long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), unsigned short, unsigned short, unsigned short>(long, long, long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), ggml_bf16_t, float, float>(long, long, long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), unsigned short, float, unsigned short>(long, long, long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_add(float, float)), unsigned short, float, float>(long, long, long, float, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), float, float, float>(long, long, long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), unsigned short, unsigned short, unsigned short>(long, long, long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), ggml_bf16_t, float, float>(long, long, long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), unsigned short, float, unsigned short>(long, long, long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_sub(float, float)), unsigned short, float, float>(long, long, long, float, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), float, float, float>(long, long, long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), unsigned short, unsigned short, unsigned short>(long, long, long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), ggml_bf16_t, float, float>(long, long, long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), unsigned short, float, unsigned short>(long, long, long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_mul(float, float)), unsigned short, float, float>(long, long, long, float, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), float, float, float>(long, long, long, float, float const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), unsigned short, unsigned short, unsigned short>(long, long, long, unsigned short, unsigned short const, unsigned short const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, ggml_bf16_t const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), ggml_bf16_t, float, ggml_bf16_t>(long, long, long, ggml_bf16_t, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), ggml_bf16_t, float, float>(long, long, long, float, ggml_bf16_t const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), unsigned short, float, unsigned short>(long, long, long, unsigned short, unsigned short const, float const) Unexecuted instantiation: binary-ops.cpp:void vec_binary_op_non_contiguous<&(op_div(float, float)), unsigned short, float, float>(long, long, long, float, unsigned short const, float const)
48
49		template <float (*op)(float, float), typename src0_t, typename src1_t, typename dst_t>
50	0	static void apply_binary_op(const ggml_compute_params * params, ggml_tensor * dst) {
51	0	const ggml_tensor * src0 = dst->src[0];
52	0	const ggml_tensor * src1 = dst->src[1];
53
54	0	GGML_ASSERT(ggml_can_repeat(src1, src0) && ggml_are_same_shape(src0, dst));
55
56	0	GGML_TENSOR_BINARY_OP_LOCALS
57
58	0	GGML_ASSERT( nb0 == sizeof(dst_t));
59	0	GGML_ASSERT(nb00 == sizeof(src0_t));
60
61	0	const auto [ir0, ir1] = get_thread_range(params, src0);
62	0	const bool is_src1_contiguous = (nb10 == sizeof(src1_t));
63
64	0	if (!is_src1_contiguous) { // broadcast not implemented yet for non-contiguous
65	0	GGML_ASSERT(ggml_are_same_shape(src0, src1));
66	0	}
67
68		#ifdef GGML_USE_ACCELERATE
69		vDSP_fn_t vDSP_op = nullptr;
70		// TODO - avoid the f32-only check using type 'trait' lookup tables and row-based src-to-float conversion functions
71		if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
72		if (op == op_add) {
73		vDSP_op = vDSP_vadd;
74		} else if (op == op_sub) {
75		vDSP_op = vDSP_vsub;
76		} else if (op == op_mul) {
77		vDSP_op = vDSP_vmul;
78		} else if (op == op_div) {
79		vDSP_op = vDSP_vdiv;
80		}
81		}
82		#endif
83
84	0	for (int64_t ir = ir0; ir < ir1; ++ir) {
85	0	const int64_t i03 = ir/(ne02*ne01);
86	0	const int64_t i02 = (ir - i03ne02ne01)/ne01;
87	0	const int64_t i01 = (ir - i03ne02ne01 - i02*ne01);
88
89	0	const int64_t i13 = i03 % ne13;
90	0	const int64_t i12 = i02 % ne12;
91	0	const int64_t i11 = i01 % ne11;
92
93	0	dst_t * dst_ptr = (dst_t ) ((char ) dst->data + i03nb3 + i02nb2 + i01*nb1 );
94	0	const src0_t * src0_ptr = (const src0_t ) ((const char ) src0->data + i03nb03 + i02nb02 + i01*nb01);
95	0	const src1_t * src1_ptr = (const src1_t ) ((const char ) src1->data + i13nb13 + i12nb12 + i11*nb11);
96
97	0	if (is_src1_contiguous) {
98		// src1 is broadcastable across src0 and dst in i1, i2, i3
99	0	const int64_t nr0 = ne00 / ne10;
100
101	0	for (int64_t r = 0; r < nr0; ++r) {
102		#ifdef GGML_USE_ACCELERATE
103		if constexpr (std::is_same_v<src0_t, float> && std::is_same_v<src1_t, float> && std::is_same_v<dst_t, float>) {
104		if (vDSP_op != nullptr) {
105		vDSP_op(src1_ptr, 1, src0_ptr + rne10, 1, dst_ptr + rne10, 1, ne10);
106		continue;
107		}
108		}
109		#endif
110	0	vec_binary_op_contiguous<op>(ne10, dst_ptr + rne10, src0_ptr + rne10, src1_ptr);
111	0	}
112	0	} else {
113	0	vec_binary_op_non_contiguous<op>(ne0, ne10, nb10, dst_ptr, src0_ptr, src1_ptr);
114	0	}
115	0	}
116	0	} Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), float, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), unsigned short, unsigned short, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), ggml_bf16_t, float, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), ggml_bf16_t, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), unsigned short, float, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_add(float, float)), unsigned short, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), float, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), unsigned short, unsigned short, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), ggml_bf16_t, float, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), ggml_bf16_t, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), unsigned short, float, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_sub(float, float)), unsigned short, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), float, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), unsigned short, unsigned short, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), ggml_bf16_t, float, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), ggml_bf16_t, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), unsigned short, float, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_mul(float, float)), unsigned short, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), float, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), unsigned short, unsigned short, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), ggml_bf16_t, float, ggml_bf16_t>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), ggml_bf16_t, float, float>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), unsigned short, float, unsigned short>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void apply_binary_op<&(op_div(float, float)), unsigned short, float, float>(ggml_compute_params const, ggml_tensor)
117
118		// TODO: Use the 'traits' lookup table (for type conversion fns), instead of a mass of 'if' conditions with long templates
119		template <float (*op)(float, float)>
120	0	static void binary_op(const ggml_compute_params * params, ggml_tensor * dst) {
121	0	const ggml_tensor * src0 = dst->src[0];
122	0	const ggml_tensor * src1 = dst->src[1];
123
124	0	/* */ if (src0->type == GGML_TYPE_F32 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) { // all f32
125	0	apply_binary_op<op, float, float, float>(params, dst);
126	0	} else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F16) { // all f16
127	0	apply_binary_op<op, ggml_fp16_t, ggml_fp16_t, ggml_fp16_t>(params, dst);
128	0	} else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_BF16 && dst->type == GGML_TYPE_BF16) { // all bf16
129	0	apply_binary_op<op, ggml_bf16_t, ggml_bf16_t, ggml_bf16_t>(params, dst);
130	0	} else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_BF16) {
131	0	apply_binary_op<op, ggml_bf16_t, float, ggml_bf16_t>(params, dst);
132	0	} else if (src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
133	0	apply_binary_op<op, ggml_bf16_t, float, float>(params, dst);
134	0	} else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) {
135	0	apply_binary_op<op, ggml_fp16_t, float, ggml_fp16_t>(params, dst);
136	0	} else if (src0->type == GGML_TYPE_F16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32) {
137	0	apply_binary_op<op, ggml_fp16_t, float, float>(params, dst);
138	0	} else {
139	0	GGML_ABORT("%s: unsupported types: dst: %s, src0: %s, src1: %s\n", __func__,
140	0	ggml_type_name(dst->type), ggml_type_name(src0->type), ggml_type_name(src1->type));
141	0	}
142	0	} Unexecuted instantiation: binary-ops.cpp:void binary_op<&(op_add(float, float))>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void binary_op<&(op_sub(float, float))>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void binary_op<&(op_mul(float, float))>(ggml_compute_params const, ggml_tensor) Unexecuted instantiation: binary-ops.cpp:void binary_op<&(op_div(float, float))>(ggml_compute_params const, ggml_tensor)
143
144	0	void ggml_compute_forward_add_non_quantized(const ggml_compute_params * params, ggml_tensor * dst) {
145	0	binary_op<op_add>(params, dst);
146	0	}
147
148	0	void ggml_compute_forward_sub(const ggml_compute_params * params, ggml_tensor * dst) {
149	0	binary_op<op_sub>(params, dst);
150	0	}
151
152	0	void ggml_compute_forward_mul(const ggml_compute_params * params, ggml_tensor * dst) {
153	0	binary_op<op_mul>(params, dst);
154	0	}
155
156	0	void ggml_compute_forward_div(const ggml_compute_params * params, ggml_tensor * dst) {
157	0	binary_op<op_div>(params, dst);
158	0	}

Coverage Report

Created: 2026-01-13 06:16