#define _CRT_SECURE_NO_DEPRECATE // Disables "unsafe" warnings on Windows

#define _USE_MATH_DEFINES // For M_PI on MSVC

#include "ggml-backend.h"

#include "ggml-impl.h"

#include "ggml-threading.h"

#include "ggml-cpu.h"

#include "ggml.h"

// FIXME: required here for quantization functions

#include "ggml-quants.h"

#ifdef GGML_USE_CPU_HBM

#include <hbwmalloc.h>

#endif

#if defined(_MSC_VER) || defined(__MINGW32__)

#include <malloc.h> // using malloc.h with MSC/MINGW

#elif !defined(__FreeBSD__) && !defined(__NetBSD__) && !defined(__OpenBSD__)

#include <alloca.h>

#endif

#include <assert.h>

#include <errno.h>

#include <time.h>

#include <math.h>

#include <stdlib.h>

#include <string.h>

#include <stdint.h>

#include <inttypes.h>

#include <stdio.h>

#include <float.h>

#include <limits.h>

#include <stdarg.h>

#include <signal.h>

#if defined(__gnu_linux__)

#include <syscall.h>

#endif

#if defined(__APPLE__)

#include <unistd.h>

#include <mach/mach.h>

#include <TargetConditionals.h>

#endif

#if defined(_WIN32)

#define WIN32_LEAN_AND_MEAN

#ifndef NOMINMAX

    #define NOMINMAX

#endif

#include <windows.h>

#endif

#define UNUSED GGML_UNUSED

#if defined(_MSC_VER)

#define m512bh(p) p

#define m512i(p) p

#else

#define m512bh(p) (__m512bh)(p)

#define m512i(p) (__m512i)(p)

#endif

#if defined(__linux__) || \

    defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) || \

    (defined(__APPLE__) && !TARGET_OS_TV && !TARGET_OS_WATCH)

#include <unistd.h>

#include <sys/types.h>

#include <sys/stat.h>

#include <sys/wait.h>

#if defined(__linux__)

#include <sys/prctl.h>

#endif

#if defined(__ANDROID__)

#include <unwind.h>

#include <dlfcn.h>

#include <stdio.h>

struct backtrace_state {

    void ** current;

    void ** end;

};

static _Unwind_Reason_Code unwind_callback(struct _Unwind_Context* context, void* arg) {

    struct backtrace_state * state = (struct backtrace_state *)arg;

    uintptr_t pc = _Unwind_GetIP(context);

    if (pc) {

        if (state->current == state->end) {

            return _URC_END_OF_STACK;

        } else {

            *state->current++ = (void*)pc;

        }

    }

    return _URC_NO_REASON;

}

static void ggml_print_backtrace_symbols(void) {

    const int max = 100;

    void* buffer[max];

    struct backtrace_state state = {buffer, buffer + max};

    _Unwind_Backtrace(unwind_callback, &state);

    int count = state.current - buffer;

    for (int idx = 0; idx < count; ++idx) {

        const void * addr = buffer[idx];

        const char * symbol = "";

        Dl_info info;

        if (dladdr(addr, &info) && info.dli_sname) {

            symbol = info.dli_sname;

        }

        fprintf(stderr, "%d: %p %s\n", idx, addr, symbol);

    }

}

#elif defined(__linux__) && defined(__GLIBC__)

#include <execinfo.h>

static void ggml_print_backtrace_symbols(void) {

    void * trace[100];

    int nptrs = backtrace(trace, sizeof(trace)/sizeof(trace[0]));

    backtrace_symbols_fd(trace, nptrs, STDERR_FILENO);

}

#else

static void ggml_print_backtrace_symbols(void) {

    // platform not supported

}

#endif

void ggml_print_backtrace(void) {

    const char * GGML_NO_BACKTRACE = getenv("GGML_NO_BACKTRACE");

    if (GGML_NO_BACKTRACE) {

        return;

    }

#if defined(__linux__)

    FILE * f = fopen("/proc/self/status", "r");

    size_t size = 0;

    char * line = NULL;

    ssize_t length = 0;

    while ((length = getline(&line, &size, f)) > 0) {

        if (!strncmp(line, "TracerPid:", sizeof("TracerPid:") - 1) &&

            (length != sizeof("TracerPid:\t0\n") - 1 || line[length - 2] != '0')) {

            // Already being debugged, and the breakpoint is the later abort()

            free(line);

            fclose(f);

            return;

        }

    }

    free(line);

    fclose(f);

    int lock[2] = { -1, -1 };

    (void) !pipe(lock); // Don't start gdb until after PR_SET_PTRACER

#endif

    const int parent_pid = getpid();

    const int child_pid = fork();

    if (child_pid < 0) { // error

#if defined(__linux__)

        close(lock[1]);

        close(lock[0]);

#endif

        return;

    } else if (child_pid == 0) { // child

        char attach[32];

        snprintf(attach, sizeof(attach), "attach %d", parent_pid);

#if defined(__linux__)

        close(lock[1]);

        (void) !read(lock[0], lock, 1);

        close(lock[0]);

#endif

        // try gdb

        execlp("gdb", "gdb", "--batch",

            "-ex", "set style enabled on",

            "-ex", attach,

            "-ex", "bt -frame-info source-and-location",

            "-ex", "detach",

            "-ex", "quit",

            (char *) NULL);

        // try lldb

        execlp("lldb", "lldb", "--batch",

            "-o", "bt",

            "-o", "quit",

            "-p", &attach[sizeof("attach ") - 1],

            (char *) NULL);

        // gdb failed, fallback to backtrace_symbols

        ggml_print_backtrace_symbols();

        _Exit(0);

    } else { // parent

#if defined(__linux__)

        prctl(PR_SET_PTRACER, child_pid);

        close(lock[1]);

        close(lock[0]);

#endif

        waitpid(child_pid, NULL, 0);

    }

}

#else

void ggml_print_backtrace(void) {

    // platform not supported

}

#endif

static ggml_abort_callback_t g_abort_callback = NULL;

// Set the abort callback (passing null will restore original abort functionality: printing a message to stdout)

GGML_API ggml_abort_callback_t ggml_set_abort_callback(ggml_abort_callback_t callback) {

    ggml_abort_callback_t ret_val = g_abort_callback;

    g_abort_callback = callback;

    return ret_val;

}

void ggml_abort(const char * file, int line, const char * fmt, ...) {

    fflush(stdout);

    char message[2048];

    int offset = snprintf(message, sizeof(message), "%s:%d: ", file, line);

    va_list args;

    va_start(args, fmt);

    vsnprintf(message + offset, sizeof(message) - offset, fmt, args);

    va_end(args);

    if (g_abort_callback) {

        g_abort_callback(message);

    } else {

        // default: print error and backtrace to stderr

        fprintf(stderr, "%s\n", message);

    }

    abort();

}

// ggml_print_backtrace is registered with std::set_terminate by ggml.cpp

//

// logging

//

struct ggml_logger_state {

    ggml_log_callback log_callback;

    void * log_callback_user_data;

};

static struct ggml_logger_state g_logger_state = {ggml_log_callback_default, NULL};

static void ggml_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {

    if (format == NULL) {

        return;

    }

    va_list args_copy;

    va_copy(args_copy, args);

    char buffer[128];

    int len = vsnprintf(buffer, 128, format, args);

    if (len < 128) {

        g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);

    } else {

        char * buffer2 = (char *) calloc(len + 1, sizeof(char));

        vsnprintf(buffer2, len + 1, format, args_copy);

        buffer2[len] = 0;

        g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);

        free(buffer2);

    }

    va_end(args_copy);

}

void ggml_log_internal(enum ggml_log_level level, const char * format, ...) {

    va_list args;

    va_start(args, format);

    ggml_log_internal_v(level, format, args);

    va_end(args);

}

void ggml_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {

    (void) level;

    (void) user_data;

    fputs(text, stderr);

    fflush(stderr);

}

//

// end of logging block

//

#ifdef GGML_USE_ACCELERATE

// uncomment to use vDSP for soft max computation

// note: not sure if it is actually faster

//#define GGML_SOFT_MAX_ACCELERATE

#endif

void * ggml_aligned_malloc(size_t size) {

#if defined(__s390x__)

    const int alignment = 256;

#else

    const int alignment = 64;

#endif

#if defined(_MSC_VER) || defined(__MINGW32__)

    return _aligned_malloc(size, alignment);

#else

    if (size == 0) {

        GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_aligned_malloc!\n");

        return NULL;

    }

    void * aligned_memory = NULL;

  #ifdef GGML_USE_CPU_HBM

    int result = hbw_posix_memalign(&aligned_memory, alignment, size);

  #elif TARGET_OS_OSX

    GGML_UNUSED(alignment);

    kern_return_t alloc_status = vm_allocate((vm_map_t) mach_task_self(), (vm_address_t *) &aligned_memory, size, VM_FLAGS_ANYWHERE);

    int result = EFAULT;

    switch (alloc_status) {

        case KERN_SUCCESS:

            result = 0;

            break;

        case KERN_INVALID_ADDRESS:

            result = EINVAL;

            break;

        case KERN_NO_SPACE:

            result = ENOMEM;

            break;

        default:

            result = EFAULT;

            break;

    }

  #else

    int result = posix_memalign(&aligned_memory, alignment, size);

  #endif

    if (result != 0) {

        // Handle allocation failure

        const char *error_desc = "unknown allocation error";

        switch (result) {

            case EINVAL:

                error_desc = "invalid alignment value";

                break;

            case ENOMEM:

                error_desc = "insufficient memory";

                break;

        }

        GGML_LOG_ERROR("%s: %s (attempted to allocate %6.2f MB)\n", __func__, error_desc, size/(1024.0*1024.0));

        return NULL;

    }

    return aligned_memory;

#endif

}

void ggml_aligned_free(void * ptr, size_t size) {

    GGML_UNUSED(size);

#if defined(_MSC_VER) || defined(__MINGW32__)

    _aligned_free(ptr);

#elif GGML_USE_CPU_HBM

    if (ptr != NULL) {

        hbw_free(ptr);

    }

#elif TARGET_OS_OSX

    if (ptr != NULL) {

        vm_deallocate((vm_map_t)mach_task_self(), (vm_address_t)ptr, size);

    }

#else

    free(ptr);

#endif

}

inline static void * ggml_malloc(size_t size) {

    if (size == 0) {

        GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_malloc!\n");

        return NULL;

    }

    void * result = malloc(size);

    if (result == NULL) {

        GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));

        GGML_ABORT("fatal error");

    }

    return result;

}

// calloc

inline static void * ggml_calloc(size_t num, size_t size) {

if ((num * size) > 9000000) {GGML_ABORT("calloc err");}

    if (num == 0 || size == 0) {

        GGML_LOG_WARN("Behavior may be unexpected when allocating 0 bytes for ggml_calloc!\n");

        return NULL;

    }

    void * result = calloc(num, size);

    if (result == NULL) {

        GGML_LOG_ERROR("%s: failed to allocate %6.2f MB\n", __func__, size/(1024.0*1024.0));

        GGML_ABORT("fatal error");

    }

    return result;

}

#define GGML_MALLOC(size)      ggml_malloc(size)

#define GGML_CALLOC(num, size) ggml_calloc(num, size)

#define GGML_FREE(ptr) free(ptr)

const char * ggml_status_to_string(enum ggml_status status) {

    switch (status) {

        case GGML_STATUS_ALLOC_FAILED: return "GGML status: error (failed to allocate memory)";

        case GGML_STATUS_FAILED:       return "GGML status: error (operation failed)";

        case GGML_STATUS_SUCCESS:      return "GGML status: success";

        case GGML_STATUS_ABORTED:      return "GGML status: warning (operation aborted)";

    }

    return "GGML status: unknown";

}

float ggml_fp16_to_fp32(ggml_fp16_t x) {

#define ggml_fp16_to_fp32 do_not_use__ggml_fp16_to_fp32__in_ggml

    return GGML_FP16_TO_FP32(x);

}

ggml_fp16_t ggml_fp32_to_fp16(float x) {

#define ggml_fp32_to_fp16 do_not_use__ggml_fp32_to_fp16__in_ggml

    return GGML_FP32_TO_FP16(x);

}

float ggml_bf16_to_fp32(ggml_bf16_t x) {

#define ggml_bf16_to_fp32 do_not_use__ggml_bf16_to_fp32__in_ggml

    return GGML_BF16_TO_FP32(x);  // it just left shifts

}

ggml_bf16_t ggml_fp32_to_bf16(float x) {

#define ggml_fp32_to_bf16 do_not_use__ggml_fp32_to_bf16__in_ggml

    return GGML_FP32_TO_BF16(x);

}

void ggml_fp16_to_fp32_row(const ggml_fp16_t * x, float * y, int64_t n) {

    for (int64_t i = 0; i < n; i++) {

        y[i] = GGML_FP16_TO_FP32(x[i]);

    }

}

void ggml_fp32_to_fp16_row(const float * x, ggml_fp16_t * y, int64_t n) {

    int i = 0;

    for (; i < n; ++i) {

        y[i] = GGML_FP32_TO_FP16(x[i]);

    }

}

void ggml_bf16_to_fp32_row(const ggml_bf16_t * x, float * y, int64_t n) {

    int i = 0;

    for (; i < n; ++i) {

        y[i] = GGML_BF16_TO_FP32(x[i]);

    }

}

void ggml_fp32_to_bf16_row_ref(const float * x, ggml_bf16_t * y, int64_t n) {

    for (int i = 0; i < n; i++) {

        y[i] = ggml_compute_fp32_to_bf16(x[i]);

    }

}

void ggml_fp32_to_bf16_row(const float * x, ggml_bf16_t * y, int64_t n) {

  int i = 0;

#if defined(__AVX512BF16__)

  // subnormals are flushed to zero on this platform

  for (; i + 32 <= n; i += 32) {

        _mm512_storeu_si512(

            (__m512i *)(y + i),

            m512i(_mm512_cvtne2ps_pbh(_mm512_loadu_ps(x + i + 16),

                                _mm512_loadu_ps(x + i))));

  }

#endif

    for (; i < n; i++) {

        y[i] = GGML_FP32_TO_BF16(x[i]);

    }

}

bool ggml_guid_matches(ggml_guid_t guid_a, ggml_guid_t guid_b) {

    return memcmp(guid_a, guid_b, sizeof(ggml_guid)) == 0;

}

const char * ggml_version(void) {

    return GGML_VERSION;

}

const char * ggml_commit(void) {

    return GGML_COMMIT;

}

//

// timing

//

#if defined(_MSC_VER) || defined(__MINGW32__)

static int64_t timer_freq, timer_start;

void ggml_time_init(void) {

    LARGE_INTEGER t;

    QueryPerformanceFrequency(&t);

    timer_freq = t.QuadPart;

    // The multiplication by 1000 or 1000000 below can cause an overflow if timer_freq

    // and the uptime is high enough.

    // We subtract the program start time to reduce the likelihood of that happening.

    QueryPerformanceCounter(&t);

    timer_start = t.QuadPart;

}

int64_t ggml_time_ms(void) {

    LARGE_INTEGER t;

    QueryPerformanceCounter(&t);

    return ((t.QuadPart-timer_start) * 1000) / timer_freq;

}

int64_t ggml_time_us(void) {

    LARGE_INTEGER t;

    QueryPerformanceCounter(&t);

    return ((t.QuadPart-timer_start) * 1000000) / timer_freq;

}

#else

void ggml_time_init(void) {}

int64_t ggml_time_ms(void) {

    struct timespec ts;

    clock_gettime(CLOCK_MONOTONIC, &ts);

    return (int64_t)ts.tv_sec*1000 + (int64_t)ts.tv_nsec/1000000;

}

int64_t ggml_time_us(void) {

    struct timespec ts;

    clock_gettime(CLOCK_MONOTONIC, &ts);

    return (int64_t)ts.tv_sec*1000000 + (int64_t)ts.tv_nsec/1000;

}

#endif

int64_t ggml_cycles(void) {

    return clock();

}

int64_t ggml_cycles_per_ms(void) {

    return CLOCKS_PER_SEC/1000;

}

//

// cross-platform UTF-8 file paths

//

#ifdef _WIN32

static wchar_t * ggml_mbstowcs(const char * mbs) {

    int wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, NULL, 0);

    if (!wlen) {

        errno = EINVAL;

        return NULL;

    }

    wchar_t * wbuf = GGML_MALLOC(wlen * sizeof(wchar_t));

    wlen = MultiByteToWideChar(CP_UTF8, 0, mbs, -1, wbuf, wlen);

    if (!wlen) {

        GGML_FREE(wbuf);

        errno = EINVAL;

        return NULL;

    }

    return wbuf;

}

#endif

FILE * ggml_fopen(const char * fname, const char * mode) {

#ifdef _WIN32

    FILE * file = NULL;

    // convert fname (UTF-8)

    wchar_t * wfname = ggml_mbstowcs(fname);

    if (wfname) {

        // convert mode (ANSI)

        wchar_t * wmode = GGML_MALLOC((strlen(mode) + 1) * sizeof(wchar_t));

        wchar_t * wmode_p = wmode;

        do {

            *wmode_p++ = (wchar_t)*mode;

        } while (*mode++);

        // open file

        file = _wfopen(wfname, wmode);

        GGML_FREE(wfname);

        GGML_FREE(wmode);

    }

    return file;

#else

    return fopen(fname, mode);

#endif

}

static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {

    [GGML_TYPE_I8] = {

        .type_name                = "i8",

        .blck_size                = 1,

        .type_size                = sizeof(int8_t),

        .is_quantized             = false,

    },

    [GGML_TYPE_I16] = {

        .type_name                = "i16",

        .blck_size                = 1,

        .type_size                = sizeof(int16_t),

        .is_quantized             = false,

    },

    [GGML_TYPE_I32] = {

        .type_name                = "i32",

        .blck_size                = 1,

        .type_size                = sizeof(int32_t),

        .is_quantized             = false,

    },

    [GGML_TYPE_I64] = {

        .type_name                = "i64",

        .blck_size                = 1,

        .type_size                = sizeof(int64_t),

        .is_quantized             = false,

    },

    [GGML_TYPE_F64] = {

        .type_name                = "f64",

        .blck_size                = 1,

        .type_size                = sizeof(double),

        .is_quantized             = false,

    },

    [GGML_TYPE_F32] = {

        .type_name                = "f32",

        .blck_size                = 1,

        .type_size                = sizeof(float),

        .is_quantized             = false,

    },

    [GGML_TYPE_F16] = {

        .type_name                = "f16",

        .blck_size                = 1,

        .type_size                = sizeof(ggml_fp16_t),

        .is_quantized             = false,

        .to_float                 = (ggml_to_float_t) ggml_fp16_to_fp32_row,

        .from_float_ref           = (ggml_from_float_t) ggml_fp32_to_fp16_row,

    },

    [GGML_TYPE_Q4_0] = {

        .type_name                = "q4_0",

        .blck_size                = QK4_0,

        .type_size                = sizeof(block_q4_0),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q4_0,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q4_0_ref,

    },

    [GGML_TYPE_Q4_1] = {

        .type_name                = "q4_1",

        .blck_size                = QK4_1,

        .type_size                = sizeof(block_q4_1),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q4_1,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q4_1_ref,

    },

    [4] = { // GGML_TYPE_Q4_2

        .type_name                = "DEPRECATED",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [5] = { // GGML_TYPE_Q4_3

        .type_name                = "DEPRECATED",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [GGML_TYPE_Q5_0] = {

        .type_name                = "q5_0",

        .blck_size                = QK5_0,

        .type_size                = sizeof(block_q5_0),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q5_0,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q5_0_ref,

    },

    [GGML_TYPE_Q5_1] = {

        .type_name                = "q5_1",

        .blck_size                = QK5_1,

        .type_size                = sizeof(block_q5_1),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q5_1,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q5_1_ref,

    },

    [GGML_TYPE_Q8_0] = {

        .type_name                = "q8_0",

        .blck_size                = QK8_0,

        .type_size                = sizeof(block_q8_0),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q8_0,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q8_0_ref,

    },

    [GGML_TYPE_Q8_1] = {

        .type_name                = "q8_1",

        .blck_size                = QK8_1,

        .type_size                = sizeof(block_q8_1),

        .is_quantized             = true,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q8_1_ref,

    },

    [GGML_TYPE_MXFP4] = {

        .type_name                = "mxfp4",

        .blck_size                = QK_MXFP4,

        .type_size                = sizeof(block_mxfp4),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_mxfp4,

        .from_float_ref           = (ggml_from_float_t)quantize_row_mxfp4_ref,

    },

    [GGML_TYPE_Q2_K] = {

        .type_name                = "q2_K",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_q2_K),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q2_K,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q2_K_ref,

    },

    [GGML_TYPE_Q3_K] = {

        .type_name                = "q3_K",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_q3_K),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q3_K,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q3_K_ref,

    },

    [GGML_TYPE_Q4_K] = {

        .type_name                = "q4_K",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_q4_K),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q4_K,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q4_K_ref,

    },

    [GGML_TYPE_Q5_K] = {

        .type_name                = "q5_K",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_q5_K),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q5_K,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q5_K_ref,

    },

    [GGML_TYPE_Q6_K] = {

        .type_name                = "q6_K",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_q6_K),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_q6_K,

        .from_float_ref           = (ggml_from_float_t) quantize_row_q6_K_ref,

    },

    [GGML_TYPE_IQ2_XXS] = {

        .type_name                = "iq2_xxs",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq2_xxs),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq2_xxs,

        .from_float_ref           = NULL,

    },

    [GGML_TYPE_IQ2_XS] = {

        .type_name                = "iq2_xs",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq2_xs),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq2_xs,

        .from_float_ref           = NULL,

    },

    [GGML_TYPE_IQ3_XXS] = {

        .type_name                = "iq3_xxs",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq3_xxs),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq3_xxs,

        .from_float_ref           = (ggml_from_float_t)quantize_row_iq3_xxs_ref,

    },

    [GGML_TYPE_IQ3_S] = {

        .type_name                = "iq3_s",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq3_s),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq3_s,

        .from_float_ref           = (ggml_from_float_t)quantize_row_iq3_s_ref,

    },

    [GGML_TYPE_IQ2_S] = {

        .type_name                = "iq2_s",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq2_s),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq2_s,

        .from_float_ref           = (ggml_from_float_t)quantize_row_iq2_s_ref,

    },

    [GGML_TYPE_IQ1_S] = {

        .type_name                = "iq1_s",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq1_s),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq1_s,

        .from_float_ref           = NULL,

    },

    [GGML_TYPE_IQ1_M] = {

        .type_name                = "iq1_m",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq1_m),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq1_m,

        .from_float_ref           = NULL,

    },

    [GGML_TYPE_IQ4_NL] = {

        .type_name                = "iq4_nl",

        .blck_size                = QK4_NL,

        .type_size                = sizeof(block_iq4_nl),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq4_nl,

        .from_float_ref           = (ggml_from_float_t)quantize_row_iq4_nl_ref,

    },

    [GGML_TYPE_IQ4_XS] = {

        .type_name                = "iq4_xs",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_iq4_xs),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_iq4_xs,

        .from_float_ref           = (ggml_from_float_t)quantize_row_iq4_xs_ref,

    },

    [GGML_TYPE_Q8_K] = {

        .type_name                = "q8_K",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_q8_K),

        .is_quantized             = true,

    },

    [GGML_TYPE_BF16] = {

        .type_name                = "bf16",

        .blck_size                = 1,

        .type_size                = sizeof(ggml_bf16_t),

        .is_quantized             = false,

        .to_float                 = (ggml_to_float_t) ggml_bf16_to_fp32_row,

        .from_float_ref           = (ggml_from_float_t) ggml_fp32_to_bf16_row_ref,

    },

    [31] = { // GGML_TYPE_Q4_0_4_4

        .type_name                = "TYPE_Q4_0_4_4 REMOVED, use Q4_0 with runtime repacking",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [32] = { // GGML_TYPE_Q4_0_4_8

        .type_name                = "TYPE_Q4_0_4_8 REMOVED, use Q4_0 with runtime repacking",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [33] = { // GGML_TYPE_Q4_0_8_8

        .type_name                = "TYPE_Q4_0_8_8 REMOVED, use Q4_0 with runtime repacking",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [GGML_TYPE_TQ1_0] = {

        .type_name                = "tq1_0",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_tq1_0),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_tq1_0,

        .from_float_ref           = (ggml_from_float_t) quantize_row_tq1_0_ref,

    },

    [GGML_TYPE_TQ2_0] = {

        .type_name                = "tq2_0",

        .blck_size                = QK_K,

        .type_size                = sizeof(block_tq2_0),

        .is_quantized             = true,

        .to_float                 = (ggml_to_float_t) dequantize_row_tq2_0,

        .from_float_ref           = (ggml_from_float_t) quantize_row_tq2_0_ref,

    },

    [36] = { // GGML_TYPE_IQ4_NL_4_4

        .type_name                = "TYPE_IQ4_NL_4_4 REMOVED, use IQ4_NL with runtime repacking",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [37] = { // GGML_TYPE_IQ4_NL_4_8

        .type_name                = "TYPE_IQ4_NL_4_8 REMOVED, use IQ4_NL with runtime repacking",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

    [38] = { // GGML_TYPE_IQ4_NL_8_8

        .type_name                = "TYPE_IQ4_NL_8_8 REMOVED, use IQ4_NL with runtime repacking",

        .blck_size                = 0,

        .type_size                = 0,

        .is_quantized             = false,

    },

};

const struct ggml_type_traits * ggml_get_type_traits(enum ggml_type type) {

    GGML_ASSERT(type < GGML_TYPE_COUNT);

    return &type_traits[type];

}

//

// ggml object

//

struct ggml_object {

    size_t offs;

    size_t size;

    struct ggml_object * next;

    enum ggml_object_type type;

    char padding[4];

};

static const size_t GGML_OBJECT_SIZE = sizeof(struct ggml_object);

//

// ggml context

//

struct ggml_context {

    size_t mem_size;

    void * mem_buffer;

    bool   mem_buffer_owned;

    bool   no_alloc;

    int    n_objects;

    struct ggml_object * objects_begin;

    struct ggml_object * objects_end;

};

//

// data types

//

static const char * GGML_OP_NAME[GGML_OP_COUNT] = {

    "NONE",

    "DUP",

    "ADD",

    "ADD_ID",

    "ADD1",

    "ACC",

    "SUB",

    "MUL",

    "DIV",

    "SQR",

    "SQRT",

    "LOG",

    "SIN",

    "COS",

    "SUM",

    "SUM_ROWS",

    "CUMSUM",

    "MEAN",

    "ARGMAX",

    "COUNT_EQUAL",

    "REPEAT",

    "REPEAT_BACK",

    "CONCAT",

    "SILU_BACK",

    "NORM",

    "RMS_NORM",

    "RMS_NORM_BACK",

    "GROUP_NORM",

    "L2_NORM",

    "MUL_MAT",

    "MUL_MAT_ID",

    "OUT_PROD",

    "SCALE",

    "SET",

    "CPY",

    "CONT",

    "RESHAPE",

    "VIEW",

    "PERMUTE",

    "TRANSPOSE",

    "GET_ROWS",

    "GET_ROWS_BACK",

    "SET_ROWS",

    "DIAG",

    "DIAG_MASK_INF",

    "DIAG_MASK_ZERO",

    "SOFT_MAX",

    "SOFT_MAX_BACK",

    "ROPE",

    "ROPE_BACK",

    "CLAMP",

    "CONV_TRANSPOSE_1D",

    "IM2COL",

    "IM2COL_BACK",

    "IM2COL_3D",

    "CONV_2D",

    "CONV_3D",

    "CONV_2D_DW",

    "CONV_TRANSPOSE_2D",

    "POOL_1D",

    "POOL_2D",

    "POOL_2D_BACK",

    "UPSCALE",

    "PAD",

    "PAD_REFLECT_1D",

    "ROLL",

    "ARANGE",

    "TIMESTEP_EMBEDDING",

    "ARGSORT",

    "LEAKY_RELU",

    "TRI",

    "FILL",

    "FLASH_ATTN_EXT",

    "FLASH_ATTN_BACK",

    "SSM_CONV",

    "SSM_SCAN",

    "WIN_PART",

    "WIN_UNPART",

    "GET_REL_POS",

    "ADD_REL_POS",

    "RWKV_WKV6",

    "GATED_LINEAR_ATTN",

    "RWKV_WKV7",

    "SOLVE_TRI",

    "UNARY",

    "MAP_CUSTOM1",

    "MAP_CUSTOM2",

    "MAP_CUSTOM3",

    "CUSTOM",

    "CROSS_ENTROPY_LOSS",

    "CROSS_ENTROPY_LOSS_BACK",

    "OPT_STEP_ADAMW",

    "OPT_STEP_SGD",

    "GLU",

};

static_assert(GGML_OP_COUNT == 94, "GGML_OP_COUNT != 94");

static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {

    "none",

    "x",

    "x+y",

    "x[i]+y",

    "x+y",

    "view(x,nb,offset)+=y->x",

    "x-y",

    "x*y",

    "x/y",

    "x^2",

    "√x",

    "log(x)",

    "sin(x)",

    "cos(x)",

    "Σx",

    "Σx_k",

    "cumsum(x)",

    "Σx/n",

    "argmax(x)",

    "count_equal(x)",

    "repeat(x)",

    "repeat_back(x)",

    "concat(x, y)",

    "silu_back(x)",