// Copyright 2023 Google LLC

//

// This source code is licensed under the BSD-style license found in the

// LICENSE file in the root directory of this source tree.

#include <assert.h>

#include <stddef.h>

#include <stdint.h>

#include "include/xnnpack.h"

#include "src/xnnpack/common.h"

#include "src/xnnpack/config.h"

#include "src/xnnpack/gemm.h"

#include "src/xnnpack/hardware-config.h"

#include "src/xnnpack/igemm.h"

#include "src/xnnpack/init-once.h"

#include "src/xnnpack/log.h"

#include "src/xnnpack/math.h"

#include "src/xnnpack/microfnptr.h"

#include "src/xnnpack/microparams-init.h"

#include "src/xnnpack/pack.h"

#include "src/xnnpack/packw.h"

#if XNN_ARCH_WASMSIMD

  #include <emscripten.h>

#endif

#define XNN_MR_TO_INDEX(MR) (MR-1)

// UARCH 0 is big core.  1 is medium or little core.

#ifndef XNN_UARCH_INDEX

#define XNN_UARCH_INDEX 0

#endif

static const int default_config = 0;

static const int consistent_config = 1;

static struct xnn_gemm_config bf16_f32_gemm_config = {0};

static struct xnn_gemm_config f16_gemm_config = {0};

static struct xnn_gemm_config f32_gemm_config[2] = {0};

static struct xnn_gemm_config f32_igemm_config = {0};

static struct xnn_gemm_config f32_gemm_nr2_config[2] = {0};

static struct xnn_gemm_config f32_qc4w_gemm_config = {0};

static struct xnn_gemm_config f32_qc8w_gemm_config = {0};

static struct xnn_gemm_config pf16_gemm_config = {0};

static struct xnn_gemm_config pf32_gemm_config = {0};

static struct xnn_gemm_config pqs8_qc8w_gemm_config = {0};

static struct xnn_gemm_config qd8_f16_qb4w_gemm_config = {0};

static struct xnn_gemm_config qd8_f16_qc4w_gemm_config = {0};

static struct xnn_gemm_config qd8_f16_qc8w_gemm_config = {0};

static struct xnn_gemm_config qd8_f16_qc8w_igemm_config = {0};

static struct xnn_gemm_config qd8_f32_qb4w_gemm_config = {0};

static struct xnn_gemm_config qd8_f32_qc4w_gemm_config = {0};

static struct xnn_gemm_config qd8_f32_qc2w_gemm_config = {0};

static struct xnn_gemm_config qd8_f32_qc8w_gemm_config = {0};

static struct xnn_gemm_config qp8_f32_qc4w_gemm_config = {0};

static struct xnn_gemm_config qp8_f32_qc8w_gemm_config = {0};

static struct xnn_gemm_config qp8_f32_qb4w_gemm_config = {0};

static struct xnn_gemm_config qdu8_f32_qc4w_gemm_config = {0};

static struct xnn_gemm_config qdu8_f16_qc8w_gemm_config = {0};

static struct xnn_gemm_config qdu8_f32_qc8w_igemm_config = {0};

static struct xnn_gemm_config qdu8_f32_qc8w_gemm_config = {0};

static struct xnn_gemm_config qdu8_f32_qb4w_gemm_config = {0};

static struct xnn_gemm_config qdu8_f16_qc4w_gemm_config = {0};

static struct xnn_gemm_config qs8_qc4w_gemm_config = {0};

static struct xnn_gemm_config qs8_qc8w_gemm_config = {0};

static struct xnn_gemm_config qu8_gemm_config = {0};

XNN_INIT_ONCE_GUARD(bf16_f32_gemm);

XNN_INIT_ONCE_GUARD(f16_gemm);

XNN_INIT_ONCE_GUARD(f32_igemm);

XNN_INIT_ONCE_GUARD(f32_gemm);

XNN_INIT_ONCE_GUARD(f32_gemm_nr2);

XNN_INIT_ONCE_GUARD(f32_qc4w_gemm);

XNN_INIT_ONCE_GUARD(f32_qc8w_gemm);

XNN_INIT_ONCE_GUARD(pf16_gemm);

XNN_INIT_ONCE_GUARD(pf32_gemm);

XNN_INIT_ONCE_GUARD(pqs8_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f16_qb4w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f16_qc4w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f16_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f16_qc8w_igemm);

XNN_INIT_ONCE_GUARD(qd8_f32_qb4w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f32_qc4w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f32_qc2w_gemm);

XNN_INIT_ONCE_GUARD(qd8_f32_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qp8_f32_qc4w_gemm);

XNN_INIT_ONCE_GUARD(qp8_f32_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qp8_f32_qb4w_gemm);

XNN_INIT_ONCE_GUARD(qdu8_f32_qc4w_gemm);

XNN_INIT_ONCE_GUARD(qdu8_f16_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qdu8_f32_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qdu8_f32_qc8w_igemm);

XNN_INIT_ONCE_GUARD(qdu8_f32_qb4w_gemm);

XNN_INIT_ONCE_GUARD(qdu8_f16_qc4w_gemm);

XNN_INIT_ONCE_GUARD(qs8_qc4w_gemm);

XNN_INIT_ONCE_GUARD(qs8_qc8w_gemm);

XNN_INIT_ONCE_GUARD(qu8_gemm);

// Macros to log the microkernel names if and when they are registered.

#define XNN_INIT_GEMM_UKERNEL(ukernel) \

  (xnn_gemm_ukernel_fn) ukernel;       \

  xnn_log_info("Using gemm microkernel '%s'.", #ukernel);

#define XNN_INIT_HMP_GEMM_UKERNEL(ukernel)                 \

  xnn_init_hmp_gemm_ukernel((xnn_gemm_ukernel_fn)ukernel); \

  xnn_log_info("Using gemm microkernel '%s'.", #ukernel);

#define XNN_INIT_IGEMM_UKERNEL(ukernel) \

  (xnn_igemm_ukernel_fn) ukernel;       \

  xnn_log_info("Using igemm microkernel '%s'.", #ukernel);

#define XNN_INIT_HMP_IGEMM_UKERNEL(ukernel)                  \

  xnn_init_hmp_igemm_ukernel((xnn_igemm_ukernel_fn)ukernel); \

  xnn_log_info("Using igemm microkernel '%s'.", #ukernel);

#define XNN_INIT_DQGEMM_UKERNEL(ukernel) \

  (xnn_dqgemm_ukernel_fn) ukernel;       \

  xnn_log_info("Using dqgemm microkernel '%s'.", #ukernel);

#define XNN_INIT_HMP_DQGEMM_UKERNEL(ukernel)                   \

  xnn_init_hmp_dqgemm_ukernel((xnn_dqgemm_ukernel_fn)ukernel); \

  xnn_log_info("Using dqgemm microkernel '%s'.", #ukernel);

#define XNN_INIT_DQIGEMM_UKERNEL(ukernel) \

  (xnn_dqigemm_ukernel_fn) ukernel;       \

  xnn_log_info("Using dqigemm microkernel '%s'.", #ukernel);

#define XNN_INIT_HMP_DQIGEMM_UKERNEL(ukernel)                    \

  xnn_init_hmp_dqigemm_ukernel((xnn_dqigemm_ukernel_fn)ukernel); \

  xnn_log_info("Using dqigemm microkernel '%s'.", #ukernel);

#define XNN_INIT_HMP_QP8GEMM_UKERNEL(ukernel)            \

  xnn_init_hmp_qp8gemm_ukernel(                          \

      (xnn_qp8_f32_qc4w_gemm_minmax_ukernel_fn)ukernel); \

  xnn_log_info("Using qp8gemm microkernel '%s'.", #ukernel);

#define XNN_INIT_HMP_QP8GEMM_BL_UKERNEL(ukernel)         \

  xnn_init_hmp_qp8gemm_bl_ukernel(                       \

      (xnn_qp8_f32_qb4w_gemm_minmax_ukernel_fn)ukernel); \

  xnn_log_info("Using qp8gemm_bl microkernel '%s'.", #ukernel);

static void init_f16_gemm_config(void) {

  // Common parameters.

  f16_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_FLOAT16;

  f16_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_FLOAT16;

  f16_gemm_config.log2_filter_element_bit_size = XNN_LOG2_SIZEOF_FLOAT16 + 3;

  f16_gemm_config.bias_element_size = sizeof(xnn_float16);

  // Arch-specific parameters.

  #if XNN_ARCH_ARM && XNN_ENABLE_ARM_FP16_VECTOR && XNN_ENABLE_ARM_FP16_SCALAR

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->arch_flags & xnn_arch_arm_neon_fp16_arith) {

      f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x8__neonfp16arith_ld64);

      f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x8__neonfp16arith_ld64);

      f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x8__neonfp16arith_ld64);

      f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x8__neonfp16arith_ld64);

      f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

      f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

      f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x8__neon_ld4lane_u8_prfm;

      f16_gemm_config.mr = 6;

      f16_gemm_config.nr = 8;

    }

  #elif XNN_ENABLE_ARM_FP16_VECTOR && XNN_ARCH_ARM64

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->arch_flags & xnn_arch_arm_neon_fp16_arith) {

      #if XNN_ENABLE_ASSEMBLY

        switch (hardware_config->uarch[XNN_UARCH_INDEX]) {

          case xnn_uarch_cortex_a55:

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55);

            f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

            f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

            f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm;

            f16_gemm_config.mr = 6;

            f16_gemm_config.nr = 16;

            break;

          case xnn_uarch_cortex_a55r0:

          case xnn_uarch_cortex_a75:

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55r0);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55r0);

            f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

            f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

            f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm;

            f16_gemm_config.mr = 6;

            f16_gemm_config.nr = 16;

            break;

          case xnn_uarch_exynos_m5:

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_4x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_4x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

            f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

            f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm;

            f16_gemm_config.mr = 4;

            f16_gemm_config.nr = 16;

            break;

          case xnn_uarch_exynos_m4:

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

            f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

            f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm;

            f16_gemm_config.mr = 6;

            f16_gemm_config.nr = 16;

            break;

          default:

          case xnn_uarch_cortex_a76:

          case xnn_uarch_cortex_a77:

          case xnn_uarch_cortex_a78:

          case xnn_uarch_cortex_a510:

          case xnn_uarch_cortex_a710:

          case xnn_uarch_cortex_a715:

          case xnn_uarch_cortex_x1:

          case xnn_uarch_cortex_x2:

          case xnn_uarch_cortex_x3:

          case xnn_uarch_cortex_x4:

          case xnn_uarch_oryon:

          case xnn_uarch_neoverse_n1:

          case xnn_uarch_neoverse_n2:

          case xnn_uarch_neoverse_v1:

          case xnn_uarch_neoverse_v2:

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a75);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x16__asm_aarch64_neonfp16arith_ld64);

            f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a75);

            f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

            f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

            f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm;

            f16_gemm_config.mr = 6;

            f16_gemm_config.nr = 16;

            break;

        }

        #if XNN_MAX_UARCH_TYPES > 1

        {

          /* Choose micro-kernels for little cores according to micro-kernel specification for the big core */

          const uint32_t mr = f16_gemm_config.mr;

          const uint32_t nr = f16_gemm_config.nr;

          for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {

            switch (hardware_config->uarch[i]) {

              case xnn_uarch_cortex_a55:

                if (mr == 6 && nr == 16) {

                  f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55);

                  f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55);

                }

                break;

              case xnn_uarch_cortex_a55r0:

              case xnn_uarch_cortex_a75:

                if (mr == 6 && nr == 16) {

                  f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55r0);

                  f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__asm_aarch64_neonfp16arith_cortex_a55r0);

                }

                break;

              default:

                break;

            }

          }

        }

        #endif  // XNN_MAX_UARCH_TYPES > 1

      #else  // XNN_ENABLE_ASSEMBLY

        f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_1x16__neonfp16arith_ld64);

        f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_gemm_minmax_ukernel_6x16__neonfp16arith_ld64);

        f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_1x16__neonfp16arith_ld64);

        f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_igemm_minmax_ukernel_6x16__neonfp16arith_ld64);

        f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

        f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

        f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__neon_ld4lane_u8_prfm;

        f16_gemm_config.mr = 6;

        f16_gemm_config.nr = 16;

      #endif  // XNN_ENABLE_ASSEMBLY

    }

  #elif XNN_ARCH_X86 || XNN_ARCH_X86_64

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->arch_flags & xnn_arch_x86_avx2) {

      f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_f32acc_gemm_minmax_ukernel_1x16__avx2_broadcast);

      f16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f16_f32acc_gemm_minmax_ukernel_4x16__avx2_broadcast);

      f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_f32acc_igemm_minmax_ukernel_1x16__avx2_broadcast);

      f16_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f16_f32acc_igemm_minmax_ukernel_4x16__avx2_broadcast);

      f16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

      f16_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f16_gemm_gio_w;

      #if XNN_ENABLE_AVX2

        f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_packw_gemm_goi_ukernel_x16__avx2_u16_prfm;

      #else

        f16_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_pack_f16_gemm_goi_w;

      #endif

      f16_gemm_config.mr = 4;

      f16_gemm_config.nr = 16;

    }

  #endif

  assert(f16_gemm_config.mr <= XNN_MAX_MR);

}

#if XNN_ARCH_WASMSIMD

  EM_JS(int, hardware_concurrency, (void), {

    var concurrency = 1;

    try {

      concurrency = self.navigator.hardwareConcurrency;

    } catch(e) {

      // d8 environment doesn't provide `self`, thus we keep the default

    }

    return concurrency;

  });

  // A cpu with more than `kCoreCountThresholdForAdaptiveAvxOptimization` is

  // assumed to support AVX instructions.

  const int kCoreCountThresholdForAdaptiveAvxOptimization = 4;

#endif

static void init_pf16_gemm_config(void) {

  // Common parameters.

  pf16_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_FLOAT16;

  pf16_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_FLOAT16;

  pf16_gemm_config.log2_filter_element_bit_size = XNN_LOG2_SIZEOF_FLOAT16 + 3;

  pf16_gemm_config.bias_element_size = sizeof(xnn_float16);

#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

  const struct xnn_hardware_config* hardware_config =

      xnn_init_hardware_config();

  assert(hardware_config != NULL);

  if (XNN_ENABLE_ARM_SME2 && (hardware_config->arch_flags & xnn_arch_arm_sme2)) {

    #if XNN_ENABLE_ARM_SME2

      const size_t mr = xnn_pf16_gemm_minmax_ukernel_32x32c2__neonsme2_get_mr();

      const size_t nr = xnn_pf16_gemm_minmax_ukernel_32x32c2__neonsme2_get_nr();

      pf16_gemm_config.arch = xnn_arch_arm_sme2;

      pf16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pf16_gemm_minmax_ukernel_1x32c2__neonsme2);

      pf16_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(mr)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pf16_gemm_minmax_ukernel_32x32c2__neonsme2);

      pf16_gemm_config.init.f16 = xnn_init_f16_minmax_scalar_params;

      pf16_gemm_config.pack_weights_and_biases = xnn_pack_kai_f16_weights_and_biases;

      pf16_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_kai_f16_weights_and_biases;

      pf16_gemm_config.mr = mr;

      pf16_gemm_config.mr_packed = mr;

      pf16_gemm_config.nr = nr;

      pf16_gemm_config.log2_kr = 1;

    #endif  // XNN_ENABLE_ARM_SME2

  }

#endif  // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

}

static void init_bf16_f32_gemm_config(void) {

  // Common parameters.

  bf16_f32_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_BFLOAT16;

  bf16_f32_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_BFLOAT16;

  bf16_f32_gemm_config.log2_filter_element_bit_size = XNN_LOG2_SIZEOF_BFLOAT16 + 3;

  bf16_f32_gemm_config.bias_element_size = sizeof(float);

  // Arch-specific parameters.

#if XNN_ARCH_X86_64

  const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

  assert(hardware_config != NULL);

  (void) hardware_config;  // May be unused.

  if (XNN_ENABLE_AVX512BF16 && (hardware_config->arch_flags & xnn_arch_x86_avx512bf16)) {

    #if XNN_ENABLE_AVX512BF16 && XNN_ENABLE_ASSEMBLY

      bf16_f32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_bf16_f32_gemm_minmax_ukernel_1x32c2__asm_amd64_avx512bf16_broadcast);

      bf16_f32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(11)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_bf16_f32_gemm_minmax_ukernel_11x32c2__asm_amd64_avx512bf16_broadcast);

      bf16_f32_gemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

      bf16_f32_gemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x16_x32_packw_gemm_goi_ukernel_x32c2__scalar;

      bf16_f32_gemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x16_x32_packw_gemm_gio_ukernel_x32c2__scalar;

      bf16_f32_gemm_config.mr = 11;

      bf16_f32_gemm_config.nr = 32;

      bf16_f32_gemm_config.log2_kr = 1;

    #endif  // XNN_ENABLE_AVX512BF16

  }

  assert(bf16_f32_gemm_config.mr <= XNN_MAX_MR);

#endif  // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

}

static void init_pf32_gemm_config(void) {

  // Common parameters.

  pf32_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_FLOAT;

  pf32_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_FLOAT;

  pf32_gemm_config.log2_filter_element_bit_size = XNN_LOG2_SIZEOF_FLOAT + 3;

  pf32_gemm_config.bias_element_size = sizeof(float);

  // Arch-specific parameters.

#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

  const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

  assert(hardware_config != NULL);

  (void) hardware_config;  // May be unused.

  if (XNN_ENABLE_ARM_SME2 && (hardware_config->arch_flags & xnn_arch_arm_sme2)) {

    #if XNN_ENABLE_ARM_SME2

      const size_t mr = xnn_pf32_gemm_minmax_ukernel_32x32__neonsme2_get_mr();

      const size_t nr = xnn_pf32_gemm_minmax_ukernel_32x32__neonsme2_get_nr();

      pf32_gemm_config.arch = xnn_arch_arm_sme2;

      pf32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pf32_gemm_minmax_ukernel_1x32__neonsme2);

      pf32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(mr)] =XNN_INIT_HMP_GEMM_UKERNEL(xnn_pf32_gemm_minmax_ukernel_32x32__neonsme2);

      pf32_gemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

      pf32_gemm_config.pack_weights_and_biases = xnn_pack_kai_f32_weights_and_biases;

      pf32_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_kai_f32_weights_and_biases;

      pf32_gemm_config.mr = mr;

      pf32_gemm_config.mr_packed = mr;

      pf32_gemm_config.nr = nr;

    #endif  // XNN_ENABLE_ARM_SME2

  } else if (XNN_ENABLE_ARM_SME &&

             (hardware_config->arch_flags & xnn_arch_arm_sme)) {

#if XNN_ENABLE_ARM_SME

    const size_t mr = xnn_pf32_gemm_minmax_ukernel_32x32__neonsme_get_mr();

    const size_t nr = xnn_pf32_gemm_minmax_ukernel_32x32__neonsme_get_nr();

    pf32_gemm_config.arch = xnn_arch_arm_sme;

    pf32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pf32_gemm_minmax_ukernel_1x32__neonsme);

    pf32_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(mr)] =XNN_INIT_HMP_GEMM_UKERNEL(xnn_pf32_gemm_minmax_ukernel_32x32__neonsme);

    pf32_gemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

    pf32_gemm_config.pack_weights_and_biases = xnn_pack_kai_f32_weights_and_biases;

    pf32_gemm_config.packed_stride_weights_and_biases = xnn_packed_stride_kai_f32_weights_and_biases;

    pf32_gemm_config.mr = mr;

    pf32_gemm_config.mr_packed = mr;

    pf32_gemm_config.nr = nr;

#endif  // XNN_ENABLE_ARM_SME

  } else {

    /* No Action */

  }

  assert(pf32_gemm_config.mr <= XNN_MAX_MR);

#endif  // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

}

static void init_pqs8_qc8w_gemm_config(void) {

  // Common parameters.

  pqs8_qc8w_gemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_INT8_T;

  pqs8_qc8w_gemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_INT8_T;

  pqs8_qc8w_gemm_config.log2_filter_element_bit_size = XNN_LOG2_SIZEOF_INT8_T + 3;

  pqs8_qc8w_gemm_config.bias_element_size = sizeof(int32_t);

  // Arch-specific parameters.

#if XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

  const struct xnn_hardware_config* hardware_config =

      xnn_init_hardware_config();

  assert(hardware_config != NULL);

  (void)hardware_config;  // May be unused.

  if (XNN_ENABLE_ARM_SME2 && (hardware_config->arch_flags & xnn_arch_arm_sme2)) {

#if XNN_ENABLE_ARM_SME2

    const size_t mr =

        xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme2_get_mr();

    const size_t nr =

        xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme2_get_nr();

    pqs8_qc8w_gemm_config.arch = xnn_arch_arm_sme2;

    pqs8_qc8w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(mr)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pqs8_qc8w_gemm_minmax_ukernel_32x32c4__neonsme2);

    pqs8_qc8w_gemm_config.minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_pqs8_qc8w_gemm_minmax_ukernel_1x32c4__neonsme2);

    pqs8_qc8w_gemm_config.minmax.igemm[XNN_MR_TO_INDEX(mr)] =

        xnn_init_hmp_packed_igemm_ukernel(

            (xnn_packed_lhs_igemm_ukernel_fn)

                xnn_pqs8_qc8w_igemm_minmax_fp32_ukernel_32x32c4__neonsme2);

    pqs8_qc8w_gemm_config.init.qs8_qc8w =

        xnn_init_qs8_qc8w_conv_minmax_fp32_scalar_params;

    pqs8_qc8w_gemm_config.pack_weights_and_biases =

        xnn_pack_kai_qs8_qc8w_weights_and_biases_sme2;

    pqs8_qc8w_gemm_config.packed_stride_weights_and_biases =

        xnn_packed_stride_kai_qs8_qc8w_weights_and_biases_sme2;

    pqs8_qc8w_gemm_config.pack_igemm_goki =

        (xnn_pack_conv_goki_w_fn)xnn_pack_kai_qs8_conv_goki_w_sme2;

    pqs8_qc8w_gemm_config.pack_igemm_kgo =

        (xnn_pack_conv_kgo_w_fn)xnn_pack_qs8_conv_kgo_w;

    pqs8_qc8w_gemm_config.pack_deconv_goki =

        (xnn_pack_deconv_goki_w_fn)xnn_pack_qs8_deconv_goki_w;

    pqs8_qc8w_gemm_config.mr = mr;

    pqs8_qc8w_gemm_config.mr_packed = mr;

    pqs8_qc8w_gemm_config.nr = nr;

    pqs8_qc8w_gemm_config.log2_kr = 2;

#endif  // XNN_ENABLE_ARM_SME2

  }

  assert(pqs8_qc8w_gemm_config.mr <= XNN_MAX_MR);

#endif  // XNN_ARCH_ARM64 && XNN_ENABLE_KLEIDIAI

}

static void init_f32_gemm_config_impl(struct xnn_gemm_config* f32_gemm_config, bool consistent_arithmetic) {

  // Common parameters.

  f32_gemm_config->log2_input_element_size = XNN_LOG2_SIZEOF_FLOAT;

  f32_gemm_config->log2_filter_element_size = XNN_LOG2_SIZEOF_FLOAT;

  f32_gemm_config->log2_filter_element_bit_size = XNN_LOG2_SIZEOF_FLOAT + 3;

  f32_gemm_config->bias_element_size = sizeof(float);

  // Arch-specific parameters.

  #if XNN_ARCH_ARM

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->arch_flags & xnn_arch_arm_neon) {

      #if XNN_ENABLE_ASSEMBLY

        switch (hardware_config->uarch[XNN_UARCH_INDEX]) {

          case xnn_uarch_cortex_a5:

          case xnn_uarch_cortex_a7:

          case xnn_uarch_krait:

          case xnn_uarch_kryo:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a7);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a53:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a55r0:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a53);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a32:

          case xnn_uarch_cortex_a35:

          case xnn_uarch_cortex_a55:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a55);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a57:

          case xnn_uarch_cortex_a72:

          case xnn_uarch_cortex_a73:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a75_prfm);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          default:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a75);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

        }

        #if XNN_MAX_UARCH_TYPES > 1

        {

          /* Choose micro-kernels for little cores according to micro-kernel specification for the big core */

          const uint32_t mr = f32_gemm_config->mr;

          const uint32_t nr = f32_gemm_config->nr;

          for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {

            switch (hardware_config->uarch[i]) {

              case xnn_uarch_cortex_a53:

                if (mr == 4 && nr == 8) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a53_prfm);

                }

                break;

              case xnn_uarch_cortex_a55r0:

                if (mr == 4 && nr == 8) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a53);

                }

                break;

              case xnn_uarch_cortex_a55:

                if (mr == 4 && nr == 8) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a55);

                }

                break;

              default:

                break;

            }

          }

        }

        #endif  // XNN_MAX_UARCH_TYPES > 1

      #else  // XNN_ENABLE_ASSEMBLY

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__neon_lane_ld64);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__neon_lane_ld128);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

        f32_gemm_config->mr = 4;

        f32_gemm_config->nr = 8;

      #endif  // XNN_ENABLE_ASSEMBLY

    } else {

      f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x4__scalar);

      f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_4x4__scalar);

      f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x4__scalar);

      f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x4__scalar);

      f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x4__scalar);

      f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_4x4__scalar);

      f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

      f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x4__scalar;

      f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x4__scalar_float_u4;

      f32_gemm_config->mr = 4;

      f32_gemm_config->nr = 4;

    }

  #elif XNN_ARCH_ARM64

    #if XNN_ENABLE_ASSEMBLY && !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC

        const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

        assert(hardware_config);

        switch (hardware_config->uarch[XNN_UARCH_INDEX]) {

          case xnn_uarch_cortex_a72:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a75_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75_prfm);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a57:

          case xnn_uarch_cortex_a75:

          case xnn_uarch_cortex_a76:

          case xnn_uarch_exynos_m3:

          case xnn_uarch_exynos_m4:

          case xnn_uarch_neoverse_n1:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a75_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a75_prfm);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 6;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_exynos_m1:

          case xnn_uarch_exynos_m2:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8s4__neonfma);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8s4__neonfma);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8s4__neonfma);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8s4__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 6;

            f32_gemm_config->nr = 8;

            f32_gemm_config->log2_sr = 2;

            break;

          case xnn_uarch_cortex_a53:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a53_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a53_prfm);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 6;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a55r0:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a53);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a53);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 6;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a35:

          case xnn_uarch_cortex_a55:

          case xnn_uarch_kryo:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a55);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a55);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 6;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a73:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a75_prfm);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a73);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 6;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_a77:

          case xnn_uarch_exynos_m5:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a75);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a75);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x8__neon_u2;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_cortex_x3:

          case xnn_uarch_neoverse_v2:

            // TODO(fbarchard): Implement asm with indexed inputs

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_ld128_2);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(8)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_8x8__asm_aarch64_neonfma_ld128_2);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x8__neon_u2;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 8;

            f32_gemm_config->nr = 8;

            break;

          case xnn_uarch_oryon:

          case xnn_uarch_cortex_x4:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x16__aarch64_neonfma_lane_ld128);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x16__aarch64_neonfma_lane_ld128);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x16__neon_u2;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x16__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 4;

            f32_gemm_config->nr = 16;

            break;

          case xnn_uarch_cortex_a78:

          case xnn_uarch_cortex_a510:

          case xnn_uarch_cortex_a710:

          case xnn_uarch_cortex_a715:

          case xnn_uarch_cortex_x1:

          case xnn_uarch_cortex_x2:

          case xnn_uarch_neoverse_n2:

          case xnn_uarch_neoverse_v1:

          default:

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_ld128_2);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_ld128_2);

            f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_7x8__asm_aarch64_neonfma_ld128_2);

            f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_gemm_config->mr = 7;

            f32_gemm_config->nr = 8;

            break;

        }

        #if XNN_MAX_UARCH_TYPES > 1

        {

          /* Choose micro-kernels for little cores according to micro-kernel specification for the big core */

          const uint32_t mr = f32_gemm_config->mr;

          const uint32_t nr = f32_gemm_config->nr;

          const uint32_t log2_sr = f32_gemm_config->log2_sr;

          for (size_t i = 1; i < XNN_MAX_UARCH_TYPES; i++) {

            switch (hardware_config->uarch[i]) {

              case xnn_uarch_cortex_a53:

                if (mr == 6 && nr == 8 && log2_sr == 0) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53_prfm);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a53_prfm);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a53_prfm);

                } else if (mr == 4 && nr == 8 && log2_sr == 0) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53_prfm);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a53_prfm);

                }

                break;

              case xnn_uarch_cortex_a55r0:

                if (mr == 6 && nr == 8 && log2_sr == 0) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a53);

                } else if (mr == 4 && nr == 8 && log2_sr == 0) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a53);

                }

                break;

              case xnn_uarch_cortex_a55:

                if (mr == 6 && nr == 8 && log2_sr == 0) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a55);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_cortex_a55);

                } else if (mr == 4 && nr == 8 && log2_sr == 0) {

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_cortex_a53);

                  f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)].function[i] = XNN_INIT_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__asm_aarch64_neonfma_cortex_a55);

                }

                break;

              default:

                break;

            }

          }

        }

      #endif  // XNN_MAX_UARCH_TYPES > 1

    #else  // XNN_ENABLE_ASSEMBLY && !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC

      #if XNN_ENABLE_ASSEMBLY

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__asm_aarch64_neonfma_ld128_acc4);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__asm_aarch64_neonfma_ld128);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

        f32_gemm_config->mr = 6;

        f32_gemm_config->nr = 8;

      #else  // !XNN_ENABLE_ASSEMBLY

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__aarch64_neonfma_lane_ld128);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__aarch64_neonfma_lane_ld128);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

        f32_gemm_config->mr = 6;

        f32_gemm_config->nr = 8;

       #endif  // XNN_ENABLE_ASSEMBLY

    #endif  // XNN_ENABLE_ASSEMBLY && !XNN_PLATFORM_IOS && !XNN_PLATFORM_MAC

  #elif XNN_ARCH_X86 || XNN_ARCH_X86_64

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    // TODO(b/460806642): Don't enable this without a fast packing kernel.

    #if false && XNN_ENABLE_AVX512F && XNN_ARCH_X86_64 && !XNN_PLATFORM_WINDOWS && XNN_ENABLE_ASSEMBLY

      if (!consistent_arithmetic && hardware_config->arch_flags & xnn_arch_x86_avx512f) {

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x32c2__asm_amd64_avx512f_broadcast);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_5x32c2__asm_amd64_avx512f_broadcast);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x32c2__avx512f_u4_prfm;

        f32_gemm_config->mr = 5;

        f32_gemm_config->nr = 32;

        f32_gemm_config->log2_kr = 1;

        f32_gemm_config->log2_sr = 0;

      } else

    #endif

    #if XNN_ENABLE_AVX512F

      if (hardware_config->arch_flags & xnn_arch_x86_avx512f) {

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x32__avx512f_broadcast);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_7x32__avx512f_broadcast);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x32__avx512f_u8;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x32__avx512f_u4_prfm;

        f32_gemm_config->mr = 7;

        f32_gemm_config->nr = 32;

      } else

    #endif

    #if XNN_ENABLE_FMA3

      if (hardware_config->arch_flags & xnn_arch_x86_fma3) {

        switch (hardware_config->uarch[XNN_UARCH_INDEX]) {

          #if XNN_ENABLE_FMA3 && XNN_ENABLE_AVX

            case xnn_uarch_zen:

            case xnn_uarch_dhyana:

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x16s4__fma3_broadcast);

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x16s4__fma3_broadcast);

              f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

              f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

              f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x16s4__avx_u4;

              f32_gemm_config->mr = 4;

              f32_gemm_config->nr = 16;

              f32_gemm_config->log2_sr = 2;

              break;

          #endif

          default:

            #if XNN_ARCH_X86_64 && !XNN_PLATFORM_WINDOWS && XNN_ENABLE_ASSEMBLY && XNN_ENABLE_FMA3 && XNN_ENABLE_AVX

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x16__asm_amd64_fma3_broadcast);

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(2)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_2x16__asm_amd64_fma3_broadcast);

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x16__asm_amd64_fma3_broadcast);

              f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

              f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x16__avx_u8;

              f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x16__avx_u4;

              f32_gemm_config->mr = 6;

              f32_gemm_config->nr = 16;

            #elif XNN_ENABLE_FMA3 && XNN_ENABLE_AVX

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x16__fma3_broadcast);

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(2)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_2x16__fma3_broadcast);

              f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_5x16__fma3_broadcast);

              f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

              f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x16__avx_u8;

              f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x16__avx_u4;

              f32_gemm_config->mr = 5;

              f32_gemm_config->nr = 16;

            #endif

            break;

        }

      } else

    #endif

    #if XNN_ENABLE_AVX

      if (hardware_config->arch_flags & xnn_arch_x86_avx) {

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x16__avx_broadcast);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_5x16__avx_broadcast);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x16__avx_u8;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x16__avx_u4;

        f32_gemm_config->mr = 5;

        f32_gemm_config->nr = 16;

      } else

    #endif

    #if XNN_ENABLE_SSE2

      if (hardware_config->arch_flags & xnn_arch_x86_sse2) {

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__sse_load1);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__sse_load1);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__sse2_u4;

        f32_gemm_config->mr = 4;

        f32_gemm_config->nr = 8;

      }

    #endif

  #elif XNN_ARCH_WASMSIMD || XNN_ARCH_WASMRELAXEDSIMD

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->is_x86) {

      #if XNN_ARCH_WASMRELAXEDSIMD

        f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x8__wasmrelaxedsimd_fma_loadsplat);

        f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_4x8__wasmrelaxedsimd_fma_loadsplat);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__wasmrelaxedsimd_fma_loadsplat);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__wasmrelaxedsimd_fma_loadsplat);

        f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x8__wasmrelaxedsimd_fma_loadsplat);

        f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_4x8__wasmrelaxedsimd_fma_loadsplat);

      #else

        if (hardware_concurrency() > kCoreCountThresholdForAdaptiveAvxOptimization) {

          f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x8__wasmsimd_loadsplat);

          f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_4x8__wasmsimd_loadsplat);

          f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_x86_loadsplat);

          f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_x86_loadsplat);

          f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_loadsplat);

          f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_4x8__wasmsimd_loadsplat);

        } else {

          f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x8__wasmsimd_splat);

          f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_4x8__wasmsimd_splat);

          f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_x86_splat);

          f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x8__wasmsimd_x86_splat);

          f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_splat);

          f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_4x8__wasmsimd_splat);

        }

      #endif

      f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

      f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

      f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__wasmsimd_u4;

      f32_gemm_config->mr = 4;

      f32_gemm_config->nr = 8;

    } else {

      #if XNN_ARCH_WASMRELAXEDSIMD

        f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x8__wasmrelaxedsimd_fma_splat);

        f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_6x8__wasmrelaxedsimd_fma_splat);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__wasmrelaxedsimd_fma_splat);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_6x8__wasmrelaxedsimd_fma_splat);

        f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x8__wasmrelaxedsimd_fma_splat);

        f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(6)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_6x8__wasmrelaxedsimd_fma_splat);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__wasmsimd_u4;

        f32_gemm_config->mr = 6;

        f32_gemm_config->nr = 8;

      #else

        f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x8__wasmsimd_splat);

        f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_5x8__wasmsimd_splat);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x8__wasmsimd_arm_splat);

        f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_5x8__wasmsimd_arm_splat);

        f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x8__wasmsimd_splat);

        f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_5x8__wasmsimd_splat);

        f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

        f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

        f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__wasmsimd_u4;

        f32_gemm_config->mr = 5;

        f32_gemm_config->nr = 8;

      #endif

    }

  #elif XNN_ARCH_RISCV && XNN_ENABLE_RISCV_VECTOR

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->arch_flags & xnn_arch_riscv_vector) {

      f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x4v__rvv);

      f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(7)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_7x4v__rvv);

      f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

      f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

      f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x4v__rvv_u8;

      f32_gemm_config->mr = 7;

      // nr is set to vlen * 4 / sizeof(float) = 4 * VLENB * 8 / 32 = VLENB

      f32_gemm_config->nr = hardware_config->vlenb;

    }

  #elif XNN_ARCH_HEXAGON && XNN_ENABLE_HVX

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    if (hardware_config->arch_flags & xnn_arch_hvx) {

      f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x64__hvx_broadcast);

      f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(5)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_5x64__hvx_broadcast);

      f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

      f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x64__hvx_u2;

      f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x64__scalar_int_u2;

      f32_gemm_config->mr = 5;

      f32_gemm_config->nr = 64;

    }

  #else

    f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_1x4__scalar);

    f32_gemm_config->linear.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_ukernel_4x4__scalar);

    f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_1x4__scalar);

    f32_gemm_config->minmax.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_minmax_ukernel_4x4__scalar);

    f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_1x4__scalar);

    f32_gemm_config->relu.gemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_GEMM_UKERNEL(xnn_f32_gemm_relu_ukernel_4x4__scalar);

    f32_gemm_config->init.f32 = xnn_init_f32_minmax_scalar_params;

    f32_gemm_config->pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_x32_packw_gemm_gio_ukernel_x4__scalar;

    f32_gemm_config->pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x4__scalar_float_u4;

    f32_gemm_config->mr = 4;

    f32_gemm_config->nr = 4;

  #endif

  assert(f32_gemm_config->mr <= XNN_MAX_MR);

}

static void init_f32_gemm_config() {

  init_f32_gemm_config_impl(&f32_gemm_config[default_config], false);

  init_f32_gemm_config_impl(&f32_gemm_config[consistent_config], true);

}

static void init_f32_igemm_config(void) {

  // Common parameters.

  f32_igemm_config.log2_input_element_size = XNN_LOG2_SIZEOF_FLOAT;

  f32_igemm_config.log2_filter_element_size = XNN_LOG2_SIZEOF_FLOAT;

  f32_igemm_config.log2_filter_element_bit_size = XNN_LOG2_SIZEOF_FLOAT + 3;

  f32_igemm_config.bias_element_size = sizeof(float);

  // Arch-specific parameters.

  #if XNN_ARCH_ARM

    const struct xnn_hardware_config* hardware_config = xnn_init_hardware_config();

    assert(hardware_config != NULL);

    (void) hardware_config;  // May be unused.

    if (hardware_config->arch_flags & xnn_arch_arm_neon) {

      #if XNN_ENABLE_ASSEMBLY

        switch (hardware_config->uarch[XNN_UARCH_INDEX]) {

          case xnn_uarch_cortex_a5:

          case xnn_uarch_cortex_a7:

          case xnn_uarch_krait:

          case xnn_uarch_kryo:

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a7);

            f32_igemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_igemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_igemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_igemm_config.mr = 4;

            f32_igemm_config.nr = 8;

            break;

          case xnn_uarch_cortex_a53:

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_igemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_igemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_igemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_igemm_config.mr = 4;

            f32_igemm_config.nr = 8;

            break;

          case xnn_uarch_cortex_a55r0:

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a53);

            f32_igemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_igemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_igemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_igemm_config.mr = 4;

            f32_igemm_config.nr = 8;

            break;

          case xnn_uarch_cortex_a32:

          case xnn_uarch_cortex_a35:

          case xnn_uarch_cortex_a55:

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a55);

            f32_igemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_igemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_igemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_igemm_config.mr = 4;

            f32_igemm_config.nr = 8;

            break;

          case xnn_uarch_cortex_a57:

          case xnn_uarch_cortex_a72:

          case xnn_uarch_cortex_a73:

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53_prfm);

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(4)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_4x8__asm_aarch32_neon_cortex_a75_prfm);

            f32_igemm_config.init.f32 = xnn_init_f32_minmax_scalar_params;

            f32_igemm_config.pack_gemm_gio = (xnn_packw_gemm_gio_ukernel_fn) xnn_pack_f32_gemm_gio_w;

            f32_igemm_config.pack_gemm_goi = (xnn_packw_gemm_goi_ukernel_fn) xnn_x32_packw_gemm_goi_ukernel_x8__neon_ld4lane_u4_prfm;

            f32_igemm_config.mr = 4;

            f32_igemm_config.nr = 8;

            break;

          default:

            f32_igemm_config.minmax.igemm[XNN_MR_TO_INDEX(1)] = XNN_INIT_HMP_IGEMM_UKERNEL(xnn_f32_igemm_minmax_ukernel_1x8__asm_aarch32_neon_cortex_a53);