// Copyright 2022 Google LLC

//

// This source code is licensed under the BSD-style license found in the

// LICENSE file in the root directory of this source tree.

#include <assert.h>

#include <stddef.h>

#include <stdint.h>

#if XNN_ENABLE_CPUINFO

#include <cpuinfo.h>

#endif  // XNN_ENABLE_CPUINFO

#include "src/xnnpack/common.h"

#include "src/xnnpack/init-once.h"

#if _WIN32

  #include <windows.h>

  #ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE

    #define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43

  #endif

#endif

#if XNN_ARCH_X86_64 && defined(__linux__) && !defined(CHROMIUM)

#include <sys/syscall.h>

#include <unistd.h>

#define XFEATURE_XTILEDATA 18

#define ARCH_REQ_XCOMP_PERM 0x1023

#endif

#if XNN_ENABLE_CPUINFO

  #include <cpuinfo.h>

#endif  // XNN_ENABLE_CPUINFO

#if XNN_ARCH_HEXAGON

// TODO - use CPUINFO

#include <HAP_compute_res.h>

#include <qurt.h>

#include "HAP_farf.h"

#include "HAP_power.h"  // NOLINT - part of the hexagon SDK

#endif

#if XNN_ARCH_RISCV

#include <inttypes.h>

#include <sys/auxv.h>

#define COMPAT_HWCAP_ISA_V (1 << ('V' - 'A'))

#endif

#if XNN_ARCH_PPC64

  #include <sys/auxv.h>

  #if defined(__FreeBSD__)

    #include <machine/cpu.h>

  #endif

#endif

#if XNN_ARCH_WASM || XNN_ARCH_WASMSIMD || XNN_ARCH_WASMRELAXEDSIMD

#include <math.h>

#endif

#if XNN_ARCH_WASMRELAXEDSIMD

#include <wasm_simd128.h>

#endif

#include "src/xnnpack/hardware-config.h"

#include "src/xnnpack/init-once.h"

#include "src/xnnpack/log.h"

#if XNN_ARCH_X86_64 && defined(__linux__) && !defined(CHROMIUM)

ssize_t xnn_syscall(size_t rax, size_t rdi, size_t rsi, size_t rdx) {

  __asm (

    "syscall"

    : "+a" (rax)

    : "D"(rdi), "S"(rsi), "d"(rdx)

    : "rcx", "r11", "memory"

  );

  return rax;

}

#endif

#if XNN_ENABLE_CPUINFO

static enum xnn_uarch cpuinfo_to_xnn_uarch(enum cpuinfo_uarch uarch) {

  switch (uarch) {

    case cpuinfo_uarch_broadwell: return xnn_uarch_broadwell;

    case cpuinfo_uarch_coyote_cove: return xnn_uarch_coyote_cove;

    case cpuinfo_uarch_crestmont: return xnn_uarch_crestmont;

    case cpuinfo_uarch_darkmont: return xnn_uarch_darkmont;

    case cpuinfo_uarch_dhyana: return xnn_uarch_dhyana;

    case cpuinfo_uarch_golden_cove: return xnn_uarch_golden_cove;

    case cpuinfo_uarch_gracemont: return xnn_uarch_gracemont;

    case cpuinfo_uarch_haswell: return xnn_uarch_haswell;

    case cpuinfo_uarch_ivy_bridge: return xnn_uarch_ivy_bridge;

    case cpuinfo_uarch_palm_cove: return xnn_uarch_palm_cove;

    case cpuinfo_uarch_raptor_cove: return xnn_uarch_raptor_cove;

    case cpuinfo_uarch_redwood_cove: return xnn_uarch_redwood_cove;

    case cpuinfo_uarch_sky_lake: return xnn_uarch_sky_lake;

    case cpuinfo_uarch_sunny_cove: return xnn_uarch_sunny_cove;

    case cpuinfo_uarch_willow_cove: return xnn_uarch_willow_cove;

    case cpuinfo_uarch_zen: return xnn_uarch_zen;

    case cpuinfo_uarch_zen2: return xnn_uarch_zen2;

    case cpuinfo_uarch_zen3: return xnn_uarch_zen3;

    case cpuinfo_uarch_zen4: return xnn_uarch_zen4;

    case cpuinfo_uarch_zen5: return xnn_uarch_zen5;

    case cpuinfo_uarch_zen6: return xnn_uarch_zen6;

    case cpuinfo_uarch_avalanche: return xnn_uarch_avalanche;

    case cpuinfo_uarch_blizzard: return xnn_uarch_blizzard;

    case cpuinfo_uarch_coll_everest: return xnn_uarch_coll_everest;

    case cpuinfo_uarch_coll_sawtooth: return xnn_uarch_coll_sawtooth;

    case cpuinfo_uarch_cortex_a5: return xnn_uarch_cortex_a5;

    case cpuinfo_uarch_cortex_a7: return xnn_uarch_cortex_a7;

    case cpuinfo_uarch_cortex_a32: return xnn_uarch_cortex_a32;

    case cpuinfo_uarch_cortex_a35: return xnn_uarch_cortex_a35;

    case cpuinfo_uarch_cortex_a53: return xnn_uarch_cortex_a53;

    case cpuinfo_uarch_cortex_a55r0: return xnn_uarch_cortex_a55r0;

    case cpuinfo_uarch_cortex_a55: return xnn_uarch_cortex_a55;

    case cpuinfo_uarch_cortex_a57: return xnn_uarch_cortex_a57;

    case cpuinfo_uarch_cortex_a72: return xnn_uarch_cortex_a72;

    case cpuinfo_uarch_cortex_a73: return xnn_uarch_cortex_a73;

    case cpuinfo_uarch_cortex_a75: return xnn_uarch_cortex_a75;

    case cpuinfo_uarch_cortex_a76: return xnn_uarch_cortex_a76;

    case cpuinfo_uarch_cortex_a77: return xnn_uarch_cortex_a77;

    case cpuinfo_uarch_cortex_a78: return xnn_uarch_cortex_a78;

    case cpuinfo_uarch_cortex_a320: return xnn_uarch_cortex_a320;

    case cpuinfo_uarch_cortex_a510: return xnn_uarch_cortex_a510;

    case cpuinfo_uarch_cortex_a520: return xnn_uarch_cortex_a520;

    case cpuinfo_uarch_cortex_a710: return xnn_uarch_cortex_a710;

    case cpuinfo_uarch_cortex_a715: return xnn_uarch_cortex_a715;

    case cpuinfo_uarch_cortex_a720: return xnn_uarch_cortex_a720;

    case cpuinfo_uarch_cortex_a725: return xnn_uarch_cortex_a725;

    case cpuinfo_uarch_cortex_x1: return xnn_uarch_cortex_x1;

    case cpuinfo_uarch_cortex_x2: return xnn_uarch_cortex_x2;

    case cpuinfo_uarch_cortex_x3: return xnn_uarch_cortex_x3;

    case cpuinfo_uarch_cortex_x4: return xnn_uarch_cortex_x4;

    case cpuinfo_uarch_cortex_x925: return xnn_uarch_cortex_x925;

    case cpuinfo_uarch_donan_everest: return xnn_uarch_donan_everest;

    case cpuinfo_uarch_donan_sawtooth: return xnn_uarch_donan_sawtooth;

    case cpuinfo_uarch_everest: return xnn_uarch_everest;

    case cpuinfo_uarch_exynos_m1: return xnn_uarch_exynos_m1;

    case cpuinfo_uarch_exynos_m2: return xnn_uarch_exynos_m2;

    case cpuinfo_uarch_exynos_m3: return xnn_uarch_exynos_m3;

    case cpuinfo_uarch_exynos_m4: return xnn_uarch_exynos_m4;

    case cpuinfo_uarch_exynos_m5: return xnn_uarch_exynos_m5;

    case cpuinfo_uarch_firestorm: return xnn_uarch_firestorm;

    case cpuinfo_uarch_icestorm: return xnn_uarch_icestorm;

    case cpuinfo_uarch_krait: return xnn_uarch_krait;

    case cpuinfo_uarch_kryo: return xnn_uarch_kryo;

    case cpuinfo_uarch_lumex_c1_nano: return xnn_uarch_lumex_c1_nano;

    case cpuinfo_uarch_lumex_c1_premium: return xnn_uarch_lumex_c1_premium;

    case cpuinfo_uarch_lumex_c1_pro: return xnn_uarch_lumex_c1_pro;

    case cpuinfo_uarch_lumex_c1_ultra: return xnn_uarch_lumex_c1_ultra;

    case cpuinfo_uarch_neoverse_n1: return xnn_uarch_neoverse_n1;

    case cpuinfo_uarch_neoverse_n2: return xnn_uarch_neoverse_n2;

    case cpuinfo_uarch_neoverse_v1: return xnn_uarch_neoverse_v1;

    case cpuinfo_uarch_neoverse_v2: return xnn_uarch_neoverse_v2;

    case cpuinfo_uarch_oryon: return xnn_uarch_oryon;

    case cpuinfo_uarch_oryon_v3: return xnn_uarch_oryon_v3;

    case cpuinfo_uarch_sawtooth: return xnn_uarch_sawtooth;

    case cpuinfo_uarch_sotra_performance: return xnn_uarch_sotra_performance;

    case cpuinfo_uarch_sotra_super: return xnn_uarch_sotra_super;

    case cpuinfo_uarch_tahiti_everest: return xnn_uarch_tahiti_everest;

    case cpuinfo_uarch_tahiti_sawtooth: return xnn_uarch_tahiti_sawtooth;

    case cpuinfo_uarch_tilos_everest: return xnn_uarch_tilos_everest;

    case cpuinfo_uarch_tilos_sawtooth: return xnn_uarch_tilos_sawtooth;

    case cpuinfo_uarch_tupai_everest: return xnn_uarch_tupai_everest;

    case cpuinfo_uarch_tupai_sawtooth: return xnn_uarch_tupai_sawtooth;

    default: return xnn_uarch_unknown;

  }

}

#endif  // XNN_ENABLE_CPUINFO

#if XNN_ENABLE_CPUINFO && XNN_ARCH_ARM

static bool cpuinfo_uarch_to_dot(enum cpuinfo_uarch uarch) {

  switch (uarch) {

    case cpuinfo_uarch_cortex_a76:       return true;

    case cpuinfo_uarch_cortex_a77:       return true;

    case cpuinfo_uarch_cortex_a78:       return true;

    case cpuinfo_uarch_cortex_a320:      return true;

    case cpuinfo_uarch_cortex_a510:      return true;

    case cpuinfo_uarch_cortex_a520:      return true;

    case cpuinfo_uarch_cortex_a710:      return true;

    case cpuinfo_uarch_cortex_a715:      return true;

    case cpuinfo_uarch_cortex_a720:      return true;

    case cpuinfo_uarch_cortex_a725:      return true;

    case cpuinfo_uarch_cortex_x1:        return true;

    case cpuinfo_uarch_cortex_x2:        return true;

    case cpuinfo_uarch_cortex_x3:        return true;

    case cpuinfo_uarch_cortex_x4:        return true;

    case cpuinfo_uarch_cortex_x925:      return true;

    case cpuinfo_uarch_lumex_c1_nano:    return true;

    case cpuinfo_uarch_lumex_c1_premium: return true;

    case cpuinfo_uarch_lumex_c1_pro:     return true;

    case cpuinfo_uarch_lumex_c1_ultra:   return true;

    case cpuinfo_uarch_neoverse_n2:      return true;

    case cpuinfo_uarch_neoverse_v2:      return true;

    case cpuinfo_uarch_oryon:            return true;

    case cpuinfo_uarch_oryon_v3:         return true;

    default: return false;

  }

}

#endif  // XNN_ENABLE_CPUINFO && XNN_ARCH_ARM

#if XNN_ARCH_RISCV

#include <unistd.h>

#include <sys/syscall.h>

extern long syscall(long number, ...);

/* Constants for kernels where headers might be missing */

#ifndef __NR_riscv_hwprobe

  #define __NR_riscv_hwprobe 258

#endif

#ifndef RISCV_HWPROBE_KEY_MVENDORID

  #define RISCV_HWPROBE_KEY_MVENDORID 0

#endif

#ifndef RISCV_HWPROBE_KEY_MARCHID

  #define RISCV_HWPROBE_KEY_MARCHID 1

#endif

#ifndef RISCV_HWPROBE_KEY_MIMPID

  #define RISCV_HWPROBE_KEY_MIMPID 2

#endif

#ifndef RISCV_HWPROBE_KEY_IMA_EXT_0

  #define RISCV_HWPROBE_KEY_IMA_EXT_0 4

#endif

#ifndef RISCV_HWPROBE_EXT_ZFH

  #define RISCV_HWPROBE_EXT_ZFH (1ULL << 27)

#endif

#ifndef RISCV_HWPROBE_EXT_ZVFH

  /* Bit 30 represents the Zvfh (Vector Float16) extension */

  #define RISCV_HWPROBE_EXT_ZVFH (1ULL << 30)

#endif

struct riscv_hwprobe {

  int64_t key;

  uint64_t value;

};

enum riscv_chipset_vendor {

  riscv_chipset_vendor_unknown = 0,

  riscv_chipset_vendor_sifive = 0x489,

};

static enum xnn_uarch riscv_decode_vendor_uarch(

    uint32_t vendor_id,

    uint32_t arch_id,

    uint32_t imp_id) {

  switch (vendor_id) {

    case riscv_chipset_vendor_sifive:

      switch (arch_id & 0x7FFFFFFFU) {

        case 0x00000907:

        case 0x00FFFB07:

          return xnn_uarch_sifive_x280;

        case 0x00001007:

        case 0x00FFFC07:

          return xnn_uarch_sifive_x390;

        default:

          if (imp_id == 0x07230908) {

            return xnn_uarch_sifive_x280;

          }

          break;

      }

      break;

    default:

      break;

  }

  return xnn_uarch_unknown;

}

static bool riscv_uarch_to_fp16(enum xnn_uarch uarch) {

  switch (uarch) {

    case xnn_uarch_sifive_x280:

    case xnn_uarch_sifive_x390:

      return true;

    default:

      return false;

  }

}

static void riscv_decode_hwprobe(

    enum xnn_uarch* uarch,

    bool* supports_fp16) {

  struct riscv_hwprobe pairs[] = {

    {.key = RISCV_HWPROBE_KEY_MVENDORID},

    {.key = RISCV_HWPROBE_KEY_MARCHID},

    {.key = RISCV_HWPROBE_KEY_MIMPID},

    {.key = RISCV_HWPROBE_KEY_IMA_EXT_0},

  };

  const size_t pairs_count = sizeof(pairs) / sizeof(struct riscv_hwprobe);

  *uarch = xnn_uarch_unknown;

  *supports_fp16 = false;

  if (syscall(__NR_riscv_hwprobe, pairs, pairs_count, 0, NULL, 0) != 0) {

    return;

  }

  uint32_t vendor_id = 0;

  uint32_t arch_id = 0;

  uint32_t imp_id = 0;

  uint64_t ima_ext_0 = 0;

  for (size_t pair = 0; pair < pairs_count; pair++) {

    switch (pairs[pair].key) {

      case RISCV_HWPROBE_KEY_MVENDORID:

        vendor_id = (uint32_t) pairs[pair].value;

        break;

      case RISCV_HWPROBE_KEY_MARCHID:

        arch_id = (uint32_t) pairs[pair].value;

        break;

      case RISCV_HWPROBE_KEY_MIMPID:

        imp_id = (uint32_t) pairs[pair].value;

        break;

      case RISCV_HWPROBE_KEY_IMA_EXT_0:

        ima_ext_0 = pairs[pair].value;

        break;

      default:

        break;

    }

  }

  xnn_log_debug(

      "riscv_hwprobe: vendor_id=0x%" PRIx32 ", arch_id=0x%" PRIx32

      ", imp_id=0x%" PRIx32 ", ima_ext_0=0x%" PRIx64,

      vendor_id, arch_id, imp_id, ima_ext_0);

  *uarch = riscv_decode_vendor_uarch(vendor_id, arch_id, imp_id);

  *supports_fp16 =

      ((ima_ext_0 & RISCV_HWPROBE_EXT_ZVFH) != 0) || riscv_uarch_to_fp16(*uarch);

}

#endif  // XNN_ARCH_RISCV

static struct xnn_hardware_config hardware_config = {0};

static struct xnn_hardware_config original_hardware_config = {0};

static bool original_hardware_config_saved = false;

XNN_INIT_ONCE_GUARD(hardware);

static void set_arch_flag(uint64_t flag, bool value) {

  if (value) {

    hardware_config.arch_flags |= flag;

  } else {

    hardware_config.arch_flags &= ~flag;

  }

}

static void init_hardware_config(void) {

  if (original_hardware_config_saved) {

    return;

  }

  hardware_config.arch_flags = 0;

  for (size_t i = 0; i < XNN_MAX_UARCH_TYPES; ++i) {

    hardware_config.uarch[i] = xnn_uarch_unknown;

  }

#if (XNN_ARCH_ARM64 || XNN_ARCH_ARM) && XNN_ENABLE_CPUINFO

#if XNN_PLATFORM_WINDOWS

  SYSTEM_INFO system_info;

  GetSystemInfo(&system_info);

  switch (system_info.wProcessorLevel) {

    case 0x803:  // Kryo 385 Silver

      set_arch_flag(xnn_arch_arm_neon_fp16_arith, true);

      break;

    default:

      // Assume that Dot Product support implies FP16 support.

      // ARM manuals don't guarantee that, but it holds in practice.

      set_arch_flag(xnn_arch_arm_neon_fp16_arith,

          !!IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE));

      break;

  }

  set_arch_flag(xnn_arch_arm_fp16_arith, hardware_config.arch_flags & xnn_arch_arm_neon_fp16_arith);

  set_arch_flag(xnn_arch_arm_neon_bf16, false);

  set_arch_flag(xnn_arch_arm_neon_dot, !!IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE));

#else

  set_arch_flag(xnn_arch_arm_fp16_arith, cpuinfo_has_arm_fp16_arith());

  set_arch_flag(xnn_arch_arm_neon_fp16_arith, cpuinfo_has_arm_neon_fp16_arith());

  set_arch_flag(xnn_arch_arm_neon_bf16, cpuinfo_has_arm_neon_bf16());

#endif

  set_arch_flag(xnn_arch_arm_vfpv3, cpuinfo_has_arm_vfpv3());

  set_arch_flag(xnn_arch_arm_neon, cpuinfo_has_arm_neon());

  set_arch_flag(xnn_arch_arm_neon_fp16, cpuinfo_has_arm_neon_fp16());

  set_arch_flag(xnn_arch_arm_neon_fma, cpuinfo_has_arm_neon_fma());

  set_arch_flag(xnn_arch_arm_neon_v8, cpuinfo_has_arm_neon_v8());

#endif

#if XNN_ARCH_ARM && XNN_ENABLE_CPUINFO

  set_arch_flag(xnn_arch_arm_v6, cpuinfo_has_arm_v6());

  set_arch_flag(xnn_arch_arm_vfpv2, cpuinfo_has_arm_vfpv2());

  // TODO(b/435053808)  Fix dot product detect on arm

  const struct cpuinfo_uarch_info* uarch = cpuinfo_get_uarch(0);

  bool uarch_dot = uarch ? cpuinfo_uarch_to_dot(uarch->uarch) : false;

  set_arch_flag(xnn_arch_arm_neon_dot, uarch_dot);

#endif

#if XNN_ARCH_ARM64 && XNN_ENABLE_CPUINFO

  set_arch_flag(xnn_arch_arm_neon_dot, cpuinfo_has_arm_neon_dot());

  set_arch_flag(xnn_arch_arm_neon_i8mm, cpuinfo_has_arm_i8mm());

#if !XNN_COMPILER_HAS_FEATURE(memory_sanitizer)

  // msan (understandably) does not support SVE/SME (b/494230133).

  set_arch_flag(xnn_arch_arm_sve, cpuinfo_has_arm_sve());

  set_arch_flag(xnn_arch_arm_sve2, cpuinfo_has_arm_sve2());

  // TODO(b/581442477): Remove once

  // https://github.com/pytorch/cpuinfo/issues/442 is fixed and smelen is

  // implemented for Windows ARM.

  const bool use_arm_sme =

      cpuinfo_has_arm_sme() && cpuinfo_get_max_arm_sme_length() > 0;

  set_arch_flag(xnn_arch_arm_sme, use_arm_sme);

  set_arch_flag(xnn_arch_arm_sme2, use_arm_sme && cpuinfo_has_arm_sme2());

  set_arch_flag(xnn_arch_arm_sme2p1, use_arm_sme && cpuinfo_has_arm_sme2p1());

#endif  // !XNN_COMPILER_HAS_FEATURE(memory_sanitizer)

#endif

#if (XNN_ARCH_X86 || XNN_ARCH_X86_64) && XNN_ENABLE_CPUINFO

  const bool use_x86_avx512skx = XNN_ENABLE_AVX512SKX && cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl();

  const bool use_x86_avx512vnni = XNN_ENABLE_AVX512VNNI && use_x86_avx512skx && cpuinfo_has_x86_avx512vnni();

  const bool use_x86_avx512vnnigfni = XNN_ENABLE_AVX512VNNIGFNI && use_x86_avx512vnni && cpuinfo_has_x86_gfni();

  const bool use_x86_avx512amx = XNN_ENABLE_AVX512AMX && XNN_ARCH_X86_64 && use_x86_avx512vnnigfni && cpuinfo_has_x86_amx_int8();

  const bool use_x86_avx512amxfp16 = XNN_ENABLE_AVX512AMXFP16 && XNN_ARCH_X86_64 && use_x86_avx512vnnigfni && cpuinfo_has_x86_avx512fp16() && cpuinfo_has_x86_amx_fp16();

#if XNN_ENABLE_ACE && XNN_ARCH_X86_64 && \

    (defined(__clang__) || defined(__GNUC__))

  // ACE microkernels use GCC/Clang inline asm; other compilers get stubs.

  const bool use_x86_ace = use_x86_avx512vnnigfni &&

                           cpuinfo_has_x86_avx10_2() &&

                           cpuinfo_has_x86_amx_tile() && cpuinfo_has_x86_ace();

#else

  const bool use_x86_ace = false;

#endif

  const bool use_x86_avx256skx = XNN_ENABLE_AVX256SKX && cpuinfo_has_x86_avx512f() && cpuinfo_has_x86_avx512bw() && cpuinfo_has_x86_avx512dq() && cpuinfo_has_x86_avx512vl();

  const bool use_x86_avx256vnni = XNN_ENABLE_AVX256VNNI && use_x86_avx256skx && cpuinfo_has_x86_avx512vnni();

  set_arch_flag(xnn_arch_x86_sse, XNN_ENABLE_SSE && cpuinfo_has_x86_sse());

  set_arch_flag(xnn_arch_x86_sse2, XNN_ENABLE_SSE2 && cpuinfo_has_x86_sse2());

  set_arch_flag(xnn_arch_x86_ssse3, XNN_ENABLE_SSSE3 && cpuinfo_has_x86_ssse3());

  set_arch_flag(xnn_arch_x86_sse4_1, XNN_ENABLE_SSE41 && cpuinfo_has_x86_sse4_1());

  set_arch_flag(xnn_arch_x86_avx, XNN_ENABLE_AVX && cpuinfo_has_x86_avx());

  set_arch_flag(xnn_arch_x86_f16c, XNN_ENABLE_F16C && cpuinfo_has_x86_f16c());

  set_arch_flag(xnn_arch_x86_fma3, XNN_ENABLE_FMA3 && cpuinfo_has_x86_fma3());

  set_arch_flag(xnn_arch_x86_avx2, XNN_ENABLE_AVX2 && cpuinfo_has_x86_avx2());

  set_arch_flag(xnn_arch_x86_avx512f, XNN_ENABLE_AVX512F && cpuinfo_has_x86_avx512f());

  set_arch_flag(xnn_arch_x86_avx512skx, use_x86_avx512skx);

  set_arch_flag(xnn_arch_x86_avx512vbmi, XNN_ENABLE_AVX512VBMI && use_x86_avx512skx && cpuinfo_has_x86_avx512vbmi());

  set_arch_flag(xnn_arch_x86_avx512vnni, use_x86_avx512vnni);

  set_arch_flag(xnn_arch_x86_avx512vnnigfni, use_x86_avx512vnnigfni);

  set_arch_flag(xnn_arch_x86_avx512fp16, XNN_ENABLE_AVX512FP16 && cpuinfo_has_x86_avx512fp16());

  set_arch_flag(xnn_arch_x86_avx512bf16, XNN_ENABLE_AVX512BF16 && cpuinfo_has_x86_avx512bf16());

  set_arch_flag(xnn_arch_x86_avx512amx, use_x86_avx512amx);

  set_arch_flag(xnn_arch_x86_avx512amxfp16, use_x86_avx512amxfp16);

  set_arch_flag(xnn_arch_x86_ace, use_x86_ace);

  set_arch_flag(xnn_arch_x86_avxvnni, XNN_ENABLE_AVXVNNI && cpuinfo_has_x86_avx2() && cpuinfo_has_x86_avxvnni() && cpuinfo_has_x86_gfni());

  set_arch_flag(xnn_arch_x86_avxvnniint8, XNN_ENABLE_AVXVNNIINT8 && cpuinfo_has_x86_avx2() && cpuinfo_has_x86_avx_vnni_int8() && cpuinfo_has_x86_gfni());

  set_arch_flag(xnn_arch_x86_avx256skx, use_x86_avx256skx);

  set_arch_flag(xnn_arch_x86_avx256vnni, use_x86_avx256vnni);

  set_arch_flag(xnn_arch_x86_avx256vnnigfni, XNN_ENABLE_AVX256VNNIGFNI && use_x86_avx256vnni && cpuinfo_has_x86_gfni());

#if XNN_ARCH_X86_64 && defined(__linux__) && !defined(CHROMIUM)

  if (use_x86_avx512amx || use_x86_avx512amxfp16 || use_x86_ace) {

    size_t status = xnn_syscall(SYS_arch_prctl, ARCH_REQ_XCOMP_PERM, XFEATURE_XTILEDATA, 0);

    if (status) {

      xnn_log_info("XFEATURE_XTILEDATA setup is failed, TMUL usage is not allowed");

      set_arch_flag(xnn_arch_x86_avx512amx, 0);

      set_arch_flag(xnn_arch_x86_avx512amxfp16, 0);

      set_arch_flag(xnn_arch_x86_ace, 0);

    }

  }

#endif

#endif  // !XNN_ARCH_X86 && !XNN_ARCH_X86_64

#if (XNN_ARCH_X86 || XNN_ARCH_X86_64) && !XNN_ENABLE_CPUINFO

  // Without cpuinfo respect the build flags

  set_arch_flag(xnn_arch_x86_sse, XNN_ENABLE_SSE);

  set_arch_flag(xnn_arch_x86_sse2, XNN_ENABLE_SSE2);

  set_arch_flag(xnn_arch_x86_ssse3, XNN_ENABLE_SSSE3);

  set_arch_flag(xnn_arch_x86_sse4_1, XNN_ENABLE_SSE41);

  set_arch_flag(xnn_arch_x86_avx, XNN_ENABLE_AVX);

  set_arch_flag(xnn_arch_x86_f16c, XNN_ENABLE_F16C);

  set_arch_flag(xnn_arch_x86_fma3, XNN_ENABLE_FMA3);

  set_arch_flag(xnn_arch_x86_avx2, XNN_ENABLE_AVX2);

#endif

#if XNN_ARCH_HEXAGON

  qurt_arch_version_t vers = {0};

  unsigned int arch_version = 0;

  if (qurt_sysenv_get_arch_version(&vers) == QURT_EOK) {

    // Lower 8 bits represents the version number in hex form

    arch_version = vers.arch_version & 0xff;

  }

  // TODO: use xnn_log_info

  FARF(ALWAYS, "HEXAGON UARCH VERSION %x\n", arch_version);

  // v69 is in Samsung S22

  if (arch_version >= 0x69) {

    set_arch_flag(xnn_arch_hvx, XNN_ENABLE_HVX);

  }

#if XNN_ENABLE_HMX

  // HMX FP16 is supported on Hexagon V73+ (Snapdragon 8 Gen 2 and newer) when

  // the HMX hardware unit is present, can be powered on, and can be locked in

  // the current protection domain.

  if (arch_version >= 0x73) {

    int power_ctx = 0;

    HAP_power_request_t request = {0};

    request.type = HAP_power_set_HMX;

    request.hmx.power_up = 1;

    if (HAP_power_set(&power_ctx, &request) == AEE_SUCCESS) {

      compute_res_attr_t compute_res;

      HAP_compute_res_attr_init(&compute_res);

      HAP_compute_res_attr_set_hmx_param(&compute_res, 1);

      unsigned int context_id = HAP_compute_res_acquire(&compute_res, 10000);

      if (context_id != 0) {

        if (HAP_compute_res_hmx_lock(context_id) == 0) {

          set_arch_flag(xnn_arch_hmx, true);

          HAP_compute_res_hmx_unlock(context_id);

        }

        HAP_compute_res_release(context_id);

      }

      request.hmx.power_up = 0;

      HAP_power_set(&power_ctx, &request);

    }

    HAP_power_destroy(&power_ctx);

  }

#endif  // XNN_ENABLE_HMX

  unsigned int max_hthreads = 1;

  qurt_sysenv_max_hthreads_t hardware_threads = {0};

  if (qurt_sysenv_get_max_hw_threads(&hardware_threads) == QURT_EOK) {

    max_hthreads = hardware_threads.max_hthreads;

  }

  FARF(ALWAYS, "HEXAGON HTHREADS %d\n", max_hthreads);

  unsigned int clkFreqHz = 0;

  HAP_power_response_t response = {.type = HAP_power_get_clk_Freq};

  if (HAP_power_get(NULL, &response) == AEE_SUCCESS) {

    clkFreqHz = response.clkFreqHz;

  }

  FARF(ALWAYS, "HEXAGON CLKFREQHZ %d\n", clkFreqHz);

#endif  // XNN_ARCH_HEXAGON

  #if XNN_ARCH_RISCV

    // TODO: Use cpuinfo when fp16 detect works

    const long hwcap = getauxval(AT_HWCAP);

    xnn_log_debug("getauxval(AT_HWCAP) = %08lX", hwcap);

    const bool use_riscv_vector = (hwcap & COMPAT_HWCAP_ISA_V) != 0;

    set_arch_flag(xnn_arch_riscv_vector, use_riscv_vector);

    enum xnn_uarch uarch = xnn_uarch_unknown;

    bool supports_fp16 = false;

    riscv_decode_hwprobe(&uarch, &supports_fp16);

    hardware_config.uarch[0] = uarch;

    set_arch_flag(xnn_arch_riscv_vector_fp16_arith, XNN_ENABLE_RISCV_FP16_VECTOR && use_riscv_vector && supports_fp16);

    if (use_riscv_vector) {

      register uint32_t vlenb __asm__ ("t0");

      __asm__(".word 0xC22022F3"  /* CSRR t0, vlenb */ : "=r" (vlenb));

      hardware_config.vlenb = vlenb;

      xnn_log_info("RISC-V VLENB: %" PRIu32, vlenb);

    }

  #endif

  #if XNN_ARCH_PPC64

    #if defined(__FreeBSD__)

      unsigned long HWCAPs = 0, HWCAPs_2 = 0;

      elf_aux_info(AT_HWCAP, &HWCAPs, sizeof(HWCAPs));

      elf_aux_info(AT_HWCAP2, &HWCAPs_2, sizeof(HWCAPs_2));

    #else

      const unsigned long HWCAPs = getauxval(AT_HWCAP);

      const unsigned long HWCAPs_2 = getauxval(AT_HWCAP2);

    #endif

    if (HWCAPs & PPC_FEATURE_HAS_VSX) {

      set_arch_flag(xnn_arch_vsx, 1);

    }

    #if defined PPC_FEATURE2_ARCH_3_1

      if (HWCAPs_2 & PPC_FEATURE2_ARCH_3_1) {

        set_arch_flag(xnn_arch_vsx3, 1);

      }

      if (HWCAPs_2 & PPC_FEATURE2_MMA) {

        set_arch_flag(xnn_arch_mma, 1);

      }

    #endif

  #endif

  #if XNN_ARCH_WASM || XNN_ARCH_WASMSIMD || XNN_ARCH_WASMRELAXEDSIMD

    // Unlike most other architectures, on x86/x86-64 when floating-point instructions

    // have no NaN arguments, but produce NaN output, the output NaN has sign bit set.

    // We use it to distinguish x86/x86-64 from other architectures, by doing subtraction

    // of two infinities (must produce NaN per IEEE 754 standard).

    static const volatile float inf = INFINITY;

    hardware_config.is_x86 = signbit(inf - inf);

  #endif  // XNN_ARCH_WASM || XNN_ARCH_WASMSIMD || XNN_ARCH_WASMRELAXEDSIMD

  #if XNN_ARCH_WASMRELAXEDSIMD

    {

      // Check if out-of-bounds behavior of Relaxed Swizzle is consistent with PSHUFB.

      const v128_t table = wasm_i8x16_const(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16);

      const v128_t index_mask = wasm_i8x16_const_splat(INT8_C(0x8F));

      const volatile v128_t index_increment = wasm_i8x16_const_splat(16);  // volatile to confuse Clang which otherwise mis-compiles

      v128_t index = wasm_i8x16_const(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);

      v128_t diff = wasm_i8x16_const_splat(0);

      for (uint32_t i = 16; i != 0; i--) {

        const v128_t pshufb_result = wasm_i8x16_swizzle(table, wasm_v128_and(index, index_mask));

        const v128_t relaxed_result = wasm_i8x16_relaxed_swizzle(table, index);

        diff = wasm_v128_or(diff, wasm_v128_xor(pshufb_result, relaxed_result));

        index = wasm_i8x16_add(index, index_increment);

      }

      set_arch_flag(xnn_arch_wasm_pshufb, !wasm_v128_any_true(diff));

    }

    {

      // Check out-of-bounds behaviour of Relaxed Integer Dot Product with Accumulation.

      const v128_t int8_input = wasm_i8x16_const(0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0);

      const volatile v128_t xint8_input = wasm_i8x16_const(0, 0, 0, -128, 0, 0, -128, 0, 0, -128, 0, 0, -128, 0, 0, 0);  // volatile to confuse Clang which otherwise ICE's

      const v128_t xint8_output = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(int8_input, xint8_input, wasm_i8x16_const_splat(0));

      const volatile v128_t overflow_input = wasm_i8x16_const(-128, -128, -128, -128, -128, -128, -1, -1, -1, -1, -128, -128, -1, -1, -1, -1);  // volatile to confuse Clang which otherwise ICE's

      const v128_t overflow_output = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_i8x16_const_splat(-128), overflow_input, wasm_i8x16_const_splat(0));

      set_arch_flag(xnn_arch_wasm_sdot, !wasm_v128_any_true(wasm_v128_or(

        wasm_v128_xor(xint8_output, wasm_i32x4_const_splat(-128)),

        wasm_v128_xor(overflow_output, wasm_i32x4_const(65536, 33024, 33024, 512)))));

    }

    {

      // Check out-of-bounds behaviour of Relaxed Integer Dot Product with Accumulation with signed and unsigned input (e.g. vpdpbusd).

      const v128_t int8_input = wasm_i8x16_const(0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0);

      const volatile v128_t xint8_input = wasm_i8x16_const(0, 0, 0, -128, 0, 0, -128, 0, 0, -128, 0, 0, -128, 0, 0, 0);  // volatile to confuse Clang which otherwise ICE's

      const v128_t xint8_output = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(int8_input, xint8_input, wasm_i8x16_const_splat(0));

      const volatile v128_t overflow_input = wasm_i8x16_const(-128, -128, -128, -128, -128, -128, -1, -1, -1, -1, -128, -128, -1, -1, -1, -1);  // volatile to confuse Clang which otherwise ICE's

      const v128_t overflow_output = wasm_i32x4_relaxed_dot_i8x16_i7x16_add(wasm_i8x16_const_splat(-128), overflow_input, wasm_i8x16_const_splat(0));

      set_arch_flag(xnn_arch_wasm_usdot, !wasm_v128_any_true(wasm_v128_or(

        wasm_v128_xor(xint8_output, wasm_i32x4_const_splat(128)),

        wasm_v128_xor(overflow_output, wasm_i32x4_const(-65536, -98048, -98048, -130560)))));

    }

    {

      const v128_t input1 = wasm_i32x4_const(0xF0F0F0F0, 0xAAAAAAAA, 0xCCCCCCCC, 0x99999999);

      const v128_t input2 = wasm_i32x4_const(0x0F0F0F0F, 0x55555555, 0x33333333, 0x66666666);

      v128_t diff = wasm_i8x16_const_splat(0);

      for (uint32_t shift = 0; shift < 32; ++shift) {

        const uint32_t mask = UINT32_C(1) << shift;

        const volatile v128_t vmask = wasm_u32x4_splat(mask);

        const v128_t blendvps_result = wasm_v128_bitselect(input1, input2, wasm_i32x4_shr(vmask, 31));

        const v128_t relaxed_result = wasm_i32x4_relaxed_laneselect(input1, input2, vmask);

        diff = wasm_v128_or(diff, wasm_v128_xor(blendvps_result, relaxed_result));

      }

      set_arch_flag(xnn_arch_wasm_blendvps, !wasm_v128_any_true(diff));

    }

    {

      const v128_t input1 = wasm_f32x4_const(16777218.f, 0.f, 0.f, 0.f);

      const v128_t input2 = wasm_f32x4_const(3.f, 0.f, 0.f, 0.f);

      const v128_t input3 = wasm_f32x4_const(3.f, 0.f, 0.f, 0.f);

      v128_t diff = wasm_i8x16_const_splat(0);

      const v128_t relaxed_result = wasm_f32x4_relaxed_madd(input1, input2, input3);

      const v128_t mul_result = wasm_f32x4_add(input3, wasm_f32x4_mul(input1, input2));

      diff = wasm_v128_or(diff, wasm_v128_xor(mul_result, relaxed_result));

      set_arch_flag(xnn_arch_wasm_fma, !wasm_v128_any_true(diff));

    }

  #endif  // XNN_ARCH_WASMRELAXEDSIMD

#if XNN_ENABLE_CPUINFO

    // Set the size of the L1 and L2 data caches.

    if (!cpuinfo_initialize()) {

      xnn_log_warning(

          "Failed to initialize cpuinfo, unable to determine L1/L2 data cache "

          "properties.");

    } else {

      const struct cpuinfo_processor* proc_info = cpuinfo_get_processor(0);

      if (proc_info != NULL) {

        // Get the L1 cache information.

        const struct cpuinfo_cache* l1_data_cache = proc_info->cache.l1d;

        if (l1_data_cache != NULL) {

          hardware_config.l1_data_cache_bytes = l1_data_cache->size;

          hardware_config.l1_data_cache_line_size = l1_data_cache->line_size;

          hardware_config.l1_data_cache_associativity =

              l1_data_cache->associativity;

          hardware_config.l1_data_cache_num_sets = l1_data_cache->sets;

          xnn_log_info(

              "l1_data_cache_bytes=%zu, l1_data_cache_line_size=%zu, "

              "l1_data_cache_associativity=%zu, l1_data_cache_num_sets=%zu.",

              hardware_config.l1_data_cache_bytes,

              hardware_config.l1_data_cache_line_size,

              hardware_config.l1_data_cache_associativity,

              hardware_config.l1_data_cache_num_sets);

        } else {

          xnn_log_warning("Unable to determine L1 data cache properties.");

        }

        // Get the L2 cache information.

        const struct cpuinfo_cache* l2_data_cache = proc_info->cache.l2;

        if (l2_data_cache != NULL) {

          hardware_config.l2_data_cache_bytes = l2_data_cache->size;

          hardware_config.l2_data_cache_line_size = l2_data_cache->line_size;

          hardware_config.l2_data_cache_associativity =

              l2_data_cache->associativity;

          hardware_config.l2_data_cache_num_sets = l2_data_cache->sets;

          xnn_log_info(

              "l2_data_cache_bytes=%zu, l2_data_cache_line_size=%zu, "

              "l2_data_cache_associativity=%zu, l2_data_cache_num_sets=%zu.",

              hardware_config.l2_data_cache_bytes,

              hardware_config.l2_data_cache_line_size,

              hardware_config.l2_data_cache_associativity,

              hardware_config.l2_data_cache_num_sets);

        } else {

          xnn_log_warning("Unable to determine L2 data cache properties.");

        }

      } else {

        xnn_log_warning("Unable to determine L1/L2 data cache properties.");

      }

    }

#if XNN_MAX_UARCH_TYPES > 1

    // Print what we think we know about the microarchs.

    xnn_log_info("cpuinfo_get_uarchs_count: %u.", cpuinfo_get_uarchs_count());

    for (int i = 0; i < cpuinfo_get_uarchs_count(); i++) {

      xnn_log_info("cpu_get_uarch(%i): 0x%x", i, cpuinfo_get_uarch(i)->uarch);

    }

#endif  // XNN_MAX_UARCH_TYPES > 1

#if !XNN_ARCH_RISCV

    for (size_t i = 0; i < XNN_MAX_UARCH_TYPES; ++i) {

      const struct cpuinfo_uarch_info* uarch = cpuinfo_get_uarch(i);

      hardware_config.uarch[i] = uarch ? cpuinfo_to_xnn_uarch(uarch->uarch) : xnn_uarch_unknown;

    }

#endif  // !XNN_ARCH_RISCV

#else

  xnn_log_warning("Unable to determine L1/L2 data cache properties.");

#endif  // XNN_ENABLE_CPUINFO

}

const struct xnn_hardware_config* xnn_init_hardware_config() {

  #if !XNN_PLATFORM_WEB && !XNN_ARCH_RISCV && !XNN_ARCH_PPC64 && XNN_ENABLE_CPUINFO

    if (!cpuinfo_initialize()) {

      xnn_log_error("failed to initialize cpuinfo");

      return NULL;

    }

  #endif  // !XNN_PLATFORM_WEB && !XNN_ARCH_RISCV && !XNN_ARCH_PPC64

  #if XNN_ARCH_ARM

    if (!cpuinfo_has_arm_v6()) {

      xnn_log_debug("unsupported hardware: ARMv6 not detected");

      return NULL;

    }

    if (!cpuinfo_has_arm_vfpv2() && !cpuinfo_has_arm_vfpv3()) {

      xnn_log_debug("unsupported hardware: VFP FPU not detected");

      return NULL;

    }

  #endif  // XNN_ARCH_ARM

  #if XNN_ARCH_X86

    if (!cpuinfo_has_x86_sse2()) {

      xnn_log_debug("unsupported hardware: SSE2 not detected");

      return NULL;

    }

  #endif  // XNN_ARCH_X86

  XNN_INIT_ONCE(hardware);

  return &hardware_config;

}

void xnn_set_hardware_config(const struct xnn_hardware_config* config) {

  const struct xnn_hardware_config* current = xnn_init_hardware_config();

  if (current != NULL && !original_hardware_config_saved) {

    original_hardware_config = hardware_config;

    original_hardware_config_saved = true;

  }

  if (config != NULL) {

    hardware_config = *config;

  }

  xnn_reset_all_init_guards();

}

void xnn_reset_hardware_config(void) {

  if (original_hardware_config_saved) {

    hardware_config = original_hardware_config;

    original_hardware_config_saved = false;

  } else {

    xnn_init_hardware_config();

  }

  xnn_reset_all_init_guards();

}