// Copyright 2020 Google LLC

// SPDX-License-Identifier: Apache-2.0

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

//      http://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.

// Main header required before using vector types.

// IWYU pragma: begin_exports

#include "hwy/base.h"

#include "hwy/detect_compiler_arch.h"

#include "hwy/detect_targets.h"

#include "hwy/highway_export.h"

#include "hwy/targets.h"

// IWYU pragma: end_exports

#if HWY_CXX_LANG < 201703L

#define HWY_DISPATCH_MAP 1

#else

#define HWY_DISPATCH_MAP 0

#endif

// This include guard is checked by foreach_target, so avoid the usual _H_

// suffix to prevent copybara from renaming it. NOTE: ops/*-inl.h are included

// after/outside this include guard.

#ifndef HWY_HIGHWAY_INCLUDED

#define HWY_HIGHWAY_INCLUDED

namespace hwy {

//------------------------------------------------------------------------------

// Shorthand for tags (defined in shared-inl.h) used to select overloads.

// Note that ScalableTag<T> is preferred over HWY_FULL, and CappedTag<T, N> over

// HWY_CAPPED(T, N).

// HWY_FULL(T[,LMUL=1]) is a native vector/group. LMUL is the number of

// registers in the group, and is ignored on targets that do not support groups.

#define HWY_FULL1(T) hwy::HWY_NAMESPACE::ScalableTag<T>

#define HWY_FULL2(T, LMUL) \

  hwy::HWY_NAMESPACE::ScalableTag<T, hwy::CeilLog2(HWY_MAX(0, LMUL))>

#define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3

// Workaround for MSVC grouping __VA_ARGS__ into a single argument

#define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren

// Trailing comma avoids -pedantic false alarm

#define HWY_CHOOSE_FULL(...) \

  HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, ))

#define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__)

// Vector of up to MAX_N lanes. It's better to use full vectors where possible.

#define HWY_CAPPED(T, MAX_N) hwy::HWY_NAMESPACE::CappedTag<T, MAX_N>

//------------------------------------------------------------------------------

// Export user functions for static/dynamic dispatch

// Evaluates to 0 inside a translation unit if it is generating anything but the

// static target (the last one if multiple targets are enabled). Used to prevent

// redefinitions of HWY_EXPORT. Unless foreach_target.h is included, we only

// compile once anyway, so this is 1 unless it is or has been included.

#ifndef HWY_ONCE

#define HWY_ONCE 1

#endif

// HWY_STATIC_DISPATCH(FUNC_NAME) is the namespace-qualified FUNC_NAME for

// HWY_STATIC_TARGET (the only defined namespace unless HWY_TARGET_INCLUDE is

// defined), and can be used to deduce the return type of Choose*.

#if HWY_STATIC_TARGET == HWY_SCALAR

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_EMU128

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_EMU128::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_RVV

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_WASM_EMU256

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM_EMU256::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_WASM

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_NEON_WITHOUT_AES

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON_WITHOUT_AES::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_NEON

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_NEON_BF16

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON_BF16::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SVE

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SVE2

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SVE_256

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE_256::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SVE2_128

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2_128::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_PPC8

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_PPC9

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC9::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_PPC10

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC10::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_Z14

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_Z14::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_Z15

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_Z15::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SSE2

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE2::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SSSE3

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_SSE4

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_AVX2

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_AVX3

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_AVX3_DL

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_AVX3_ZEN4

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_ZEN4::FUNC_NAME

#elif HWY_STATIC_TARGET == HWY_AVX3_SPR

#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_SPR::FUNC_NAME

#endif

// HWY_CHOOSE_*(FUNC_NAME) expands to the function pointer for that target or

// nullptr is that target was not compiled.

#if HWY_TARGETS & HWY_EMU128

#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_EMU128::FUNC_NAME

#elif HWY_TARGETS & HWY_SCALAR

#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &N_SCALAR::FUNC_NAME

#else

// When HWY_SCALAR/HWY_EMU128 are not present and other targets were disabled at

// runtime, fall back to the baseline with HWY_STATIC_DISPATCH().

#define HWY_CHOOSE_FALLBACK(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)

#endif

#if HWY_TARGETS & HWY_WASM_EMU256

#define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) &N_WASM_EMU256::FUNC_NAME

#else

#define HWY_CHOOSE_WASM_EMU256(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_WASM

#define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME

#else

#define HWY_CHOOSE_WASM(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_RVV

#define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME

#else

#define HWY_CHOOSE_RVV(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_NEON_WITHOUT_AES

#define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) &N_NEON_WITHOUT_AES::FUNC_NAME

#else

#define HWY_CHOOSE_NEON_WITHOUT_AES(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_NEON

#define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME

#else

#define HWY_CHOOSE_NEON(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_NEON_BF16

#define HWY_CHOOSE_NEON_BF16(FUNC_NAME) &N_NEON_BF16::FUNC_NAME

#else

#define HWY_CHOOSE_NEON_BF16(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SVE

#define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME

#else

#define HWY_CHOOSE_SVE(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SVE2

#define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME

#else

#define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SVE_256

#define HWY_CHOOSE_SVE_256(FUNC_NAME) &N_SVE_256::FUNC_NAME

#else

#define HWY_CHOOSE_SVE_256(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SVE2_128

#define HWY_CHOOSE_SVE2_128(FUNC_NAME) &N_SVE2_128::FUNC_NAME

#else

#define HWY_CHOOSE_SVE2_128(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_PPC8

#define HWY_CHOOSE_PPC8(FUNC_NAME) &N_PPC8::FUNC_NAME

#else

#define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_PPC9

#define HWY_CHOOSE_PPC9(FUNC_NAME) &N_PPC9::FUNC_NAME

#else

#define HWY_CHOOSE_PPC9(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_PPC10

#define HWY_CHOOSE_PPC10(FUNC_NAME) &N_PPC10::FUNC_NAME

#else

#define HWY_CHOOSE_PPC10(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_Z14

#define HWY_CHOOSE_Z14(FUNC_NAME) &N_Z14::FUNC_NAME

#else

#define HWY_CHOOSE_Z14(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_Z15

#define HWY_CHOOSE_Z15(FUNC_NAME) &N_Z15::FUNC_NAME

#else

#define HWY_CHOOSE_Z15(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SSE2

#define HWY_CHOOSE_SSE2(FUNC_NAME) &N_SSE2::FUNC_NAME

#else

#define HWY_CHOOSE_SSE2(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SSSE3

#define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME

#else

#define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_SSE4

#define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME

#else

#define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_AVX2

#define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME

#else

#define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_AVX3

#define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME

#else

#define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_AVX3_DL

#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME

#else

#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_AVX3_ZEN4

#define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) &N_AVX3_ZEN4::FUNC_NAME

#else

#define HWY_CHOOSE_AVX3_ZEN4(FUNC_NAME) nullptr

#endif

#if HWY_TARGETS & HWY_AVX3_SPR

#define HWY_CHOOSE_AVX3_SPR(FUNC_NAME) &N_AVX3_SPR::FUNC_NAME

#else

#define HWY_CHOOSE_AVX3_SPR(FUNC_NAME) nullptr

#endif

// MSVC 2017 workaround: the non-type template parameter to ChooseAndCall

// apparently cannot be an array. Use a function pointer instead, which has the

// disadvantage that we call the static (not best) target on the first call to

// any HWY_DYNAMIC_DISPATCH.

#if (HWY_COMPILER_MSVC && HWY_COMPILER_MSVC < 1915) || \

    (HWY_COMPILER_GCC_ACTUAL && HWY_COMPILER_GCC_ACTUAL < 700)

#define HWY_DISPATCH_WORKAROUND 1

#else

#define HWY_DISPATCH_WORKAROUND 0

#endif

#if HWY_DISPATCH_MAP

struct AllExports {

  template <class FuncPtr, class ExportsKey, uint64_t kHash>

  static const FuncPtr*& GetRefToExportsPtr() {

    static const FuncPtr* s_exports = nullptr;

    return s_exports;

  }

};

#endif

// Provides a static member function which is what is called during the first

// HWY_DYNAMIC_DISPATCH, where GetIndex is still zero, and instantiations of

// this function are the first entry in the tables created by HWY_EXPORT[_T].

template <typename RetType, typename... Args>

struct FunctionCache {

 public:

  typedef RetType(FuncType)(Args...);

  using FuncPtr = FuncType*;

  // A template function that when instantiated has the same signature as the

  // function being called. This function initializes the bit array of targets

  // supported by the current CPU and then calls the appropriate entry within

  // the HWY_EXPORT table. Subsequent calls via HWY_DYNAMIC_DISPATCH to any

  // exported functions, even those defined by different translation units,

  // will dispatch directly to the best available target.

#if HWY_DISPATCH_MAP

  template <class ExportsKey, uint64_t kHash>

  static RetType ChooseAndCall(Args... args) {

    ChosenTarget& chosen_target = GetChosenTarget();

    chosen_target.Update(SupportedTargets());

    const FuncPtr* table = AllExports::template GetRefToExportsPtr<

        FuncPtr, RemoveCvRef<ExportsKey>, kHash>();

    HWY_ASSERT(table);

    return (table[chosen_target.GetIndex()])(args...);

  }

#if !HWY_DISPATCH_WORKAROUND

  template <const FuncPtr* table>

  static RetType TableChooseAndCall(Args... args) {

    ChosenTarget& chosen_target = GetChosenTarget();

    chosen_target.Update(SupportedTargets());

    return (table[chosen_target.GetIndex()])(args...);

  }

#endif  // !HWY_DISPATCH_WORKAROUND

#else   // !HWY_DISPATCH_MAP: zero-overhead, but requires C++17

  template <const FuncPtr* table>

  static RetType ChooseAndCall(Args... args) {

    ChosenTarget& chosen_target = GetChosenTarget();

    chosen_target.Update(SupportedTargets());

    return (table[chosen_target.GetIndex()])(args...);

  }

Unexecuted instantiation: enc_cluster.cc:_ZN3hwy13FunctionCacheIvJRKN3jxl9HistogramEEE13ChooseAndCallIXadsoKPFvS4_EL_ZNS1_L36HistogramEntropyHighwayDispatchTableEEEEEEvS4_

Unexecuted instantiation: enc_cluster.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNSt3__16vectorINS1_9HistogramENS3_9allocatorIS5_EEEEmPS8_PNS4_IjNS6_IjEEEEEE13ChooseAndCallIXadsoKPFS2_SA_mSB_SE_EL_ZNS1_L41FastClusterHistogramsHighwayDispatchTableEEEEEES2_SA_mSB_SE_

Unexecuted instantiation: enc_detect_dots.cc:_ZN3hwy13FunctionCacheIN3jxl8StatusOrINS1_5PlaneIfEEEEJRKNS1_6Image3IfEES9_PNS1_10ThreadPoolEEE13ChooseAndCallIXadsoKPFS5_S9_S9_SB_EL_ZNS1_L42SumOfSquareDifferencesHighwayDispatchTableEEEEEES5_S9_S9_SB_

Unexecuted instantiation: enc_convolve_separable5.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_5PlaneIfEERKNS1_5RectTImEERKNS1_17WeightsSeparable5EPNS1_10ThreadPoolEPS4_EE13ChooseAndCallIXadsoKPFS2_S6_SA_SD_SF_SG_EL_ZNS1_L30Separable5HighwayDispatchTableEEEEEES2_S6_SA_SD_SF_SG_

Unexecuted instantiation: enc_xyb.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_13ColorEncodingEfPKNS1_5PlaneIfEEPNS1_10ThreadPoolEPNS1_6Image3IfEERK15JxlCmsInterfaceSE_EE13ChooseAndCallIXadsoKPFS2_S5_fS9_SB_SE_SH_SE_EL_ZNS1_L25ToXYBHighwayDispatchTableEEEEEES2_S5_fS9_SB_SE_SH_SE_

Unexecuted instantiation: enc_xyb.cc:_ZN3hwy13FunctionCacheIvJPfS1_S1_PKfmEE13ChooseAndCallIXadsoKPFvS1_S1_S1_S3_mEL_ZN3jxlL37LinearRGBRowToXYBHighwayDispatchTableEEEEEEvS1_S1_S1_S3_m

Unexecuted instantiation: enc_xyb.cc:_ZN3hwy13FunctionCacheIvJfPfEE13ChooseAndCallIXadsoKPFvfS1_EL_ZN3jxlL39ComputePremulAbsorbHighwayDispatchTableEEEEEEvfS1_

Unexecuted instantiation: enc_xyb.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_5PlaneIfEES6_S6_PS4_S7_S7_PNS1_10ThreadPoolEEE13ChooseAndCallIXadsoKPFS2_S6_S6_S6_S7_S7_S7_S9_EL_ZNS1_L30RgbToYcbcrHighwayDispatchTableEEEEEES2_S6_S6_S6_S7_S7_S7_S9_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_6Image3IfEERKNS1_17ButteraugliParamsEPS4_PNS1_8BlurTempESA_EE13ChooseAndCallIXadsoKPFS2_S6_S9_SA_SC_SA_EL_ZNS1_L38OpsinDynamicsImageHighwayDispatchTableEEEEEES2_S6_S9_SA_SC_SA_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJmmRKNS1_17ButteraugliParamsEPNS1_8BlurTempERKNS1_6Image3IfEERNS1_11PsychoImageEEE13ChooseAndCallIXadsoKPFS2_mmS5_S7_SB_SD_EL_ZNS1_L39SeparateFrequenciesHighwayDispatchTableEEEEEES2_mmS5_S7_SB_SD_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_11PsychoImageES5_mmRKNS1_17ButteraugliParamsEPNS1_8BlurTempEPNS1_5PlaneIfEESD_EE13ChooseAndCallIXadsoKPFS2_S5_S5_mmS8_SA_SD_SD_EL_ZNS1_L35MaskPsychoImageHighwayDispatchTableEEEEEES2_S5_S5_mmS8_SA_SD_SD_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_5PlaneIfEES6_dddPS4_S7_EE13ChooseAndCallIXadsoKPFS2_S6_S6_dddS7_S7_EL_ZNS1_L32MaltaDiffMapHighwayDispatchTableEEEEEES2_S6_S6_dddS7_S7_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_5PlaneIfEES6_dddPS4_S7_EE13ChooseAndCallIXadsoKPFS2_S6_S6_dddS7_S7_EL_ZNS1_L34MaltaDiffMapLFHighwayDispatchTableEEEEEES2_S6_S6_dddS7_S7_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIvJRKN3jxl5PlaneIfEES5_ffPS3_EE13ChooseAndCallIXadsoKPFvS5_S5_ffS6_EL_ZNS1_L36L2DiffAsymmetricHighwayDispatchTableEEEEEEvS5_S5_ffS6_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIvJRKN3jxl5PlaneIfEES5_fPS3_EE13ChooseAndCallIXadsoKPFvS5_S5_fS6_EL_ZNS1_L26L2DiffHighwayDispatchTableEEEEEEvS5_S5_fS6_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIvJRKN3jxl5PlaneIfEES5_fPS3_EE13ChooseAndCallIXadsoKPFvS5_S5_fS6_EL_ZNS1_L29SetL2DiffHighwayDispatchTableEEEEEEvS5_S5_fS6_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_5PlaneIfEERKNS1_6Image3IfEESA_fPS4_EE13ChooseAndCallIXadsoKPFS2_S6_SA_SA_fSB_EL_ZNS1_L44CombineChannelsToDiffmapHighwayDispatchTableEEEEEES2_S6_SA_SA_fSB_

Unexecuted instantiation: butteraugli.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRNS1_6Image3IfEES5_RKNS1_17ButteraugliParamsERNS1_5PlaneIfEEEE13ChooseAndCallIXadsoKPFS2_S5_S5_S8_SB_EL_ZNS1_L45ButteraugliDiffmapInPlaceHighwayDispatchTableEEEEEES2_S5_S5_S8_SB_

Unexecuted instantiation: enc_adaptive_quantization.cc:_ZN3hwy13FunctionCacheIN3jxl8StatusOrINS1_5PlaneIfEEEEJfRKNS1_6Image3IfEERKNS1_5RectTImEEfPNS1_10ThreadPoolEPS4_SG_EE13ChooseAndCallIXadsoKPFS5_fS9_SD_fSF_SG_SG_EL_ZNS1_L43AdaptiveQuantizationMapHighwayDispatchTableEEEEEES5_fS9_SD_fSF_SG_SG_

Unexecuted instantiation: enc_group.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJmPNS1_18PassesEncoderStateERKNS1_6Image3IfEERKNS1_5RectTImEEPS6_EE13ChooseAndCallIXadsoKPFS2_mS4_S8_SC_SD_EL_ZNS1_L39ComputeCoefficientsHighwayDispatchTableEEEEEES2_mS4_S8_SC_SD_

Unexecuted instantiation: enc_chroma_from_luma.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJP22JxlMemoryManagerStructmPNS1_5PlaneIfEEEE13ChooseAndCallIXadsoKPFS2_S4_mS7_EL_ZNS1_L33InitDCStorageHighwayDispatchTableEEEEEES2_S4_mS7_

Unexecuted instantiation: enc_chroma_from_luma.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_6Image3IfEERKNS1_5RectTImEERKNS1_15DequantMatricesEPKNS1_15AcStrategyImageEPKNS1_5PlaneIiEEPKNS1_9QuantizerESA_bbPNSH_IaEESP_PNSH_IfEENS1_4SpanIfEEEE13ChooseAndCallIXadsoKPFS2_S6_SA_SD_SG_SK_SN_SA_bbSP_SP_SR_ST_EL_ZNS1_L31ComputeTileHighwayDispatchTableEEEEEES2_S6_SA_SD_SG_SK_SN_SA_bbSP_SP_SR_ST_

Unexecuted instantiation: enc_ac_strategy.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_14CompressParamsERKNS1_9ACSConfigERKNS1_5RectTImEERKNS1_19ColorCorrelationMapEPfPjPNS1_15AcStrategyImageEEE13ChooseAndCallIXadsoKPFS2_S5_S8_SC_SF_SG_SH_SJ_EL_ZNS1_L34ProcessRectACSHighwayDispatchTableEEEEEES2_S5_S8_SC_SF_SG_SH_SJ_

Unexecuted instantiation: enc_entropy_coder.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJPKjRKNS1_5RectTImEEPrPKiRKNS1_15AcStrategyImageERKNS1_22YCbCrChromaSubsamplingEPNS1_6Image3IiEEPNSt3__16vectorINS1_5TokenENSM_9allocatorISO_EEEERKNS1_5PlaneIhEERKNST_IiEERKNS1_11BlockCtxMapEEE13ChooseAndCallIXadsoKPFS2_S4_S8_SC_SF_SI_SL_SS_SW_SZ_S12_EL_ZNS1_L40TokenizeCoefficientsHighwayDispatchTableEEEEEES2_S4_S8_SC_SF_SI_SL_SS_SW_SZ_S12_

Unexecuted instantiation: jxl_cms.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJPvmPKfPfmEE13ChooseAndCallIXadsoKPFS2_S3_mS5_S6_mEL_ZNS1_12_GLOBAL__N_141DoColorSpaceTransformHighwayDispatchTableEEEEEES2_S3_mS5_S6_m

Unexecuted instantiation: enc_ma.cc:_ZN3hwy13FunctionCacheIvJRN3jxl11TreeSamplesEfRKNSt3__16vectorINS1_21ModularMultiplierInfoENS4_9allocatorIS6_EEEENS4_5arrayINSC_IjLm2EEELm2EEEfPNS5_INS1_20PropertyDecisionNodeENS7_ISF_EEEEEE13ChooseAndCallIXadsoKPFvS3_fSB_SE_fSI_EL_ZNS1_L33FindBestSplitHighwayDispatchTableEEEEEEvS3_fSB_SE_fSI_

Unexecuted instantiation: compressed_dc.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJP22JxlMemoryManagerStructPKfPNS1_6Image3IfEEPNS1_10ThreadPoolEEE13ChooseAndCallIXadsoKPFS2_S4_S6_S9_SB_EL_ZNS1_L39AdaptiveDCSmoothingHighwayDispatchTableEEEEEES2_S4_S6_S9_SB_

Unexecuted instantiation: compressed_dc.cc:_ZN3hwy13FunctionCacheIvJRKN3jxl5RectTImEEPNS1_6Image3IfEEPNS1_5PlaneIhEERKNS1_5ImageEPKffSG_RKNS1_22YCbCrChromaSubsamplingERKNS1_11BlockCtxMapEEE13ChooseAndCallIXadsoKPFvS5_S8_SB_SE_SG_fSG_SJ_SM_EL_ZNS1_L29DequantDCHighwayDispatchTableEEEEEEvS5_S8_SB_SE_SG_fSG_SJ_SM_

Unexecuted instantiation: convolve_symmetric5.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_5PlaneIfEERKNS1_5RectTImEERKNS1_17WeightsSymmetric5EPNS1_10ThreadPoolEPS4_SA_EE13ChooseAndCallIXadsoKPFS2_S6_SA_SD_SF_SG_SA_EL_ZNS1_L30Symmetric5HighwayDispatchTableEEEEEES2_S6_SA_SD_SF_SG_SA_

Unexecuted instantiation: dec_external_image.cc:_ZN3hwy13FunctionCacheIvJPKfPNS_9float16_tEmEE13ChooseAndCallIXadsoKPFvS2_S4_mEL_ZN3jxlL30FloatToF16HighwayDispatchTableEEEEEEvS2_S4_m

Unexecuted instantiation: dec_external_image.cc:_ZN3hwy13FunctionCacheIvJPKfPjmfmEE13ChooseAndCallIXadsoKPFvS2_S3_mfmEL_ZN3jxlL30FloatToU32HighwayDispatchTableEEEEEEvS2_S3_mfm

Unexecuted instantiation: dec_group.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_11FrameHeaderEPNS1_8GetBlockEPNS1_13GroupDecCacheEPNS1_18PassesDecoderStateEmmRNS1_19RenderPipelineInputEPNS1_4jpeg8JPEGDataENS1_8DrawModeEEE13ChooseAndCallIXadsoKPFS2_S5_S7_S9_SB_mmSD_SG_SH_EL_ZNS1_12_GLOBAL__N_135DecodeGroupImplHighwayDispatchTableEEEEEES2_S5_S7_S9_SB_mmSD_SG_SH_

Unexecuted instantiation: dec_modular.cc:_ZN3hwy13FunctionCacheIvJmPKifPfEE13ChooseAndCallIXadsoKPFvmS2_fS3_EL_ZN3jxlL36SingleFromSingleHighwayDispatchTableEEEEEEvmS2_fS3_

Unexecuted instantiation: dec_modular.cc:_ZN3hwy13FunctionCacheIvJmPKiS2_fPfEE13ChooseAndCallIXadsoKPFvmS2_S2_fS3_EL_ZN3jxlL31MultiplySumHighwayDispatchTableEEEEEEvmS2_S2_fS3_

Unexecuted instantiation: dec_modular.cc:_ZN3hwy13FunctionCacheIvJmPKifPfS3_S3_EE13ChooseAndCallIXadsoKPFvmS2_fS3_S3_S3_EL_ZN3jxlL33RgbFromSingleHighwayDispatchTableEEEEEEvmS2_fS3_S3_S3_

Unexecuted instantiation: dec_noise.cc:_ZN3hwy13FunctionCacheIvJmmmmRKNSt3__14pairIPN3jxl5PlaneIfEENS3_5RectTImEEEESB_SB_EE13ChooseAndCallIXadsoKPFvmmmmSB_SB_SB_EL_ZNS3_12_GLOBAL__N_133Random3PlanesHighwayDispatchTableEEEEEEvmmmmSB_SB_SB_

Unexecuted instantiation: dec_xyb.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_6Image3IfEERKNS1_5RectTImEEPNS1_10ThreadPoolEPS4_RKNS1_11OpsinParamsEEE13ChooseAndCallIXadsoKPFS2_S6_SA_SC_SD_SG_EL_ZNS1_L33OpsinToLinearHighwayDispatchTableEEEEEES2_S6_SA_SC_SD_SG_

Unexecuted instantiation: squeeze.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRNS1_5ImageERKNSt3__16vectorINS1_13SqueezeParamsENS5_9allocatorIS7_EEEEPNS1_10ThreadPoolEEE13ChooseAndCallIXadsoKPFS2_S4_SC_SE_EL_ZNS1_L30InvSqueezeHighwayDispatchTableEEEEEES2_S4_SC_SE_

Unexecuted instantiation: rct.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRNS1_5ImageEmmPNS1_10ThreadPoolEEE13ChooseAndCallIXadsoKPFS2_S4_mmS6_EL_ZNS1_L26InvRCTHighwayDispatchTableEEEEEES2_S4_mmS6_

Unexecuted instantiation: quant_weights.cc:_ZN3hwy13FunctionCacheIN3jxl6StatusEJRKNS1_13QuantEncodingEPfS6_mNS1_10QuantTableEPmEE13ChooseAndCallIXadsoKPFS2_S5_S6_S6_mS7_S8_EL_ZNS1_12_GLOBAL__N_137ComputeQuantTableHighwayDispatchTableEEEEEES2_S5_S6_S6_mS7_S8_

Unexecuted instantiation: stage_blending.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_11FrameHeaderEPKNS3_18PassesDecoderStateERKNS3_13ColorEncodingEEE13ChooseAndCallIXadsoKPFS7_SA_SD_SG_EL_ZNS3_L36GetBlendingStageHighwayDispatchTableEEEEEES7_SA_SD_SG_

Unexecuted instantiation: stage_chroma_upsampling.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJmbEE13ChooseAndCallIXadsoKPFS7_mbEL_ZNS3_L44GetChromaUpsamplingStageHighwayDispatchTableEEEEEES7_mb

Unexecuted instantiation: stage_cms.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_18OutputEncodingInfoEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L31GetCmsStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_epf.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_10LoopFilterERKNS3_5PlaneIfEEEE13ChooseAndCallIXadsoKPFS7_SA_SE_EL_ZNS3_L32GetEPFStage0HighwayDispatchTableEEEEEES7_SA_SE_

Unexecuted instantiation: stage_epf.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_10LoopFilterERKNS3_5PlaneIfEEEE13ChooseAndCallIXadsoKPFS7_SA_SE_EL_ZNS3_L32GetEPFStage1HighwayDispatchTableEEEEEES7_SA_SE_

Unexecuted instantiation: stage_epf.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_10LoopFilterERKNS3_5PlaneIfEEEE13ChooseAndCallIXadsoKPFS7_SA_SE_EL_ZNS3_L32GetEPFStage2HighwayDispatchTableEEEEEES7_SA_SE_

Unexecuted instantiation: stage_from_linear.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_18OutputEncodingInfoEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L38GetFromLinearStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_gaborish.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_10LoopFilterEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L36GetGaborishStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_noise.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_11NoiseParamsERKNS3_16ColorCorrelationEmEE13ChooseAndCallIXadsoKPFS7_SA_SD_mEL_ZNS3_L36GetAddNoiseStageHighwayDispatchTableEEEEEES7_SA_SD_m

Unexecuted instantiation: stage_noise.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJmEE13ChooseAndCallIXadsoKPFS7_mEL_ZNS3_L41GetConvolveNoiseStageHighwayDispatchTableEEEEEES7_m

Unexecuted instantiation: stage_splines.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJPKNS3_7SplinesEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L34GetSplineStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_to_linear.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_18OutputEncodingInfoEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L36GetToLinearStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_tone_mapping.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_18OutputEncodingInfoEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L39GetToneMappingStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_upsampling.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_19CustomTransformDataEmmEE13ChooseAndCallIXadsoKPFS7_SA_mmEL_ZNS3_L38GetUpsamplingStageHighwayDispatchTableEEEEEES7_SA_mm

Unexecuted instantiation: stage_write.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_11ImageOutputEmmbbmNS3_11OrientationERNS1_6vectorIS8_NS1_9allocatorIS8_EEEEP22JxlMemoryManagerStructEE13ChooseAndCallIXadsoKPFS7_SA_mmbbmSB_SG_SI_EL_ZNS3_L41GetWriteToOutputStageHighwayDispatchTableEEEEEES7_SA_mmbbmSB_SG_SI_

Unexecuted instantiation: stage_xyb.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJRKNS3_18OutputEncodingInfoEEE13ChooseAndCallIXadsoKPFS7_SA_EL_ZNS3_L31GetXYBStageHighwayDispatchTableEEEEEES7_SA_

Unexecuted instantiation: stage_ycbcr.cc:_ZN3hwy13FunctionCacheINSt3__110unique_ptrIN3jxl19RenderPipelineStageENS1_14default_deleteIS4_EEEEJEE13ChooseAndCallIXadsoKPFS7_vEL_ZNS3_L33GetYCbCrStageHighwayDispatchTableEEEEEES7_v

simd_util.cc:_ZN3hwy13FunctionCacheImJEE13ChooseAndCallIXadsoKPFmvEL_ZN3jxlL33MaxVectorSizeHighwayDispatchTableEEEEEEmv

Line

Count

Source

337

2

  static RetType ChooseAndCall(Args... args) {

338

2

    ChosenTarget& chosen_target = GetChosenTarget();

339

2

    chosen_target.Update(SupportedTargets());

340

2

    return (table[chosen_target.GetIndex()])(args...);

341

2

  }

Unexecuted instantiation: splines.cc:_ZN3hwy13FunctionCacheIvJRKN3jxl6SplineERKNSt3__16vectorINS5_4pairINS2_5PointEfEENS5_9allocatorIS9_EEEEfRNS6_INS1_13SplineSegmentENSA_ISF_EEEERNS6_INS7_ImmEENSA_ISJ_EEEEEE13ChooseAndCallIXadsoKPFvS4_SE_fSI_SM_EL_ZNS1_L38SegmentsFromPointsHighwayDispatchTableEEEEEEvS4_SE_fSI_SM_

Unexecuted instantiation: splines.cc:_ZN3hwy13FunctionCacheIvJPfS1_S1_mmmbPKN3jxl13SplineSegmentEPKmS7_EE13ChooseAndCallIXadsoKPFvS1_S1_S1_mmmbS5_S7_S7_EL_ZNS2_L32DrawSegmentsHighwayDispatchTableEEEEEEvS1_S1_S1_mmmbS5_S7_S7_

#endif  // HWY_DISPATCH_MAP

};

// Used to deduce the template parameters RetType and Args from a function.

template <typename RetType, typename... Args>

FunctionCache<RetType, Args...> DeduceFunctionCache(RetType (*)(Args...)) {

  return FunctionCache<RetType, Args...>();

}

#define HWY_DISPATCH_TABLE(FUNC_NAME) \

  HWY_CONCAT(FUNC_NAME, HighwayDispatchTable)

// HWY_EXPORT(FUNC_NAME); expands to a static array that is used by

// HWY_DYNAMIC_DISPATCH() to call the appropriate function at runtime.

// After being exported, it can be called from other parts of the same source

// file using HWY_DYNAMIC_DISPATCH(), in particular from a function wrapper

// like in the following example:

//

//   #include "hwy/highway.h"

//   HWY_BEFORE_NAMESPACE();

//   namespace skeleton {

//   namespace HWY_NAMESPACE {

//

//   void MyFunction(int a, char b, const char* c) { ... }

//

//   // NOLINTNEXTLINE(google-readability-namespace-comments)

//   }  // namespace HWY_NAMESPACE

//   }  // namespace skeleton

//   HWY_AFTER_NAMESPACE();

//

//   namespace skeleton {

//   HWY_EXPORT(MyFunction);  // Defines the dispatch table in this scope.

//

//   void MyFunction(int a, char b, const char* c) {

//     return HWY_DYNAMIC_DISPATCH(MyFunction)(a, b, c);

//   }

//   }  // namespace skeleton

//

// For templated code with a single type parameter, instead use HWY_EXPORT_T and

// its HWY_DYNAMIC_DISPATCH_T counterpart:

//

//   template <typename T>

//   void MyFunctionCaller(T ...) {

//     // First argument to both HWY_EXPORT_T and HWY_DYNAMIC_DISPATCH_T is an

//     // arbitrary table name; you must provide the same name for each call.

//     // It is fine to have multiple HWY_EXPORT_T in a function, but a 64-bit

//     // FNV hash collision among *any* table names will trigger HWY_ABORT.

//     HWY_EXPORT_T(Table1, MyFunction<T>)

//     HWY_DYNAMIC_DISPATCH_T(Table1)(a, b, c);

//   }

//

// Note that HWY_EXPORT_T must be invoked inside a template (in the above

// example: `MyFunctionCaller`), so that a separate table will be created for

// each template instantiation. For convenience, we also provide a macro that

// combines both steps and avoids the need to pick a table name:

//

//   template <typename T>

//   void MyFunctionCaller(T ...) {

//     // Table name is automatically chosen. Note that this variant must be

//     // called in statement context; it is not a valid expression.

//     HWY_EXPORT_AND_DYNAMIC_DISPATCH_T(MyFunction<T>)(a, b, c);

//   }

// Simplified version for IDE or the dynamic dispatch case with only one target.

#if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)

// We use a table to provide the same compile error conditions as with the

// non-simplified case, but the table only has a single entry.

#define HWY_EXPORT_T(TABLE_NAME, FUNC_NAME)                               \

  HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const \

  HWY_DISPATCH_TABLE(TABLE_NAME)[1] = {&HWY_STATIC_DISPATCH(FUNC_NAME)}

// Use the table, not just STATIC_DISPATCH as in DYNAMIC_DISPATCH, because

// TABLE_NAME might not match the function name.

#define HWY_DYNAMIC_POINTER_T(TABLE_NAME) (HWY_DISPATCH_TABLE(TABLE_NAME)[0])

#define HWY_DYNAMIC_DISPATCH_T(TABLE_NAME) \

  (*(HWY_DYNAMIC_POINTER_T(TABLE_NAME)))

#define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME)

#define HWY_DYNAMIC_POINTER(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME)

#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME)

#else  // not simplified: full table

// Pre-C++17 workaround: non-type template arguments must have linkage, which

// means we cannot pass &table as a template argument to ChooseAndCall.

// ChooseAndCall must find a way to access the table in order to dispatch to the

// chosen target:

// 0) Skipping this by dispatching to the static target would be surprising to

//    users and may have serious performance implications.

// 1) An extra function parameter would be unacceptable because it changes the

//    user-visible function signature.

// 2) Declaring a table, then defining a pointer to it would work, but requires

//    an additional DECLARE step outside the function so that the pointer has

//    linkage, which breaks existing code.

// 3) We instead associate the function with the table using an instance of an

//    unnamed struct and the hash of the table name as the key. Because

//    ChooseAndCall has the type information, it can then cast to the function

//    pointer type. However, we cannot simply pass the name as a template

//    argument to ChooseAndCall because this requires char*, which hits the same

//    linkage problem. We instead hash the table name, which assumes the

//    function names do not have collisions.

#if HWY_DISPATCH_MAP

static constexpr uint64_t FNV(const char* name) {

  return *name ? static_cast<uint64_t>(static_cast<uint8_t>(*name)) ^

                     (0x100000001b3ULL * FNV(name + 1))

               : 0xcbf29ce484222325ULL;

}

template <uint64_t kHash>

struct AddExport {

  template <class ExportsKey, class FuncPtr>

  AddExport(ExportsKey /*exports_key*/, const char* table_name,

            const FuncPtr* table) {

    using FuncCache = decltype(DeduceFunctionCache(hwy::DeclVal<FuncPtr>()));

    static_assert(

        hwy::IsSame<RemoveCvRef<FuncPtr>, typename FuncCache::FuncPtr>(),

        "FuncPtr should be same type as FuncCache::FuncPtr");

    const FuncPtr*& exports_ptr = AllExports::template GetRefToExportsPtr<

        RemoveCvRef<FuncPtr>, RemoveCvRef<ExportsKey>, kHash>();

    if (exports_ptr && exports_ptr != table) {

      HWY_ABORT("Hash collision for %s, rename the function\n", table_name);

    } else {

      exports_ptr = table;

    }

  }

};

// Dynamic dispatch: defines table of function pointers. This must be invoked

// from inside the function template that calls the template we are exporting.

// TABLE_NAME must match the one passed to HWY_DYNAMIC_DISPATCH_T. This

// argument allows multiple exports within one function.

#define HWY_EXPORT_T(TABLE_NAME, FUNC_NAME)                                   \

  static const struct {                                                       \

  } HWY_CONCAT(TABLE_NAME, HighwayDispatchExportsKey) = {};                   \

  static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE(  \

      TABLE_NAME)[static_cast<size_t>(HWY_MAX_DYNAMIC_TARGETS + 2)] = {       \

      /* The first entry in the table initializes the global cache and        \

       * calls the appropriate function. */                                   \

      &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME)))::  \

          template ChooseAndCall<decltype(HWY_CONCAT(                         \

                                     TABLE_NAME, HighwayDispatchExportsKey)), \

                                 hwy::FNV(#TABLE_NAME)>,                      \

      HWY_CHOOSE_TARGET_LIST(FUNC_NAME),                                      \

      HWY_CHOOSE_FALLBACK(FUNC_NAME),                                         \

  };                                                                          \

  HWY_MAYBE_UNUSED static hwy::AddExport<hwy::FNV(#TABLE_NAME)> HWY_CONCAT(   \

      HighwayAddTable, __LINE__)(                                             \

      HWY_CONCAT(TABLE_NAME, HighwayDispatchExportsKey), #TABLE_NAME,         \

      HWY_DISPATCH_TABLE(TABLE_NAME))

// For non-template functions. Not necessarily invoked within a function, hence

// we derive the string and variable names from FUNC_NAME, not HWY_FUNCTION.

#if HWY_DISPATCH_WORKAROUND

#define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME)

#else

#define HWY_EXPORT(FUNC_NAME)                                                \

  static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \

      FUNC_NAME)[static_cast<size_t>(HWY_MAX_DYNAMIC_TARGETS + 2)] = {       \

      /* The first entry in the table initializes the global cache and       \

       * calls the appropriate function. */                                  \

      &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME))):: \

          template TableChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>,        \

      HWY_CHOOSE_TARGET_LIST(FUNC_NAME),                                     \

      HWY_CHOOSE_FALLBACK(FUNC_NAME),                                        \

  }

#endif  // HWY_DISPATCH_WORKAROUND

#else  // !HWY_DISPATCH_MAP

// Zero-overhead, but requires C++17 for non-type template arguments without

// linkage, because HWY_EXPORT_T tables are local static variables.

#define HWY_EXPORT_T(TABLE_NAME, FUNC_NAME)                                  \

  static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) const HWY_DISPATCH_TABLE( \

      TABLE_NAME)[static_cast<size_t>(HWY_MAX_DYNAMIC_TARGETS + 2)] = {      \

      /* The first entry in the table initializes the global cache and       \

       * calls the appropriate function. */                                  \

      &decltype(hwy::DeduceFunctionCache(&HWY_STATIC_DISPATCH(FUNC_NAME))):: \

          template ChooseAndCall<HWY_DISPATCH_TABLE(TABLE_NAME)>,            \

      HWY_CHOOSE_TARGET_LIST(FUNC_NAME),                                     \

      HWY_CHOOSE_FALLBACK(FUNC_NAME),                                        \

  }

#define HWY_EXPORT(FUNC_NAME) HWY_EXPORT_T(FUNC_NAME, FUNC_NAME)

#endif  // HWY_DISPATCH_MAP

// HWY_DISPATCH_MAP only affects how tables are created, not their usage.

// Evaluates to the function pointer for the chosen target.

#define HWY_DYNAMIC_POINTER(FUNC_NAME) \

  (HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::GetChosenTarget().GetIndex()])

// Calls the function pointer for the chosen target.

#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) (*(HWY_DYNAMIC_POINTER(FUNC_NAME)))

// Same as DISPATCH, but provide a different arg name to clarify usage.

#define HWY_DYNAMIC_DISPATCH_T(TABLE_NAME) HWY_DYNAMIC_DISPATCH(TABLE_NAME)

#define HWY_DYNAMIC_POINTER_T(TABLE_NAME) HWY_DYNAMIC_POINTER(TABLE_NAME)

#endif  // HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0)

// Returns the name of an anonymous dispatch table that is only shared with

// macro invocations coming from the same source line.

#define HWY_DISPATCH_TABLE_T() HWY_CONCAT(HighwayDispatchTableT, __LINE__)

// For templated code, combines export and dispatch using an anonymous table.

#define HWY_EXPORT_AND_DYNAMIC_DISPATCH_T(FUNC_NAME) \

  HWY_EXPORT_T(HWY_DISPATCH_TABLE_T(), FUNC_NAME);   \

  HWY_DYNAMIC_DISPATCH_T(HWY_DISPATCH_TABLE_T())

// DEPRECATED names; please use HWY_HAVE_* instead.

#define HWY_CAP_INTEGER64 HWY_HAVE_INTEGER64

#define HWY_CAP_FLOAT16 HWY_HAVE_FLOAT16

#define HWY_CAP_FLOAT64 HWY_HAVE_FLOAT64

}  // namespace hwy

#endif  // HWY_HIGHWAY_INCLUDED

//------------------------------------------------------------------------------

// NOTE: the following definitions and ops/*.h depend on HWY_TARGET, so we want

// to include them once per target, which is ensured by the toggle check.

// Because ops/*.h are included under it, they do not need their own guard.

#if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE)

#ifdef HWY_HIGHWAY_PER_TARGET

#undef HWY_HIGHWAY_PER_TARGET

#else

#define HWY_HIGHWAY_PER_TARGET

#endif

// These define ops inside namespace hwy::HWY_NAMESPACE.

#if HWY_TARGET == HWY_SSE2 || HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4

#include "hwy/ops/x86_128-inl.h"

#elif HWY_TARGET == HWY_AVX2

#include "hwy/ops/x86_256-inl.h"

#elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL || \

    HWY_TARGET == HWY_AVX3_ZEN4 || HWY_TARGET == HWY_AVX3_SPR

#include "hwy/ops/x86_512-inl.h"

#elif HWY_TARGET == HWY_Z14 || HWY_TARGET == HWY_Z15 || \

    (HWY_TARGET & HWY_ALL_PPC)

#include "hwy/ops/ppc_vsx-inl.h"

#elif HWY_TARGET & HWY_ALL_NEON

#include "hwy/ops/arm_neon-inl.h"

#elif HWY_TARGET & HWY_ALL_SVE

#include "hwy/ops/arm_sve-inl.h"

#elif HWY_TARGET == HWY_WASM_EMU256

#include "hwy/ops/wasm_256-inl.h"

#elif HWY_TARGET == HWY_WASM

#include "hwy/ops/wasm_128-inl.h"

#elif HWY_TARGET == HWY_RVV

#include "hwy/ops/rvv-inl.h"

#elif HWY_TARGET == HWY_EMU128

#include "hwy/ops/emu128-inl.h"

#elif HWY_TARGET == HWY_SCALAR

#include "hwy/ops/scalar-inl.h"

#else

#pragma message("HWY_TARGET does not match any known target")

#endif  // HWY_TARGET

#include "hwy/ops/generic_ops-inl.h"

#endif  // HWY_HIGHWAY_PER_TARGET