/src/zlib-ng/functable.c

Source
/* functable.c -- Choose relevant optimized functions at runtime
 * Copyright (C) 2017 Hans Kristian Rosbach
 * For conditions of distribution and use, see copyright notice in zlib.h
 */
#ifndef DISABLE_RUNTIME_CPU_DETECTION

#include "zbuild.h"

#if defined(_MSC_VER)
#  include <intrin.h>
#endif

#include "functable.h"
#include "cpu_features.h"
#include "arch_functions.h"

/* Platform has pointer size atomic store */
#if defined(__GNUC__) || defined(__clang__)
#  define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \
    __atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST)
#  define FUNCTABLE_BARRIER() __atomic_thread_fence(__ATOMIC_SEQ_CST)
#elif defined(_MSC_VER)
#  define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \
    _InterlockedExchangePointer((void * volatile *)&(functable.FUNC_NAME), (void *)(VAR.FUNC_NAME))
#  ifdef ARCH_ARM
#    define FUNCTABLE_BARRIER() do { \
    _ReadWriteBarrier();  \
    __dmb(0xB); /* _ARM_BARRIER_ISH */ \
    _ReadWriteBarrier(); \
} while (0)
#  else
#    define FUNCTABLE_BARRIER() _ReadWriteBarrier()
#  endif
#else
#  warning Unable to detect atomic intrinsic support.
#  define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \
    *((void * volatile *)&(functable.FUNC_NAME)) = (void *)(VAR.FUNC_NAME)
#  define FUNCTABLE_BARRIER() do { /* Empty */ } while (0)
#endif

/* Verify all pointers are valid before assigning, return 1 on failure
 * This allows inflateinit/deflateinit functions to gracefully return Z_VERSION_ERROR
 * if functable initialization fails.
 */
#define FUNCTABLE_VERIFY_ASSIGN(VAR, FUNC_NAME) \
    if (!VAR.FUNC_NAME) { \
        fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \
        return 1; \
    } \
    FUNCTABLE_ASSIGN(VAR, FUNC_NAME);

/* Functable init & abort on failure.
 * Abort is needed because some stub functions are reachable without first
 * calling any inflateinit/deflateinit functions, and have no error propagation.
 */
#define FUNCTABLE_INIT_ABORT \
    if (init_functable()) { \
        fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \
        abort(); \
    };

// Empty stub, used when functable has already been initialized
static int force_init_empty(void) {
    return 0;
}

/* Functable initialization.
 * Selects the best available optimized functions appropriate for the runtime cpu.
 */
static int init_functable(void) {
    struct functable_s ft;
    struct cpu_features cf;

    memset(&ft, 0, sizeof(struct functable_s));
    cpu_check_features(&cf);
    ft.force_init = &force_init_empty;

    // Only use necessary generic functions when no suitable simd versions are available.
#ifdef ADLER32_FALLBACK
    ft.adler32 = &adler32_c;
    ft.adler32_copy = &adler32_copy_c;
#endif
#ifdef CHUNKSET_FALLBACK
    ft.chunkmemset_safe = &chunkmemset_safe_c;
    ft.inflate_fast = &inflate_fast_c;
#endif
#ifdef COMPARE256_FALLBACK
    ft.compare256 = &compare256_c;
    ft.longest_match = &longest_match_c;
    ft.longest_match_roll = &longest_match_roll_c;
#endif
#ifdef CRC32_BRAID_FALLBACK
    ft.crc32 = &crc32_braid;
    ft.crc32_copy = &crc32_copy_braid;
#endif
#ifdef SLIDE_HASH_FALLBACK
    ft.slide_hash = &slide_hash_c;
    ft.slide_hash_head = &slide_hash_head_c;
#endif

    // Select arch-optimized functions
#ifdef WITH_OPTIM

    // Chorba generic C fallback
#ifdef CRC32_CHORBA_FALLBACK
    ft.crc32 = &crc32_chorba;
    ft.crc32_copy = &crc32_copy_chorba;
#endif

    // X86 - SSE2
#ifdef X86_SSE2
#  ifndef X86_SSE2_NATIVE
    if (cf.x86.has_sse2)
#  endif
    {
#  ifndef X86_AVX2_NATIVE
        ft.chunkmemset_safe = &chunkmemset_safe_sse2;
        ft.compare256 = &compare256_sse2;
        ft.inflate_fast = &inflate_fast_sse2;
        ft.longest_match = &longest_match_sse2;
        ft.longest_match_roll = &longest_match_roll_sse2;
        ft.slide_hash = &slide_hash_sse2;
        ft.slide_hash_head = &slide_hash_head_sse2;
#  endif
#  if defined(CRC32_CHORBA_SSE_FALLBACK) && !defined(X86_SSE41_NATIVE) && !defined(X86_PCLMULQDQ_NATIVE)
        ft.crc32 = &crc32_chorba_sse2;
        ft.crc32_copy = &crc32_copy_chorba_sse2;
#  endif
    }
#endif
    // X86 - SSSE3
#ifdef X86_SSSE3
#  ifndef X86_SSSE3_NATIVE
    if (cf.x86.has_ssse3)
#  endif
    {
        ft.adler32 = &adler32_ssse3;
        ft.adler32_copy = &adler32_copy_ssse3;
#  ifndef X86_AVX2_NATIVE
        ft.chunkmemset_safe = &chunkmemset_safe_ssse3;
        ft.inflate_fast = &inflate_fast_ssse3;
#  endif
    }
#endif

    // X86 - SSE4.1
#if defined(X86_SSE41) && !defined(X86_PCLMULQDQ_NATIVE)
#  ifndef X86_SSE41_NATIVE
    if (cf.x86.has_sse41)
#  endif
    {
#  ifdef CRC32_CHORBA_SSE_FALLBACK
        ft.crc32 = &crc32_chorba_sse41;
        ft.crc32_copy = &crc32_copy_chorba_sse41;
#  endif
    }
#endif

    // X86 - SSE4.2
#if defined(X86_SSE42) && !defined(X86_AVX512_NATIVE)
#  ifndef X86_SSE42_NATIVE
    if (cf.x86.has_sse42)
#  endif
    {
        ft.adler32_copy = &adler32_copy_sse42;
    }
#endif
    // X86 - PCLMUL
#if defined(X86_PCLMULQDQ_CRC) && !defined(X86_VPCLMULQDQ_NATIVE)
#  ifndef X86_PCLMULQDQ_NATIVE
    if (cf.x86.has_pclmulqdq)
#  endif
    {
        ft.crc32 = &crc32_pclmulqdq;
        ft.crc32_copy = &crc32_copy_pclmulqdq;
    }
#endif
    // X86 - AVX2
#ifdef X86_AVX2
    /* BMI2 support is all but implicit with AVX2 but let's sanity check this just in case. Enabling BMI2 allows for
     * flagless shifts, resulting in fewer flag stalls for the pipeline, and allows us to set destination registers
     * for the shift results as an operand, eliminating several register-register moves when the original value needs
     * to remain intact. They also allow for a count operand that isn't the CL register, avoiding contention there */
#  ifndef X86_AVX2_NATIVE
    if (cf.x86.has_avx2 && cf.x86.has_bmi2)
#  endif
    {
#  ifndef X86_AVX512_NATIVE
        ft.adler32 = &adler32_avx2;
        ft.adler32_copy = &adler32_copy_avx2;
        ft.chunkmemset_safe = &chunkmemset_safe_avx2;
        ft.compare256 = &compare256_avx2;
        ft.inflate_fast = &inflate_fast_avx2;
        ft.longest_match = &longest_match_avx2;
        ft.longest_match_roll = &longest_match_roll_avx2;
#  endif
        ft.slide_hash = &slide_hash_avx2;
        ft.slide_hash_head = &slide_hash_head_avx2;
    }
#endif
#ifdef X86_AVX2VNNI
    if (cf.x86.has_avx2vnni) {
        ft.adler32 = &adler32_avx2_vnni;
        //ft.adler32_copy = &adler32_copy_avx2_vnni;
    }
#endif
    // X86 - AVX512 (F,DQ,BW,Vl)
#ifdef X86_AVX512
#  ifndef X86_AVX512_NATIVE
    if (cf.x86.has_avx512_common)
#  endif
    {
#  ifndef X86_AVX512VNNI_NATIVE
        ft.adler32 = &adler32_avx512;
        ft.adler32_copy = &adler32_copy_avx512;
#  endif
        ft.chunkmemset_safe = &chunkmemset_safe_avx512;
        ft.compare256 = &compare256_avx512;
        ft.inflate_fast = &inflate_fast_avx512;
        ft.longest_match = &longest_match_avx512;
        ft.longest_match_roll = &longest_match_roll_avx512;
    }
#endif
#ifdef X86_AVX512VNNI
#  ifndef X86_AVX512VNNI_NATIVE
    if (cf.x86.has_avx512vnni)
#  endif
    {
        ft.adler32 = &adler32_avx512_vnni;
        ft.adler32_copy = &adler32_copy_avx512_vnni;
    }
#endif
    // X86 - VPCLMULQDQ (AVX2)
#ifdef X86_VPCLMULQDQ_AVX2
#  ifndef X86_VPCLMULQDQ_AVX2_NATIVE
    if (cf.x86.has_pclmulqdq && cf.x86.has_avx2 && cf.x86.has_vpclmulqdq)
#  endif
    {
        ft.crc32 = &crc32_vpclmulqdq_avx2;
        ft.crc32_copy = &crc32_copy_vpclmulqdq_avx2;
    }
#endif
    // X86 - VPCLMULQDQ (AVX-512)
#ifdef X86_VPCLMULQDQ_AVX512
#  ifndef X86_VPCLMULQDQ_AVX512_NATIVE
    if (cf.x86.has_pclmulqdq && cf.x86.has_avx512_common && cf.x86.has_vpclmulqdq)
#  endif
    {
        ft.crc32 = &crc32_vpclmulqdq_avx512;
        ft.crc32_copy = &crc32_copy_vpclmulqdq_avx512;
    }
#endif


    // ARM - SIMD
#if defined(ARM_SIMD) && !defined(ARM_NEON_NATIVE)
#  ifndef ARM_SIMD_NATIVE
    if (cf.arm.has_simd)
#  endif
    {
        ft.slide_hash = &slide_hash_armv6;
        ft.slide_hash_head = &slide_hash_head_armv6;
    }
#endif
    // ARM - NEON
#ifdef ARM_NEON
#  ifndef ARM_NEON_NATIVE
    if (cf.arm.has_neon)
#  endif
    {
        ft.adler32 = &adler32_neon;
        ft.adler32_copy = &adler32_copy_neon;
        ft.chunkmemset_safe = &chunkmemset_safe_neon;
        ft.compare256 = &compare256_neon;
        ft.inflate_fast = &inflate_fast_neon;
        ft.longest_match = &longest_match_neon;
        ft.longest_match_roll = &longest_match_roll_neon;
        ft.slide_hash = &slide_hash_neon;
        ft.slide_hash_head = &slide_hash_head_neon;
    }
#endif
    // ARM - NEON DotProd
#ifdef ARM_NEON_DOTPROD
#  ifndef ARM_NEON_DOTPROD_NATIVE
    if (cf.arm.has_neon && cf.arm.has_dotprod)
#  endif
    {
        ft.adler32 = &adler32_neon_dotprod;
        ft.adler32_copy = &adler32_copy_neon_dotprod;
    }
#endif
    // ARM - CRC32
#if defined(ARM_CRC32) && !defined(ARM_PMULL_EOR3_NATIVE)
#  ifndef ARM_CRC32_NATIVE
    if (cf.arm.has_crc32)
#  endif
    {
        ft.crc32 = &crc32_armv8;
        ft.crc32_copy = &crc32_copy_armv8;
    }
#endif
    // ARM - PMULL EOR3
#ifdef ARM_PMULL_EOR3
#  ifndef ARM_PMULL_EOR3_NATIVE
    if (cf.arm.has_crc32 && cf.arm.has_pmull && cf.arm.has_eor3 && cf.arm.has_fast_pmull)
#  endif
    {
        ft.crc32 = &crc32_armv8_pmull_eor3;
        ft.crc32_copy = &crc32_copy_armv8_pmull_eor3;
    }
#endif

    // Power - VMX
#ifdef PPC_VMX
#  ifndef PPC_VMX_NATIVE
    if (cf.power.has_altivec)
#  endif
    {
        ft.adler32 = &adler32_vmx;
        ft.adler32_copy = &adler32_copy_vmx;
        ft.slide_hash = &slide_hash_vmx;
        ft.slide_hash_head = &slide_hash_head_vmx;
    }
#endif
    // Power8 - VSX
#ifdef POWER8_VSX
#  ifndef POWER8_VSX_NATIVE
    if (cf.power.has_arch_2_07)
#  endif
    {
        ft.adler32 = &adler32_power8;
        ft.adler32_copy = &adler32_copy_power8;
        ft.chunkmemset_safe = &chunkmemset_safe_power8;
        ft.inflate_fast = &inflate_fast_power8;
        ft.slide_hash = &slide_hash_power8;
        ft.slide_hash_head = &slide_hash_head_power8;
    }
#endif
#ifdef POWER8_VSX_CRC32
#  ifndef POWER8_VSX_CRC32_NATIVE
    if (cf.power.has_arch_2_07)
#  endif
    {
        ft.crc32 = &crc32_power8;
        ft.crc32_copy = &crc32_copy_power8;
    }
#endif
    // Power9
#ifdef POWER9
#  ifndef POWER9_NATIVE
    if (cf.power.has_arch_3_00)
#  endif
    {
        ft.compare256 = &compare256_power9;
        ft.longest_match = &longest_match_power9;
        ft.longest_match_roll = &longest_match_roll_power9;
    }
#endif


    // RISCV - RVV
#ifdef RISCV_RVV
#  ifndef RISCV_RVV_NATIVE
    if (cf.riscv.has_rvv)
#  endif
    {
        ft.adler32 = &adler32_rvv;
        ft.adler32_copy = &adler32_copy_rvv;
        ft.chunkmemset_safe = &chunkmemset_safe_rvv;
        ft.compare256 = &compare256_rvv;
        ft.inflate_fast = &inflate_fast_rvv;
        ft.longest_match = &longest_match_rvv;
        ft.longest_match_roll = &longest_match_roll_rvv;
        ft.slide_hash = &slide_hash_rvv;
        ft.slide_hash_head = &slide_hash_head_rvv;
    }
#endif

    // RISCV - ZBC
#ifdef RISCV_CRC32_ZBC
#  ifndef RISCV_ZBC_NATIVE
    if (cf.riscv.has_zbc)
#  endif
    {
        ft.crc32 = &crc32_riscv64_zbc;
        ft.crc32_copy = &crc32_copy_riscv64_zbc;
    }
#endif

    // S390
#ifdef S390_VX
#  ifndef S390_VX_NATIVE
    if (cf.s390.has_vx)
#  endif
    {
        ft.crc32 = &crc32_s390_vx;
        ft.crc32_copy = &crc32_copy_s390_vx;
        ft.slide_hash = &slide_hash_vx;
        ft.slide_hash_head = &slide_hash_head_vx;
    }
#endif

    // LOONGARCH
#ifdef LOONGARCH_CRC
#  ifndef LOONGARCH_CRC_NATIVE
    if (cf.loongarch.has_crc)
#  endif
    {
        ft.crc32 = &crc32_loongarch64;
        ft.crc32_copy = &crc32_copy_loongarch64;
    }
#endif
#if defined(LOONGARCH_LSX) && !defined(LOONGARCH_LASX_NATIVE)
#  ifndef LOONGARCH_LSX_NATIVE
    if (cf.loongarch.has_lsx)
#  endif
    {
        ft.adler32 = &adler32_lsx;
        ft.adler32_copy = &adler32_copy_lsx;
        ft.chunkmemset_safe = &chunkmemset_safe_lsx;
        ft.compare256 = &compare256_lsx;
        ft.inflate_fast = &inflate_fast_lsx;
        ft.longest_match = &longest_match_lsx;
        ft.longest_match_roll = &longest_match_roll_lsx;
        ft.slide_hash = &slide_hash_lsx;
        ft.slide_hash_head = &slide_hash_head_lsx;
    }
#endif
#ifdef LOONGARCH_LASX
#  ifndef LOONGARCH_LASX_NATIVE
    if (cf.loongarch.has_lasx)
#  endif
    {
        ft.adler32 = &adler32_lasx;
        ft.adler32_copy = &adler32_copy_lasx;
        ft.chunkmemset_safe = &chunkmemset_safe_lasx;
        ft.compare256 = &compare256_lasx;
        ft.inflate_fast = &inflate_fast_lasx;
        ft.longest_match = &longest_match_lasx;
        ft.longest_match_roll = &longest_match_roll_lasx;
        ft.slide_hash = &slide_hash_lasx;
        ft.slide_hash_head = &slide_hash_head_lasx;
    }
#endif

#endif // WITH_OPTIM

    // Assign function pointers individually for atomic operation
    FUNCTABLE_ASSIGN(ft, force_init);
    FUNCTABLE_VERIFY_ASSIGN(ft, adler32);
    FUNCTABLE_VERIFY_ASSIGN(ft, adler32_copy);
    FUNCTABLE_VERIFY_ASSIGN(ft, chunkmemset_safe);
    FUNCTABLE_VERIFY_ASSIGN(ft, compare256);
    FUNCTABLE_VERIFY_ASSIGN(ft, crc32);
    FUNCTABLE_VERIFY_ASSIGN(ft, crc32_copy);
    FUNCTABLE_VERIFY_ASSIGN(ft, inflate_fast);
    FUNCTABLE_VERIFY_ASSIGN(ft, longest_match);
    FUNCTABLE_VERIFY_ASSIGN(ft, longest_match_roll);
    FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash);
    FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash_head);

    // Memory barrier for weak memory order CPUs
    FUNCTABLE_BARRIER();

    return Z_OK;
}

#if !defined(_WIN32) && (defined(__GNUC__) || defined(__clang__))
static void __attribute__((constructor)) functable_constructor(void) {
    FUNCTABLE_INIT_ABORT;
}
#endif

/* stub functions */
static int force_init_stub(void) {
    return init_functable();
}

static uint32_t adler32_stub(uint32_t adler, const uint8_t* buf, size_t len) {
    FUNCTABLE_INIT_ABORT;
    return functable.adler32(adler, buf, len);
}

static uint32_t adler32_copy_stub(uint32_t adler, uint8_t* dst, const uint8_t* src, size_t len) {
    FUNCTABLE_INIT_ABORT;
    return functable.adler32_copy(adler, dst, src, len);
}

static uint8_t* chunkmemset_safe_stub(uint8_t* out, uint8_t *from, size_t len, size_t left) {
    FUNCTABLE_INIT_ABORT;
    return functable.chunkmemset_safe(out, from, len, left);
}

static uint32_t compare256_stub(const uint8_t* src0, const uint8_t* src1) {
    FUNCTABLE_INIT_ABORT;
    return functable.compare256(src0, src1);
}

static uint32_t crc32_stub(uint32_t crc, const uint8_t* buf, size_t len) {
    FUNCTABLE_INIT_ABORT;
    return functable.crc32(crc, buf, len);
}

static uint32_t crc32_copy_stub(uint32_t crc, uint8_t *dst, const uint8_t *src, size_t len) {
    FUNCTABLE_INIT_ABORT;
    return functable.crc32_copy(crc, dst, src, len);
}

static void inflate_fast_stub(PREFIX3(stream) *strm, uint32_t start, int safe_mode) {
    FUNCTABLE_INIT_ABORT;
    functable.inflate_fast(strm, start, safe_mode);
}

static uint32_t longest_match_stub(deflate_state* const s, uint32_t cur_match) {
    FUNCTABLE_INIT_ABORT;
    return functable.longest_match(s, cur_match);
}

static uint32_t longest_match_roll_stub(deflate_state* const s, uint32_t cur_match) {
    FUNCTABLE_INIT_ABORT;
    return functable.longest_match_roll(s, cur_match);
}

static void slide_hash_stub(deflate_state* s) {
    FUNCTABLE_INIT_ABORT;
    functable.slide_hash(s);
}

static void slide_hash_head_stub(deflate_state *s) {
    FUNCTABLE_INIT_ABORT;
    functable.slide_hash_head(s);
}

/* functable init */
Z_INTERNAL struct functable_s functable = {
    force_init_stub,
    adler32_stub,
    adler32_copy_stub,
    chunkmemset_safe_stub,
    compare256_stub,
    crc32_stub,
    crc32_copy_stub,
    inflate_fast_stub,
    longest_match_stub,
    longest_match_roll_stub,
    slide_hash_stub,
    slide_hash_head_stub,
};

#endif

Coverage Report

Created: 2026-07-25 06:58

Line	Count	Source
1		/* functable.c -- Choose relevant optimized functions at runtime
2		* Copyright (C) 2017 Hans Kristian Rosbach
3		* For conditions of distribution and use, see copyright notice in zlib.h
4		*/
5		#ifndef DISABLE_RUNTIME_CPU_DETECTION
6
7		#include "zbuild.h"
8
9		#if defined(_MSC_VER)
10		# include <intrin.h>
11		#endif
12
13		#include "functable.h"
14		#include "cpu_features.h"
15		#include "arch_functions.h"
16
17		/* Platform has pointer size atomic store */
18		#if defined(__GNUC__) \|\| defined(__clang__)
19		# define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \
20	24	__atomic_store(&(functable.FUNC_NAME), &(VAR.FUNC_NAME), __ATOMIC_SEQ_CST)
21	2	# define FUNCTABLE_BARRIER() __atomic_thread_fence(__ATOMIC_SEQ_CST)
22		#elif defined(_MSC_VER)
23		# define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \
24		_InterlockedExchangePointer((void * volatile )&(functable.FUNC_NAME), (void )(VAR.FUNC_NAME))
25		# ifdef ARCH_ARM
26		# define FUNCTABLE_BARRIER() do { \
27		_ReadWriteBarrier(); \
28		__dmb(0xB); /* _ARM_BARRIER_ISH */ \
29		_ReadWriteBarrier(); \
30		} while (0)
31		# else
32		# define FUNCTABLE_BARRIER() _ReadWriteBarrier()
33		# endif
34		#else
35		# warning Unable to detect atomic intrinsic support.
36		# define FUNCTABLE_ASSIGN(VAR, FUNC_NAME) \
37		((void volatile )&(functable.FUNC_NAME)) = (void )(VAR.FUNC_NAME)
38		# define FUNCTABLE_BARRIER() do { /* Empty */ } while (0)
39		#endif
40
41		/* Verify all pointers are valid before assigning, return 1 on failure
42		* This allows inflateinit/deflateinit functions to gracefully return Z_VERSION_ERROR
43		* if functable initialization fails.
44		*/
45		#define FUNCTABLE_VERIFY_ASSIGN(VAR, FUNC_NAME) \
46	22	if (!VAR.FUNC_NAME) { \
47	0	fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \
48	0	return 1; \
49	0	} \
50	22	FUNCTABLE_ASSIGN(VAR, FUNC_NAME);
51
52		/* Functable init & abort on failure.
53		* Abort is needed because some stub functions are reachable without first
54		* calling any inflateinit/deflateinit functions, and have no error propagation.
55		*/
56		#define FUNCTABLE_INIT_ABORT \
57	2	if (init_functable()) { \
58	0	fprintf(stderr, "Zlib-ng functable failed initialization!\n"); \
59	0	abort(); \
60	2	};
61
62		// Empty stub, used when functable has already been initialized
63	7.19k	static int force_init_empty(void) {
64	7.19k	return 0;
65	7.19k	}
66
67		/* Functable initialization.
68		* Selects the best available optimized functions appropriate for the runtime cpu.
69		*/
70	2	static int init_functable(void) {
71	2	struct functable_s ft;
72	2	struct cpu_features cf;
73
74	2	memset(&ft, 0, sizeof(struct functable_s));
75	2	cpu_check_features(&cf);
76	2	ft.force_init = &force_init_empty;
77
78		// Only use necessary generic functions when no suitable simd versions are available.
79	2	#ifdef ADLER32_FALLBACK
80	2	ft.adler32 = &adler32_c;
81	2	ft.adler32_copy = &adler32_copy_c;
82	2	#endif
83		#ifdef CHUNKSET_FALLBACK
84		ft.chunkmemset_safe = &chunkmemset_safe_c;
85		ft.inflate_fast = &inflate_fast_c;
86		#endif
87		#ifdef COMPARE256_FALLBACK
88		ft.compare256 = &compare256_c;
89		ft.longest_match = &longest_match_c;
90		ft.longest_match_roll = &longest_match_roll_c;
91		#endif
92	2	#ifdef CRC32_BRAID_FALLBACK
93	2	ft.crc32 = &crc32_braid;
94	2	ft.crc32_copy = &crc32_copy_braid;
95	2	#endif
96		#ifdef SLIDE_HASH_FALLBACK
97		ft.slide_hash = &slide_hash_c;
98		ft.slide_hash_head = &slide_hash_head_c;
99		#endif
100
101		// Select arch-optimized functions
102	2	#ifdef WITH_OPTIM
103
104		// Chorba generic C fallback
105	2	#ifdef CRC32_CHORBA_FALLBACK
106	2	ft.crc32 = &crc32_chorba;
107	2	ft.crc32_copy = &crc32_copy_chorba;
108	2	#endif
109
110		// X86 - SSE2
111	2	#ifdef X86_SSE2
112		# ifndef X86_SSE2_NATIVE
113		if (cf.x86.has_sse2)
114		# endif
115	2	{
116	2	# ifndef X86_AVX2_NATIVE
117	2	ft.chunkmemset_safe = &chunkmemset_safe_sse2;
118	2	ft.compare256 = &compare256_sse2;
119	2	ft.inflate_fast = &inflate_fast_sse2;
120	2	ft.longest_match = &longest_match_sse2;
121	2	ft.longest_match_roll = &longest_match_roll_sse2;
122	2	ft.slide_hash = &slide_hash_sse2;
123	2	ft.slide_hash_head = &slide_hash_head_sse2;
124	2	# endif
125	2	# if defined(CRC32_CHORBA_SSE_FALLBACK) && !defined(X86_SSE41_NATIVE) && !defined(X86_PCLMULQDQ_NATIVE)
126	2	ft.crc32 = &crc32_chorba_sse2;
127	2	ft.crc32_copy = &crc32_copy_chorba_sse2;
128	2	# endif
129	2	}
130	2	#endif
131		// X86 - SSSE3
132	2	#ifdef X86_SSSE3
133	2	# ifndef X86_SSSE3_NATIVE
134	2	if (cf.x86.has_ssse3)
135	2	# endif
136	2	{
137	2	ft.adler32 = &adler32_ssse3;
138	2	ft.adler32_copy = &adler32_copy_ssse3;
139	2	# ifndef X86_AVX2_NATIVE
140	2	ft.chunkmemset_safe = &chunkmemset_safe_ssse3;
141	2	ft.inflate_fast = &inflate_fast_ssse3;
142	2	# endif
143	2	}
144	2	#endif
145
146		// X86 - SSE4.1
147	2	#if defined(X86_SSE41) && !defined(X86_PCLMULQDQ_NATIVE)
148	2	# ifndef X86_SSE41_NATIVE
149	2	if (cf.x86.has_sse41)
150	2	# endif
151	2	{
152	2	# ifdef CRC32_CHORBA_SSE_FALLBACK
153	2	ft.crc32 = &crc32_chorba_sse41;
154	2	ft.crc32_copy = &crc32_copy_chorba_sse41;
155	2	# endif
156	2	}
157	2	#endif
158
159		// X86 - SSE4.2
160	2	#if defined(X86_SSE42) && !defined(X86_AVX512_NATIVE)
161	2	# ifndef X86_SSE42_NATIVE
162	2	if (cf.x86.has_sse42)
163	2	# endif
164	2	{
165	2	ft.adler32_copy = &adler32_copy_sse42;
166	2	}
167	2	#endif
168		// X86 - PCLMUL
169	2	#if defined(X86_PCLMULQDQ_CRC) && !defined(X86_VPCLMULQDQ_NATIVE)
170	2	# ifndef X86_PCLMULQDQ_NATIVE
171	2	if (cf.x86.has_pclmulqdq)
172	2	# endif
173	2	{
174	2	ft.crc32 = &crc32_pclmulqdq;
175	2	ft.crc32_copy = &crc32_copy_pclmulqdq;
176	2	}
177	2	#endif
178		// X86 - AVX2
179	2	#ifdef X86_AVX2
180		/* BMI2 support is all but implicit with AVX2 but let's sanity check this just in case. Enabling BMI2 allows for
181		* flagless shifts, resulting in fewer flag stalls for the pipeline, and allows us to set destination registers
182		* for the shift results as an operand, eliminating several register-register moves when the original value needs
183		* to remain intact. They also allow for a count operand that isn't the CL register, avoiding contention there */
184	2	# ifndef X86_AVX2_NATIVE
185	2	if (cf.x86.has_avx2 && cf.x86.has_bmi2)
186	2	# endif
187	2	{
188	2	# ifndef X86_AVX512_NATIVE
189	2	ft.adler32 = &adler32_avx2;
190	2	ft.adler32_copy = &adler32_copy_avx2;
191	2	ft.chunkmemset_safe = &chunkmemset_safe_avx2;
192	2	ft.compare256 = &compare256_avx2;
193	2	ft.inflate_fast = &inflate_fast_avx2;
194	2	ft.longest_match = &longest_match_avx2;
195	2	ft.longest_match_roll = &longest_match_roll_avx2;
196	2	# endif
197	2	ft.slide_hash = &slide_hash_avx2;
198	2	ft.slide_hash_head = &slide_hash_head_avx2;
199	2	}
200	2	#endif
201	2	#ifdef X86_AVX2VNNI
202	2	if (cf.x86.has_avx2vnni) {
203	0	ft.adler32 = &adler32_avx2_vnni;
204		//ft.adler32_copy = &adler32_copy_avx2_vnni;
205	0	}
206	2	#endif
207		// X86 - AVX512 (F,DQ,BW,Vl)
208	2	#ifdef X86_AVX512
209	2	# ifndef X86_AVX512_NATIVE
210	2	if (cf.x86.has_avx512_common)
211	0	# endif
212	0	{
213	0	# ifndef X86_AVX512VNNI_NATIVE
214	0	ft.adler32 = &adler32_avx512;
215	0	ft.adler32_copy = &adler32_copy_avx512;
216	0	# endif
217	0	ft.chunkmemset_safe = &chunkmemset_safe_avx512;
218	0	ft.compare256 = &compare256_avx512;
219	0	ft.inflate_fast = &inflate_fast_avx512;
220	0	ft.longest_match = &longest_match_avx512;
221	0	ft.longest_match_roll = &longest_match_roll_avx512;
222	0	}
223	2	#endif
224	2	#ifdef X86_AVX512VNNI
225	2	# ifndef X86_AVX512VNNI_NATIVE
226	2	if (cf.x86.has_avx512vnni)
227	0	# endif
228	0	{
229	0	ft.adler32 = &adler32_avx512_vnni;
230	0	ft.adler32_copy = &adler32_copy_avx512_vnni;
231	0	}
232	2	#endif
233		// X86 - VPCLMULQDQ (AVX2)
234	2	#ifdef X86_VPCLMULQDQ_AVX2
235	2	# ifndef X86_VPCLMULQDQ_AVX2_NATIVE
236	2	if (cf.x86.has_pclmulqdq && cf.x86.has_avx2 && cf.x86.has_vpclmulqdq)
237	0	# endif
238	0	{
239	0	ft.crc32 = &crc32_vpclmulqdq_avx2;
240	0	ft.crc32_copy = &crc32_copy_vpclmulqdq_avx2;
241	0	}
242	2	#endif
243		// X86 - VPCLMULQDQ (AVX-512)
244	2	#ifdef X86_VPCLMULQDQ_AVX512
245	2	# ifndef X86_VPCLMULQDQ_AVX512_NATIVE
246	2	if (cf.x86.has_pclmulqdq && cf.x86.has_avx512_common && cf.x86.has_vpclmulqdq)
247	0	# endif
248	0	{
249	0	ft.crc32 = &crc32_vpclmulqdq_avx512;
250	0	ft.crc32_copy = &crc32_copy_vpclmulqdq_avx512;
251	0	}
252	2	#endif
253
254
255		// ARM - SIMD
256		#if defined(ARM_SIMD) && !defined(ARM_NEON_NATIVE)
257		# ifndef ARM_SIMD_NATIVE
258		if (cf.arm.has_simd)
259		# endif
260		{
261		ft.slide_hash = &slide_hash_armv6;
262		ft.slide_hash_head = &slide_hash_head_armv6;
263		}
264		#endif
265		// ARM - NEON
266		#ifdef ARM_NEON
267		# ifndef ARM_NEON_NATIVE
268		if (cf.arm.has_neon)
269		# endif
270		{
271		ft.adler32 = &adler32_neon;
272		ft.adler32_copy = &adler32_copy_neon;
273		ft.chunkmemset_safe = &chunkmemset_safe_neon;
274		ft.compare256 = &compare256_neon;
275		ft.inflate_fast = &inflate_fast_neon;
276		ft.longest_match = &longest_match_neon;
277		ft.longest_match_roll = &longest_match_roll_neon;
278		ft.slide_hash = &slide_hash_neon;
279		ft.slide_hash_head = &slide_hash_head_neon;
280		}
281		#endif
282		// ARM - NEON DotProd
283		#ifdef ARM_NEON_DOTPROD
284		# ifndef ARM_NEON_DOTPROD_NATIVE
285		if (cf.arm.has_neon && cf.arm.has_dotprod)
286		# endif
287		{
288		ft.adler32 = &adler32_neon_dotprod;
289		ft.adler32_copy = &adler32_copy_neon_dotprod;
290		}
291		#endif
292		// ARM - CRC32
293		#if defined(ARM_CRC32) && !defined(ARM_PMULL_EOR3_NATIVE)
294		# ifndef ARM_CRC32_NATIVE
295		if (cf.arm.has_crc32)
296		# endif
297		{
298		ft.crc32 = &crc32_armv8;
299		ft.crc32_copy = &crc32_copy_armv8;
300		}
301		#endif
302		// ARM - PMULL EOR3
303		#ifdef ARM_PMULL_EOR3
304		# ifndef ARM_PMULL_EOR3_NATIVE
305		if (cf.arm.has_crc32 && cf.arm.has_pmull && cf.arm.has_eor3 && cf.arm.has_fast_pmull)
306		# endif
307		{
308		ft.crc32 = &crc32_armv8_pmull_eor3;
309		ft.crc32_copy = &crc32_copy_armv8_pmull_eor3;
310		}
311		#endif
312
313		// Power - VMX
314		#ifdef PPC_VMX
315		# ifndef PPC_VMX_NATIVE
316		if (cf.power.has_altivec)
317		# endif
318		{
319		ft.adler32 = &adler32_vmx;
320		ft.adler32_copy = &adler32_copy_vmx;
321		ft.slide_hash = &slide_hash_vmx;
322		ft.slide_hash_head = &slide_hash_head_vmx;
323		}
324		#endif
325		// Power8 - VSX
326		#ifdef POWER8_VSX
327		# ifndef POWER8_VSX_NATIVE
328		if (cf.power.has_arch_2_07)
329		# endif
330		{
331		ft.adler32 = &adler32_power8;
332		ft.adler32_copy = &adler32_copy_power8;
333		ft.chunkmemset_safe = &chunkmemset_safe_power8;
334		ft.inflate_fast = &inflate_fast_power8;
335		ft.slide_hash = &slide_hash_power8;
336		ft.slide_hash_head = &slide_hash_head_power8;
337		}
338		#endif
339		#ifdef POWER8_VSX_CRC32
340		# ifndef POWER8_VSX_CRC32_NATIVE
341		if (cf.power.has_arch_2_07)
342		# endif
343		{
344		ft.crc32 = &crc32_power8;
345		ft.crc32_copy = &crc32_copy_power8;
346		}
347		#endif
348		// Power9
349		#ifdef POWER9
350		# ifndef POWER9_NATIVE
351		if (cf.power.has_arch_3_00)
352		# endif
353		{
354		ft.compare256 = &compare256_power9;
355		ft.longest_match = &longest_match_power9;
356		ft.longest_match_roll = &longest_match_roll_power9;
357		}
358		#endif
359
360
361		// RISCV - RVV
362		#ifdef RISCV_RVV
363		# ifndef RISCV_RVV_NATIVE
364		if (cf.riscv.has_rvv)
365		# endif
366		{
367		ft.adler32 = &adler32_rvv;
368		ft.adler32_copy = &adler32_copy_rvv;
369		ft.chunkmemset_safe = &chunkmemset_safe_rvv;
370		ft.compare256 = &compare256_rvv;
371		ft.inflate_fast = &inflate_fast_rvv;
372		ft.longest_match = &longest_match_rvv;
373		ft.longest_match_roll = &longest_match_roll_rvv;
374		ft.slide_hash = &slide_hash_rvv;
375		ft.slide_hash_head = &slide_hash_head_rvv;
376		}
377		#endif
378
379		// RISCV - ZBC
380		#ifdef RISCV_CRC32_ZBC
381		# ifndef RISCV_ZBC_NATIVE
382		if (cf.riscv.has_zbc)
383		# endif
384		{
385		ft.crc32 = &crc32_riscv64_zbc;
386		ft.crc32_copy = &crc32_copy_riscv64_zbc;
387		}
388		#endif
389
390		// S390
391		#ifdef S390_VX
392		# ifndef S390_VX_NATIVE
393		if (cf.s390.has_vx)
394		# endif
395		{
396		ft.crc32 = &crc32_s390_vx;
397		ft.crc32_copy = &crc32_copy_s390_vx;
398		ft.slide_hash = &slide_hash_vx;
399		ft.slide_hash_head = &slide_hash_head_vx;
400		}
401		#endif
402
403		// LOONGARCH
404		#ifdef LOONGARCH_CRC
405		# ifndef LOONGARCH_CRC_NATIVE
406		if (cf.loongarch.has_crc)
407		# endif
408		{
409		ft.crc32 = &crc32_loongarch64;
410		ft.crc32_copy = &crc32_copy_loongarch64;
411		}
412		#endif
413		#if defined(LOONGARCH_LSX) && !defined(LOONGARCH_LASX_NATIVE)
414		# ifndef LOONGARCH_LSX_NATIVE
415		if (cf.loongarch.has_lsx)
416		# endif
417		{
418		ft.adler32 = &adler32_lsx;
419		ft.adler32_copy = &adler32_copy_lsx;
420		ft.chunkmemset_safe = &chunkmemset_safe_lsx;
421		ft.compare256 = &compare256_lsx;
422		ft.inflate_fast = &inflate_fast_lsx;
423		ft.longest_match = &longest_match_lsx;
424		ft.longest_match_roll = &longest_match_roll_lsx;
425		ft.slide_hash = &slide_hash_lsx;
426		ft.slide_hash_head = &slide_hash_head_lsx;
427		}
428		#endif
429		#ifdef LOONGARCH_LASX
430		# ifndef LOONGARCH_LASX_NATIVE
431		if (cf.loongarch.has_lasx)
432		# endif
433		{
434		ft.adler32 = &adler32_lasx;
435		ft.adler32_copy = &adler32_copy_lasx;
436		ft.chunkmemset_safe = &chunkmemset_safe_lasx;
437		ft.compare256 = &compare256_lasx;
438		ft.inflate_fast = &inflate_fast_lasx;
439		ft.longest_match = &longest_match_lasx;
440		ft.longest_match_roll = &longest_match_roll_lasx;
441		ft.slide_hash = &slide_hash_lasx;
442		ft.slide_hash_head = &slide_hash_head_lasx;
443		}
444		#endif
445
446	2	#endif // WITH_OPTIM
447
448		// Assign function pointers individually for atomic operation
449	2	FUNCTABLE_ASSIGN(ft, force_init);
450	2	FUNCTABLE_VERIFY_ASSIGN(ft, adler32);
451	2	FUNCTABLE_VERIFY_ASSIGN(ft, adler32_copy);
452	2	FUNCTABLE_VERIFY_ASSIGN(ft, chunkmemset_safe);
453	2	FUNCTABLE_VERIFY_ASSIGN(ft, compare256);
454	2	FUNCTABLE_VERIFY_ASSIGN(ft, crc32);
455	2	FUNCTABLE_VERIFY_ASSIGN(ft, crc32_copy);
456	2	FUNCTABLE_VERIFY_ASSIGN(ft, inflate_fast);
457	2	FUNCTABLE_VERIFY_ASSIGN(ft, longest_match);
458	2	FUNCTABLE_VERIFY_ASSIGN(ft, longest_match_roll);
459	2	FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash);
460	2	FUNCTABLE_VERIFY_ASSIGN(ft, slide_hash_head);
461
462		// Memory barrier for weak memory order CPUs
463	2	FUNCTABLE_BARRIER();
464
465	2	return Z_OK;
466	2	}
467
468		#if !defined(_WIN32) && (defined(__GNUC__) \|\| defined(__clang__))
469	2	static void __attribute__((constructor)) functable_constructor(void) {
470	2	FUNCTABLE_INIT_ABORT;
471	2	}
472		#endif
473
474		/* stub functions */
475	0	static int force_init_stub(void) {
476	0	return init_functable();
477	0	}
478
479	0	static uint32_t adler32_stub(uint32_t adler, const uint8_t* buf, size_t len) {
480	0	FUNCTABLE_INIT_ABORT;
481	0	return functable.adler32(adler, buf, len);
482	0	}
483
484	0	static uint32_t adler32_copy_stub(uint32_t adler, uint8_t* dst, const uint8_t* src, size_t len) {
485	0	FUNCTABLE_INIT_ABORT;
486	0	return functable.adler32_copy(adler, dst, src, len);
487	0	}
488
489	0	static uint8_t* chunkmemset_safe_stub(uint8_t* out, uint8_t *from, size_t len, size_t left) {
490	0	FUNCTABLE_INIT_ABORT;
491	0	return functable.chunkmemset_safe(out, from, len, left);
492	0	}
493
494	0	static uint32_t compare256_stub(const uint8_t* src0, const uint8_t* src1) {
495	0	FUNCTABLE_INIT_ABORT;
496	0	return functable.compare256(src0, src1);
497	0	}
498
499	0	static uint32_t crc32_stub(uint32_t crc, const uint8_t* buf, size_t len) {
500	0	FUNCTABLE_INIT_ABORT;
501	0	return functable.crc32(crc, buf, len);
502	0	}
503
504	0	static uint32_t crc32_copy_stub(uint32_t crc, uint8_t dst, const uint8_t src, size_t len) {
505	0	FUNCTABLE_INIT_ABORT;
506	0	return functable.crc32_copy(crc, dst, src, len);
507	0	}
508
509	0	static void inflate_fast_stub(PREFIX3(stream) *strm, uint32_t start, int safe_mode) {
510	0	FUNCTABLE_INIT_ABORT;
511	0	functable.inflate_fast(strm, start, safe_mode);
512	0	}
513
514	0	static uint32_t longest_match_stub(deflate_state* const s, uint32_t cur_match) {
515	0	FUNCTABLE_INIT_ABORT;
516	0	return functable.longest_match(s, cur_match);
517	0	}
518
519	0	static uint32_t longest_match_roll_stub(deflate_state* const s, uint32_t cur_match) {
520	0	FUNCTABLE_INIT_ABORT;
521	0	return functable.longest_match_roll(s, cur_match);
522	0	}
523
524	0	static void slide_hash_stub(deflate_state* s) {
525	0	FUNCTABLE_INIT_ABORT;
526	0	functable.slide_hash(s);
527	0	}
528
529	0	static void slide_hash_head_stub(deflate_state *s) {
530	0	FUNCTABLE_INIT_ABORT;
531	0	functable.slide_hash_head(s);
532	0	}
533
534		/* functable init */
535		Z_INTERNAL struct functable_s functable = {
536		force_init_stub,
537		adler32_stub,
538		adler32_copy_stub,
539		chunkmemset_safe_stub,
540		compare256_stub,
541		crc32_stub,
542		crc32_copy_stub,
543		inflate_fast_stub,
544		longest_match_stub,
545		longest_match_roll_stub,
546		slide_hash_stub,
547		slide_hash_head_stub,
548		};
549
550		#endif