/src/c-blosc2/blosc/bitshuffle-constant.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2026 Blosc Development Team <blosc@blosc.org> |
3 | | * License: BSD 3-Clause (see LICENSE.txt) |
4 | | * |
5 | | * A verified constant prefix can be emitted without transposing its bits. |
6 | | * Returning a multiple of 128 keeps the caller's vector loop aligned to its |
7 | | * normal tile size. A full constant plane may include an eight-byte tail. |
8 | | */ |
9 | | #ifndef BLOSC_BITSHUFFLE_CONSTANT_H |
10 | | #define BLOSC_BITSHUFFLE_CONSTANT_H |
11 | | |
12 | | #include <stddef.h> |
13 | | #include <stdint.h> |
14 | | #include <string.h> |
15 | | |
16 | | #if defined(__AVX2__) |
17 | | #include <immintrin.h> |
18 | | #elif defined(__SSE2__) |
19 | | #include <emmintrin.h> |
20 | | #elif defined(__aarch64__) || defined(_M_ARM64) |
21 | | #include <arm_neon.h> |
22 | | #endif |
23 | | |
24 | 372k | static inline size_t bshuf_constant_prefix(const uint8_t* input, size_t nbyte) { |
25 | | /* Eight row fills do not amortize their setup on tiny byte planes. */ |
26 | 372k | if (nbyte < 1024) return 0; |
27 | 251k | const uint8_t value = input[0]; |
28 | | #if defined(__AVX2__) |
29 | | const __m256i v = _mm256_set1_epi8((char)value); |
30 | | __m256i diff = _mm256_or_si256( |
31 | | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)input), v), |
32 | | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input + nbyte - 32)), v)); |
33 | 251k | if (!_mm256_testz_si256(diff, diff)) return 0; |
34 | | #elif defined(__SSE2__) |
35 | | const __m128i v = _mm_set1_epi8((char)value); |
36 | | __m128i diff = _mm_or_si128( |
37 | | _mm_xor_si128(_mm_loadu_si128((const __m128i*)input), v), |
38 | | _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input + nbyte - 16)), v)); |
39 | 0 | if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return 0; |
40 | | #elif defined(__aarch64__) || defined(_M_ARM64) |
41 | | const uint8x16_t v = vdupq_n_u8(value); |
42 | | uint8x16_t diff = vorrq_u8(veorq_u8(vld1q_u8(input), v), |
43 | | veorq_u8(vld1q_u8(input + nbyte - 16), v)); |
44 | | if (vmaxvq_u8(diff) != 0) return 0; |
45 | | #else |
46 | | return 0; |
47 | | #endif |
48 | 39.4k | size_t j = 0; |
49 | | /* Stop at the first nonconstant tile. Its predecessors can still be emitted |
50 | | * directly, so an unsuccessful scan never requires retransposing that prefix. */ |
51 | 562k | for (; j + 127 < nbyte; j += 128) { |
52 | | #if defined(__AVX2__) |
53 | | __m256i a = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j)), v), |
54 | | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+32)), v)); |
55 | | __m256i b = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+64)), v), |
56 | | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+96)), v)); |
57 | | diff = _mm256_or_si256(a, b); |
58 | 549k | if (!_mm256_testz_si256(diff, diff)) return j; |
59 | | #elif defined(__SSE2__) |
60 | | diff = _mm_setzero_si128(); |
61 | 0 | for (size_t k = 0; k < 128; k += 16) { |
62 | 0 | diff = _mm_or_si128(diff, _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input+j+k)), v)); |
63 | 0 | } |
64 | 0 | if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return j; |
65 | | #elif defined(__aarch64__) || defined(_M_ARM64) |
66 | | diff = vdupq_n_u8(0); |
67 | | for (size_t k = 0; k < 128; k += 16) { |
68 | | diff = vorrq_u8(diff, veorq_u8(vld1q_u8(input+j+k), v)); |
69 | | } |
70 | | if (vmaxvq_u8(diff) != 0) return j; |
71 | | #endif |
72 | 549k | } |
73 | 15.1k | for (size_t k = j; k < nbyte; ++k) { |
74 | 2.55k | if (input[k] != value) return j; |
75 | 2.55k | } |
76 | 12.5k | return nbyte; |
77 | 12.5k | } Unexecuted instantiation: bitshuffle-sse2.c:bshuf_constant_prefix bitshuffle-avx2.c:bshuf_constant_prefix Line | Count | Source | 24 | 372k | static inline size_t bshuf_constant_prefix(const uint8_t* input, size_t nbyte) { | 25 | | /* Eight row fills do not amortize their setup on tiny byte planes. */ | 26 | 372k | if (nbyte < 1024) return 0; | 27 | 251k | const uint8_t value = input[0]; | 28 | 251k | #if defined(__AVX2__) | 29 | 251k | const __m256i v = _mm256_set1_epi8((char)value); | 30 | 251k | __m256i diff = _mm256_or_si256( | 31 | 251k | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)input), v), | 32 | 251k | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input + nbyte - 32)), v)); | 33 | 251k | if (!_mm256_testz_si256(diff, diff)) return 0; | 34 | | #elif defined(__SSE2__) | 35 | | const __m128i v = _mm_set1_epi8((char)value); | 36 | | __m128i diff = _mm_or_si128( | 37 | | _mm_xor_si128(_mm_loadu_si128((const __m128i*)input), v), | 38 | | _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input + nbyte - 16)), v)); | 39 | | if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return 0; | 40 | | #elif defined(__aarch64__) || defined(_M_ARM64) | 41 | | const uint8x16_t v = vdupq_n_u8(value); | 42 | | uint8x16_t diff = vorrq_u8(veorq_u8(vld1q_u8(input), v), | 43 | | veorq_u8(vld1q_u8(input + nbyte - 16), v)); | 44 | | if (vmaxvq_u8(diff) != 0) return 0; | 45 | | #else | 46 | | return 0; | 47 | | #endif | 48 | 39.4k | size_t j = 0; | 49 | | /* Stop at the first nonconstant tile. Its predecessors can still be emitted | 50 | | * directly, so an unsuccessful scan never requires retransposing that prefix. */ | 51 | 562k | for (; j + 127 < nbyte; j += 128) { | 52 | 549k | #if defined(__AVX2__) | 53 | 549k | __m256i a = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j)), v), | 54 | 549k | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+32)), v)); | 55 | 549k | __m256i b = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+64)), v), | 56 | 549k | _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+96)), v)); | 57 | 549k | diff = _mm256_or_si256(a, b); | 58 | 549k | if (!_mm256_testz_si256(diff, diff)) return j; | 59 | | #elif defined(__SSE2__) | 60 | | diff = _mm_setzero_si128(); | 61 | | for (size_t k = 0; k < 128; k += 16) { | 62 | | diff = _mm_or_si128(diff, _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input+j+k)), v)); | 63 | | } | 64 | | if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return j; | 65 | | #elif defined(__aarch64__) || defined(_M_ARM64) | 66 | | diff = vdupq_n_u8(0); | 67 | | for (size_t k = 0; k < 128; k += 16) { | 68 | | diff = vorrq_u8(diff, veorq_u8(vld1q_u8(input+j+k), v)); | 69 | | } | 70 | | if (vmaxvq_u8(diff) != 0) return j; | 71 | | #endif | 72 | 549k | } | 73 | 15.1k | for (size_t k = j; k < nbyte; ++k) { | 74 | 2.55k | if (input[k] != value) return j; | 75 | 2.55k | } | 76 | 12.5k | return nbyte; | 77 | 12.5k | } |
|
78 | | |
79 | 372k | static inline size_t bshuf_emit_constant_prefix(const void* in, void* out, size_t nbyte) { |
80 | 372k | const uint8_t* input = (const uint8_t*)in; |
81 | 372k | size_t prefix = bshuf_constant_prefix(input, nbyte); |
82 | 372k | if (prefix != 0) { |
83 | 278k | for (size_t bit = 0; bit < 8; ++bit) { |
84 | 247k | memset((uint8_t*)out + bit * (nbyte / 8), (input[0] >> bit) & 1 ? 0xff : 0, prefix / 8); |
85 | 247k | } |
86 | 30.9k | } |
87 | 372k | return prefix; |
88 | 372k | } Unexecuted instantiation: bitshuffle-sse2.c:bshuf_emit_constant_prefix bitshuffle-avx2.c:bshuf_emit_constant_prefix Line | Count | Source | 79 | 372k | static inline size_t bshuf_emit_constant_prefix(const void* in, void* out, size_t nbyte) { | 80 | 372k | const uint8_t* input = (const uint8_t*)in; | 81 | 372k | size_t prefix = bshuf_constant_prefix(input, nbyte); | 82 | 372k | if (prefix != 0) { | 83 | 278k | for (size_t bit = 0; bit < 8; ++bit) { | 84 | 247k | memset((uint8_t*)out + bit * (nbyte / 8), (input[0] >> bit) & 1 ? 0xff : 0, prefix / 8); | 85 | 247k | } | 86 | 30.9k | } | 87 | 372k | return prefix; | 88 | 372k | } |
|
89 | | #endif |