Coverage Report

Created: 2026-11-19 07:17

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/c-blosc2/blosc/bitshuffle-constant.h
Line
Count
Source
1
/*
2
 * Copyright (c) 2026 Blosc Development Team <blosc@blosc.org>
3
 * License: BSD 3-Clause (see LICENSE.txt)
4
 *
5
 * A verified constant prefix can be emitted without transposing its bits.
6
 * Returning a multiple of 128 keeps the caller's vector loop aligned to its
7
 * normal tile size. A full constant plane may include an eight-byte tail.
8
 */
9
#ifndef BLOSC_BITSHUFFLE_CONSTANT_H
10
#define BLOSC_BITSHUFFLE_CONSTANT_H
11
12
#include <stddef.h>
13
#include <stdint.h>
14
#include <string.h>
15
16
#if defined(__AVX2__)
17
#include <immintrin.h>
18
#elif defined(__SSE2__)
19
#include <emmintrin.h>
20
#elif defined(__aarch64__) || defined(_M_ARM64)
21
#include <arm_neon.h>
22
#endif
23
24
372k
static inline size_t bshuf_constant_prefix(const uint8_t* input, size_t nbyte) {
25
  /* Eight row fills do not amortize their setup on tiny byte planes. */
26
372k
  if (nbyte < 1024) return 0;
27
251k
  const uint8_t value = input[0];
28
#if defined(__AVX2__)
29
  const __m256i v = _mm256_set1_epi8((char)value);
30
  __m256i diff = _mm256_or_si256(
31
      _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)input), v),
32
      _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input + nbyte - 32)), v));
33
251k
  if (!_mm256_testz_si256(diff, diff)) return 0;
34
#elif defined(__SSE2__)
35
  const __m128i v = _mm_set1_epi8((char)value);
36
  __m128i diff = _mm_or_si128(
37
      _mm_xor_si128(_mm_loadu_si128((const __m128i*)input), v),
38
      _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input + nbyte - 16)), v));
39
0
  if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return 0;
40
#elif defined(__aarch64__) || defined(_M_ARM64)
41
  const uint8x16_t v = vdupq_n_u8(value);
42
  uint8x16_t diff = vorrq_u8(veorq_u8(vld1q_u8(input), v),
43
                            veorq_u8(vld1q_u8(input + nbyte - 16), v));
44
  if (vmaxvq_u8(diff) != 0) return 0;
45
#else
46
  return 0;
47
#endif
48
39.4k
  size_t j = 0;
49
  /* Stop at the first nonconstant tile. Its predecessors can still be emitted
50
   * directly, so an unsuccessful scan never requires retransposing that prefix. */
51
562k
  for (; j + 127 < nbyte; j += 128) {
52
#if defined(__AVX2__)
53
    __m256i a = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j)), v),
54
                               _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+32)), v));
55
    __m256i b = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+64)), v),
56
                               _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+96)), v));
57
    diff = _mm256_or_si256(a, b);
58
549k
    if (!_mm256_testz_si256(diff, diff)) return j;
59
#elif defined(__SSE2__)
60
    diff = _mm_setzero_si128();
61
0
    for (size_t k = 0; k < 128; k += 16) {
62
0
      diff = _mm_or_si128(diff, _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input+j+k)), v));
63
0
    }
64
0
    if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return j;
65
#elif defined(__aarch64__) || defined(_M_ARM64)
66
    diff = vdupq_n_u8(0);
67
    for (size_t k = 0; k < 128; k += 16) {
68
      diff = vorrq_u8(diff, veorq_u8(vld1q_u8(input+j+k), v));
69
    }
70
    if (vmaxvq_u8(diff) != 0) return j;
71
#endif
72
549k
  }
73
15.1k
  for (size_t k = j; k < nbyte; ++k) {
74
2.55k
    if (input[k] != value) return j;
75
2.55k
  }
76
12.5k
  return nbyte;
77
12.5k
}
Unexecuted instantiation: bitshuffle-sse2.c:bshuf_constant_prefix
bitshuffle-avx2.c:bshuf_constant_prefix
Line
Count
Source
24
372k
static inline size_t bshuf_constant_prefix(const uint8_t* input, size_t nbyte) {
25
  /* Eight row fills do not amortize their setup on tiny byte planes. */
26
372k
  if (nbyte < 1024) return 0;
27
251k
  const uint8_t value = input[0];
28
251k
#if defined(__AVX2__)
29
251k
  const __m256i v = _mm256_set1_epi8((char)value);
30
251k
  __m256i diff = _mm256_or_si256(
31
251k
      _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)input), v),
32
251k
      _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input + nbyte - 32)), v));
33
251k
  if (!_mm256_testz_si256(diff, diff)) return 0;
34
#elif defined(__SSE2__)
35
  const __m128i v = _mm_set1_epi8((char)value);
36
  __m128i diff = _mm_or_si128(
37
      _mm_xor_si128(_mm_loadu_si128((const __m128i*)input), v),
38
      _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input + nbyte - 16)), v));
39
  if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return 0;
40
#elif defined(__aarch64__) || defined(_M_ARM64)
41
  const uint8x16_t v = vdupq_n_u8(value);
42
  uint8x16_t diff = vorrq_u8(veorq_u8(vld1q_u8(input), v),
43
                            veorq_u8(vld1q_u8(input + nbyte - 16), v));
44
  if (vmaxvq_u8(diff) != 0) return 0;
45
#else
46
  return 0;
47
#endif
48
39.4k
  size_t j = 0;
49
  /* Stop at the first nonconstant tile. Its predecessors can still be emitted
50
   * directly, so an unsuccessful scan never requires retransposing that prefix. */
51
562k
  for (; j + 127 < nbyte; j += 128) {
52
549k
#if defined(__AVX2__)
53
549k
    __m256i a = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j)), v),
54
549k
                               _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+32)), v));
55
549k
    __m256i b = _mm256_or_si256(_mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+64)), v),
56
549k
                               _mm256_xor_si256(_mm256_loadu_si256((const __m256i*)(input+j+96)), v));
57
549k
    diff = _mm256_or_si256(a, b);
58
549k
    if (!_mm256_testz_si256(diff, diff)) return j;
59
#elif defined(__SSE2__)
60
    diff = _mm_setzero_si128();
61
    for (size_t k = 0; k < 128; k += 16) {
62
      diff = _mm_or_si128(diff, _mm_xor_si128(_mm_loadu_si128((const __m128i*)(input+j+k)), v));
63
    }
64
    if (_mm_movemask_epi8(_mm_cmpeq_epi8(diff, _mm_setzero_si128())) != 0xffff) return j;
65
#elif defined(__aarch64__) || defined(_M_ARM64)
66
    diff = vdupq_n_u8(0);
67
    for (size_t k = 0; k < 128; k += 16) {
68
      diff = vorrq_u8(diff, veorq_u8(vld1q_u8(input+j+k), v));
69
    }
70
    if (vmaxvq_u8(diff) != 0) return j;
71
#endif
72
549k
  }
73
15.1k
  for (size_t k = j; k < nbyte; ++k) {
74
2.55k
    if (input[k] != value) return j;
75
2.55k
  }
76
12.5k
  return nbyte;
77
12.5k
}
78
79
372k
static inline size_t bshuf_emit_constant_prefix(const void* in, void* out, size_t nbyte) {
80
372k
  const uint8_t* input = (const uint8_t*)in;
81
372k
  size_t prefix = bshuf_constant_prefix(input, nbyte);
82
372k
  if (prefix != 0) {
83
278k
    for (size_t bit = 0; bit < 8; ++bit) {
84
247k
      memset((uint8_t*)out + bit * (nbyte / 8), (input[0] >> bit) & 1 ? 0xff : 0, prefix / 8);
85
247k
    }
86
30.9k
  }
87
372k
  return prefix;
88
372k
}
Unexecuted instantiation: bitshuffle-sse2.c:bshuf_emit_constant_prefix
bitshuffle-avx2.c:bshuf_emit_constant_prefix
Line
Count
Source
79
372k
static inline size_t bshuf_emit_constant_prefix(const void* in, void* out, size_t nbyte) {
80
372k
  const uint8_t* input = (const uint8_t*)in;
81
372k
  size_t prefix = bshuf_constant_prefix(input, nbyte);
82
372k
  if (prefix != 0) {
83
278k
    for (size_t bit = 0; bit < 8; ++bit) {
84
247k
      memset((uint8_t*)out + bit * (nbyte / 8), (input[0] >> bit) & 1 ? 0xff : 0, prefix / 8);
85
247k
    }
86
30.9k
  }
87
372k
  return prefix;
88
372k
}
89
#endif