Coverage Report

Created: 2026-11-19 07:17

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/c-blosc2/blosc/shuffle-bytedelta-ssse3.c
Line
Count
Source
1
/*********************************************************************
2
  Blosc - Blocked Shuffling and Compression Library
3
  Copyright (c) 2026 Blosc Development Team <blosc@blosc.org>
4
  License: BSD 3-Clause (see LICENSE.txt)
5
**********************************************************************/
6
7
/* This translation unit alone is compiled for SSSE3. The baseline shuffle
8
 * dispatcher must check CPUID before calling either entry point. */
9
#include "../plugins/filters/bytedelta/bytedelta.h"
10
11
#include <tmmintrin.h>
12
#include <string.h>
13
14
0
static inline void load_planes16(const uint8_t *input, __m128i *planes, int typesize) {
15
0
  if (typesize == 2) {
16
0
    const __m128i mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15);
17
0
    __m128i a = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)input), mask);
18
0
    __m128i b = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(input + 16)), mask);
19
0
    planes[0] = _mm_unpacklo_epi64(a, b);
20
0
    planes[1] = _mm_unpackhi_epi64(a, b);
21
0
  }
22
0
  else if (typesize == 4) {
23
0
    const __m128i mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
24
0
    __m128i a[4], b[4];
25
0
    for (int p = 0; p < 4; p++) {
26
0
      a[p] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(input + p * 16)), mask);
27
0
    }
28
0
    b[0] = _mm_unpacklo_epi32(a[0], a[1]);
29
0
    b[1] = _mm_unpackhi_epi32(a[0], a[1]);
30
0
    b[2] = _mm_unpacklo_epi32(a[2], a[3]);
31
0
    b[3] = _mm_unpackhi_epi32(a[2], a[3]);
32
0
    planes[0] = _mm_unpacklo_epi64(b[0], b[2]);
33
0
    planes[1] = _mm_unpackhi_epi64(b[0], b[2]);
34
0
    planes[2] = _mm_unpacklo_epi64(b[1], b[3]);
35
0
    planes[3] = _mm_unpackhi_epi64(b[1], b[3]);
36
0
  }
37
0
  else {
38
0
    const __m128i mask = _mm_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15);
39
0
    __m128i a[8], b[8];
40
0
    for (int p = 0; p < 8; p++) {
41
0
      b[p] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(input + p * 16)), mask);
42
0
    }
43
0
    for (int p = 0; p < 4; p++) {
44
0
      a[p * 2] = _mm_unpacklo_epi16(b[p * 2], b[p * 2 + 1]);
45
0
      a[p * 2 + 1] = _mm_unpackhi_epi16(b[p * 2], b[p * 2 + 1]);
46
0
    }
47
0
    for (int p = 0; p < 4; p++) {
48
0
      int k = p < 2 ? p : p + 2;
49
0
      b[p * 2] = _mm_unpacklo_epi32(a[k], a[k + 2]);
50
0
      b[p * 2 + 1] = _mm_unpackhi_epi32(a[k], a[k + 2]);
51
0
    }
52
0
    for (int p = 0; p < 4; p++) {
53
0
      planes[p * 2] = _mm_unpacklo_epi64(b[p], b[p + 4]);
54
0
      planes[p * 2 + 1] = _mm_unpackhi_epi64(b[p], b[p + 4]);
55
0
    }
56
0
  }
57
0
}
58
59
206k
static inline void store_planes16(uint8_t *output, const __m128i *planes, int typesize) {
60
206k
  if (typesize == 2) {
61
175k
    _mm_storeu_si128((__m128i *)output, _mm_unpacklo_epi8(planes[0], planes[1]));
62
175k
    _mm_storeu_si128((__m128i *)(output + 16), _mm_unpackhi_epi8(planes[0], planes[1]));
63
175k
  }
64
31.3k
  else if (typesize == 4) {
65
26.9k
    __m128i a[4];
66
26.9k
    a[0] = _mm_unpacklo_epi8(planes[0], planes[1]);
67
26.9k
    a[1] = _mm_unpackhi_epi8(planes[0], planes[1]);
68
26.9k
    a[2] = _mm_unpacklo_epi8(planes[2], planes[3]);
69
26.9k
    a[3] = _mm_unpackhi_epi8(planes[2], planes[3]);
70
26.9k
    _mm_storeu_si128((__m128i *)output, _mm_unpacklo_epi16(a[0], a[2]));
71
26.9k
    _mm_storeu_si128((__m128i *)(output + 16), _mm_unpackhi_epi16(a[0], a[2]));
72
26.9k
    _mm_storeu_si128((__m128i *)(output + 32), _mm_unpacklo_epi16(a[1], a[3]));
73
26.9k
    _mm_storeu_si128((__m128i *)(output + 48), _mm_unpackhi_epi16(a[1], a[3]));
74
26.9k
  }
75
4.38k
  else {
76
4.38k
    __m128i a[8], b[8];
77
21.9k
    for (int p = 0; p < 4; p++) {
78
17.5k
      a[p * 2] = _mm_unpacklo_epi8(planes[p * 2], planes[p * 2 + 1]);
79
17.5k
      a[p * 2 + 1] = _mm_unpackhi_epi8(planes[p * 2], planes[p * 2 + 1]);
80
17.5k
    }
81
13.1k
    for (int p = 0; p < 2; p++) {
82
8.77k
      b[p * 2] = _mm_unpacklo_epi16(a[p], a[p + 2]);
83
8.77k
      b[p * 2 + 1] = _mm_unpackhi_epi16(a[p], a[p + 2]);
84
8.77k
      b[p * 2 + 4] = _mm_unpacklo_epi16(a[p + 4], a[p + 6]);
85
8.77k
      b[p * 2 + 5] = _mm_unpackhi_epi16(a[p + 4], a[p + 6]);
86
8.77k
    }
87
21.9k
    for (int p = 0; p < 4; p++) {
88
17.5k
      _mm_storeu_si128((__m128i *)(output + p * 32), _mm_unpacklo_epi32(b[p], b[p + 4]));
89
17.5k
      _mm_storeu_si128((__m128i *)(output + p * 32 + 16), _mm_unpackhi_epi32(b[p], b[p + 4]));
90
17.5k
    }
91
4.38k
  }
92
206k
}
93
94
494k
static inline __m128i prefix_sum16(__m128i value) {
95
494k
  value = _mm_add_epi8(value, _mm_slli_si128(value, 1));
96
494k
  value = _mm_add_epi8(value, _mm_slli_si128(value, 2));
97
494k
  value = _mm_add_epi8(value, _mm_slli_si128(value, 4));
98
494k
  return _mm_add_epi8(value, _mm_slli_si128(value, 8));
99
494k
}
100
101
/* Compile-time plane counts keep transpose/delta stages together and allow
102
 * unrolling without a variable-count plane loop inside every SIMD tile. */
103
#define DEFINE_PAIR(N) \
104
0
static int forward##N(const uint8_t *input, uint8_t *output, int32_t length) { \
105
0
  int count = length / N, vector_count = count / 16 * 16; \
106
0
  __m128i previous[N], planes[N]; \
107
0
  for (int p = 0; p < N; p++) previous[p] = _mm_setzero_si128(); \
108
0
  for (int i = 0; i < vector_count; i += 16) { \
109
0
    load_planes16(input + i * N, planes, N); \
110
0
    for (int p = 0; p < N; p++) { \
111
0
      __m128i delta = _mm_sub_epi8(planes[p], _mm_alignr_epi8(planes[p], previous[p], 15)); \
112
0
      _mm_storeu_si128((__m128i *)(output + p * count + i), delta); \
113
0
      previous[p] = planes[p]; \
114
0
    } \
115
0
  } \
116
0
  for (int p = 0; p < N; p++) { \
117
0
    uint8_t carry = (uint8_t)(_mm_extract_epi16(previous[p], 7) >> 8); \
118
0
    for (int i = vector_count; i < count; i++) { \
119
0
      uint8_t value = input[i * N + p]; \
120
0
      output[p * count + i] = (uint8_t)(value - carry); \
121
0
      carry = value; \
122
0
    } \
123
0
  } \
124
0
  memcpy(output + count * N, input + count * N, (size_t)(length % N)); \
125
0
  return 1; \
126
0
} \
127
1.21k
static int backward##N(const uint8_t *input, uint8_t *output, int32_t length) { \
128
1.21k
  int count = length / N, vector_count = count / 16 * 16; \
129
1.21k
  __m128i previous[N], planes[N]; \
130
6.79k
  for (int p = 0; p < N; p++) previous[p] = _mm_setzero_si128(); \
131
208k
  for (int i = 0; i < vector_count; i += 16) { \
132
701k
    for (int p = 0; p < N; p++) { \
133
494k
      __m128i delta = _mm_loadu_si128((const __m128i *)(input + p * count + i)); \
134
494k
      planes[p] = _mm_add_epi8(prefix_sum16(delta), _mm_shuffle_epi8(previous[p], _mm_set1_epi8(15))); \
135
494k
      previous[p] = planes[p]; \
136
494k
    } \
137
206k
    store_planes16(output + i * N, planes, N); \
138
206k
  } \
139
6.79k
  for (int p = 0; p < N; p++) { \
140
5.57k
    uint8_t carry = (uint8_t)(_mm_extract_epi16(previous[p], 7) >> 8); \
141
25.3k
    for (int i = vector_count; i < count; i++) { \
142
19.7k
      carry = (uint8_t)(carry + input[p * count + i]); \
143
19.7k
      output[i * N + p] = carry; \
144
19.7k
    } \
145
5.57k
  } \
146
1.21k
  memcpy(output + count * N, input + count * N, (size_t)(length % N)); \
147
1.21k
  return 1; \
148
1.21k
}
149
150
385
DEFINE_PAIR(2)
Unexecuted instantiation: shuffle-bytedelta-ssse3.c:forward2
shuffle-bytedelta-ssse3.c:backward2
Line
Count
Source
150
DEFINE_PAIR(2)
151
454
DEFINE_PAIR(4)
Unexecuted instantiation: shuffle-bytedelta-ssse3.c:forward4
shuffle-bytedelta-ssse3.c:backward4
Line
Count
Source
151
DEFINE_PAIR(4)
152
374
DEFINE_PAIR(8)
Unexecuted instantiation: shuffle-bytedelta-ssse3.c:forward8
shuffle-bytedelta-ssse3.c:backward8
Line
Count
Source
152
DEFINE_PAIR(8)
153
#undef DEFINE_PAIR
154
155
0
int shuffle_bytedelta_ssse3_forward(const uint8_t *input, uint8_t *output, int32_t length, int typesize) {
156
0
  switch (typesize) {
157
0
    case 2: return forward2(input, output, length);
158
0
    case 4: return forward4(input, output, length);
159
0
    case 8: return forward8(input, output, length);
160
0
    default: return 0;
161
0
  }
162
0
}
163
164
1.21k
int shuffle_bytedelta_ssse3_backward(const uint8_t *input, uint8_t *output, int32_t length, int typesize) {
165
1.21k
  switch (typesize) {
166
385
    case 2: return backward2(input, output, length);
167
454
    case 4: return backward4(input, output, length);
168
374
    case 8: return backward8(input, output, length);
169
0
    default: return 0;
170
1.21k
  }
171
1.21k
}