/src/c-blosc2/blosc/shuffle-bytedelta-ssse3.c
Line | Count | Source |
1 | | /********************************************************************* |
2 | | Blosc - Blocked Shuffling and Compression Library |
3 | | Copyright (c) 2026 Blosc Development Team <blosc@blosc.org> |
4 | | License: BSD 3-Clause (see LICENSE.txt) |
5 | | **********************************************************************/ |
6 | | |
7 | | /* This translation unit alone is compiled for SSSE3. The baseline shuffle |
8 | | * dispatcher must check CPUID before calling either entry point. */ |
9 | | #include "../plugins/filters/bytedelta/bytedelta.h" |
10 | | |
11 | | #include <tmmintrin.h> |
12 | | #include <string.h> |
13 | | |
14 | 0 | static inline void load_planes16(const uint8_t *input, __m128i *planes, int typesize) { |
15 | 0 | if (typesize == 2) { |
16 | 0 | const __m128i mask = _mm_setr_epi8(0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15); |
17 | 0 | __m128i a = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)input), mask); |
18 | 0 | __m128i b = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(input + 16)), mask); |
19 | 0 | planes[0] = _mm_unpacklo_epi64(a, b); |
20 | 0 | planes[1] = _mm_unpackhi_epi64(a, b); |
21 | 0 | } |
22 | 0 | else if (typesize == 4) { |
23 | 0 | const __m128i mask = _mm_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); |
24 | 0 | __m128i a[4], b[4]; |
25 | 0 | for (int p = 0; p < 4; p++) { |
26 | 0 | a[p] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(input + p * 16)), mask); |
27 | 0 | } |
28 | 0 | b[0] = _mm_unpacklo_epi32(a[0], a[1]); |
29 | 0 | b[1] = _mm_unpackhi_epi32(a[0], a[1]); |
30 | 0 | b[2] = _mm_unpacklo_epi32(a[2], a[3]); |
31 | 0 | b[3] = _mm_unpackhi_epi32(a[2], a[3]); |
32 | 0 | planes[0] = _mm_unpacklo_epi64(b[0], b[2]); |
33 | 0 | planes[1] = _mm_unpackhi_epi64(b[0], b[2]); |
34 | 0 | planes[2] = _mm_unpacklo_epi64(b[1], b[3]); |
35 | 0 | planes[3] = _mm_unpackhi_epi64(b[1], b[3]); |
36 | 0 | } |
37 | 0 | else { |
38 | 0 | const __m128i mask = _mm_setr_epi8(0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15); |
39 | 0 | __m128i a[8], b[8]; |
40 | 0 | for (int p = 0; p < 8; p++) { |
41 | 0 | b[p] = _mm_shuffle_epi8(_mm_loadu_si128((const __m128i *)(input + p * 16)), mask); |
42 | 0 | } |
43 | 0 | for (int p = 0; p < 4; p++) { |
44 | 0 | a[p * 2] = _mm_unpacklo_epi16(b[p * 2], b[p * 2 + 1]); |
45 | 0 | a[p * 2 + 1] = _mm_unpackhi_epi16(b[p * 2], b[p * 2 + 1]); |
46 | 0 | } |
47 | 0 | for (int p = 0; p < 4; p++) { |
48 | 0 | int k = p < 2 ? p : p + 2; |
49 | 0 | b[p * 2] = _mm_unpacklo_epi32(a[k], a[k + 2]); |
50 | 0 | b[p * 2 + 1] = _mm_unpackhi_epi32(a[k], a[k + 2]); |
51 | 0 | } |
52 | 0 | for (int p = 0; p < 4; p++) { |
53 | 0 | planes[p * 2] = _mm_unpacklo_epi64(b[p], b[p + 4]); |
54 | 0 | planes[p * 2 + 1] = _mm_unpackhi_epi64(b[p], b[p + 4]); |
55 | 0 | } |
56 | 0 | } |
57 | 0 | } |
58 | | |
59 | 206k | static inline void store_planes16(uint8_t *output, const __m128i *planes, int typesize) { |
60 | 206k | if (typesize == 2) { |
61 | 175k | _mm_storeu_si128((__m128i *)output, _mm_unpacklo_epi8(planes[0], planes[1])); |
62 | 175k | _mm_storeu_si128((__m128i *)(output + 16), _mm_unpackhi_epi8(planes[0], planes[1])); |
63 | 175k | } |
64 | 31.3k | else if (typesize == 4) { |
65 | 26.9k | __m128i a[4]; |
66 | 26.9k | a[0] = _mm_unpacklo_epi8(planes[0], planes[1]); |
67 | 26.9k | a[1] = _mm_unpackhi_epi8(planes[0], planes[1]); |
68 | 26.9k | a[2] = _mm_unpacklo_epi8(planes[2], planes[3]); |
69 | 26.9k | a[3] = _mm_unpackhi_epi8(planes[2], planes[3]); |
70 | 26.9k | _mm_storeu_si128((__m128i *)output, _mm_unpacklo_epi16(a[0], a[2])); |
71 | 26.9k | _mm_storeu_si128((__m128i *)(output + 16), _mm_unpackhi_epi16(a[0], a[2])); |
72 | 26.9k | _mm_storeu_si128((__m128i *)(output + 32), _mm_unpacklo_epi16(a[1], a[3])); |
73 | 26.9k | _mm_storeu_si128((__m128i *)(output + 48), _mm_unpackhi_epi16(a[1], a[3])); |
74 | 26.9k | } |
75 | 4.38k | else { |
76 | 4.38k | __m128i a[8], b[8]; |
77 | 21.9k | for (int p = 0; p < 4; p++) { |
78 | 17.5k | a[p * 2] = _mm_unpacklo_epi8(planes[p * 2], planes[p * 2 + 1]); |
79 | 17.5k | a[p * 2 + 1] = _mm_unpackhi_epi8(planes[p * 2], planes[p * 2 + 1]); |
80 | 17.5k | } |
81 | 13.1k | for (int p = 0; p < 2; p++) { |
82 | 8.77k | b[p * 2] = _mm_unpacklo_epi16(a[p], a[p + 2]); |
83 | 8.77k | b[p * 2 + 1] = _mm_unpackhi_epi16(a[p], a[p + 2]); |
84 | 8.77k | b[p * 2 + 4] = _mm_unpacklo_epi16(a[p + 4], a[p + 6]); |
85 | 8.77k | b[p * 2 + 5] = _mm_unpackhi_epi16(a[p + 4], a[p + 6]); |
86 | 8.77k | } |
87 | 21.9k | for (int p = 0; p < 4; p++) { |
88 | 17.5k | _mm_storeu_si128((__m128i *)(output + p * 32), _mm_unpacklo_epi32(b[p], b[p + 4])); |
89 | 17.5k | _mm_storeu_si128((__m128i *)(output + p * 32 + 16), _mm_unpackhi_epi32(b[p], b[p + 4])); |
90 | 17.5k | } |
91 | 4.38k | } |
92 | 206k | } |
93 | | |
94 | 494k | static inline __m128i prefix_sum16(__m128i value) { |
95 | 494k | value = _mm_add_epi8(value, _mm_slli_si128(value, 1)); |
96 | 494k | value = _mm_add_epi8(value, _mm_slli_si128(value, 2)); |
97 | 494k | value = _mm_add_epi8(value, _mm_slli_si128(value, 4)); |
98 | 494k | return _mm_add_epi8(value, _mm_slli_si128(value, 8)); |
99 | 494k | } |
100 | | |
101 | | /* Compile-time plane counts keep transpose/delta stages together and allow |
102 | | * unrolling without a variable-count plane loop inside every SIMD tile. */ |
103 | | #define DEFINE_PAIR(N) \ |
104 | 0 | static int forward##N(const uint8_t *input, uint8_t *output, int32_t length) { \ |
105 | 0 | int count = length / N, vector_count = count / 16 * 16; \ |
106 | 0 | __m128i previous[N], planes[N]; \ |
107 | 0 | for (int p = 0; p < N; p++) previous[p] = _mm_setzero_si128(); \ |
108 | 0 | for (int i = 0; i < vector_count; i += 16) { \ |
109 | 0 | load_planes16(input + i * N, planes, N); \ |
110 | 0 | for (int p = 0; p < N; p++) { \ |
111 | 0 | __m128i delta = _mm_sub_epi8(planes[p], _mm_alignr_epi8(planes[p], previous[p], 15)); \ |
112 | 0 | _mm_storeu_si128((__m128i *)(output + p * count + i), delta); \ |
113 | 0 | previous[p] = planes[p]; \ |
114 | 0 | } \ |
115 | 0 | } \ |
116 | 0 | for (int p = 0; p < N; p++) { \ |
117 | 0 | uint8_t carry = (uint8_t)(_mm_extract_epi16(previous[p], 7) >> 8); \ |
118 | 0 | for (int i = vector_count; i < count; i++) { \ |
119 | 0 | uint8_t value = input[i * N + p]; \ |
120 | 0 | output[p * count + i] = (uint8_t)(value - carry); \ |
121 | 0 | carry = value; \ |
122 | 0 | } \ |
123 | 0 | } \ |
124 | 0 | memcpy(output + count * N, input + count * N, (size_t)(length % N)); \ |
125 | 0 | return 1; \ |
126 | 0 | } \ |
127 | 1.21k | static int backward##N(const uint8_t *input, uint8_t *output, int32_t length) { \ |
128 | 1.21k | int count = length / N, vector_count = count / 16 * 16; \ |
129 | 1.21k | __m128i previous[N], planes[N]; \ |
130 | 6.79k | for (int p = 0; p < N; p++) previous[p] = _mm_setzero_si128(); \ |
131 | 208k | for (int i = 0; i < vector_count; i += 16) { \ |
132 | 701k | for (int p = 0; p < N; p++) { \ |
133 | 494k | __m128i delta = _mm_loadu_si128((const __m128i *)(input + p * count + i)); \ |
134 | 494k | planes[p] = _mm_add_epi8(prefix_sum16(delta), _mm_shuffle_epi8(previous[p], _mm_set1_epi8(15))); \ |
135 | 494k | previous[p] = planes[p]; \ |
136 | 494k | } \ |
137 | 206k | store_planes16(output + i * N, planes, N); \ |
138 | 206k | } \ |
139 | 6.79k | for (int p = 0; p < N; p++) { \ |
140 | 5.57k | uint8_t carry = (uint8_t)(_mm_extract_epi16(previous[p], 7) >> 8); \ |
141 | 25.3k | for (int i = vector_count; i < count; i++) { \ |
142 | 19.7k | carry = (uint8_t)(carry + input[p * count + i]); \ |
143 | 19.7k | output[i * N + p] = carry; \ |
144 | 19.7k | } \ |
145 | 5.57k | } \ |
146 | 1.21k | memcpy(output + count * N, input + count * N, (size_t)(length % N)); \ |
147 | 1.21k | return 1; \ |
148 | 1.21k | } |
149 | | |
150 | 385 | DEFINE_PAIR(2) Unexecuted instantiation: shuffle-bytedelta-ssse3.c:forward2 shuffle-bytedelta-ssse3.c:backward2 Line | Count | Source | 150 | | DEFINE_PAIR(2) |
|
151 | 454 | DEFINE_PAIR(4) Unexecuted instantiation: shuffle-bytedelta-ssse3.c:forward4 shuffle-bytedelta-ssse3.c:backward4 Line | Count | Source | 151 | | DEFINE_PAIR(4) |
|
152 | 374 | DEFINE_PAIR(8) Unexecuted instantiation: shuffle-bytedelta-ssse3.c:forward8 shuffle-bytedelta-ssse3.c:backward8 Line | Count | Source | 152 | | DEFINE_PAIR(8) |
|
153 | | #undef DEFINE_PAIR |
154 | | |
155 | 0 | int shuffle_bytedelta_ssse3_forward(const uint8_t *input, uint8_t *output, int32_t length, int typesize) { |
156 | 0 | switch (typesize) { |
157 | 0 | case 2: return forward2(input, output, length); |
158 | 0 | case 4: return forward4(input, output, length); |
159 | 0 | case 8: return forward8(input, output, length); |
160 | 0 | default: return 0; |
161 | 0 | } |
162 | 0 | } |
163 | | |
164 | 1.21k | int shuffle_bytedelta_ssse3_backward(const uint8_t *input, uint8_t *output, int32_t length, int typesize) { |
165 | 1.21k | switch (typesize) { |
166 | 385 | case 2: return backward2(input, output, length); |
167 | 454 | case 4: return backward4(input, output, length); |
168 | 374 | case 8: return backward8(input, output, length); |
169 | 0 | default: return 0; |
170 | 1.21k | } |
171 | 1.21k | } |