/src/c-blosc2/blosc/bitshuffle-sse2.c
Line | Count | Source |
1 | | /********************************************************************* |
2 | | Blosc - Blocked Shuffling and Compression Library |
3 | | |
4 | | Copyright (c) 2021 Blosc Development Team <blosc@blosc.org> |
5 | | https://blosc.org |
6 | | License: BSD 3-Clause (see LICENSE.txt) |
7 | | |
8 | | See LICENSE.txt for details about copyright and rights to use. |
9 | | **********************************************************************/ |
10 | | |
11 | | /********************************************************************* |
12 | | Bitshuffle - Filter for improving compression of typed binary data. |
13 | | |
14 | | Author: Kiyoshi Masui <kiyo@physics.ubc.ca> |
15 | | Website: https://github.com/kiyo-masui/bitshuffle |
16 | | |
17 | | Note: Adapted for c-blosc by Francesc Alted. |
18 | | |
19 | | See LICENSES/BITSHUFFLE.txt file for details about copyright and |
20 | | rights to use. |
21 | | **********************************************************************/ |
22 | | |
23 | | |
24 | | #include "bitshuffle-sse2.h" |
25 | | #include "bitshuffle-generic.h" |
26 | | #include "shuffle-sse2.h" |
27 | | #include <string.h> |
28 | | #include <stdlib.h> |
29 | | |
30 | | /* Make sure SSE2 is available for the compilation target and compiler. */ |
31 | | #if defined(__SSE2__) |
32 | | |
33 | | #include <emmintrin.h> |
34 | | #include "bitshuffle-constant.h" |
35 | | |
36 | | /* The next is useful for debugging purposes */ |
37 | | #if 0 |
38 | | #include <stdio.h> |
39 | | #include <string.h> |
40 | | |
41 | | |
42 | | static void printxmm(__m128i xmm0) |
43 | | { |
44 | | uint8_t buf[32]; |
45 | | |
46 | | ((__m128i *)buf)[0] = xmm0; |
47 | | printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n", |
48 | | buf[0], buf[1], buf[2], buf[3], |
49 | | buf[4], buf[5], buf[6], buf[7], |
50 | | buf[8], buf[9], buf[10], buf[11], |
51 | | buf[12], buf[13], buf[14], buf[15]); |
52 | | } |
53 | | #endif |
54 | | |
55 | | |
56 | | /* ---- Worker code that requires SSE2. Intel Petium 4 (2000) and later. ---- */ |
57 | | |
58 | | |
59 | | /* Transpose bytes within elements for 16 bit elements. */ |
60 | 0 | int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) { |
61 | |
|
62 | 0 | size_t ii; |
63 | 0 | const char *in_b = (const char*) in; |
64 | 0 | char *out_b = (char*) out; |
65 | 0 | __m128i a0, b0, a1, b1; |
66 | |
|
67 | 0 | for (ii=0; ii + 15 < size; ii += 16) { |
68 | 0 | a0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 0*16]); |
69 | 0 | b0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 1*16]); |
70 | |
|
71 | 0 | a1 = _mm_unpacklo_epi8(a0, b0); |
72 | 0 | b1 = _mm_unpackhi_epi8(a0, b0); |
73 | |
|
74 | 0 | a0 = _mm_unpacklo_epi8(a1, b1); |
75 | 0 | b0 = _mm_unpackhi_epi8(a1, b1); |
76 | |
|
77 | 0 | a1 = _mm_unpacklo_epi8(a0, b0); |
78 | 0 | b1 = _mm_unpackhi_epi8(a0, b0); |
79 | |
|
80 | 0 | a0 = _mm_unpacklo_epi8(a1, b1); |
81 | 0 | b0 = _mm_unpackhi_epi8(a1, b1); |
82 | |
|
83 | 0 | _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); |
84 | 0 | _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); |
85 | 0 | } |
86 | 0 | return bshuf_trans_byte_elem_remainder(in, out, size, 2, |
87 | 0 | size - size % 16); |
88 | 0 | } |
89 | | |
90 | | |
91 | | /* Transpose bytes within elements for 32 bit elements. */ |
92 | 0 | int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) { |
93 | |
|
94 | 0 | size_t ii; |
95 | 0 | const char *in_b; |
96 | 0 | char *out_b; |
97 | 0 | in_b = (const char*) in; |
98 | 0 | out_b = (char*) out; |
99 | 0 | __m128i a0, b0, c0, d0, a1, b1, c1, d1; |
100 | |
|
101 | 0 | for (ii=0; ii + 15 < size; ii += 16) { |
102 | 0 | a0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 0*16]); |
103 | 0 | b0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 1*16]); |
104 | 0 | c0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 2*16]); |
105 | 0 | d0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 3*16]); |
106 | |
|
107 | 0 | a1 = _mm_unpacklo_epi8(a0, b0); |
108 | 0 | b1 = _mm_unpackhi_epi8(a0, b0); |
109 | 0 | c1 = _mm_unpacklo_epi8(c0, d0); |
110 | 0 | d1 = _mm_unpackhi_epi8(c0, d0); |
111 | |
|
112 | 0 | a0 = _mm_unpacklo_epi8(a1, b1); |
113 | 0 | b0 = _mm_unpackhi_epi8(a1, b1); |
114 | 0 | c0 = _mm_unpacklo_epi8(c1, d1); |
115 | 0 | d0 = _mm_unpackhi_epi8(c1, d1); |
116 | |
|
117 | 0 | a1 = _mm_unpacklo_epi8(a0, b0); |
118 | 0 | b1 = _mm_unpackhi_epi8(a0, b0); |
119 | 0 | c1 = _mm_unpacklo_epi8(c0, d0); |
120 | 0 | d1 = _mm_unpackhi_epi8(c0, d0); |
121 | |
|
122 | 0 | a0 = _mm_unpacklo_epi64(a1, c1); |
123 | 0 | b0 = _mm_unpackhi_epi64(a1, c1); |
124 | 0 | c0 = _mm_unpacklo_epi64(b1, d1); |
125 | 0 | d0 = _mm_unpackhi_epi64(b1, d1); |
126 | |
|
127 | 0 | _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); |
128 | 0 | _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); |
129 | 0 | _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); |
130 | 0 | _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); |
131 | 0 | } |
132 | 0 | return bshuf_trans_byte_elem_remainder(in, out, size, 4, |
133 | 0 | size - size % 16); |
134 | 0 | } |
135 | | |
136 | | |
137 | | /* Transpose bytes within elements for 64 bit elements. */ |
138 | 0 | int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) { |
139 | |
|
140 | 0 | size_t ii; |
141 | 0 | const char* in_b = (const char*) in; |
142 | 0 | char* out_b = (char*) out; |
143 | 0 | __m128i a0, b0, c0, d0, e0, f0, g0, h0; |
144 | 0 | __m128i a1, b1, c1, d1, e1, f1, g1, h1; |
145 | |
|
146 | 0 | for (ii=0; ii + 15 < size; ii += 16) { |
147 | 0 | a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]); |
148 | 0 | b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]); |
149 | 0 | c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]); |
150 | 0 | d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]); |
151 | 0 | e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]); |
152 | 0 | f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]); |
153 | 0 | g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]); |
154 | 0 | h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]); |
155 | |
|
156 | 0 | a1 = _mm_unpacklo_epi8(a0, b0); |
157 | 0 | b1 = _mm_unpackhi_epi8(a0, b0); |
158 | 0 | c1 = _mm_unpacklo_epi8(c0, d0); |
159 | 0 | d1 = _mm_unpackhi_epi8(c0, d0); |
160 | 0 | e1 = _mm_unpacklo_epi8(e0, f0); |
161 | 0 | f1 = _mm_unpackhi_epi8(e0, f0); |
162 | 0 | g1 = _mm_unpacklo_epi8(g0, h0); |
163 | 0 | h1 = _mm_unpackhi_epi8(g0, h0); |
164 | |
|
165 | 0 | a0 = _mm_unpacklo_epi8(a1, b1); |
166 | 0 | b0 = _mm_unpackhi_epi8(a1, b1); |
167 | 0 | c0 = _mm_unpacklo_epi8(c1, d1); |
168 | 0 | d0 = _mm_unpackhi_epi8(c1, d1); |
169 | 0 | e0 = _mm_unpacklo_epi8(e1, f1); |
170 | 0 | f0 = _mm_unpackhi_epi8(e1, f1); |
171 | 0 | g0 = _mm_unpacklo_epi8(g1, h1); |
172 | 0 | h0 = _mm_unpackhi_epi8(g1, h1); |
173 | |
|
174 | 0 | a1 = _mm_unpacklo_epi32(a0, c0); |
175 | 0 | b1 = _mm_unpackhi_epi32(a0, c0); |
176 | 0 | c1 = _mm_unpacklo_epi32(b0, d0); |
177 | 0 | d1 = _mm_unpackhi_epi32(b0, d0); |
178 | 0 | e1 = _mm_unpacklo_epi32(e0, g0); |
179 | 0 | f1 = _mm_unpackhi_epi32(e0, g0); |
180 | 0 | g1 = _mm_unpacklo_epi32(f0, h0); |
181 | 0 | h1 = _mm_unpackhi_epi32(f0, h0); |
182 | |
|
183 | 0 | a0 = _mm_unpacklo_epi64(a1, e1); |
184 | 0 | b0 = _mm_unpackhi_epi64(a1, e1); |
185 | 0 | c0 = _mm_unpacklo_epi64(b1, f1); |
186 | 0 | d0 = _mm_unpackhi_epi64(b1, f1); |
187 | 0 | e0 = _mm_unpacklo_epi64(c1, g1); |
188 | 0 | f0 = _mm_unpackhi_epi64(c1, g1); |
189 | 0 | g0 = _mm_unpacklo_epi64(d1, h1); |
190 | 0 | h0 = _mm_unpackhi_epi64(d1, h1); |
191 | |
|
192 | 0 | _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0); |
193 | 0 | _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0); |
194 | 0 | _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0); |
195 | 0 | _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0); |
196 | 0 | _mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0); |
197 | 0 | _mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0); |
198 | 0 | _mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0); |
199 | 0 | _mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0); |
200 | 0 | } |
201 | 0 | return bshuf_trans_byte_elem_remainder(in, out, size, 8, |
202 | 0 | size - size % 16); |
203 | 0 | } |
204 | | |
205 | | |
206 | | /* Transpose bytes within elements using the best SSE algorithm available. */ |
207 | | int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size, |
208 | 7.46k | const size_t elem_size, void* tmp_buf) { |
209 | | |
210 | 7.46k | int64_t count; |
211 | | |
212 | | // Trivial cases: power of 2 bytes. |
213 | 7.46k | switch (elem_size) { |
214 | 0 | case 1: |
215 | 0 | count = bshuf_copy(in, out, size, elem_size); |
216 | 0 | return count; |
217 | 0 | case 2: |
218 | 0 | count = bshuf_trans_byte_elem_SSE_16(in, out, size); |
219 | 0 | return count; |
220 | 0 | case 4: |
221 | 0 | count = bshuf_trans_byte_elem_SSE_32(in, out, size); |
222 | 0 | return count; |
223 | 0 | case 8: |
224 | 0 | count = bshuf_trans_byte_elem_SSE_64(in, out, size); |
225 | 0 | return count; |
226 | 7.46k | } |
227 | | |
228 | | // Worst case: odd number of bytes. Turns out that this is faster for |
229 | | // (odd * 2) byte elements as well (hence % 4). |
230 | 7.46k | if (elem_size % 4) { |
231 | 7.46k | count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); |
232 | 7.46k | return count; |
233 | 7.46k | } |
234 | | |
235 | | // Multiple of power of 2: transpose hierarchically. |
236 | 0 | { |
237 | 0 | size_t nchunk_elem; |
238 | |
|
239 | 0 | if ((elem_size % 8) == 0) { |
240 | 0 | nchunk_elem = elem_size / 8; |
241 | 0 | TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t); |
242 | 0 | count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf, |
243 | 0 | size * nchunk_elem); |
244 | 0 | bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size); |
245 | 0 | } else if ((elem_size % 4) == 0) { |
246 | 0 | nchunk_elem = elem_size / 4; |
247 | 0 | TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t); |
248 | 0 | count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf, |
249 | 0 | size * nchunk_elem); |
250 | 0 | bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size); |
251 | 0 | } else { |
252 | | // Not used since scalar algorithm is faster. |
253 | 0 | nchunk_elem = elem_size / 2; |
254 | 0 | TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t); |
255 | 0 | count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf, |
256 | 0 | size * nchunk_elem); |
257 | 0 | bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size); |
258 | 0 | } |
259 | |
|
260 | 0 | return count; |
261 | 7.46k | } |
262 | 7.46k | } |
263 | | |
264 | | |
265 | | /* Transpose bits within bytes. */ |
266 | | int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size, |
267 | 0 | const size_t elem_size) { |
268 | |
|
269 | 0 | size_t ii, kk; |
270 | 0 | const char* in_b = (const char*) in; |
271 | 0 | char* out_b = (char*) out; |
272 | |
|
273 | 0 | int64_t count; |
274 | |
|
275 | 0 | size_t nbyte = elem_size * size; |
276 | |
|
277 | 0 | CHECK_MULT_EIGHT(nbyte); |
278 | |
|
279 | 0 | __m128i xmm; |
280 | 0 | int32_t bt; |
281 | |
|
282 | 0 | ii = bshuf_emit_constant_prefix(in, out, nbyte); |
283 | 0 | if (ii == nbyte) return nbyte; |
284 | 0 | for (; ii + 15 < nbyte; ii += 16) { |
285 | 0 | xmm = _mm_loadu_si128((__m128i *) &in_b[ii]); |
286 | 0 | for (kk = 0; kk < 8; kk++) { |
287 | 0 | bt = _mm_movemask_epi8(xmm); |
288 | 0 | xmm = _mm_slli_epi16(xmm, 1); |
289 | 0 | uint16_t bits = (uint16_t)bt; |
290 | 0 | memcpy(out_b + ((7 - kk) * nbyte + ii) / 8, &bits, sizeof(bits)); |
291 | 0 | } |
292 | 0 | } |
293 | 0 | count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size, |
294 | 0 | nbyte - nbyte % 16); |
295 | 0 | return count; |
296 | 0 | } |
297 | | |
298 | | |
299 | | /* Transpose bits within elements. */ |
300 | | int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size, |
301 | 0 | const size_t elem_size, void* tmp_buf) { |
302 | |
|
303 | 0 | int64_t count; |
304 | |
|
305 | 0 | CHECK_MULT_EIGHT(size); |
306 | |
|
307 | 0 | if ((elem_size == 1 || elem_size == 2 || elem_size == 4 || elem_size == 8 || elem_size == 16) |
308 | 0 | && size <= INT32_MAX / elem_size) { |
309 | 0 | const uint8_t* bytes = (const uint8_t*)in; |
310 | 0 | if (elem_size != 1) { |
311 | 0 | shuffle_sse2((int32_t)elem_size, (int32_t)(size * elem_size), in, tmp_buf); |
312 | 0 | bytes = (const uint8_t*)tmp_buf; |
313 | 0 | } |
314 | | /* Bit rows of each byte plane are already in their final wire order. */ |
315 | 0 | for (size_t j = 0; j < elem_size; ++j) { |
316 | 0 | count = bshuf_trans_bit_byte_SSE(bytes + j * size, (uint8_t*)out + j * size, size, 1); |
317 | 0 | CHECK_ERR(count); |
318 | 0 | } |
319 | 0 | return size * elem_size; |
320 | 0 | } |
321 | 0 | count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size, tmp_buf); |
322 | 0 | CHECK_ERR(count); |
323 | 0 | count = bshuf_trans_bit_byte_SSE(out, tmp_buf, size, elem_size); |
324 | 0 | CHECK_ERR(count); |
325 | 0 | count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); |
326 | |
|
327 | 0 | return count; |
328 | 0 | } |
329 | | |
330 | | |
331 | | /* For data organized into a row for each bit (8 * elem_size rows), transpose |
332 | | * the bytes. */ |
333 | | static inline __m128i trans_bit_8x8_SSE(__m128i x); |
334 | | |
335 | | /* transpose_bits is only true for a single byte plane (elem_size == 1). */ |
336 | | static int64_t trans_byte_bitrow_SSE(const void* in, void* out, const size_t size, |
337 | 110k | const size_t elem_size, const bool transpose_bits) { |
338 | | |
339 | 110k | size_t ii, jj; |
340 | 110k | const char* in_b = (const char*) in; |
341 | 110k | char* out_b = (char*) out; |
342 | | |
343 | 110k | CHECK_MULT_EIGHT(size); |
344 | | |
345 | 110k | size_t nrows = 8 * elem_size; |
346 | 110k | size_t nbyte_row = size / 8; |
347 | | |
348 | 110k | __m128i a0, b0, c0, d0, e0, f0, g0, h0; |
349 | 110k | __m128i a1, b1, c1, d1, e1, f1, g1, h1; |
350 | 110k | __m128 *as, *bs, *cs, *ds, *es, *fs, *gs, *hs; |
351 | | |
352 | 271k | for (ii = 0; ii + 7 < nrows; ii += 8) { |
353 | 30.9M | for (jj = 0; jj + 15 < nbyte_row; jj += 16) { |
354 | 30.7M | a0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 0)*nbyte_row + jj]); |
355 | 30.7M | b0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 1)*nbyte_row + jj]); |
356 | 30.7M | c0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 2)*nbyte_row + jj]); |
357 | 30.7M | d0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 3)*nbyte_row + jj]); |
358 | 30.7M | e0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 4)*nbyte_row + jj]); |
359 | 30.7M | f0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 5)*nbyte_row + jj]); |
360 | 30.7M | g0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 6)*nbyte_row + jj]); |
361 | 30.7M | h0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 7)*nbyte_row + jj]); |
362 | | |
363 | | |
364 | 30.7M | a1 = _mm_unpacklo_epi8(a0, b0); |
365 | 30.7M | b1 = _mm_unpacklo_epi8(c0, d0); |
366 | 30.7M | c1 = _mm_unpacklo_epi8(e0, f0); |
367 | 30.7M | d1 = _mm_unpacklo_epi8(g0, h0); |
368 | 30.7M | e1 = _mm_unpackhi_epi8(a0, b0); |
369 | 30.7M | f1 = _mm_unpackhi_epi8(c0, d0); |
370 | 30.7M | g1 = _mm_unpackhi_epi8(e0, f0); |
371 | 30.7M | h1 = _mm_unpackhi_epi8(g0, h0); |
372 | | |
373 | | |
374 | 30.7M | a0 = _mm_unpacklo_epi16(a1, b1); |
375 | 30.7M | b0 = _mm_unpacklo_epi16(c1, d1); |
376 | 30.7M | c0 = _mm_unpackhi_epi16(a1, b1); |
377 | 30.7M | d0 = _mm_unpackhi_epi16(c1, d1); |
378 | | |
379 | 30.7M | e0 = _mm_unpacklo_epi16(e1, f1); |
380 | 30.7M | f0 = _mm_unpacklo_epi16(g1, h1); |
381 | 30.7M | g0 = _mm_unpackhi_epi16(e1, f1); |
382 | 30.7M | h0 = _mm_unpackhi_epi16(g1, h1); |
383 | | |
384 | | |
385 | 30.7M | a1 = _mm_unpacklo_epi32(a0, b0); |
386 | 30.7M | b1 = _mm_unpackhi_epi32(a0, b0); |
387 | | |
388 | 30.7M | c1 = _mm_unpacklo_epi32(c0, d0); |
389 | 30.7M | d1 = _mm_unpackhi_epi32(c0, d0); |
390 | | |
391 | 30.7M | e1 = _mm_unpacklo_epi32(e0, f0); |
392 | 30.7M | f1 = _mm_unpackhi_epi32(e0, f0); |
393 | | |
394 | 30.7M | g1 = _mm_unpacklo_epi32(g0, h0); |
395 | 30.7M | h1 = _mm_unpackhi_epi32(g0, h0); |
396 | | |
397 | 30.7M | if (transpose_bits) { |
398 | 109M | #define STORE_BYTES(v, k) _mm_storeu_si128((__m128i*)(out_b + (jj + (k)) * 8), trans_bit_8x8_SSE(v)) |
399 | 13.7M | STORE_BYTES(a1, 0); STORE_BYTES(b1, 2); |
400 | 13.7M | STORE_BYTES(c1, 4); STORE_BYTES(d1, 6); |
401 | 13.7M | STORE_BYTES(e1, 8); STORE_BYTES(f1, 10); |
402 | 13.7M | STORE_BYTES(g1, 12); STORE_BYTES(h1, 14); |
403 | 13.7M | #undef STORE_BYTES |
404 | 13.7M | continue; |
405 | 13.7M | } |
406 | | // We don't have a storeh instruction for integers, so interpret |
407 | | // as a float. Have a storel (_mm_storel_epi64). |
408 | 17.0M | as = (__m128 *) &a1; |
409 | 17.0M | bs = (__m128 *) &b1; |
410 | 17.0M | cs = (__m128 *) &c1; |
411 | 17.0M | ds = (__m128 *) &d1; |
412 | 17.0M | es = (__m128 *) &e1; |
413 | 17.0M | fs = (__m128 *) &f1; |
414 | 17.0M | gs = (__m128 *) &g1; |
415 | 17.0M | hs = (__m128 *) &h1; |
416 | | |
417 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 0) * nrows + ii], *as); |
418 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 2) * nrows + ii], *bs); |
419 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 4) * nrows + ii], *cs); |
420 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 6) * nrows + ii], *ds); |
421 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 8) * nrows + ii], *es); |
422 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 10) * nrows + ii], *fs); |
423 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 12) * nrows + ii], *gs); |
424 | 17.0M | _mm_storel_pi((__m64 *) &out_b[(jj + 14) * nrows + ii], *hs); |
425 | | |
426 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 1) * nrows + ii], *as); |
427 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 3) * nrows + ii], *bs); |
428 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 5) * nrows + ii], *cs); |
429 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 7) * nrows + ii], *ds); |
430 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 9) * nrows + ii], *es); |
431 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 11) * nrows + ii], *fs); |
432 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 13) * nrows + ii], *gs); |
433 | 17.0M | _mm_storeh_pi((__m64 *) &out_b[(jj + 15) * nrows + ii], *hs); |
434 | 17.0M | } |
435 | 555k | for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) { |
436 | 394k | out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj]; |
437 | 394k | out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj]; |
438 | 394k | out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj]; |
439 | 394k | out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj]; |
440 | 394k | out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj]; |
441 | 394k | out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj]; |
442 | 394k | out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj]; |
443 | 394k | out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj]; |
444 | 394k | if (transpose_bits) { |
445 | 174k | uint64_t x, t; |
446 | 174k | memcpy(&x, out_b + jj * 8, 8); |
447 | 174k | TRANS_BIT_8X8(x, t); |
448 | 174k | memcpy(out_b + jj * 8, &x, 8); |
449 | 174k | } |
450 | 394k | } |
451 | 160k | } |
452 | 110k | return size * elem_size; |
453 | 110k | } |
454 | | |
455 | | int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size, |
456 | 7.46k | const size_t elem_size) { |
457 | 7.46k | return trans_byte_bitrow_SSE(in, out, size, elem_size, false); |
458 | 7.46k | } |
459 | | |
460 | | /* Decode one byte plane without materializing the transposed bit rows. */ |
461 | 103k | int64_t bshuf_untrans_bit_byte_SSE(const void* in, void* out, const size_t size) { |
462 | 103k | return trans_byte_bitrow_SSE(in, out, size, 1, true); |
463 | 103k | } |
464 | | |
465 | | /* Transpose the 8x8 bit matrix in each 64-bit lane (as TRANS_BIT_8X8). */ |
466 | 109M | static inline __m128i trans_bit_8x8_SSE(__m128i x) { |
467 | 109M | __m128i t; |
468 | 109M | t = _mm_and_si128(_mm_xor_si128(x, _mm_srli_epi64(x, 7)), |
469 | 109M | _mm_set1_epi64x(0x00AA00AA00AA00AALL)); |
470 | 109M | x = _mm_xor_si128(x, _mm_xor_si128(t, _mm_slli_epi64(t, 7))); |
471 | 109M | t = _mm_and_si128(_mm_xor_si128(x, _mm_srli_epi64(x, 14)), |
472 | 109M | _mm_set1_epi64x(0x0000CCCC0000CCCCLL)); |
473 | 109M | x = _mm_xor_si128(x, _mm_xor_si128(t, _mm_slli_epi64(t, 14))); |
474 | 109M | t = _mm_and_si128(_mm_xor_si128(x, _mm_srli_epi64(x, 28)), |
475 | 109M | _mm_set1_epi64x(0x00000000F0F0F0F0LL)); |
476 | 109M | x = _mm_xor_si128(x, _mm_xor_si128(t, _mm_slli_epi64(t, 28))); |
477 | 109M | return x; |
478 | 109M | } |
479 | | |
480 | | |
481 | | /* Shuffle bits within the bytes of eight element blocks. */ |
482 | | int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size, |
483 | 7.46k | const size_t elem_size) { |
484 | | |
485 | 7.46k | CHECK_MULT_EIGHT(size); |
486 | | |
487 | | // With a bit of care, this could be written such that such that it is |
488 | | // in_buf = out_buf safe. |
489 | 7.46k | const char* in_b = (const char*) in; |
490 | 7.46k | char* out_b = (char*) out; |
491 | 7.46k | uint16_t* out_ui16 = (uint16_t*) out; |
492 | | |
493 | 7.46k | size_t ii, jj, kk; |
494 | 7.46k | size_t nbyte = elem_size * size; |
495 | | |
496 | 7.46k | __m128i xmm; |
497 | 7.46k | int32_t bt; |
498 | | |
499 | 7.46k | if (elem_size <= 2) { |
500 | | // Each 8-byte word holds the 8 bit rows of 8 one-byte elements, or of |
501 | | // the low or the high bytes of 8 two-byte elements: transposing its |
502 | | // bits gives those bytes. |
503 | 0 | for (ii = 0; ii + 15 < nbyte; ii += 16) { |
504 | 0 | xmm = trans_bit_8x8_SSE(_mm_loadu_si128((__m128i *) &in_b[ii])); |
505 | 0 | if (elem_size == 2) { |
506 | | // interleave the low bytes (first word) and high bytes (second word) |
507 | 0 | xmm = _mm_unpacklo_epi8(xmm, _mm_unpackhi_epi64(xmm, xmm)); |
508 | 0 | } |
509 | 0 | _mm_storeu_si128((__m128i *) &out_b[ii], xmm); |
510 | 0 | } |
511 | 0 | if (ii < nbyte) { |
512 | 0 | bshuf_shuffle_bit_eightelem_scal(&in_b[ii], &out_b[ii], |
513 | 0 | (nbyte - ii) / elem_size, elem_size); |
514 | 0 | } |
515 | 7.46k | } else if (elem_size % 2) { |
516 | 5.47k | bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size); |
517 | 5.47k | } else { |
518 | 9.81M | for (ii = 0; ii + 8 * elem_size - 1 < nbyte; |
519 | 9.81M | ii += 8 * elem_size) { |
520 | 94.4M | for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) { |
521 | 84.5M | xmm = _mm_loadu_si128((__m128i *) &in_b[ii + jj]); |
522 | 761M | for (kk = 0; kk < 8; kk++) { |
523 | 676M | bt = _mm_movemask_epi8(xmm); |
524 | 676M | xmm = _mm_slli_epi16(xmm, 1); |
525 | 676M | size_t ind = (ii + jj / 8 + (7 - kk) * elem_size); |
526 | 676M | out_ui16[ind / 2] = bt; |
527 | 676M | } |
528 | 84.5M | } |
529 | 9.81M | } |
530 | 1.99k | } |
531 | 7.46k | return size * elem_size; |
532 | 7.46k | } |
533 | | |
534 | | |
535 | | /* Untranspose bits within elements. */ |
536 | | int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size, |
537 | 0 | const size_t elem_size, void* tmp_buf) { |
538 | |
|
539 | 0 | int64_t count; |
540 | |
|
541 | 0 | CHECK_MULT_EIGHT(size); |
542 | |
|
543 | 0 | if (elem_size == 1) return bshuf_untrans_bit_byte_SSE(in, out, size); |
544 | 0 | if ((elem_size == 2 || elem_size == 4 || elem_size == 8 || elem_size == 16) |
545 | 0 | && size <= INT32_MAX / elem_size) { |
546 | 0 | for (size_t j = 0; j < elem_size; ++j) { |
547 | 0 | bshuf_untrans_bit_byte_SSE((const uint8_t*)in + j * size, (uint8_t*)tmp_buf + j * size, size); |
548 | 0 | } |
549 | 0 | unshuffle_sse2((int32_t)elem_size, (int32_t)(size * elem_size), tmp_buf, out); |
550 | 0 | return size * elem_size; |
551 | 0 | } |
552 | 0 | count = bshuf_trans_byte_bitrow_SSE(in, tmp_buf, size, elem_size); |
553 | 0 | CHECK_ERR(count); |
554 | 0 | count = bshuf_shuffle_bit_eightelem_SSE(tmp_buf, out, size, elem_size); |
555 | |
|
556 | 0 | return count; |
557 | 0 | } |
558 | | |
559 | | const bool is_bshuf_SSE = true; |
560 | | |
561 | | #else /* defined(__SSE2__) */ |
562 | | |
563 | | const bool is_bshuf_SSE = false; |
564 | | |
565 | | int64_t bshuf_untrans_bit_byte_SSE(const void* in, void* out, const size_t size) { |
566 | | abort(); |
567 | | } |
568 | | |
569 | | int64_t |
570 | | bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size, |
571 | | const size_t elem_size, void* tmp_buf) { |
572 | | abort(); |
573 | | } |
574 | | |
575 | | int64_t |
576 | | bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size, |
577 | | const size_t elem_size, void* tmp_buf) { |
578 | | abort(); |
579 | | } |
580 | | |
581 | | #endif /* defined(__SSE2__) */ |