Coverage Report

Created: 2026-11-19 07:17

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/c-blosc2/blosc/bitshuffle-sse2.c
Line
Count
Source
1
/*********************************************************************
2
  Blosc - Blocked Shuffling and Compression Library
3
4
  Copyright (c) 2021  Blosc Development Team <blosc@blosc.org>
5
  https://blosc.org
6
  License: BSD 3-Clause (see LICENSE.txt)
7
8
  See LICENSE.txt for details about copyright and rights to use.
9
**********************************************************************/
10
11
/*********************************************************************
12
  Bitshuffle - Filter for improving compression of typed binary data.
13
14
  Author: Kiyoshi Masui <kiyo@physics.ubc.ca>
15
  Website: https://github.com/kiyo-masui/bitshuffle
16
17
  Note: Adapted for c-blosc by Francesc Alted.
18
19
  See LICENSES/BITSHUFFLE.txt file for details about copyright and
20
  rights to use.
21
**********************************************************************/
22
23
24
#include "bitshuffle-sse2.h"
25
#include "bitshuffle-generic.h"
26
#include "shuffle-sse2.h"
27
#include <string.h>
28
#include <stdlib.h>
29
30
/* Make sure SSE2 is available for the compilation target and compiler. */
31
#if defined(__SSE2__)
32
33
#include <emmintrin.h>
34
#include "bitshuffle-constant.h"
35
36
/* The next is useful for debugging purposes */
37
#if 0
38
#include <stdio.h>
39
#include <string.h>
40
41
42
static void printxmm(__m128i xmm0)
43
{
44
  uint8_t buf[32];
45
46
  ((__m128i *)buf)[0] = xmm0;
47
  printf("%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x,%x\n",
48
          buf[0], buf[1], buf[2], buf[3],
49
          buf[4], buf[5], buf[6], buf[7],
50
          buf[8], buf[9], buf[10], buf[11],
51
          buf[12], buf[13], buf[14], buf[15]);
52
}
53
#endif
54
55
56
/* ---- Worker code that requires SSE2. Intel Petium 4 (2000) and later. ---- */
57
58
59
/* Transpose bytes within elements for 16 bit elements. */
60
0
int64_t bshuf_trans_byte_elem_SSE_16(const void* in, void* out, const size_t size) {
61
62
0
  size_t ii;
63
0
  const char *in_b = (const char*) in;
64
0
  char *out_b = (char*) out;
65
0
  __m128i a0, b0, a1, b1;
66
67
0
  for (ii=0; ii + 15 < size; ii += 16) {
68
0
    a0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 0*16]);
69
0
    b0 = _mm_loadu_si128((__m128i *) &in_b[2*ii + 1*16]);
70
71
0
    a1 = _mm_unpacklo_epi8(a0, b0);
72
0
    b1 = _mm_unpackhi_epi8(a0, b0);
73
74
0
    a0 = _mm_unpacklo_epi8(a1, b1);
75
0
    b0 = _mm_unpackhi_epi8(a1, b1);
76
77
0
    a1 = _mm_unpacklo_epi8(a0, b0);
78
0
    b1 = _mm_unpackhi_epi8(a0, b0);
79
80
0
    a0 = _mm_unpacklo_epi8(a1, b1);
81
0
    b0 = _mm_unpackhi_epi8(a1, b1);
82
83
0
    _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
84
0
    _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
85
0
  }
86
0
  return bshuf_trans_byte_elem_remainder(in, out, size, 2,
87
0
                                         size - size % 16);
88
0
}
89
90
91
/* Transpose bytes within elements for 32 bit elements. */
92
0
int64_t bshuf_trans_byte_elem_SSE_32(const void* in, void* out, const size_t size) {
93
94
0
  size_t ii;
95
0
  const char *in_b;
96
0
  char *out_b;
97
0
  in_b = (const char*) in;
98
0
  out_b = (char*) out;
99
0
  __m128i a0, b0, c0, d0, a1, b1, c1, d1;
100
101
0
  for (ii=0; ii + 15 < size; ii += 16) {
102
0
    a0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 0*16]);
103
0
    b0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 1*16]);
104
0
    c0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 2*16]);
105
0
    d0 = _mm_loadu_si128((__m128i *) &in_b[4*ii + 3*16]);
106
107
0
    a1 = _mm_unpacklo_epi8(a0, b0);
108
0
    b1 = _mm_unpackhi_epi8(a0, b0);
109
0
    c1 = _mm_unpacklo_epi8(c0, d0);
110
0
    d1 = _mm_unpackhi_epi8(c0, d0);
111
112
0
    a0 = _mm_unpacklo_epi8(a1, b1);
113
0
    b0 = _mm_unpackhi_epi8(a1, b1);
114
0
    c0 = _mm_unpacklo_epi8(c1, d1);
115
0
    d0 = _mm_unpackhi_epi8(c1, d1);
116
117
0
    a1 = _mm_unpacklo_epi8(a0, b0);
118
0
    b1 = _mm_unpackhi_epi8(a0, b0);
119
0
    c1 = _mm_unpacklo_epi8(c0, d0);
120
0
    d1 = _mm_unpackhi_epi8(c0, d0);
121
122
0
    a0 = _mm_unpacklo_epi64(a1, c1);
123
0
    b0 = _mm_unpackhi_epi64(a1, c1);
124
0
    c0 = _mm_unpacklo_epi64(b1, d1);
125
0
    d0 = _mm_unpackhi_epi64(b1, d1);
126
127
0
    _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
128
0
    _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
129
0
    _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0);
130
0
    _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0);
131
0
  }
132
0
  return bshuf_trans_byte_elem_remainder(in, out, size, 4,
133
0
                                         size - size % 16);
134
0
}
135
136
137
/* Transpose bytes within elements for 64 bit elements. */
138
0
int64_t bshuf_trans_byte_elem_SSE_64(const void* in, void* out, const size_t size) {
139
140
0
  size_t ii;
141
0
  const char* in_b = (const char*) in;
142
0
  char* out_b = (char*) out;
143
0
  __m128i a0, b0, c0, d0, e0, f0, g0, h0;
144
0
  __m128i a1, b1, c1, d1, e1, f1, g1, h1;
145
146
0
  for (ii=0; ii + 15 < size; ii += 16) {
147
0
    a0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 0*16]);
148
0
    b0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 1*16]);
149
0
    c0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 2*16]);
150
0
    d0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 3*16]);
151
0
    e0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 4*16]);
152
0
    f0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 5*16]);
153
0
    g0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 6*16]);
154
0
    h0 = _mm_loadu_si128((__m128i *) &in_b[8*ii + 7*16]);
155
156
0
    a1 = _mm_unpacklo_epi8(a0, b0);
157
0
    b1 = _mm_unpackhi_epi8(a0, b0);
158
0
    c1 = _mm_unpacklo_epi8(c0, d0);
159
0
    d1 = _mm_unpackhi_epi8(c0, d0);
160
0
    e1 = _mm_unpacklo_epi8(e0, f0);
161
0
    f1 = _mm_unpackhi_epi8(e0, f0);
162
0
    g1 = _mm_unpacklo_epi8(g0, h0);
163
0
    h1 = _mm_unpackhi_epi8(g0, h0);
164
165
0
    a0 = _mm_unpacklo_epi8(a1, b1);
166
0
    b0 = _mm_unpackhi_epi8(a1, b1);
167
0
    c0 = _mm_unpacklo_epi8(c1, d1);
168
0
    d0 = _mm_unpackhi_epi8(c1, d1);
169
0
    e0 = _mm_unpacklo_epi8(e1, f1);
170
0
    f0 = _mm_unpackhi_epi8(e1, f1);
171
0
    g0 = _mm_unpacklo_epi8(g1, h1);
172
0
    h0 = _mm_unpackhi_epi8(g1, h1);
173
174
0
    a1 = _mm_unpacklo_epi32(a0, c0);
175
0
    b1 = _mm_unpackhi_epi32(a0, c0);
176
0
    c1 = _mm_unpacklo_epi32(b0, d0);
177
0
    d1 = _mm_unpackhi_epi32(b0, d0);
178
0
    e1 = _mm_unpacklo_epi32(e0, g0);
179
0
    f1 = _mm_unpackhi_epi32(e0, g0);
180
0
    g1 = _mm_unpacklo_epi32(f0, h0);
181
0
    h1 = _mm_unpackhi_epi32(f0, h0);
182
183
0
    a0 = _mm_unpacklo_epi64(a1, e1);
184
0
    b0 = _mm_unpackhi_epi64(a1, e1);
185
0
    c0 = _mm_unpacklo_epi64(b1, f1);
186
0
    d0 = _mm_unpackhi_epi64(b1, f1);
187
0
    e0 = _mm_unpacklo_epi64(c1, g1);
188
0
    f0 = _mm_unpackhi_epi64(c1, g1);
189
0
    g0 = _mm_unpacklo_epi64(d1, h1);
190
0
    h0 = _mm_unpackhi_epi64(d1, h1);
191
192
0
    _mm_storeu_si128((__m128i *) &out_b[0*size + ii], a0);
193
0
    _mm_storeu_si128((__m128i *) &out_b[1*size + ii], b0);
194
0
    _mm_storeu_si128((__m128i *) &out_b[2*size + ii], c0);
195
0
    _mm_storeu_si128((__m128i *) &out_b[3*size + ii], d0);
196
0
    _mm_storeu_si128((__m128i *) &out_b[4*size + ii], e0);
197
0
    _mm_storeu_si128((__m128i *) &out_b[5*size + ii], f0);
198
0
    _mm_storeu_si128((__m128i *) &out_b[6*size + ii], g0);
199
0
    _mm_storeu_si128((__m128i *) &out_b[7*size + ii], h0);
200
0
  }
201
0
  return bshuf_trans_byte_elem_remainder(in, out, size, 8,
202
0
                                         size - size % 16);
203
0
}
204
205
206
/* Transpose bytes within elements using the best SSE algorithm available. */
207
int64_t bshuf_trans_byte_elem_SSE(const void* in, void* out, const size_t size,
208
7.46k
                                  const size_t elem_size, void* tmp_buf) {
209
210
7.46k
  int64_t count;
211
212
  // Trivial cases: power of 2 bytes.
213
7.46k
  switch (elem_size) {
214
0
    case 1:
215
0
      count = bshuf_copy(in, out, size, elem_size);
216
0
      return count;
217
0
    case 2:
218
0
      count = bshuf_trans_byte_elem_SSE_16(in, out, size);
219
0
      return count;
220
0
    case 4:
221
0
      count = bshuf_trans_byte_elem_SSE_32(in, out, size);
222
0
      return count;
223
0
    case 8:
224
0
      count = bshuf_trans_byte_elem_SSE_64(in, out, size);
225
0
      return count;
226
7.46k
  }
227
228
  // Worst case: odd number of bytes. Turns out that this is faster for
229
  // (odd * 2) byte elements as well (hence % 4).
230
7.46k
  if (elem_size % 4) {
231
7.46k
    count = bshuf_trans_byte_elem_scal(in, out, size, elem_size);
232
7.46k
    return count;
233
7.46k
  }
234
235
  // Multiple of power of 2: transpose hierarchically.
236
0
  {
237
0
    size_t nchunk_elem;
238
239
0
    if ((elem_size % 8) == 0) {
240
0
      nchunk_elem = elem_size / 8;
241
0
      TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int64_t);
242
0
      count = bshuf_trans_byte_elem_SSE_64(out, tmp_buf,
243
0
                                           size * nchunk_elem);
244
0
      bshuf_trans_elem(tmp_buf, out, 8, nchunk_elem, size);
245
0
    } else if ((elem_size % 4) == 0) {
246
0
      nchunk_elem = elem_size / 4;
247
0
      TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int32_t);
248
0
      count = bshuf_trans_byte_elem_SSE_32(out, tmp_buf,
249
0
                                           size * nchunk_elem);
250
0
      bshuf_trans_elem(tmp_buf, out, 4, nchunk_elem, size);
251
0
    } else {
252
      // Not used since scalar algorithm is faster.
253
0
      nchunk_elem = elem_size / 2;
254
0
      TRANS_ELEM_TYPE(in, out, size, nchunk_elem, int16_t);
255
0
      count = bshuf_trans_byte_elem_SSE_16(out, tmp_buf,
256
0
                                           size * nchunk_elem);
257
0
      bshuf_trans_elem(tmp_buf, out, 2, nchunk_elem, size);
258
0
    }
259
260
0
    return count;
261
7.46k
  }
262
7.46k
}
263
264
265
/* Transpose bits within bytes. */
266
int64_t bshuf_trans_bit_byte_SSE(const void* in, void* out, const size_t size,
267
0
                                 const size_t elem_size) {
268
269
0
  size_t ii, kk;
270
0
  const char* in_b = (const char*) in;
271
0
  char* out_b = (char*) out;
272
273
0
  int64_t count;
274
275
0
  size_t nbyte = elem_size * size;
276
277
0
  CHECK_MULT_EIGHT(nbyte);
278
279
0
  __m128i xmm;
280
0
  int32_t bt;
281
282
0
  ii = bshuf_emit_constant_prefix(in, out, nbyte);
283
0
  if (ii == nbyte) return nbyte;
284
0
  for (; ii + 15 < nbyte; ii += 16) {
285
0
    xmm = _mm_loadu_si128((__m128i *) &in_b[ii]);
286
0
    for (kk = 0; kk < 8; kk++) {
287
0
      bt = _mm_movemask_epi8(xmm);
288
0
      xmm = _mm_slli_epi16(xmm, 1);
289
0
      uint16_t bits = (uint16_t)bt;
290
0
      memcpy(out_b + ((7 - kk) * nbyte + ii) / 8, &bits, sizeof(bits));
291
0
    }
292
0
  }
293
0
  count = bshuf_trans_bit_byte_remainder(in, out, size, elem_size,
294
0
                                         nbyte - nbyte % 16);
295
0
  return count;
296
0
}
297
298
299
/* Transpose bits within elements. */
300
int64_t bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size,
301
0
                                 const size_t elem_size, void* tmp_buf) {
302
303
0
  int64_t count;
304
305
0
  CHECK_MULT_EIGHT(size);
306
307
0
  if ((elem_size == 1 || elem_size == 2 || elem_size == 4 || elem_size == 8 || elem_size == 16)
308
0
      && size <= INT32_MAX / elem_size) {
309
0
    const uint8_t* bytes = (const uint8_t*)in;
310
0
    if (elem_size != 1) {
311
0
      shuffle_sse2((int32_t)elem_size, (int32_t)(size * elem_size), in, tmp_buf);
312
0
      bytes = (const uint8_t*)tmp_buf;
313
0
    }
314
    /* Bit rows of each byte plane are already in their final wire order. */
315
0
    for (size_t j = 0; j < elem_size; ++j) {
316
0
      count = bshuf_trans_bit_byte_SSE(bytes + j * size, (uint8_t*)out + j * size, size, 1);
317
0
      CHECK_ERR(count);
318
0
    }
319
0
    return size * elem_size;
320
0
  }
321
0
  count = bshuf_trans_byte_elem_SSE(in, out, size, elem_size, tmp_buf);
322
0
  CHECK_ERR(count);
323
0
  count = bshuf_trans_bit_byte_SSE(out, tmp_buf, size, elem_size);
324
0
  CHECK_ERR(count);
325
0
  count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size);
326
327
0
  return count;
328
0
}
329
330
331
/* For data organized into a row for each bit (8 * elem_size rows), transpose
332
 * the bytes. */
333
static inline __m128i trans_bit_8x8_SSE(__m128i x);
334
335
/* transpose_bits is only true for a single byte plane (elem_size == 1). */
336
static int64_t trans_byte_bitrow_SSE(const void* in, void* out, const size_t size,
337
110k
                                     const size_t elem_size, const bool transpose_bits) {
338
339
110k
  size_t ii, jj;
340
110k
  const char* in_b = (const char*) in;
341
110k
  char* out_b = (char*) out;
342
343
110k
  CHECK_MULT_EIGHT(size);
344
345
110k
  size_t nrows = 8 * elem_size;
346
110k
  size_t nbyte_row = size / 8;
347
348
110k
  __m128i a0, b0, c0, d0, e0, f0, g0, h0;
349
110k
  __m128i a1, b1, c1, d1, e1, f1, g1, h1;
350
110k
  __m128 *as, *bs, *cs, *ds, *es, *fs, *gs, *hs;
351
352
271k
  for (ii = 0; ii + 7 < nrows; ii += 8) {
353
30.9M
    for (jj = 0; jj + 15 < nbyte_row; jj += 16) {
354
30.7M
      a0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 0)*nbyte_row + jj]);
355
30.7M
      b0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 1)*nbyte_row + jj]);
356
30.7M
      c0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 2)*nbyte_row + jj]);
357
30.7M
      d0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 3)*nbyte_row + jj]);
358
30.7M
      e0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 4)*nbyte_row + jj]);
359
30.7M
      f0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 5)*nbyte_row + jj]);
360
30.7M
      g0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 6)*nbyte_row + jj]);
361
30.7M
      h0 = _mm_loadu_si128((__m128i *) &in_b[(ii + 7)*nbyte_row + jj]);
362
363
364
30.7M
      a1 = _mm_unpacklo_epi8(a0, b0);
365
30.7M
      b1 = _mm_unpacklo_epi8(c0, d0);
366
30.7M
      c1 = _mm_unpacklo_epi8(e0, f0);
367
30.7M
      d1 = _mm_unpacklo_epi8(g0, h0);
368
30.7M
      e1 = _mm_unpackhi_epi8(a0, b0);
369
30.7M
      f1 = _mm_unpackhi_epi8(c0, d0);
370
30.7M
      g1 = _mm_unpackhi_epi8(e0, f0);
371
30.7M
      h1 = _mm_unpackhi_epi8(g0, h0);
372
373
374
30.7M
      a0 = _mm_unpacklo_epi16(a1, b1);
375
30.7M
      b0 = _mm_unpacklo_epi16(c1, d1);
376
30.7M
      c0 = _mm_unpackhi_epi16(a1, b1);
377
30.7M
      d0 = _mm_unpackhi_epi16(c1, d1);
378
379
30.7M
      e0 = _mm_unpacklo_epi16(e1, f1);
380
30.7M
      f0 = _mm_unpacklo_epi16(g1, h1);
381
30.7M
      g0 = _mm_unpackhi_epi16(e1, f1);
382
30.7M
      h0 = _mm_unpackhi_epi16(g1, h1);
383
384
385
30.7M
      a1 = _mm_unpacklo_epi32(a0, b0);
386
30.7M
      b1 = _mm_unpackhi_epi32(a0, b0);
387
388
30.7M
      c1 = _mm_unpacklo_epi32(c0, d0);
389
30.7M
      d1 = _mm_unpackhi_epi32(c0, d0);
390
391
30.7M
      e1 = _mm_unpacklo_epi32(e0, f0);
392
30.7M
      f1 = _mm_unpackhi_epi32(e0, f0);
393
394
30.7M
      g1 = _mm_unpacklo_epi32(g0, h0);
395
30.7M
      h1 = _mm_unpackhi_epi32(g0, h0);
396
397
30.7M
      if (transpose_bits) {
398
109M
#define STORE_BYTES(v, k) _mm_storeu_si128((__m128i*)(out_b + (jj + (k)) * 8), trans_bit_8x8_SSE(v))
399
13.7M
        STORE_BYTES(a1, 0); STORE_BYTES(b1, 2);
400
13.7M
        STORE_BYTES(c1, 4); STORE_BYTES(d1, 6);
401
13.7M
        STORE_BYTES(e1, 8); STORE_BYTES(f1, 10);
402
13.7M
        STORE_BYTES(g1, 12); STORE_BYTES(h1, 14);
403
13.7M
#undef STORE_BYTES
404
13.7M
        continue;
405
13.7M
      }
406
      // We don't have a storeh instruction for integers, so interpret
407
      // as a float. Have a storel (_mm_storel_epi64).
408
17.0M
      as = (__m128 *) &a1;
409
17.0M
      bs = (__m128 *) &b1;
410
17.0M
      cs = (__m128 *) &c1;
411
17.0M
      ds = (__m128 *) &d1;
412
17.0M
      es = (__m128 *) &e1;
413
17.0M
      fs = (__m128 *) &f1;
414
17.0M
      gs = (__m128 *) &g1;
415
17.0M
      hs = (__m128 *) &h1;
416
417
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 0) * nrows + ii], *as);
418
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 2) * nrows + ii], *bs);
419
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 4) * nrows + ii], *cs);
420
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 6) * nrows + ii], *ds);
421
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 8) * nrows + ii], *es);
422
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 10) * nrows + ii], *fs);
423
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 12) * nrows + ii], *gs);
424
17.0M
      _mm_storel_pi((__m64 *) &out_b[(jj + 14) * nrows + ii], *hs);
425
426
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 1) * nrows + ii], *as);
427
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 3) * nrows + ii], *bs);
428
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 5) * nrows + ii], *cs);
429
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 7) * nrows + ii], *ds);
430
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 9) * nrows + ii], *es);
431
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 11) * nrows + ii], *fs);
432
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 13) * nrows + ii], *gs);
433
17.0M
      _mm_storeh_pi((__m64 *) &out_b[(jj + 15) * nrows + ii], *hs);
434
17.0M
    }
435
555k
    for (jj = nbyte_row - nbyte_row % 16; jj < nbyte_row; jj ++) {
436
394k
      out_b[jj * nrows + ii + 0] = in_b[(ii + 0)*nbyte_row + jj];
437
394k
      out_b[jj * nrows + ii + 1] = in_b[(ii + 1)*nbyte_row + jj];
438
394k
      out_b[jj * nrows + ii + 2] = in_b[(ii + 2)*nbyte_row + jj];
439
394k
      out_b[jj * nrows + ii + 3] = in_b[(ii + 3)*nbyte_row + jj];
440
394k
      out_b[jj * nrows + ii + 4] = in_b[(ii + 4)*nbyte_row + jj];
441
394k
      out_b[jj * nrows + ii + 5] = in_b[(ii + 5)*nbyte_row + jj];
442
394k
      out_b[jj * nrows + ii + 6] = in_b[(ii + 6)*nbyte_row + jj];
443
394k
      out_b[jj * nrows + ii + 7] = in_b[(ii + 7)*nbyte_row + jj];
444
394k
      if (transpose_bits) {
445
174k
        uint64_t x, t;
446
174k
        memcpy(&x, out_b + jj * 8, 8);
447
174k
        TRANS_BIT_8X8(x, t);
448
174k
        memcpy(out_b + jj * 8, &x, 8);
449
174k
      }
450
394k
    }
451
160k
  }
452
110k
  return size * elem_size;
453
110k
}
454
455
int64_t bshuf_trans_byte_bitrow_SSE(const void* in, void* out, const size_t size,
456
7.46k
                                  const size_t elem_size) {
457
7.46k
  return trans_byte_bitrow_SSE(in, out, size, elem_size, false);
458
7.46k
}
459
460
/* Decode one byte plane without materializing the transposed bit rows. */
461
103k
int64_t bshuf_untrans_bit_byte_SSE(const void* in, void* out, const size_t size) {
462
103k
  return trans_byte_bitrow_SSE(in, out, size, 1, true);
463
103k
}
464
465
/* Transpose the 8x8 bit matrix in each 64-bit lane (as TRANS_BIT_8X8). */
466
109M
static inline __m128i trans_bit_8x8_SSE(__m128i x) {
467
109M
  __m128i t;
468
109M
  t = _mm_and_si128(_mm_xor_si128(x, _mm_srli_epi64(x, 7)),
469
109M
                    _mm_set1_epi64x(0x00AA00AA00AA00AALL));
470
109M
  x = _mm_xor_si128(x, _mm_xor_si128(t, _mm_slli_epi64(t, 7)));
471
109M
  t = _mm_and_si128(_mm_xor_si128(x, _mm_srli_epi64(x, 14)),
472
109M
                    _mm_set1_epi64x(0x0000CCCC0000CCCCLL));
473
109M
  x = _mm_xor_si128(x, _mm_xor_si128(t, _mm_slli_epi64(t, 14)));
474
109M
  t = _mm_and_si128(_mm_xor_si128(x, _mm_srli_epi64(x, 28)),
475
109M
                    _mm_set1_epi64x(0x00000000F0F0F0F0LL));
476
109M
  x = _mm_xor_si128(x, _mm_xor_si128(t, _mm_slli_epi64(t, 28)));
477
109M
  return x;
478
109M
}
479
480
481
/* Shuffle bits within the bytes of eight element blocks. */
482
int64_t bshuf_shuffle_bit_eightelem_SSE(const void* in, void* out, const size_t size,
483
7.46k
                                        const size_t elem_size) {
484
485
7.46k
  CHECK_MULT_EIGHT(size);
486
487
  // With a bit of care, this could be written such that such that it is
488
  // in_buf = out_buf safe.
489
7.46k
  const char* in_b = (const char*) in;
490
7.46k
  char* out_b = (char*) out;
491
7.46k
  uint16_t* out_ui16 = (uint16_t*) out;
492
493
7.46k
  size_t ii, jj, kk;
494
7.46k
  size_t nbyte = elem_size * size;
495
496
7.46k
  __m128i xmm;
497
7.46k
  int32_t bt;
498
499
7.46k
  if (elem_size <= 2) {
500
    // Each 8-byte word holds the 8 bit rows of 8 one-byte elements, or of
501
    // the low or the high bytes of 8 two-byte elements: transposing its
502
    // bits gives those bytes.
503
0
    for (ii = 0; ii + 15 < nbyte; ii += 16) {
504
0
      xmm = trans_bit_8x8_SSE(_mm_loadu_si128((__m128i *) &in_b[ii]));
505
0
      if (elem_size == 2) {
506
        // interleave the low bytes (first word) and high bytes (second word)
507
0
        xmm = _mm_unpacklo_epi8(xmm, _mm_unpackhi_epi64(xmm, xmm));
508
0
      }
509
0
      _mm_storeu_si128((__m128i *) &out_b[ii], xmm);
510
0
    }
511
0
    if (ii < nbyte) {
512
0
      bshuf_shuffle_bit_eightelem_scal(&in_b[ii], &out_b[ii],
513
0
                                       (nbyte - ii) / elem_size, elem_size);
514
0
    }
515
7.46k
  } else if (elem_size % 2) {
516
5.47k
    bshuf_shuffle_bit_eightelem_scal(in, out, size, elem_size);
517
5.47k
  } else {
518
9.81M
    for (ii = 0; ii + 8 * elem_size - 1 < nbyte;
519
9.81M
         ii += 8 * elem_size) {
520
94.4M
      for (jj = 0; jj + 15 < 8 * elem_size; jj += 16) {
521
84.5M
        xmm = _mm_loadu_si128((__m128i *) &in_b[ii + jj]);
522
761M
        for (kk = 0; kk < 8; kk++) {
523
676M
          bt = _mm_movemask_epi8(xmm);
524
676M
          xmm = _mm_slli_epi16(xmm, 1);
525
676M
          size_t ind = (ii + jj / 8 + (7 - kk) * elem_size);
526
676M
          out_ui16[ind / 2] = bt;
527
676M
        }
528
84.5M
      }
529
9.81M
    }
530
1.99k
  }
531
7.46k
  return size * elem_size;
532
7.46k
}
533
534
535
/* Untranspose bits within elements. */
536
int64_t bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size,
537
0
                                   const size_t elem_size, void* tmp_buf) {
538
539
0
  int64_t count;
540
541
0
  CHECK_MULT_EIGHT(size);
542
543
0
  if (elem_size == 1) return bshuf_untrans_bit_byte_SSE(in, out, size);
544
0
  if ((elem_size == 2 || elem_size == 4 || elem_size == 8 || elem_size == 16)
545
0
      && size <= INT32_MAX / elem_size) {
546
0
    for (size_t j = 0; j < elem_size; ++j) {
547
0
      bshuf_untrans_bit_byte_SSE((const uint8_t*)in + j * size, (uint8_t*)tmp_buf + j * size, size);
548
0
    }
549
0
    unshuffle_sse2((int32_t)elem_size, (int32_t)(size * elem_size), tmp_buf, out);
550
0
    return size * elem_size;
551
0
  }
552
0
  count = bshuf_trans_byte_bitrow_SSE(in, tmp_buf, size, elem_size);
553
0
  CHECK_ERR(count);
554
0
  count =  bshuf_shuffle_bit_eightelem_SSE(tmp_buf, out, size, elem_size);
555
556
0
  return count;
557
0
}
558
559
const bool is_bshuf_SSE = true;
560
561
#else /* defined(__SSE2__) */
562
563
const bool is_bshuf_SSE = false;
564
565
int64_t bshuf_untrans_bit_byte_SSE(const void* in, void* out, const size_t size) {
566
  abort();
567
}
568
569
int64_t
570
bshuf_trans_bit_elem_SSE(const void* in, void* out, const size_t size,
571
                         const size_t elem_size, void* tmp_buf) {
572
  abort();
573
}
574
575
int64_t
576
bshuf_untrans_bit_elem_SSE(const void* in, void* out, const size_t size,
577
                           const size_t elem_size, void* tmp_buf) {
578
  abort();
579
}
580
581
#endif /* defined(__SSE2__) */