/src/c-blosc2/blosc/bitshuffle-generic.c
Line | Count | Source |
1 | | /********************************************************************* |
2 | | Blosc - Blocked Shuffling and Compression Library |
3 | | |
4 | | Copyright (c) 2021 Blosc Development Team <blosc@blosc.org> |
5 | | https://blosc.org |
6 | | License: BSD 3-Clause (see LICENSE.txt) |
7 | | |
8 | | See LICENSE.txt for details about copyright and rights to use. |
9 | | **********************************************************************/ |
10 | | |
11 | | #include "bitshuffle-generic.h" |
12 | | |
13 | | #include <stdlib.h> |
14 | | |
15 | | #ifdef _MSC_VER |
16 | | #pragma warning (push) |
17 | | #pragma warning (disable: 4146) |
18 | | #endif |
19 | | |
20 | | |
21 | | /* Memory copy with bshuf call signature. For testing and profiling. */ |
22 | | int64_t bshuf_copy(const void* in, void* out, const size_t size, |
23 | 0 | const size_t elem_size) { |
24 | |
|
25 | 0 | const char* in_b = (const char*) in; |
26 | 0 | char* out_b = (char*) out; |
27 | |
|
28 | 0 | memcpy(out_b, in_b, size * elem_size); |
29 | 0 | return size * elem_size; |
30 | 0 | } |
31 | | |
32 | | |
33 | | /* Transpose bytes within elements, starting partway through input. */ |
34 | | int64_t bshuf_trans_byte_elem_remainder(const void* in, void* out, const size_t size, |
35 | 7.46k | const size_t elem_size, const size_t start) { |
36 | | |
37 | 7.46k | size_t ii, jj, kk; |
38 | 7.46k | const char* in_b = (const char*) in; |
39 | 7.46k | char* out_b = (char*) out; |
40 | | |
41 | 7.46k | CHECK_MULT_EIGHT(start); |
42 | | |
43 | 7.46k | if (size > start) { |
44 | | // ii loop separated into 2 loops so the compiler can unroll |
45 | | // the inner one. |
46 | 26.1k | for (ii = start; ii + 7 < size; ii += 8) { |
47 | 97.5k | for (jj = 0; jj < elem_size; jj++) { |
48 | 709k | for (kk = 0; kk < 8; kk++) { |
49 | 630k | out_b[jj * size + ii + kk] |
50 | 630k | = in_b[ii * elem_size + kk * elem_size + jj]; |
51 | 630k | } |
52 | 78.8k | } |
53 | 18.6k | } |
54 | 7.46k | for (ii = size - size % 8; ii < size; ii ++) { |
55 | 0 | for (jj = 0; jj < elem_size; jj++) { |
56 | 0 | out_b[jj * size + ii] = in_b[ii * elem_size + jj]; |
57 | 0 | } |
58 | 0 | } |
59 | 7.46k | } |
60 | 7.46k | return size * elem_size; |
61 | 7.46k | } |
62 | | |
63 | | |
64 | | /* Transpose bytes within elements. */ |
65 | | int64_t bshuf_trans_byte_elem_scal(const void* in, void* out, const size_t size, |
66 | 7.46k | const size_t elem_size) { |
67 | | |
68 | 7.46k | return bshuf_trans_byte_elem_remainder(in, out, size, elem_size, 0); |
69 | 7.46k | } |
70 | | |
71 | | |
72 | | /* Transpose bits within bytes. */ |
73 | | int64_t bshuf_trans_bit_byte_remainder(const void* in, void* out, const size_t size, |
74 | 360k | const size_t elem_size, const size_t start_byte) { |
75 | | |
76 | 360k | const uint64_t* in_b = (const uint64_t*) in; |
77 | 360k | uint8_t* out_b = (uint8_t*) out; |
78 | | |
79 | 360k | uint64_t x, t; |
80 | | |
81 | 360k | size_t ii, kk; |
82 | 360k | size_t nbyte = elem_size * size; |
83 | 360k | size_t nbyte_bitrow = nbyte / 8; |
84 | | |
85 | 360k | uint64_t e=1; |
86 | 360k | const int little_endian = *(uint8_t *) &e == 1; |
87 | 360k | const size_t bit_row_skip = little_endian ? nbyte_bitrow : -nbyte_bitrow; |
88 | 360k | const int64_t bit_row_offset = little_endian ? 0 : 7 * nbyte_bitrow; |
89 | | |
90 | 360k | CHECK_MULT_EIGHT(nbyte); |
91 | 360k | CHECK_MULT_EIGHT(start_byte); |
92 | | |
93 | 542k | for (ii = start_byte / 8; ii < nbyte_bitrow; ii ++) { |
94 | 182k | x = in_b[ii]; |
95 | 182k | if (little_endian) { |
96 | 182k | TRANS_BIT_8X8(x, t); |
97 | 182k | } else { |
98 | 0 | TRANS_BIT_8X8_BE(x, t); |
99 | 0 | } |
100 | 1.63M | for (kk = 0; kk < 8; kk ++) { |
101 | 1.45M | out_b[bit_row_offset + kk * bit_row_skip + ii] = x; |
102 | 1.45M | x = x >> 8; |
103 | 1.45M | } |
104 | 182k | } |
105 | 360k | return size * elem_size; |
106 | 360k | } |
107 | | |
108 | | |
109 | | /* Transpose bits within bytes. */ |
110 | | int64_t bshuf_trans_bit_byte_scal(const void* in, void* out, const size_t size, |
111 | 0 | const size_t elem_size) { |
112 | |
|
113 | 0 | return bshuf_trans_bit_byte_remainder(in, out, size, elem_size, 0); |
114 | 0 | } |
115 | | |
116 | | |
117 | | /* General transpose of an array, optimized for large element sizes. */ |
118 | | int64_t bshuf_trans_elem(const void* in, void* out, const size_t lda, |
119 | 7.46k | const size_t ldb, const size_t elem_size) { |
120 | | |
121 | 7.46k | size_t ii, jj; |
122 | 7.46k | const char* in_b = (const char*) in; |
123 | 7.46k | char* out_b = (char*) out; |
124 | 67.1k | for(ii = 0; ii < lda; ii++) { |
125 | 304k | for(jj = 0; jj < ldb; jj++) { |
126 | 245k | memcpy(&out_b[(jj*lda + ii) * elem_size], |
127 | 245k | &in_b[(ii*ldb + jj) * elem_size], elem_size); |
128 | 245k | } |
129 | 59.6k | } |
130 | 7.46k | return lda * ldb * elem_size; |
131 | 7.46k | } |
132 | | |
133 | | |
134 | | /* Transpose rows of shuffled bits (size / 8 bytes) within groups of 8. */ |
135 | | int64_t bshuf_trans_bitrow_eight(const void* in, void* out, const size_t size, |
136 | 7.46k | const size_t elem_size) { |
137 | | |
138 | 7.46k | size_t nbyte_bitrow = size / 8; |
139 | | |
140 | 7.46k | CHECK_MULT_EIGHT(size); |
141 | | |
142 | 7.46k | return bshuf_trans_elem(in, out, 8, elem_size, nbyte_bitrow); |
143 | 7.46k | } |
144 | | |
145 | | |
146 | | /* Transpose bits within elements. */ |
147 | | int64_t bshuf_trans_bit_elem_scal(const void* in, void* out, const size_t size, |
148 | 0 | const size_t elem_size, void* tmp_buf) { |
149 | |
|
150 | 0 | int64_t count; |
151 | |
|
152 | 0 | CHECK_MULT_EIGHT(size); |
153 | |
|
154 | 0 | count = bshuf_trans_byte_elem_scal(in, out, size, elem_size); |
155 | 0 | CHECK_ERR(count); |
156 | 0 | count = bshuf_trans_bit_byte_scal(out, tmp_buf, size, elem_size); |
157 | 0 | CHECK_ERR(count); |
158 | 0 | count = bshuf_trans_bitrow_eight(tmp_buf, out, size, elem_size); |
159 | |
|
160 | 0 | return count; |
161 | 0 | } |
162 | | |
163 | | |
164 | | /* For data organized into a row for each bit (8 * elem_size rows), transpose |
165 | | * the bytes. */ |
166 | | int64_t bshuf_trans_byte_bitrow_scal(const void* in, void* out, const size_t size, |
167 | 0 | const size_t elem_size) { |
168 | 0 | size_t ii, jj, kk, nbyte_row; |
169 | 0 | const char *in_b; |
170 | 0 | char *out_b; |
171 | | |
172 | |
|
173 | 0 | in_b = (const char*) in; |
174 | 0 | out_b = (char*) out; |
175 | |
|
176 | 0 | nbyte_row = size / 8; |
177 | |
|
178 | 0 | CHECK_MULT_EIGHT(size); |
179 | |
|
180 | 0 | for (jj = 0; jj < elem_size; jj++) { |
181 | 0 | for (ii = 0; ii < nbyte_row; ii++) { |
182 | 0 | for (kk = 0; kk < 8; kk++) { |
183 | 0 | out_b[ii * 8 * elem_size + jj * 8 + kk] = \ |
184 | 0 | in_b[(jj * 8 + kk) * nbyte_row + ii]; |
185 | 0 | } |
186 | 0 | } |
187 | 0 | } |
188 | 0 | return size * elem_size; |
189 | 0 | } |
190 | | |
191 | | |
192 | | /* Shuffle bits within the bytes of eight element blocks. */ |
193 | | int64_t bshuf_shuffle_bit_eightelem_scal(const void* in, void* out, \ |
194 | 5.47k | const size_t size, const size_t elem_size) { |
195 | | |
196 | 5.47k | const char *in_b; |
197 | 5.47k | char *out_b; |
198 | 5.47k | uint64_t x, t; |
199 | 5.47k | size_t ii, jj, kk; |
200 | 5.47k | size_t nbyte, out_index; |
201 | | |
202 | 5.47k | uint64_t e=1; |
203 | 5.47k | const int little_endian = *(uint8_t *) &e == 1; |
204 | 5.47k | const size_t elem_skip = little_endian ? elem_size : -elem_size; |
205 | 5.47k | const uint64_t elem_offset = little_endian ? 0 : 7 * elem_size; |
206 | | |
207 | 5.47k | CHECK_MULT_EIGHT(size); |
208 | | |
209 | 5.47k | in_b = (const char*) in; |
210 | 5.47k | out_b = (char*) out; |
211 | | |
212 | 5.47k | nbyte = elem_size * size; |
213 | | |
214 | 46.3k | for (jj = 0; jj < 8 * elem_size; jj += 8) { |
215 | 104M | for (ii = 0; ii + 8 * elem_size - 1 < nbyte; ii += 8 * elem_size) { |
216 | 104M | x = *((uint64_t*) &in_b[ii + jj]); |
217 | 104M | if (little_endian) { |
218 | 104M | TRANS_BIT_8X8(x, t); |
219 | 104M | } else { |
220 | 0 | TRANS_BIT_8X8_BE(x, t); |
221 | 0 | } |
222 | 936M | for (kk = 0; kk < 8; kk++) { |
223 | 832M | out_index = ii + jj / 8 + elem_offset + kk * elem_skip; |
224 | 832M | *((uint8_t*) &out_b[out_index]) = x; |
225 | 832M | x = x >> 8; |
226 | 832M | } |
227 | 104M | } |
228 | 40.8k | } |
229 | 5.47k | return size * elem_size; |
230 | 5.47k | } |
231 | | |
232 | | |
233 | | /* Untranspose bits within elements. */ |
234 | | int64_t bshuf_untrans_bit_elem_scal(const void* in, void* out, const size_t size, |
235 | 0 | const size_t elem_size, void* tmp_buf) { |
236 | |
|
237 | 0 | int64_t count; |
238 | |
|
239 | 0 | CHECK_MULT_EIGHT(size); |
240 | |
|
241 | 0 | count = bshuf_trans_byte_bitrow_scal(in, tmp_buf, size, elem_size); |
242 | 0 | CHECK_ERR(count); |
243 | 0 | count = bshuf_shuffle_bit_eightelem_scal(tmp_buf, out, size, elem_size); |
244 | |
|
245 | 0 | return count; |
246 | 0 | } |
247 | | |
248 | | #ifdef _MSC_VER |
249 | | #pragma warning (pop) |
250 | | #endif |