/src/croaring/include/roaring/portability.h
Line | Count | Source |
1 | | /* |
2 | | * portability.h |
3 | | * |
4 | | * This header centralizes compiler-, platform-, and architecture-specific |
5 | | * portability definitions used throughout CRoaring. It provides feature |
6 | | * detection, calling-convention and attribute macros, intrinsic and inline |
7 | | * assembly enablement, endianness helpers, alignment annotations, atomic |
8 | | * reference-count support, and other low-level compatibility glue. |
9 | | * |
10 | | * The goal is to keep these conditional definitions in one place so the rest |
11 | | * of the codebase can rely on a more uniform interface across GCC, Clang, |
12 | | * MSVC, x86/x64, ARM/NEON, and other supported environments. |
13 | | */ |
14 | | |
15 | | /** |
16 | | * All macros should be prefixed with either CROARING or ROARING. |
17 | | * The library uses both ROARING_... |
18 | | * as well as CROARING_ as prefixes. The ROARING_ prefix is for |
19 | | * macros that are provided by the build system or that are closely |
20 | | * related to the format. The header macros may also use ROARING_. |
21 | | * The CROARING_ prefix is for internal macros that a user is unlikely |
22 | | * to ever interact with. |
23 | | */ |
24 | | |
25 | | #ifndef CROARING_INCLUDE_PORTABILITY_H_ |
26 | | #define CROARING_INCLUDE_PORTABILITY_H_ |
27 | | |
28 | | // Users who need _GNU_SOURCE should define it? |
29 | | // #ifndef _GNU_SOURCE |
30 | | // #define _GNU_SOURCE 1 |
31 | | // #endif // _GNU_SOURCE |
32 | | #ifndef __STDC_FORMAT_MACROS |
33 | | #define __STDC_FORMAT_MACROS 1 |
34 | | #endif // __STDC_FORMAT_MACROS |
35 | | |
36 | | #ifdef _MSC_VER |
37 | | #define CROARING_VISUAL_STUDIO 1 |
38 | | /** |
39 | | * We want to differentiate carefully between |
40 | | * clang under visual studio and regular visual |
41 | | * studio. |
42 | | */ |
43 | | #ifdef __clang__ |
44 | | // clang under visual studio |
45 | | #define CROARING_CLANG_VISUAL_STUDIO 1 |
46 | | #else |
47 | | // just regular visual studio (best guess) |
48 | | #define CROARING_REGULAR_VISUAL_STUDIO 1 |
49 | | #endif // __clang__ |
50 | | #endif // _MSC_VER |
51 | | #ifndef CROARING_VISUAL_STUDIO |
52 | | #define CROARING_VISUAL_STUDIO 0 |
53 | | #endif |
54 | | #ifndef CROARING_CLANG_VISUAL_STUDIO |
55 | | #define CROARING_CLANG_VISUAL_STUDIO 0 |
56 | | #endif |
57 | | #ifndef CROARING_REGULAR_VISUAL_STUDIO |
58 | | #define CROARING_REGULAR_VISUAL_STUDIO 0 |
59 | | #endif |
60 | | |
61 | | #include <stdbool.h> |
62 | | #include <stdint.h> |
63 | | #include <stdlib.h> // will provide posix_memalign with _POSIX_C_SOURCE as defined above |
64 | | #ifdef __GLIBC__ |
65 | | #include <malloc.h> // this should never be needed but there are some reports that it is needed. |
66 | | #endif |
67 | | // alignas/alignof are keywords in C++ and in C23+, where <stdalign.h> is |
68 | | // deprecated. Only include it for C11..C17. |
69 | | #if !defined(__cplusplus) && \ |
70 | | (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 202311L) |
71 | | #include <stdalign.h> |
72 | | #endif |
73 | | |
74 | | #ifdef __cplusplus |
75 | | extern "C" { // portability definitions are in global scope, not a namespace |
76 | | #endif |
77 | | |
78 | | #if defined(__SIZEOF_LONG_LONG__) && __SIZEOF_LONG_LONG__ != 8 |
79 | | #error This code assumes 64-bit long longs (by use of the GCC intrinsics). Your system is not currently supported. |
80 | | #endif |
81 | | |
82 | | #if CROARING_REGULAR_VISUAL_STUDIO |
83 | | #ifndef __restrict__ |
84 | | #define __restrict__ __restrict |
85 | | #endif // __restrict__ |
86 | | #endif // CROARING_REGULAR_VISUAL_STUDIO |
87 | | |
88 | | #if defined(__riscv) || defined(_M_RISCV32) || defined(_M_RISCV64) |
89 | | #define CROARING_IS_RISCV 1 |
90 | | |
91 | | #if (defined(__riscv_xlen) && (__riscv_xlen == 64)) || defined(_M_RISCV64) |
92 | | #define CROARING_IS_RISCV64 1 |
93 | | #endif |
94 | | |
95 | | #elif defined(__x86_64__) || defined(_M_X64) |
96 | | // we have an x64 processor |
97 | | #define CROARING_IS_X64 1 |
98 | | |
99 | | #if defined(_MSC_VER) && (_MSC_VER < 1910) |
100 | | // Old visual studio systems won't support AVX2 well. |
101 | | #undef CROARING_IS_X64 |
102 | | #endif |
103 | | |
104 | | #if defined(__clang_major__) && (__clang_major__ <= 8) && !defined(__AVX2__) |
105 | | // Older versions of clang have a bug affecting us |
106 | | // https://stackoverflow.com/questions/57228537/how-does-one-use-pragma-clang-attribute-push-with-c-namespaces |
107 | | #undef CROARING_IS_X64 |
108 | | #endif |
109 | | |
110 | | #ifdef ROARING_DISABLE_X64 |
111 | | #undef CROARING_IS_X64 |
112 | | #endif |
113 | | // we include the intrinsic header |
114 | | #if !CROARING_REGULAR_VISUAL_STUDIO |
115 | | /* Non-Microsoft C/C++-compatible compiler */ |
116 | | #include <x86intrin.h> // on some recent GCC, this will declare posix_memalign |
117 | | |
118 | | #if CROARING_CLANG_VISUAL_STUDIO |
119 | | |
120 | | /** |
121 | | * You are not supposed, normally, to include these |
122 | | * headers directly. Instead you should either include intrin.h |
123 | | * or x86intrin.h. However, when compiling with clang |
124 | | * under Windows (i.e., when _MSC_VER is set), these headers |
125 | | * only get included *if* the corresponding features are detected |
126 | | * from macros: |
127 | | * e.g., if __AVX2__ is set... in turn, we normally set these |
128 | | * macros by compiling against the corresponding architecture |
129 | | * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole |
130 | | * software with these advanced instructions. These headers would |
131 | | * normally guard against such usage, but we carefully included |
132 | | * <x86intrin.h> (or <intrin.h>) before, so the headers |
133 | | * are fooled. |
134 | | */ |
135 | | // To avoid reordering imports: |
136 | | // clang-format off |
137 | | #include <bmiintrin.h> // for _blsr_u64 |
138 | | #include <lzcntintrin.h> // for __lzcnt64 |
139 | | #include <immintrin.h> // for most things (AVX2, AVX512, _popcnt64) |
140 | | #include <smmintrin.h> |
141 | | #include <tmmintrin.h> |
142 | | #include <avxintrin.h> |
143 | | #include <avx2intrin.h> |
144 | | #include <wmmintrin.h> |
145 | | #if _MSC_VER >= 1920 |
146 | | // Important: we need the AVX-512 headers: |
147 | | #include <avx512fintrin.h> |
148 | | #include <avx512dqintrin.h> |
149 | | #include <avx512cdintrin.h> |
150 | | #include <avx512bwintrin.h> |
151 | | #include <avx512vlintrin.h> |
152 | | #include <avx512vbmiintrin.h> |
153 | | #include <avx512vbmi2intrin.h> |
154 | | #include <avx512vpopcntdqintrin.h> |
155 | | // clang-format on |
156 | | #endif // _MSC_VER >= 1920 |
157 | | // unfortunately, we may not get _blsr_u64, but, thankfully, clang |
158 | | // has it as a macro. |
159 | | #ifndef _blsr_u64 |
160 | | // we roll our own |
161 | | #define _blsr_u64(n) ((n - 1) & n) |
162 | | #endif // _blsr_u64 |
163 | | #endif // SIMDJSON_CLANG_VISUAL_STUDIO |
164 | | |
165 | | #endif // CROARING_REGULAR_VISUAL_STUDIO |
166 | | #endif // defined(__x86_64__) || defined(_M_X64) |
167 | | |
168 | | #if !defined(CROARING_USENEON) && !defined(DISABLENEON) && defined(__ARM_NEON) |
169 | | #define CROARING_USENEON |
170 | | #endif |
171 | | #if defined(CROARING_USENEON) |
172 | | #include <arm_neon.h> |
173 | | #endif |
174 | | |
175 | | #if defined(__e2k__) |
176 | | // we have an e2k (Elbrus-2000) processor |
177 | | #define CROARING_IS_E2K 1 |
178 | | #endif |
179 | | |
180 | | #if !CROARING_REGULAR_VISUAL_STUDIO && !defined(CROARING_IS_E2K) && !__FILC__ |
181 | | /* Non-Microsoft C/C++-compatible compiler, assumes that it supports inline |
182 | | * assembly */ |
183 | | #define CROARING_INLINE_ASM 1 |
184 | | #endif // _MSC_VER |
185 | | |
186 | | #if CROARING_REGULAR_VISUAL_STUDIO |
187 | | /* Microsoft C/C++-compatible compiler */ |
188 | | #include <intrin.h> |
189 | | |
190 | | #ifndef __clang__ // if one compiles with MSVC *with* clang, then these |
191 | | // intrinsics are defined!!! |
192 | | #define CROARING_INTRINSICS 1 |
193 | | // sadly there is no way to check whether we are missing these intrinsics |
194 | | // specifically. |
195 | | |
196 | | /* wrappers for Visual Studio built-ins that look like gcc built-ins |
197 | | * __builtin_ctzll */ |
198 | | /** result might be undefined when input_num is zero */ |
199 | | inline int roaring_trailing_zeroes(unsigned long long input_num) { |
200 | | unsigned long index; |
201 | | #ifdef _WIN64 // highly recommended!!! |
202 | | _BitScanForward64(&index, input_num); |
203 | | #else // if we must support 32-bit Windows |
204 | | if ((uint32_t)input_num != 0) { |
205 | | _BitScanForward(&index, (uint32_t)input_num); |
206 | | } else { |
207 | | _BitScanForward(&index, (uint32_t)(input_num >> 32)); |
208 | | index += 32; |
209 | | } |
210 | | #endif // _WIN64 |
211 | | return index; |
212 | | } |
213 | | |
214 | | /* wrappers for Visual Studio built-ins that look like gcc built-ins |
215 | | * __builtin_clzll */ |
216 | | /** result might be undefined when input_num is zero */ |
217 | | inline int roaring_leading_zeroes(unsigned long long input_num) { |
218 | | unsigned long index; |
219 | | #ifdef _WIN64 // highly recommended!!! |
220 | | _BitScanReverse64(&index, input_num); |
221 | | #else // if we must support 32-bit Windows |
222 | | if (input_num > 0xFFFFFFFF) { |
223 | | _BitScanReverse(&index, (uint32_t)(input_num >> 32)); |
224 | | index += 32; |
225 | | } else { |
226 | | _BitScanReverse(&index, (uint32_t)(input_num)); |
227 | | } |
228 | | #endif // _WIN64 |
229 | | return 63 - index; |
230 | | } |
231 | | |
232 | | /* Use #define so this is effective even under /Ob0 (no inline) */ |
233 | | #define roaring_unreachable __assume(0) |
234 | | #endif // __clang__ |
235 | | |
236 | | #endif // CROARING_REGULAR_VISUAL_STUDIO |
237 | | |
238 | | #ifndef CROARING_INTRINSICS |
239 | | #define CROARING_INTRINSICS 1 |
240 | 0 | #define roaring_unreachable __builtin_unreachable() |
241 | | /** result might be undefined when input_num is zero */ |
242 | 74.8M | inline int roaring_trailing_zeroes(unsigned long long input_num) { |
243 | 74.8M | return __builtin_ctzll(input_num); |
244 | 74.8M | } |
245 | | /** result might be undefined when input_num is zero */ |
246 | 4.55k | inline int roaring_leading_zeroes(unsigned long long input_num) { |
247 | 4.55k | return __builtin_clzll(input_num); |
248 | 4.55k | } |
249 | | #endif |
250 | | |
251 | | #if CROARING_REGULAR_VISUAL_STUDIO |
252 | | #define ALIGNED(x) __declspec(align(x)) |
253 | | #elif defined(__GNUC__) || defined(__clang__) |
254 | | #define ALIGNED(x) __attribute__((aligned(x))) |
255 | | #else |
256 | | #warning "Warning. Unrecognized compiler." |
257 | | #define ALIGNED(x) |
258 | | #endif |
259 | | |
260 | | #if defined(__GNUC__) || defined(__clang__) |
261 | | #define CROARING_WARN_UNUSED __attribute__((warn_unused_result)) |
262 | | #else |
263 | | #define CROARING_WARN_UNUSED |
264 | | #endif |
265 | | |
266 | | #define IS_BIG_ENDIAN (*(uint16_t *)"\0\xff" < 0x100) |
267 | | |
268 | | #ifdef CROARING_USENEON |
269 | | // we can always compute the popcount fast. |
270 | | #elif (defined(_M_ARM) || defined(_M_ARM64)) && \ |
271 | | ((defined(_WIN64) || defined(_WIN32)) && \ |
272 | | defined(CROARING_REGULAR_VISUAL_STUDIO) && \ |
273 | | CROARING_REGULAR_VISUAL_STUDIO) |
274 | | // we will need this function: |
275 | | static inline int roaring_hamming_backup(uint64_t x) { |
276 | | uint64_t c1 = UINT64_C(0x5555555555555555); |
277 | | uint64_t c2 = UINT64_C(0x3333333333333333); |
278 | | uint64_t c4 = UINT64_C(0x0F0F0F0F0F0F0F0F); |
279 | | x -= (x >> 1) & c1; |
280 | | x = ((x >> 2) & c2) + (x & c2); |
281 | | x = (x + (x >> 4)) & c4; |
282 | | x *= UINT64_C(0x0101010101010101); |
283 | | return x >> 56; |
284 | | } |
285 | | #endif |
286 | | |
287 | 366k | static inline int roaring_hamming(uint64_t x) { |
288 | | #if defined(_WIN64) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ |
289 | | CROARING_REGULAR_VISUAL_STUDIO |
290 | | #ifdef CROARING_USENEON |
291 | | return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); |
292 | | #elif defined(_M_ARM64) |
293 | | return roaring_hamming_backup(x); |
294 | | // (int) _CountOneBits64(x); is unavailable |
295 | | #else // _M_ARM64 |
296 | | return (int)__popcnt64(x); |
297 | | #endif // _M_ARM64 |
298 | | #elif defined(_WIN32) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ |
299 | | CROARING_REGULAR_VISUAL_STUDIO |
300 | | #ifdef _M_ARM |
301 | | return roaring_hamming_backup(x); |
302 | | // _CountOneBits is unavailable |
303 | | #else // _M_ARM |
304 | | return (int)__popcnt((unsigned int)x) + |
305 | | (int)__popcnt((unsigned int)(x >> 32)); |
306 | | #endif // _M_ARM |
307 | | #else |
308 | 366k | return __builtin_popcountll(x); |
309 | 366k | #endif |
310 | 366k | } Unexecuted instantiation: croaring_fuzzer_cc.cc:roaring_hamming(unsigned long) roaring.c:roaring_hamming Line | Count | Source | 287 | 134k | static inline int roaring_hamming(uint64_t x) { | 288 | | #if defined(_WIN64) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ | 289 | | CROARING_REGULAR_VISUAL_STUDIO | 290 | | #ifdef CROARING_USENEON | 291 | | return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); | 292 | | #elif defined(_M_ARM64) | 293 | | return roaring_hamming_backup(x); | 294 | | // (int) _CountOneBits64(x); is unavailable | 295 | | #else // _M_ARM64 | 296 | | return (int)__popcnt64(x); | 297 | | #endif // _M_ARM64 | 298 | | #elif defined(_WIN32) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ | 299 | | CROARING_REGULAR_VISUAL_STUDIO | 300 | | #ifdef _M_ARM | 301 | | return roaring_hamming_backup(x); | 302 | | // _CountOneBits is unavailable | 303 | | #else // _M_ARM | 304 | | return (int)__popcnt((unsigned int)x) + | 305 | | (int)__popcnt((unsigned int)(x >> 32)); | 306 | | #endif // _M_ARM | 307 | | #else | 308 | 134k | return __builtin_popcountll(x); | 309 | 134k | #endif | 310 | 134k | } |
Unexecuted instantiation: roaring_array.c:roaring_hamming Unexecuted instantiation: array_util.c:roaring_hamming Unexecuted instantiation: bitset_util.c:roaring_hamming Unexecuted instantiation: bitset.c:roaring_hamming Unexecuted instantiation: array.c:roaring_hamming Unexecuted instantiation: containers.c:roaring_hamming Unexecuted instantiation: convert.c:roaring_hamming mixed_intersection.c:roaring_hamming Line | Count | Source | 287 | 232k | static inline int roaring_hamming(uint64_t x) { | 288 | | #if defined(_WIN64) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ | 289 | | CROARING_REGULAR_VISUAL_STUDIO | 290 | | #ifdef CROARING_USENEON | 291 | | return vaddv_u8(vcnt_u8(vcreate_u8(input_num))); | 292 | | #elif defined(_M_ARM64) | 293 | | return roaring_hamming_backup(x); | 294 | | // (int) _CountOneBits64(x); is unavailable | 295 | | #else // _M_ARM64 | 296 | | return (int)__popcnt64(x); | 297 | | #endif // _M_ARM64 | 298 | | #elif defined(_WIN32) && defined(CROARING_REGULAR_VISUAL_STUDIO) && \ | 299 | | CROARING_REGULAR_VISUAL_STUDIO | 300 | | #ifdef _M_ARM | 301 | | return roaring_hamming_backup(x); | 302 | | // _CountOneBits is unavailable | 303 | | #else // _M_ARM | 304 | | return (int)__popcnt((unsigned int)x) + | 305 | | (int)__popcnt((unsigned int)(x >> 32)); | 306 | | #endif // _M_ARM | 307 | | #else | 308 | 232k | return __builtin_popcountll(x); | 309 | 232k | #endif | 310 | 232k | } |
Unexecuted instantiation: mixed_union.c:roaring_hamming Unexecuted instantiation: mixed_equal.c:roaring_hamming Unexecuted instantiation: mixed_subset.c:roaring_hamming Unexecuted instantiation: mixed_negation.c:roaring_hamming Unexecuted instantiation: mixed_xor.c:roaring_hamming Unexecuted instantiation: mixed_andnot.c:roaring_hamming Unexecuted instantiation: run.c:roaring_hamming Unexecuted instantiation: isadetection.c:roaring_hamming Unexecuted instantiation: croaring_fuzzer.c:roaring_hamming Unexecuted instantiation: roaring64.c:roaring_hamming Unexecuted instantiation: art.c:roaring_hamming |
311 | | |
312 | | #ifndef UINT64_C |
313 | | #define UINT64_C(c) (c##ULL) |
314 | | #endif // UINT64_C |
315 | | |
316 | | #ifndef UINT32_C |
317 | | #define UINT32_C(c) (c##UL) |
318 | | #endif // UINT32_C |
319 | | |
320 | | #ifdef __cplusplus |
321 | | } // extern "C" { |
322 | | #endif // __cplusplus |
323 | | |
324 | | // this is almost standard? |
325 | | #undef STRINGIFY_IMPLEMENTATION_ |
326 | | #undef STRINGIFY |
327 | | #define STRINGIFY_IMPLEMENTATION_(a) #a |
328 | | #define STRINGIFY(a) STRINGIFY_IMPLEMENTATION_(a) |
329 | | |
330 | | // Our fast kernels require 64-bit systems. |
331 | | // |
332 | | // On 32-bit x86, we lack 64-bit popcnt, lzcnt, blsr instructions. |
333 | | // Furthermore, the number of SIMD registers is reduced. |
334 | | // |
335 | | // On 32-bit ARM, we would have smaller registers. |
336 | | // |
337 | | // The library should still have the fallback kernel. It is |
338 | | // slower, but it should run everywhere. |
339 | | |
340 | | // |
341 | | // Enable valid runtime implementations, and select |
342 | | // CROARING_BUILTIN_IMPLEMENTATION |
343 | | // |
344 | | |
345 | | // We are going to use runtime dispatch. |
346 | | #if CROARING_IS_X64 |
347 | | #ifdef __clang__ |
348 | | // clang does not have GCC push pop |
349 | | // warning: clang attribute push can't be used within a namespace in clang up |
350 | | // til 8.0 so CROARING_TARGET_REGION and CROARING_UNTARGET_REGION must be |
351 | | // *outside* of a namespace. |
352 | | #define CROARING_TARGET_REGION(T) \ |
353 | | _Pragma(STRINGIFY(clang attribute push(__attribute__((target(T))), \ |
354 | | apply_to = function))) |
355 | | #define CROARING_UNTARGET_REGION _Pragma("clang attribute pop") |
356 | | #elif defined(__GNUC__) |
357 | | // GCC is easier |
358 | | #define CROARING_TARGET_REGION(T) \ |
359 | | _Pragma("GCC push_options") _Pragma(STRINGIFY(GCC target(T))) |
360 | | #define CROARING_UNTARGET_REGION _Pragma("GCC pop_options") |
361 | | #endif // clang then gcc |
362 | | |
363 | | #endif // CROARING_IS_X64 |
364 | | |
365 | | // Default target region macros don't do anything. |
366 | | #ifndef CROARING_TARGET_REGION |
367 | | #define CROARING_TARGET_REGION(T) |
368 | | #define CROARING_UNTARGET_REGION |
369 | | #endif |
370 | | |
371 | | #define CROARING_TARGET_AVX2 \ |
372 | | CROARING_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") |
373 | | #define CROARING_TARGET_AVX512 \ |
374 | | CROARING_TARGET_REGION( \ |
375 | | "avx2,bmi,bmi2,pclmul,lzcnt,popcnt,avx512f,avx512dq,avx512bw," \ |
376 | | "avx512vbmi2,avx512bitalg,avx512vpopcntdq") |
377 | | #define CROARING_UNTARGET_AVX2 CROARING_UNTARGET_REGION |
378 | | #define CROARING_UNTARGET_AVX512 CROARING_UNTARGET_REGION |
379 | | |
380 | | #ifdef __AVX2__ |
381 | | // No need for runtime dispatching. |
382 | | // It is unnecessary and harmful to old clang to tag regions. |
383 | | #undef CROARING_TARGET_AVX2 |
384 | | #define CROARING_TARGET_AVX2 |
385 | | #undef CROARING_UNTARGET_AVX2 |
386 | | #define CROARING_UNTARGET_AVX2 |
387 | | #endif |
388 | | |
389 | | #if defined(__AVX512F__) && defined(__AVX512DQ__) && defined(__AVX512BW__) && \ |
390 | | defined(__AVX512VBMI2__) && defined(__AVX512BITALG__) && \ |
391 | | defined(__AVX512VPOPCNTDQ__) |
392 | | // No need for runtime dispatching. |
393 | | // It is unnecessary and harmful to old clang to tag regions. |
394 | | #undef CROARING_TARGET_AVX512 |
395 | | #define CROARING_TARGET_AVX512 |
396 | | #undef CROARING_UNTARGET_AVX512 |
397 | | #define CROARING_UNTARGET_AVX512 |
398 | | #endif |
399 | | |
400 | | // Allow unaligned memory access |
401 | | #if defined(__GNUC__) || defined(__clang__) |
402 | | #define CROARING_ALLOW_UNALIGNED __attribute__((no_sanitize("alignment"))) |
403 | | #else |
404 | | #define CROARING_ALLOW_UNALIGNED |
405 | | #endif |
406 | | |
407 | | #if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) |
408 | | #define CROARING_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) |
409 | | #elif defined(_WIN32) |
410 | | #define CROARING_IS_BIG_ENDIAN 0 |
411 | | #else |
412 | | #if defined(__APPLE__) || \ |
413 | | defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined |
414 | | // __ORDER_BIG_ENDIAN__ |
415 | | #include <machine/endian.h> |
416 | | #elif defined(sun) || \ |
417 | | defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) |
418 | | #include <sys/byteorder.h> |
419 | | #else // defined(__APPLE__) || defined(__FreeBSD__) |
420 | | |
421 | | #ifdef __has_include |
422 | | #if __has_include(<endian.h>) |
423 | | #include <endian.h> |
424 | | #endif //__has_include(<endian.h>) |
425 | | #endif //__has_include |
426 | | |
427 | | #endif // defined(__APPLE__) || defined(__FreeBSD__) |
428 | | |
429 | | #ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) |
430 | | #define CROARING_IS_BIG_ENDIAN 0 |
431 | | #endif |
432 | | |
433 | | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
434 | | #define CROARING_IS_BIG_ENDIAN 0 |
435 | | #else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
436 | | #define CROARING_IS_BIG_ENDIAN 1 |
437 | | #endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ |
438 | | #endif |
439 | | |
440 | | // Host <-> big endian conversion. |
441 | | #if CROARING_IS_BIG_ENDIAN |
442 | | #define croaring_htobe64(x) (x) |
443 | | |
444 | | #elif defined(_WIN32) || defined(_WIN64) // CROARING_IS_BIG_ENDIAN |
445 | | #include <stdlib.h> |
446 | | #define croaring_htobe64(x) _byteswap_uint64(x) |
447 | | |
448 | | #elif defined(__APPLE__) // CROARING_IS_BIG_ENDIAN |
449 | | #include <libkern/OSByteOrder.h> |
450 | | #define croaring_htobe64(x) OSSwapInt64(x) |
451 | | |
452 | | #elif defined(__has_include) && \ |
453 | | __has_include( \ |
454 | | <byteswap.h>) && (defined(__linux__) || defined(__FreeBSD__)) // CROARING_IS_BIG_ENDIAN |
455 | | #include <byteswap.h> |
456 | | #if defined(__linux__) |
457 | 1.05M | #define croaring_htobe64(x) bswap_64(x) |
458 | | #elif defined(__FreeBSD__) |
459 | | #define croaring_htobe64(x) bswap64(x) |
460 | | #else |
461 | | #warning "Unknown platform, report as an error" |
462 | | #endif |
463 | | |
464 | | #else // CROARING_IS_BIG_ENDIAN |
465 | | // Gets compiled to bswap or equivalent on most compilers. |
466 | | #define croaring_htobe64(x) \ |
467 | | (((x & 0x00000000000000FFULL) << 56) | \ |
468 | | ((x & 0x000000000000FF00ULL) << 40) | \ |
469 | | ((x & 0x0000000000FF0000ULL) << 24) | \ |
470 | | ((x & 0x00000000FF000000ULL) << 8) | ((x & 0x000000FF00000000ULL) >> 8) | \ |
471 | | ((x & 0x0000FF0000000000ULL) >> 24) | \ |
472 | | ((x & 0x00FF000000000000ULL) >> 40) | \ |
473 | | ((x & 0xFF00000000000000ULL) >> 56)) |
474 | | #endif // CROARING_IS_BIG_ENDIAN |
475 | 0 | #define croaring_be64toh(x) croaring_htobe64(x) |
476 | | // End of host <-> big endian conversion. |
477 | | |
478 | | // Host <-> little-endian conversion helpers. |
479 | | // |
480 | | // The CRoaring "portable" serialization format (and the regular |
481 | | // roaring_bitmap_serialize / Roaring64Map::write formats which build on it) |
482 | | // is defined to be little-endian on the wire. Code that reads or writes |
483 | | // multi-byte integers to such buffers must convert between host and |
484 | | // little-endian byte order. On little-endian hosts these are no-ops; on |
485 | | // big-endian hosts they swap bytes. |
486 | | // |
487 | | // The "frozen" format is intentionally non-portable and uses native byte |
488 | | // order; it must not use these helpers. |
489 | | #if CROARING_IS_BIG_ENDIAN |
490 | | |
491 | | static inline uint16_t croaring_bswap16(uint16_t x) { |
492 | | return (uint16_t)((x << 8) | (x >> 8)); |
493 | | } |
494 | | |
495 | | static inline uint32_t croaring_bswap32(uint32_t x) { |
496 | | return ((x & 0x000000FFU) << 24) | ((x & 0x0000FF00U) << 8) | |
497 | | ((x & 0x00FF0000U) >> 8) | ((x & 0xFF000000U) >> 24); |
498 | | } |
499 | | |
500 | | static inline uint64_t croaring_bswap64(uint64_t x) { |
501 | | return ((x & 0x00000000000000FFULL) << 56) | |
502 | | ((x & 0x000000000000FF00ULL) << 40) | |
503 | | ((x & 0x0000000000FF0000ULL) << 24) | |
504 | | ((x & 0x00000000FF000000ULL) << 8) | |
505 | | ((x & 0x000000FF00000000ULL) >> 8) | |
506 | | ((x & 0x0000FF0000000000ULL) >> 24) | |
507 | | ((x & 0x00FF000000000000ULL) >> 40) | |
508 | | ((x & 0xFF00000000000000ULL) >> 56); |
509 | | } |
510 | | |
511 | | #define croaring_htole16(x) croaring_bswap16(x) |
512 | | #define croaring_htole32(x) croaring_bswap32(x) |
513 | | #define croaring_htole64(x) croaring_bswap64(x) |
514 | | |
515 | | #else // CROARING_IS_BIG_ENDIAN |
516 | | |
517 | 3.79M | #define croaring_htole16(x) (x) |
518 | 119k | #define croaring_htole32(x) (x) |
519 | 1.77k | #define croaring_htole64(x) (x) |
520 | | |
521 | | #endif // CROARING_IS_BIG_ENDIAN |
522 | | |
523 | 3.63M | #define croaring_letoh16(x) croaring_htole16(x) |
524 | 26.8k | #define croaring_letoh32(x) croaring_htole32(x) |
525 | 1.77k | #define croaring_letoh64(x) croaring_htole64(x) |
526 | | |
527 | | // Defines for the possible CROARING atomic implementations |
528 | | #define CROARING_ATOMIC_IMPL_NONE 1 |
529 | | #define CROARING_ATOMIC_IMPL_CPP 2 |
530 | | #define CROARING_ATOMIC_IMPL_C 3 |
531 | | #define CROARING_ATOMIC_IMPL_C_WINDOWS 4 |
532 | | |
533 | | // If the use has forced a specific implementation, use that, otherwise, |
534 | | // figure out the best implementation we can use. |
535 | | #if !defined(CROARING_ATOMIC_IMPL) |
536 | | #if defined(__cplusplus) && __cplusplus >= 201103L |
537 | | #ifdef __has_include |
538 | | #if __has_include(<atomic>) |
539 | | #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP |
540 | | #endif //__has_include(<atomic>) |
541 | | #else |
542 | | // We lack __has_include to check: |
543 | | #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_CPP |
544 | | #endif //__has_include |
545 | | #elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) |
546 | | #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C |
547 | | #elif CROARING_REGULAR_VISUAL_STUDIO |
548 | | // https://www.technetworkhub.com/c11-atomics-in-visual-studio-2022-version-17/ |
549 | | #define CROARING_ATOMIC_IMPL CROARING_ATOMIC_IMPL_C_WINDOWS |
550 | | #endif |
551 | | #endif // !defined(CROARING_ATOMIC_IMPL) |
552 | | |
553 | | #if CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C |
554 | | #include <stdatomic.h> |
555 | | typedef _Atomic(uint32_t) croaring_refcount_t; |
556 | | |
557 | 0 | static inline void croaring_refcount_inc(croaring_refcount_t *val) { |
558 | | // Increasing the reference counter can always be done with |
559 | | // memory_order_relaxed: New references to an object can only be formed from |
560 | | // an existing reference, and passing an existing reference from one thread |
561 | | // to another must already provide any required synchronization. |
562 | 0 | atomic_fetch_add_explicit(val, 1, memory_order_relaxed); |
563 | 0 | } Unexecuted instantiation: roaring.c:croaring_refcount_inc Unexecuted instantiation: roaring_array.c:croaring_refcount_inc Unexecuted instantiation: array_util.c:croaring_refcount_inc Unexecuted instantiation: bitset_util.c:croaring_refcount_inc Unexecuted instantiation: bitset.c:croaring_refcount_inc Unexecuted instantiation: array.c:croaring_refcount_inc Unexecuted instantiation: containers.c:croaring_refcount_inc Unexecuted instantiation: convert.c:croaring_refcount_inc Unexecuted instantiation: mixed_intersection.c:croaring_refcount_inc Unexecuted instantiation: mixed_union.c:croaring_refcount_inc Unexecuted instantiation: mixed_equal.c:croaring_refcount_inc Unexecuted instantiation: mixed_subset.c:croaring_refcount_inc Unexecuted instantiation: mixed_negation.c:croaring_refcount_inc Unexecuted instantiation: mixed_xor.c:croaring_refcount_inc Unexecuted instantiation: mixed_andnot.c:croaring_refcount_inc Unexecuted instantiation: run.c:croaring_refcount_inc Unexecuted instantiation: isadetection.c:croaring_refcount_inc Unexecuted instantiation: croaring_fuzzer.c:croaring_refcount_inc Unexecuted instantiation: roaring64.c:croaring_refcount_inc Unexecuted instantiation: art.c:croaring_refcount_inc |
564 | | |
565 | 0 | static inline bool croaring_refcount_dec(croaring_refcount_t *val) { |
566 | | // It is important to enforce any possible access to the object in one |
567 | | // thread (through an existing reference) to happen before deleting the |
568 | | // object in a different thread. This is achieved by a "release" operation |
569 | | // after dropping a reference (any access to the object through this |
570 | | // reference must obviously happened before), and an "acquire" operation |
571 | | // before deleting the object. |
572 | 0 | bool is_zero = atomic_fetch_sub_explicit(val, 1, memory_order_release) == 1; |
573 | 0 | if (is_zero) { |
574 | 0 | atomic_thread_fence(memory_order_acquire); |
575 | 0 | } |
576 | 0 | return is_zero; |
577 | 0 | } Unexecuted instantiation: roaring.c:croaring_refcount_dec Unexecuted instantiation: roaring_array.c:croaring_refcount_dec Unexecuted instantiation: array_util.c:croaring_refcount_dec Unexecuted instantiation: bitset_util.c:croaring_refcount_dec Unexecuted instantiation: bitset.c:croaring_refcount_dec Unexecuted instantiation: array.c:croaring_refcount_dec Unexecuted instantiation: containers.c:croaring_refcount_dec Unexecuted instantiation: convert.c:croaring_refcount_dec Unexecuted instantiation: mixed_intersection.c:croaring_refcount_dec Unexecuted instantiation: mixed_union.c:croaring_refcount_dec Unexecuted instantiation: mixed_equal.c:croaring_refcount_dec Unexecuted instantiation: mixed_subset.c:croaring_refcount_dec Unexecuted instantiation: mixed_negation.c:croaring_refcount_dec Unexecuted instantiation: mixed_xor.c:croaring_refcount_dec Unexecuted instantiation: mixed_andnot.c:croaring_refcount_dec Unexecuted instantiation: run.c:croaring_refcount_dec Unexecuted instantiation: isadetection.c:croaring_refcount_dec Unexecuted instantiation: croaring_fuzzer.c:croaring_refcount_dec Unexecuted instantiation: roaring64.c:croaring_refcount_dec Unexecuted instantiation: art.c:croaring_refcount_dec |
578 | | |
579 | 0 | static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { |
580 | 0 | return atomic_load_explicit(val, memory_order_relaxed); |
581 | 0 | } Unexecuted instantiation: roaring.c:croaring_refcount_get Unexecuted instantiation: roaring_array.c:croaring_refcount_get Unexecuted instantiation: array_util.c:croaring_refcount_get Unexecuted instantiation: bitset_util.c:croaring_refcount_get Unexecuted instantiation: bitset.c:croaring_refcount_get Unexecuted instantiation: array.c:croaring_refcount_get Unexecuted instantiation: containers.c:croaring_refcount_get Unexecuted instantiation: convert.c:croaring_refcount_get Unexecuted instantiation: mixed_intersection.c:croaring_refcount_get Unexecuted instantiation: mixed_union.c:croaring_refcount_get Unexecuted instantiation: mixed_equal.c:croaring_refcount_get Unexecuted instantiation: mixed_subset.c:croaring_refcount_get Unexecuted instantiation: mixed_negation.c:croaring_refcount_get Unexecuted instantiation: mixed_xor.c:croaring_refcount_get Unexecuted instantiation: mixed_andnot.c:croaring_refcount_get Unexecuted instantiation: run.c:croaring_refcount_get Unexecuted instantiation: isadetection.c:croaring_refcount_get Unexecuted instantiation: croaring_fuzzer.c:croaring_refcount_get Unexecuted instantiation: roaring64.c:croaring_refcount_get Unexecuted instantiation: art.c:croaring_refcount_get |
582 | | #elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_CPP |
583 | | #include <atomic> |
584 | | typedef std::atomic<uint32_t> croaring_refcount_t; |
585 | | |
586 | 0 | static inline void croaring_refcount_inc(croaring_refcount_t *val) { |
587 | 0 | val->fetch_add(1, std::memory_order_relaxed); |
588 | 0 | } |
589 | | |
590 | 0 | static inline bool croaring_refcount_dec(croaring_refcount_t *val) { |
591 | 0 | // See above comments on the c11 atomic implementation for memory ordering |
592 | 0 | bool is_zero = val->fetch_sub(1, std::memory_order_release) == 1; |
593 | 0 | if (is_zero) { |
594 | 0 | std::atomic_thread_fence(std::memory_order_acquire); |
595 | 0 | } |
596 | 0 | return is_zero; |
597 | 0 | } |
598 | | |
599 | 0 | static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { |
600 | 0 | return val->load(std::memory_order_relaxed); |
601 | 0 | } |
602 | | #elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_C_WINDOWS |
603 | | #include <intrin.h> |
604 | | #pragma intrinsic(_InterlockedIncrement) |
605 | | #pragma intrinsic(_InterlockedDecrement) |
606 | | |
607 | | // _InterlockedIncrement and _InterlockedDecrement take a (signed) long, and |
608 | | // overflow is defined to wrap, so we can pretend it is a uint32_t for our case |
609 | | typedef volatile long croaring_refcount_t; |
610 | | |
611 | | static inline void croaring_refcount_inc(croaring_refcount_t *val) { |
612 | | _InterlockedIncrement(val); |
613 | | } |
614 | | |
615 | | static inline bool croaring_refcount_dec(croaring_refcount_t *val) { |
616 | | return _InterlockedDecrement(val) == 0; |
617 | | } |
618 | | |
619 | | static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { |
620 | | // Per |
621 | | // https://learn.microsoft.com/en-us/windows/win32/sync/interlocked-variable-access |
622 | | // > Simple reads and writes to properly-aligned 32-bit variables are atomic |
623 | | // > operations. In other words, you will not end up with only one portion |
624 | | // > of the variable updated; all bits are updated in an atomic fashion. |
625 | | return *val; |
626 | | } |
627 | | #elif CROARING_ATOMIC_IMPL == CROARING_ATOMIC_IMPL_NONE |
628 | | #include <assert.h> |
629 | | typedef uint32_t croaring_refcount_t; |
630 | | |
631 | | static inline void croaring_refcount_inc(croaring_refcount_t *val) { |
632 | | *val += 1; |
633 | | } |
634 | | |
635 | | static inline bool croaring_refcount_dec(croaring_refcount_t *val) { |
636 | | assert(*val > 0); |
637 | | *val -= 1; |
638 | | return *val == 0; |
639 | | } |
640 | | |
641 | | static inline uint32_t croaring_refcount_get(const croaring_refcount_t *val) { |
642 | | return *val; |
643 | | } |
644 | | #else |
645 | | #error "Unknown atomic implementation" |
646 | | #endif |
647 | | |
648 | | #if defined(__GNUC__) || defined(__clang__) |
649 | | #define CROARING_DEPRECATED __attribute__((deprecated)) |
650 | | #elif defined(_MSC_VER) |
651 | | #define CROARING_DEPRECATED __declspec(deprecated) |
652 | | #else |
653 | | #define CROARING_DEPRECATED |
654 | | #endif // defined(__GNUC__) || defined(__clang__) |
655 | | |
656 | | // We want to initialize structs to zero portably (C and C++), without |
657 | | // warnings. We can do mystruct s = CROARING_ZERO_INITIALIZER; |
658 | | #if __cplusplus |
659 | | #define CROARING_ZERO_INITIALIZER \ |
660 | | {} |
661 | | #else |
662 | | #define CROARING_ZERO_INITIALIZER \ |
663 | 3.64k | { 0 } |
664 | | #endif |
665 | | |
666 | | #if defined(__cplusplus) |
667 | | #define CROARING_STATIC_ASSERT(x, y) static_assert(x, y) |
668 | | #else |
669 | 0 | #define CROARING_STATIC_ASSERT(x, y) _Static_assert(x, y) |
670 | | #endif |
671 | | |
672 | | // We need portability.h to be included first, |
673 | | // but we also always want isadetection.h to be |
674 | | // included (right after). |
675 | | // See https://github.com/RoaringBitmap/CRoaring/issues/394 |
676 | | // There is no scenario where we want portability.h to |
677 | | // be included, but not isadetection.h: the latter is a |
678 | | // strict requirement. |
679 | | #include <roaring/isadetection.h> // include it last! |
680 | | #endif /* INCLUDE_PORTABILITY_H_ */ |