Line data Source code
1 : #ifndef HEADER_fd_src_util_fd_util_base_h
2 : #define HEADER_fd_src_util_fd_util_base_h
3 :
4 : /* Base development environment */
5 :
6 : /* Compiler checks ****************************************************/
7 :
8 : #ifdef __cplusplus
9 :
10 : #if __cplusplus<201703L
11 : #error "Firedancer requires C++17 or later"
12 : #endif
13 :
14 : #else
15 :
16 : #if __STDC_VERSION__<201710L
17 : #error "Firedancer requires C Standard version C17 or later"
18 : #endif
19 :
20 : #endif //__cplusplus
21 :
22 : /* Build target capabilities ******************************************/
23 :
24 : /* Different build targets often have different levels of support for
25 : various language and hardware features. The presence of various
26 : features can be tested at preprocessor, compile, or run time via the
27 : below capability macros.
28 :
29 : Code that does not exploit any of these capabilities written within
30 : the base development environment should be broadly portable across a
31 : range of build targets ranging from on-chain virtual machines to
32 : commodity hosts to custom hardware.
33 :
34 : As such, highly portable yet high performance code is possible by
35 : writing generic implementations that do not exploit any of the below
36 : capabilities as a portable fallback along with build target specific
37 : optimized implementations that are invoked when the build target
38 : supports the appropriate capabilities.
39 :
40 : The base development itself provides lots of functionality to help
41 : with implementing portable fallbacks while making very minimal
42 : assumptions about the build targets and zero use of 3rd party
43 : libraries (these might make unknown additional assumptions about the
44 : build target, including availability of a quality implementation of
45 : the library on the build target). */
46 :
47 : /* FD_HAS_HOSTED: If the build target is hosted (e.g. resides on a host
48 : with a POSIX-ish environment ... practically speaking, stdio.h,
49 : stdlib.h, unistd.h, et al more or less behave normally ...
50 : pedantically XOPEN_SOURCE=700), FD_HAS_HOSTED will be 1. It will be
51 : zero otherwise. */
52 :
53 : #ifndef FD_HAS_HOSTED
54 : #define FD_HAS_HOSTED 0
55 : #endif
56 :
57 : /* FD_HAS_ATOMIC: If the build target supports atomic operations
58 : between threads accessing a common memory region (include threads
59 : that reside in different processes on a host communicating via a
60 : shared memory region with potentially different local virtual
61 : mappings). Practically speaking, does atomic compare-and-swap et al
62 : work? */
63 :
64 : #ifndef FD_HAS_ATOMIC
65 : #define FD_HAS_ATOMIC 0
66 : #endif
67 :
68 : /* FD_HAS_THREADS: If the build target supports a POSIX-ish notion of
69 : threads (e.g. practically speaking, global variables declared within
70 : a compile unit are visible to more than one thread of execution,
71 : pthread.h / threading parts of C standard, the atomics parts of the
72 : C standard, ... more or less work normally), FD_HAS_THREADS will be
73 : 1. It will be zero otherwise. FD_HAS_THREADS implies FD_HAS_HOSTED
74 : and FD_HAS_ATOMIC. */
75 :
76 : #ifndef FD_HAS_THREADS
77 : #define FD_HAS_THREADS 0
78 : #endif
79 :
80 : /* FD_HAS_INT128: If the build target supports reasonably efficient
81 : 128-bit wide integer operations, define FD_HAS_INT128 to 1 to enable
82 : use of them in implementations. */
83 :
84 : #ifndef FD_HAS_INT128
85 : #define FD_HAS_INT128 0
86 : #endif
87 :
88 : /* FD_HAS_DOUBLE: If the build target supports reasonably efficient
89 : IEEE 754 64-bit wide double precision floating point operations,
90 : define FD_HAS_DOUBLE to 1 to enable use of them in implementations.
91 : Note that even if the build target does not, va_args handling in the
92 : C / C++ language requires promotion of a float in a va_arg list to a
93 : double. Thus, C / C++ language that support IEEE 754 float also
94 : implies a minimum level of support for double (though not necessarily
95 : efficient or IEEE 754). That is, even if a target does not have
96 : FD_HAS_DOUBLE, there might still be limited use of double in va_arg
97 : list handling. */
98 :
99 : #ifndef FD_HAS_DOUBLE
100 : #define FD_HAS_DOUBLE 0
101 : #endif
102 :
103 : /* FD_HAS_ALLOCA: If the build target supports fast alloca-style
104 : dynamic stack memory allocation (e.g. alloca.h / __builtin_alloca
105 : more or less work normally), define FD_HAS_ALLOCA to 1 to enable use
106 : of it in implementations. */
107 :
108 : #ifndef FD_HAS_ALLOCA
109 : #define FD_HAS_ALLOCA 0
110 : #endif
111 :
112 : /* FD_HAS_X86: If the build target supports x86 specific features and
113 : can benefit from x86 specific optimizations, define FD_HAS_X86. Code
114 : needing more specific target features (Intel / AMD / SSE / AVX2 /
115 : AVX512 / etc) can specialize further as necessary with even more
116 : precise capabilities (that in turn imply FD_HAS_X86). */
117 :
118 : #ifndef FD_HAS_X86
119 : #define FD_HAS_X86 0
120 : #endif
121 :
122 : /* These allow even more precise targeting for X86. */
123 :
124 : /* FD_HAS_SSE indicates the target supports Intel SSE4 style SIMD
125 : (basically do the 128-bit wide parts of "x86intrin.h" work).
126 : Recommend using the simd/fd_sse.h APIs instead of raw Intel
127 : intrinsics for readability and to facilitate portability to non-x86
128 : platforms. Implies FD_HAS_X86. */
129 :
130 : #ifndef FD_HAS_SSE
131 : #define FD_HAS_SSE 0
132 : #endif
133 :
134 : /* FD_HAS_AVX indicates the target supports Intel AVX2 style SIMD
135 : (basically do the 256-bit wide parts of "x86intrin.h" work).
136 : Recommend using the simd/fd_avx.h APIs instead of raw Intel
137 : intrinsics for readability and to facilitate portability to non-x86
138 : platforms. Implies FD_HAS_SSE. */
139 :
140 : #ifndef FD_HAS_AVX
141 : #define FD_HAS_AVX 0
142 : #endif
143 :
144 : /* FD_HAS_AVX512 indicates the target supports Intel AVX-512 style SIMD
145 : (basically do the 512-bit wide parts of "x86intrin.h" work).
146 : Recommend using the simd/fd_avx512.h APIs instead of raw Intel
147 : intrinsics for readability and to facilitate portability to non-x86
148 : platforms. Implies FD_HAS_AVX. */
149 :
150 : #ifndef FD_HAS_AVX512
151 : #define FD_HAS_AVX512 0
152 : #endif
153 :
154 : /* FD_HAS_SHANI indicates that the target supports Intel SHA extensions
155 : which accelerate SHA-1 and SHA-256 computation. This extension is
156 : also called SHA-NI or SHA_NI (Secure Hash Algorithm New
157 : Instructions). Although proposed in 2013, they're only supported on
158 : Intel Ice Lake and AMD Zen CPUs and newer. Implies FD_HAS_AVX. */
159 :
160 : #ifndef FD_HAS_SHANI
161 : #define FD_HAS_SHANI 0
162 : #endif
163 :
164 : /* FD_HAS_GFNI indicates that the target supports Intel Galois Field
165 : extensions, which accelerate operations over binary extension fields,
166 : especially GF(2^8). These instructions are supported on Intel Ice
167 : Lake and newer and AMD Zen4 and newer CPUs. Implies FD_HAS_AVX. */
168 :
169 : #ifndef FD_HAS_GFNI
170 : #define FD_HAS_GFNI 0
171 : #endif
172 :
173 : /* FD_HAS_AESNI indicates that the target supports AES-NI extensions,
174 : which accelerate AES encryption and decryption. While AVX predates
175 : the original AES-NI extension, the combination of AES-NI+AVX adds
176 : additional opcodes (such as vaesenc, a more flexible variant of
177 : aesenc). Thus, implies FD_HAS_AVX. A conservative estimate for
178 : minimum platform support is Intel Haswell or AMD Zen. */
179 :
180 : #ifndef FD_HAS_AESNI
181 : #define FD_HAS_AESNI 0
182 : #endif
183 :
184 : /* FD_HAS_ARM: If the build target supports armv8-a specific features
185 : and can benefit from aarch64 specific optimizations, define
186 : FD_HAS_ARM. */
187 :
188 : #ifndef FD_HAS_ARM
189 : #define FD_HAS_ARM 0
190 : #endif
191 :
192 : #ifndef FD_HAS_RISCV
193 : #define FD_HAS_RISCV 0
194 : #endif
195 :
196 : /* FD_HAS_LZ4 indicates that the target supports LZ4 compression.
197 : Roughly, does "#include <lz4.h>" and the APIs therein work? */
198 :
199 : #ifndef FD_HAS_LZ4
200 0 : #define FD_HAS_LZ4 0
201 : #endif
202 :
203 : /* FD_HAS_COVERAGE indicates that the build target is built with coverage instrumentation. */
204 :
205 : #ifndef FD_HAS_COVERAGE
206 : #define FD_HAS_COVERAGE 0
207 : #endif
208 :
209 : /* FD_HAS_ASAN indicates that the build target is using ASAN. */
210 :
211 : #ifndef FD_HAS_ASAN
212 : #define FD_HAS_ASAN 0
213 : #endif
214 :
215 : /* FD_HAS_UBSAN indicates that the build target is using UBSAN. */
216 :
217 : #ifndef FD_HAS_UBSAN
218 : #define FD_HAS_UBSAN 0
219 : #endif
220 :
221 : /* FD_HAS_DEEPASAN indicates that the build target is using ASAN with manual
222 : memory poisoning for fd_alloc, fd_wksp, and fd_scratch. */
223 :
224 : #ifndef FD_HAS_DEEPASAN
225 : #define FD_HAS_DEEPASAN 0
226 : #endif
227 :
228 : /* Base development environment ***************************************/
229 :
230 : /* The functionality provided by these vanilla headers are always
231 : available within the base development environment. Notably, stdio.h
232 : / stdlib.h / et al are not included here as these make lots of
233 : assumptions about the build target that may not be true (especially
234 : for on-chain and custom hardware use). Code should prefer the fd
235 : util equivalents for such functionality when possible. */
236 :
237 : #include <stdalign.h>
238 : #include <string.h>
239 : #include <limits.h>
240 : #include <float.h>
241 :
242 : /* Work around some library naming irregularities */
243 : /* FIXME: Consider this for FLOAT/FLT, DOUBLE/DBL too? */
244 :
245 0 : #define SHORT_MIN SHRT_MIN
246 0 : #define SHORT_MAX SHRT_MAX
247 814241 : #define USHORT_MAX USHRT_MAX
248 :
249 : /* Primitive types ****************************************************/
250 :
251 : /* These typedefs provide single token regularized names for all the
252 : primitive types in the base development environment:
253 :
254 : char !
255 : schar ! short int long int128 !!
256 : uchar ushort uint ulong uint128 !!
257 : float
258 : double !!!
259 :
260 : ! Does not assume the sign of char. A naked char should be treated
261 : as cstr character and mathematical operations should be avoided on
262 : them. This is less than ideal as the patterns for integer types in
263 : the C/C++ language spec itself are far more consistent with a naked
264 : char naturally being treated as signed (see above). But there are
265 : lots of conflicts between architectures, languages and standard
266 : libraries about this so any use of a naked char shouldn't assume
267 : the sign ... sigh.
268 :
269 : !! Only available if FD_HAS_INT128 is defined
270 :
271 : !!! Should only be used if FD_HAS_DOUBLE is defined but see note in
272 : FD_HAS_DOUBLE about C/C++ silent promotions of float to double in
273 : va_arg lists.
274 :
275 : Note also that these token names more naturally interoperate with
276 : integer constant declarations, type generic code generation
277 : techniques, with printf-style format strings than the stdint.h /
278 : inttypes.h handling.
279 :
280 : To minimize portability issues, unexpected silent type conversion
281 : issues, align with typical developer implicit usage, align with
282 : typical build target usage, ..., assumes char / short / int / long
283 : are 8 / 16 / 32 / 64 twos complement integers and float is IEEE-754
284 : single precision. Further assumes little endian, truncating signed
285 : integer division, sign extending (arithmetic) signed right shift and
286 : signed left shift behaves the same as an unsigned left shift from bit
287 : operations point of view (technically the standard says signed left
288 : shift is undefined if the result would overflow). Also, except for
289 : int128/uint128, assumes that aligned access to these will be
290 : naturally atomic. Lastly assumes that unaligned access to these is
291 : functionally valid but does not assume that unaligned access to these
292 : is efficient or atomic.
293 :
294 : For values meant to be held in registers, code should prefer long /
295 : ulong types (improves asm generation given the prevalence of 64-bit
296 : targets and also to avoid lots of tricky bugs with silent promotions
297 : in the language ... e.g. ushort should ideally only be used for
298 : in-memory representations).
299 :
300 : These are currently not prefixed given how often they are used. If
301 : this becomes problematic prefixes can be added as necessary.
302 : Specifically, C++ allows typedefs to be defined multiple times so
303 : long as they are equivalent. Inequivalent collisions are not
304 : supported but should be rare (e.g. if a 3rd party header thinks
305 : "ulong" should be something other than "unsigned long", the 3rd party
306 : header probably should be nuked from orbit). C11 and forward also
307 : allow multiple equivalent typedefs. C99 and earlier don't but this
308 : is typically only a warning and then only if pedantic warnings are
309 : enabled. Thus, if we want to support users using C99 and earlier who
310 : want to do a strict compile and have a superfluous collision with
311 : these types in other libraries, uncomment the below (or do something
312 : equivalent for the compiler). */
313 :
314 : //#pragma GCC diagnostic push
315 : //#pragma GCC diagnostic ignored "-Wpedantic"
316 :
317 : typedef signed char schar; /* See above note of sadness */
318 :
319 : typedef unsigned char uchar;
320 : typedef unsigned short ushort;
321 : typedef unsigned int uint;
322 : typedef unsigned long ulong;
323 :
324 : #ifdef __SIZEOF_INT128__
325 :
326 : __extension__ typedef __int128 int128;
327 : __extension__ typedef unsigned __int128 uint128;
328 :
329 15787 : #define UINT128_MAX (~(uint128)0)
330 0 : #define INT128_MAX ((int128)(UINT128_MAX>>1))
331 0 : #define INT128_MIN (-INT128_MAX-(int128)1)
332 :
333 : #endif
334 :
335 : //#pragma GCC diagnostic pop
336 :
337 : /* Compiler tricks ****************************************************/
338 :
339 : /* FD_STRINGIFY,FD_CONCAT{2,3,4}: Various macros for token
340 : stringification and pasting. FD_STRINGIFY returns the argument as a
341 : cstr (e.g. FD_STRINGIFY(foo) -> "foo"). FD_CONCAT* pastes the tokens
342 : together into a single token (e.g. FD_CONCAT3(a,b,c) -> abc). The
343 : EXPAND variants first expand their arguments and then do the token
344 : operation (e.g. FD_EXPAND_THEN_STRINGIFY(__LINE__) -> "104" if done
345 : on line 104 of the source code file). */
346 :
347 0 : #define FD_STRINGIFY(x)#x
348 7214 : #define FD_CONCAT2(a,b)a##b
349 0 : #define FD_CONCAT3(a,b,c)a##b##c
350 0 : #define FD_CONCAT4(a,b,c,d)a##b##c##d
351 :
352 : #define FD_EXPAND_THEN_STRINGIFY(x)FD_STRINGIFY(x)
353 7214 : #define FD_EXPAND_THEN_CONCAT2(a,b)FD_CONCAT2(a,b)
354 1384346180 : #define FD_EXPAND_THEN_CONCAT3(a,b,c)FD_CONCAT3(a,b,c)
355 0 : #define FD_EXPAND_THEN_CONCAT4(a,b,c,d)FD_CONCAT4(a,b,c,d)
356 :
357 : /* FD_VA_ARGS_SELECT(__VA_ARGS__,e32,e31,...e1): Macro that expands to
358 : en at compile time where n is number of items in the __VA_ARGS__
359 : list. If __VA_ARGS__ is empty, returns e1. Assumes __VA_ARGS__ has
360 : at most 32 arguments. Useful for making a variadic macro whose
361 : behavior depends on the number of arguments in __VA_ARGS__. */
362 :
363 : #define FD_VA_ARGS_SELECT(A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z,a,b,c,d,e,f,_,...)_
364 :
365 : /* FD_SRC_LOCATION returns a const cstr holding the line of code where
366 : FD_SRC_LOCATION was used. */
367 :
368 : #define FD_SRC_LOCATION __FILE__ "(" FD_EXPAND_THEN_STRINGIFY(__LINE__) ")"
369 :
370 : /* FD_STATIC_ASSERT tests at compile time if c is non-zero. If not,
371 : it aborts the compile with an error. err itself should be a token
372 : (e.g. not a string, no whitespace, etc). */
373 :
374 : #ifdef __cplusplus
375 : #define FD_STATIC_ASSERT(c,err) static_assert(c, #err)
376 : #else
377 29550 : #define FD_STATIC_ASSERT(c,err) _Static_assert(c, #err)
378 : #endif
379 :
380 : /* FD_ADDRESS_OF_PACKED_MEMBER(x): Linguistically does &(x) but without
381 : recent compiler complaints that &x might be unaligned if x is a
382 : member of a packed datastructure. (Often needed for interfacing with
383 : hardware / packets / etc.) */
384 :
385 0 : #define FD_ADDRESS_OF_PACKED_MEMBER( x ) (__extension__({ \
386 0 : char * _fd_aopm = (char *)&(x); \
387 0 : __asm__( "# FD_ADDRESS_OF_PACKED_MEMBER(" #x ") @" FD_SRC_LOCATION : "+r" (_fd_aopm) :: ); \
388 0 : (__typeof__(&(x)))_fd_aopm; \
389 0 : }))
390 :
391 : /* FD_PROTOTYPES_{BEGIN,END}: Headers that might be included in C++
392 : source should encapsulate the prototypes of code and globals
393 : contained in compilation units compiled as C with a
394 : FD_PROTOTYPES_{BEGIN,END} pair. */
395 :
396 : #ifdef __cplusplus
397 : #define FD_PROTOTYPES_BEGIN extern "C" {
398 : #else
399 : #define FD_PROTOTYPES_BEGIN
400 : #endif
401 :
402 : #ifdef __cplusplus
403 : #define FD_PROTOTYPES_END }
404 : #else
405 : #define FD_PROTOTYPES_END
406 : #endif
407 :
408 : /* FD_ASM_LG_ALIGN(lg_n) expands to an alignment assembler directive
409 : appropriate for the current architecture/ABI. The resulting align
410 : is 2^(lg_n) bytes, i.e. FD_ASM_LG_ALIGN(3) aligns by 8 bytes. */
411 :
412 : #if defined(__aarch64__)
413 : #define FD_ASM_LG_ALIGN(lg_n) ".align " #lg_n "\n"
414 : #elif defined(__x86_64__) || defined(__powerpc64__) || defined(__riscv)
415 : #define FD_ASM_LG_ALIGN(lg_n) ".p2align " #lg_n "\n"
416 : #endif
417 :
418 : /* FD_IMPORT declares a variable name and initializes with the contents
419 : of the file at path (with potentially some assembly directives for
420 : additional footer info). It is equivalent to:
421 :
422 : type const name[] __attribute__((aligned(align))) = {
423 :
424 : ... code that would initialize the contents of name to the
425 : ... raw binary data found in the file at path at compile time
426 : ... (with any appended information as specified by footer)
427 :
428 : };
429 :
430 : ulong const name_sz = ... number of bytes pointed to by name;
431 :
432 : More precisely, this creates a symbol "name" in the object file that
433 : points to a read-only copy of the raw data in the file at "path" as
434 : it was at compile time. 2^lg_align specifies the minimum alignment
435 : required for the copy's first byte as an unsuffixed decimal integer.
436 : footer are assembly commands to permit additional data to be appended
437 : to the copy (use "" for footer if no footer is necessary).
438 :
439 : Then it exposes a pointer to this copy in the current compilation
440 : unit as name and the byte size as name_sz. name_sz covers the first
441 : byte of the included data to the last byte of the footer inclusive.
442 :
443 : The dummy linker symbol _fd_import_name_sz will also be created in
444 : the object file as some under the hood magic to make this work. This
445 : should not be used in any compile unit as some compilers (I'm looking
446 : at you clang-15, but apparently not clang-10) will sometimes mangle
447 : its value from what it was set to in the object file even when marked
448 : as absolute in the object file.
449 :
450 : This should only be used at global scope and should be done at most
451 : once over all object files / libraries used to make a program. If
452 : other compilation units want to make use of an import in a different
453 : compilation unit, they should declare:
454 :
455 : extern type const name[] __attribute__((aligned(align)));
456 :
457 : and/or:
458 :
459 : extern ulong const name_sz;
460 :
461 : as necessary (that is, do the usual to use name and name_sz as shown
462 : for the pseudo code above).
463 :
464 : Important safety tip! gcc -M will generally not detect the
465 : dependency this creates between the importing file and the imported
466 : file. This can cause incremental builds to miss changes to the
467 : imported file. Ideally, we would have FD_IMPORT automatically do
468 : something like:
469 :
470 : _Pragma( "GCC dependency \"" path "\" )
471 :
472 : This doesn't work as is because _Pragma needs some macro expansion
473 : hacks to accept this (this is doable). After that workaround, this
474 : still doesn't work because, due to tooling limitations, the pragma
475 : path is relative to the source file directory and the FD_IMPORT path
476 : is relative to the make directory (working around this would
477 : require a __FILE__-like directive for the source code directory base
478 : path). Even if that did exist, it might still not work because
479 : out-of-tree builds often require some substitutions to the gcc -M
480 : generated dependencies that this might not pick up (at least not
481 : without some build system surgery). And then it still wouldn't work
482 : because gcc -M seems to ignore all of this anyways (which is the
483 : actual show stopper as this pragma does something subtly different
484 : than what the name suggests and there isn't any obvious support for a
485 : "pseudo-include".) Another reminder that make clean and fast builds
486 : are our friend. */
487 :
488 : #if defined(__ELF__)
489 :
490 : #define FD_IMPORT( name, path, type, lg_align, footer ) \
491 : __asm__( ".section .rodata,\"a\",@progbits\n" \
492 : ".type " #name ",@object\n" \
493 : ".globl " #name "\n" \
494 : FD_ASM_LG_ALIGN(lg_align) \
495 : #name ":\n" \
496 : ".incbin \"" path "\"\n" \
497 : footer "\n" \
498 : ".size " #name ",. - " #name "\n" \
499 : "_fd_import_" #name "_sz = . - " #name "\n" \
500 : ".type " #name "_sz,@object\n" \
501 : ".globl " #name "_sz\n" \
502 : FD_ASM_LG_ALIGN(3) \
503 : #name "_sz:\n" \
504 : ".quad _fd_import_" #name "_sz\n" \
505 : ".size " #name "_sz,8\n" \
506 : ".previous\n" ); \
507 : extern type const name[] __attribute__((aligned(1<<(lg_align)))); \
508 : extern ulong const name##_sz
509 :
510 : #elif defined(__MACH__)
511 :
512 : #define FD_IMPORT( name, path, type, lg_align, footer ) \
513 : __asm__( ".section __DATA,__const\n" \
514 : ".globl _" #name "\n" \
515 : FD_ASM_LG_ALIGN(lg_align) \
516 : "_" #name ":\n" \
517 : ".incbin \"" path "\"\n" \
518 : footer "\n" \
519 : "_fd_import_" #name "_sz = . - _" #name "\n" \
520 : ".globl _" #name "_sz\n" \
521 : FD_ASM_LG_ALIGN(3) \
522 : "_" #name "_sz:\n" \
523 : ".quad _fd_import_" #name "_sz\n" \
524 : ".previous\n" ); \
525 : extern type const name[] __attribute__((aligned(1<<(lg_align)))); \
526 : extern ulong const name##_sz
527 :
528 : #endif
529 :
530 : /* FD_IMPORT_{BINARY,CSTR} are common cases for FD_IMPORT.
531 :
532 : In BINARY, the file is imported into the object file and exposed to
533 : the caller as a uchar binary data. name_sz will be the number of
534 : bytes in the file at time of import. name will have 128 byte
535 : alignment.
536 :
537 : In CSTR, the file is imported into the object file with a '\0'
538 : termination appended and exposed to the caller as a cstr. Assuming
539 : the file is text (i.e. has no internal '\0's), strlen(name) will be
540 : the number of bytes in the file and name_sz will be strlen(name)+1.
541 : name can have arbitrary alignment. */
542 :
543 : #ifdef FD_IMPORT
544 : #define FD_IMPORT_BINARY(name, path) FD_IMPORT( name, path, uchar, 7, "" )
545 : #define FD_IMPORT_CSTR( name, path) FD_IMPORT( name, path, char, 1, ".byte 0" )
546 : #endif
547 :
548 : /* Optimizer tricks ***************************************************/
549 :
550 : /* FD_RESTRICT is a pointer modifier to designate a pointer as
551 : restricted. Hoops jumped because C++-17 still doesn't understand
552 : restrict ... sigh */
553 :
554 : #ifndef FD_RESTRICT
555 : #ifdef __cplusplus
556 : #define FD_RESTRICT __restrict
557 : #else
558 : #define FD_RESTRICT restrict
559 : #endif
560 : #endif
561 :
562 : /* fd_type_pun(p), fd_type_pun_const(p): These allow use of type
563 : punning while keeping strict aliasing optimizations enabled (e.g.
564 : some UNIX APIs, like sockaddr related APIs are dependent on type
565 : punning). These allow these API's to be used cleanly while keeping
566 : strict aliasing optimizations enabled and strict alias checking done. */
567 :
568 : static inline void *
569 1709888 : fd_type_pun( void * p ) {
570 1709888 : __asm__( "# fd_type_pun @" FD_SRC_LOCATION : "+r" (p) :: "memory" );
571 1709888 : return p;
572 1709888 : }
573 :
574 : static inline void const *
575 1260215 : fd_type_pun_const( void const * p ) {
576 1260215 : __asm__( "# fd_type_pun_const @" FD_SRC_LOCATION : "+r" (p) :: "memory" );
577 1260215 : return p;
578 1260215 : }
579 :
580 : /* FD_{LIKELY,UNLIKELY}(c): Evaluates c and returns whether it is
581 : logical true/false as long (1L/0L). It also hints to the optimizer
582 : whether it should optimize for the case of c evaluating as
583 : true/false. */
584 :
585 81782789 : #define FD_LIKELY(c) __builtin_expect( !!(c), 1L )
586 691921304 : #define FD_UNLIKELY(c) __builtin_expect( !!(c), 0L )
587 :
588 : /* FD_FN_PURE hints to the optimizer that the function, roughly
589 : speaking, does not have side effects. As such, the compiler can
590 : replace a call to the function with the result of an earlier call to
591 : that function provided the inputs and memory used haven't changed.
592 :
593 : IMPORTANT SAFETY TIP! Recent compilers seem to take an undocumented
594 : and debatable stance that pure functions do no writes to memory.
595 : This is a sufficient condition for the above but not a necessary one.
596 :
597 : Consider, for example, the real world case of an otherwise pure
598 : function that uses pass-by-reference to return more than one value
599 : (an unpleasant practice that is sadly often necessary because C/C++,
600 : compilers and underlying platform ABIs are very bad at helping
601 : developers simply and clearly express their intent to return multiple
602 : values and then generate good assembly for such).
603 :
604 : If called multiple times sequentially, all but the first call to such
605 : a "pure" function could be optimized away because the non-volatile
606 : memory writes done in the all but the 1st call for the
607 : pass-by-reference-returns write the same value to normal memory that
608 : was written on the 1st call. That is, these calls return the same
609 : value for their direct return and do writes that do not have any
610 : visible effect.
611 :
612 : Thus, while it is safe for the compiler to eliminate all but the
613 : first call via techniques like common subexpression elimination, it
614 : is not safe for the compiler to infer that the first call did no
615 : writes.
616 :
617 : But recent compilers seem to do exactly that.
618 :
619 : Sigh ... we can't use FD_FN_PURE on such functions because of all the
620 : above linguistic, compiler, documentation and ABI infinite sadness.
621 :
622 : TL;DR To be safe against the above vagaries, recommend using
623 : FD_FN_PURE to annotate functions that do no memory writes (including
624 : trivial memory writes) and try to design HPC APIs to avoid returning
625 : multiple values as much as possible.
626 :
627 : Followup: FD_FN_PURE expands to nothing by default given additional
628 : confusion between how current languages, compilers, CI, fuzzing, and
629 : developers interpret this function attribute. We keep it around
630 : given it documents the intent of various APIs and so it can be
631 : manually enabled to find implementation surprises during bullet
632 : proofing (e.g. under compiler options like "extra-brutality").
633 : Hopefully someday, pure function attributes will someday be handled
634 : more consistently across the board. */
635 :
636 : #ifndef FD_FN_PURE
637 : #define FD_FN_PURE
638 : #endif
639 :
640 : /* FD_FN_CONST is like pure but also, even stronger, indicates that the
641 : function does not depend on the state of memory. See note above
642 : about why this expands to nothing by default. */
643 :
644 : #ifndef FD_FN_CONST
645 : #define FD_FN_CONST
646 : #endif
647 :
648 : /* FD_FN_UNUSED indicates that it is okay if the function with static
649 : linkage is not used. Allows working around -Winline in header only
650 : APIs where the compiler decides not to actually inline the function.
651 : (This belief, frequently promulgated by anti-macro cults, that "An
652 : Inline Function is As Fast As a Macro" ... an entire section in gcc's
653 : documentation devoted to it in fact ... remains among the biggest
654 : lies in computer science. Yes, an inline function is as fast as a
655 : macro ... when the compiler actually decides to treat the inline
656 : keyword more than just for entertainment purposes only. Which, as
657 : -Winline proves, it frequently doesn't. Sigh ... force_inline like
658 : compiler extensions might be an alternative here but they have their
659 : own portability issues.) */
660 :
661 0 : #define FD_FN_UNUSED __attribute__((unused))
662 :
663 : /* FD_FN_UNSANITIZED tells the compiler to disable AddressSanitizer and
664 : UndefinedBehaviorSanitizer instrumentation. For some functions, this
665 : can improve instrumented compile time by ~30x. */
666 :
667 : #if FD_HAS_MSAN
668 : #define FD_FN_UNSANITIZED __attribute__((no_sanitize("memory")))
669 : #else
670 : #define FD_FN_UNSANITIZED __attribute__((no_sanitize("address", "undefined")))
671 : #endif
672 :
673 : /* FD_FN_SENSITIVE instruments the compiler to sanitize sensitive functions.
674 : https://eprint.iacr.org/2023/1713 (Sec 3.2)
675 : - Clear all registers with __attribute__((zero_call_used_regs("all")))
676 : - Clear stack with __attribute__((strub)), available in gcc 14+ */
677 :
678 : #if __has_attribute(strub)
679 : #define FD_FN_SENSITIVE __attribute__((strub)) __attribute__((zero_call_used_regs("all")))
680 : #elif __has_attribute(zero_call_used_regs)
681 : #define FD_FN_SENSITIVE __attribute__((zero_call_used_regs("all")))
682 : #else
683 : #define FD_FN_SENSITIVE
684 : #endif
685 :
686 : /* FD_PARAM_UNUSED indicates that it is okay if the function parameter is not
687 : used. */
688 :
689 : #define FD_PARAM_UNUSED __attribute__((unused))
690 :
691 : /* FD_TYPE_PACKED indicates that a type is to be packed, resetting its
692 : alignment to 1. */
693 :
694 : #define FD_TYPE_PACKED __attribute__((packed))
695 :
696 : /* FD_WARN_UNUSED tells the compiler the result (from a function) should
697 : be checked. This is useful to force callers to either check the result
698 : or deliberately and explicitly ignore it. Good for result codes and
699 : errors */
700 :
701 : #define FD_WARN_UNUSED __attribute__ ((warn_unused_result))
702 :
703 : /* FD_FALLTHRU tells the compiler that a case in a switch falls through
704 : to the next case. This avoids the compiler complaining, in cases where
705 : it is an intentional fall through.
706 : The "while(0)" avoids a compiler complaint in the event the case
707 : has no statement, example:
708 : switch( return_code ) {
709 : case RETURN_CASE_1: FD_FALLTHRU;
710 : case RETURN_CASE_2: FD_FALLTHRU;
711 : case RETURN_CASE_3:
712 : case_123();
713 : default:
714 : case_other();
715 : }
716 :
717 : See C++17 [[fallthrough]] and gcc __attribute__((fallthrough)) */
718 :
719 : #define FD_FALLTHRU while(0) __attribute__((fallthrough))
720 :
721 : /* FD_COMPILER_FORGET(var): Tells the compiler that it shouldn't use
722 : any knowledge it has about the provided register-compatible variable
723 : var for optimizations going forward (i.e. the variable has changed in
724 : a deterministic but unknown-to-the-compiler way where the actual
725 : change is the identity operation). Useful for inhibiting various
726 : branch nest misoptimizations (compilers unfortunately tend to
727 : radically underestimate the impact in raw average performance and
728 : jitter and the probability of branch mispredicts or the cost to the
729 : CPU of having lots of branches). This is not asm volatile (use
730 : UNPREDICTABLE below for that) and has no clobbers. So if var is not
731 : used after the forget, the compiler can optimize the FORGET away
732 : (along with operations preceding it used to produce var). */
733 :
734 1287187 : #define FD_COMPILER_FORGET(var) __asm__( "# FD_COMPILER_FORGET(" #var ")@" FD_SRC_LOCATION : "+r" (var) )
735 :
736 : /* FD_COMPILER_UNPREDICTABLE(var): Same as FD_COMPILER_FORGET(var) but
737 : the provided variable has changed in a non-deterministic way from the
738 : compiler's POV (e.g. the value in the variable on output should not
739 : be treated as a compile time constant even if it is one
740 : linguistically). Useful for suppressing unwanted
741 : compile-time-const-based optimizations like hoisting operations with
742 : useful CPU side effects out of a critical loop. */
743 :
744 1554 : #define FD_COMPILER_UNPREDICTABLE(var) __asm__ __volatile__( "# FD_COMPILER_UNPREDICTABLE(" #var ")@" FD_SRC_LOCATION : "+m,r" (var) )
745 :
746 : /* Atomic tricks ******************************************************/
747 :
748 : /* FD_COMPILER_MFENCE(): Tells the compiler that it can't move any
749 : memory operations (load or store) from before the MFENCE to after the
750 : MFENCE (and vice versa). The processor itself might still reorder
751 : around the fence though (that requires platform specific fences). */
752 :
753 50555907 : #define FD_COMPILER_MFENCE() __asm__ __volatile__( "# FD_COMPILER_MFENCE()@" FD_SRC_LOCATION ::: "memory" )
754 :
755 : /* FD_HW_MFENCE(): A full hardware memory fence. All prior stores
756 : are globally visible before any subsequent loads execute. Use
757 : when a compiler fence is insufficient, e.g. epoch-based safe
758 : reclamation where a store must be visible to another core before
759 : a dependent load on this core (StoreLoad barrier). */
760 :
761 : #if FD_HAS_X86
762 1213599 : #define FD_HW_MFENCE() __asm__ __volatile__( "lock addl $0, (%%rsp)" ::: "memory", "cc" )
763 0 : #define FD_HW_MFENCE_LD() FD_COMPILER_MFENCE()
764 0 : #define FD_HW_MFENCE_ST() FD_COMPILER_MFENCE()
765 : #elif FD_HAS_ARM
766 : #define FD_HW_MFENCE() __asm__ __volatile__( "dmb ish" ::: "memory" )
767 : #define FD_HW_MFENCE_LD() __asm__ __volatile__( "dmb ishld" ::: "memory" )
768 : #define FD_HW_MFENCE_ST() __asm__ __volatile__( "dmb ishst" ::: "memory" )
769 : #elif FD_HAS_RISCV
770 : #define FD_HW_MFENCE() __asm__ __volatile__( "fence rw,rw" ::: "memory" )
771 : #define FD_HW_MFENCE_LD() __asm__ __volatile__( "fence r,rw" ::: "memory" )
772 : #define FD_HW_MFENCE_ST() __asm__ __volatile__( "fence w,w" ::: "memory" )
773 : #else
774 : #define FD_HW_MFENCE() __sync_synchronize()
775 : #define FD_HW_MFENCE_LD() __sync_synchronize()
776 : #define FD_HW_MFENCE_ST() __sync_synchronize()
777 : #endif
778 :
779 : /* FD_SPIN_PAUSE(): Yields the logical core of the calling thread to
780 : the other logical cores sharing the same underlying physical core for
781 : a few clocks without yielding it to the operating system scheduler.
782 : Typically useful for shared memory spin polling loops, especially if
783 : hyperthreading is in use. IMPORTANT SAFETY TIP! This might act as a
784 : FD_COMPILER_MFENCE on some combinations of toolchains and targets
785 : (e.g. gcc documents that __builtin_ia32_pause also does a compiler
786 : memory fence) but this should not be relied upon for portable code
787 : (consider making this a compiler memory fence on all platforms?) */
788 :
789 : #if FD_HAS_X86
790 293549570 : #define FD_SPIN_PAUSE() __builtin_ia32_pause()
791 : #elif FD_HAS_ARM
792 : #define FD_SPIN_PAUSE() __asm__ __volatile__( "yield" ::: "memory" )
793 : #else
794 : #define FD_SPIN_PAUSE() ((void)0)
795 : #endif
796 :
797 : /* FD_YIELD(): Yields the logical core of the calling thread to the
798 : operating system scheduler if a hosted target and does a spin pause
799 : otherwise. */
800 :
801 : #if FD_HAS_HOSTED
802 1237 : #define FD_YIELD() fd_yield()
803 : #else
804 : #define FD_YIELD() FD_SPIN_PAUSE()
805 : #endif
806 :
807 : /* FD_VOLATILE_CONST(x): Tells the compiler that it is not able to
808 : predict the value obtained by dereferencing x and that dereferencing
809 : x might have other side effects (e.g. maybe another thread could
810 : change the value and the compiler has no way of knowing this).
811 : Generally speaking, the volatile keyword is broken linguistically.
812 : Volatility is not a property of the variable but of the
813 : dereferencing of a variable (e.g. what is volatile from the POV of a
814 : reader of a shared variable is not necessarily volatile from the POV
815 : of a writer of that shared variable in a different thread). */
816 :
817 473987094 : #define FD_VOLATILE_CONST(x) (*((volatile const __typeof__((x)) *)&(x)))
818 :
819 : /* FD_VOLATILE(x): tells the compiler that it is not able to predict the
820 : effect of modifying x and that dereferencing x might have other side
821 : effects (e.g. maybe another thread is spinning on x waiting for its
822 : value to change and the compiler has no way of knowing this). */
823 :
824 119671748 : #define FD_VOLATILE(x) (*((volatile __typeof__((x)) *)&(x)))
825 :
826 : /* FD_ATOMIC_FETCH_AND_{ADD,SUB,OR,AND,XOR}(p,v):
827 :
828 : FD_ATOMIC_FETCH_AND_ADD(p,v) does
829 : f = *p;
830 : *p = f + v
831 : return f;
832 : as a single atomic operation. Similarly for the other variants. */
833 :
834 3632104 : #define FD_ATOMIC_FETCH_AND_ADD(p,v) __sync_fetch_and_add( (p), (v) )
835 4110418 : #define FD_ATOMIC_FETCH_AND_SUB(p,v) __sync_fetch_and_sub( (p), (v) )
836 376965 : #define FD_ATOMIC_FETCH_AND_OR( p,v) __sync_fetch_and_or( (p), (v) )
837 1105820 : #define FD_ATOMIC_FETCH_AND_AND(p,v) __sync_fetch_and_and( (p), (v) )
838 : #define FD_ATOMIC_FETCH_AND_XOR(p,v) __sync_fetch_and_xor( (p), (v) )
839 :
840 : /* FD_ATOMIC_{ADD,SUB,OR,AND,XOR}_AND_FETCH(p,v):
841 :
842 : FD_ATOMIC_{ADD,SUB,OR,AND,XOR}_AND_FETCH(p,v) does
843 : r = *p + v;
844 : *p = r;
845 : return r;
846 : as a single atomic operation. Similarly for the other variants. */
847 :
848 >1844*10^16 : #define FD_ATOMIC_ADD_AND_FETCH(p,v) __sync_add_and_fetch( (p), (v) )
849 0 : #define FD_ATOMIC_SUB_AND_FETCH(p,v) __sync_sub_and_fetch( (p), (v) )
850 : #define FD_ATOMIC_OR_AND_FETCH( p,v) __sync_or_and_fetch( (p), (v) )
851 : #define FD_ATOMIC_AND_AND_FETCH(p,v) __sync_and_and_fetch( (p), (v) )
852 : #define FD_ATOMIC_XOR_AND_FETCH(p,v) __sync_xor_and_fetch( (p), (v) )
853 :
854 : /* FD_ATOMIC_CAS(p,c,s):
855 :
856 : o = FD_ATOMIC_CAS(p,c,s) conceptually does:
857 : o = *p;
858 : if( o==c ) *p = s;
859 : return o
860 : as a single atomic operation. */
861 :
862 1240398 : #define FD_ATOMIC_CAS(p,c,s) __sync_val_compare_and_swap( (p), (c), (s) )
863 :
864 : /* FD_ATOMIC_XCHG(p,v):
865 :
866 : o = FD_ATOMIC_XCHG( p, v ) conceptually does:
867 : o = *p
868 : *p = v
869 : return o
870 : as a single atomic operation. */
871 :
872 0 : #define FD_ATOMIC_XCHG(p,v) __atomic_exchange_n( (p), (v), __ATOMIC_SEQ_CST )
873 :
874 : /* FD_TL: This indicates that the variable should be thread local.
875 :
876 : FD_ONCE_{BEGIN,END}: The block:
877 :
878 : FD_ONCE_BEGIN {
879 : ... code ...
880 : } FD_ONCE_END
881 :
882 : linguistically behaves like:
883 :
884 : do {
885 : ... code ...
886 : } while(0)
887 :
888 : But provides a low overhead guarantee that:
889 : - The block will be executed at most once over all threads
890 : in a process (i.e. the set of threads which share global
891 : variables).
892 : - No thread in a process that encounters the block will continue
893 : past it until it has executed once.
894 :
895 : This implies that caller promises a ONCE block will execute in a
896 : finite time. (Meant for doing simple lightweight initializations.)
897 :
898 : It is okay to nest ONCE blocks. The thread that executes the
899 : outermost will execute all the nested once as part of executing the
900 : outermost.
901 :
902 : A ONCE implicitly provides a compiler memory fence to reduce the risk
903 : that the compiler will assume that operations done in the once block
904 : on another thread have not been done (e.g. propagating pre-once block
905 : variable values into post-once block code). It is up to the user to
906 : provide any necessary hardware fencing (usually not necessary).
907 :
908 : FD_THREAD_ONCE_{BEGIN,END}: The block:
909 :
910 : FD_THREAD_ONCE_BEGIN {
911 : ... code ...
912 : } FD_THREAD_ONCE_END;
913 :
914 : is similar except the guarantee is that the block only covers the
915 : invoking thread and it does not provide any fencing. If a thread
916 : once begin is nested inside a once begin, that thread once begin will
917 : only be executed on the thread that executes the thread once begin.
918 : It is similarly okay to nest ONCE block inside a THREAD_ONCE block.
919 :
920 : FD_TURNSTILE_{BEGIN,BLOCKED,END} implement a turnstile for all
921 : threads in a process. Only one thread can be in the turnstile at a
922 : time. Usage:
923 :
924 : FD_TURNSTILE_BEGIN(blocking) {
925 :
926 : ... At this point, we are the only thread executing this block of
927 : ... code.
928 : ...
929 : ... Do operations that must be done by threads one-at-a-time
930 : ... here.
931 : ...
932 : ... Because compiler memory fences are done just before entering
933 : ... and after exiting this block, there is typically no need to
934 : ... use any atomics / volatile / fencing here. That is, we can
935 : ... just write "normal" code on platforms where writes to memory
936 : ... become visible to other threads in the order in which they
937 : ... were issued in the machine code (e.g. x86) as others will not
938 : ... proceed with this block until they exit it. YMMV for non-x86
939 : ... platforms (probably need additional hardware store fences in
940 : ... these macros).
941 : ...
942 : ... It is safe to use "break" and/or "continue" within this
943 : ... block. The block will exit with the appropriate compiler
944 : ... fencing and unlocking. Execution will resume immediately
945 : ... after FD_TURNSTILE_END.
946 :
947 : ... IMPORTANT SAFETY TIP! DO NOT RETURN FROM THIS BLOCK.
948 :
949 : } FD_TURNSTILE_BLOCKED {
950 :
951 : ... At this point, there was another thread in the turnstile when
952 : ... we tried to enter the turnstile.
953 : ...
954 : ... Handle blocked here.
955 : ...
956 : ... On exiting this block, if blocking was zero, we will resume
957 : ... execution immediately after FD_TURNSTILE_END. If blocking
958 : ... was non-zero, we will resume execution immediately before
959 : ... FD_TURNSTILE_BEGIN (e.g. we will retry again after a short
960 : ... spin pause).
961 : ...
962 : ... It is safe to use "break" and/or "continue" within this
963 : ... block. Both will exit this block and resume execution
964 : ... at the location indicated as per what blocking specified
965 : ... when the turnstile was entered.
966 : ...
967 : ... It is technically safe to return from this block but
968 : ... also extremely gross.
969 :
970 : } FD_TURNSTILE_END; */
971 :
972 : #if FD_HAS_THREADS /* Potentially more than one thread in the process */
973 :
974 : #ifndef FD_TL
975 : #define FD_TL __thread
976 : #endif
977 :
978 2 : #define FD_ONCE_BEGIN do { \
979 2 : FD_COMPILER_MFENCE(); \
980 2 : static volatile int _fd_once_block_state = 0; \
981 2 : for(;;) { \
982 2 : int _fd_once_block_tmp = _fd_once_block_state; \
983 2 : if( FD_LIKELY( _fd_once_block_tmp>0 ) ) break; \
984 2 : if( FD_LIKELY( !_fd_once_block_tmp ) && \
985 1 : FD_LIKELY( !FD_ATOMIC_CAS( &_fd_once_block_state, 0, -1 ) ) ) { \
986 1 : do
987 :
988 : #define FD_ONCE_END \
989 1 : while(0); \
990 1 : FD_COMPILER_MFENCE(); \
991 1 : _fd_once_block_state = 1; \
992 1 : break; \
993 1 : } \
994 1 : FD_YIELD(); \
995 0 : } \
996 2 : } while(0)
997 :
998 0 : #define FD_THREAD_ONCE_BEGIN do { \
999 0 : static FD_TL int _fd_thread_once_block_state = 0; \
1000 0 : if( FD_UNLIKELY( !_fd_thread_once_block_state ) ) { \
1001 0 : do
1002 :
1003 : #define FD_THREAD_ONCE_END \
1004 0 : while(0); \
1005 0 : _fd_thread_once_block_state = 1; \
1006 0 : } \
1007 0 : } while(0)
1008 :
1009 0 : #define FD_TURNSTILE_BEGIN(blocking) do { \
1010 0 : static volatile int _fd_turnstile_state = 0; \
1011 0 : int _fd_turnstile_blocking = (blocking); \
1012 0 : for(;;) { \
1013 0 : int _fd_turnstile_tmp = _fd_turnstile_state; \
1014 0 : if( FD_LIKELY( !_fd_turnstile_tmp ) && \
1015 0 : FD_LIKELY( !FD_ATOMIC_CAS( &_fd_turnstile_state, 0, 1 ) ) ) { \
1016 0 : FD_COMPILER_MFENCE(); \
1017 0 : do
1018 :
1019 : #define FD_TURNSTILE_BLOCKED \
1020 0 : while(0); \
1021 0 : FD_COMPILER_MFENCE(); \
1022 0 : _fd_turnstile_state = 0; \
1023 0 : FD_COMPILER_MFENCE(); \
1024 0 : break; \
1025 0 : } \
1026 0 : FD_COMPILER_MFENCE(); \
1027 0 : do
1028 :
1029 : #define FD_TURNSTILE_END \
1030 0 : while(0); \
1031 0 : FD_COMPILER_MFENCE(); \
1032 0 : if( !_fd_turnstile_blocking ) break; /* likely compile time */ \
1033 0 : FD_SPIN_PAUSE(); \
1034 0 : } \
1035 0 : } while(0)
1036 :
1037 : #else /* Only one thread in the process */
1038 :
1039 : #ifndef FD_TL
1040 : #define FD_TL /**/
1041 : #endif
1042 :
1043 : #define FD_ONCE_BEGIN do { \
1044 : static int _fd_once_block_state = 0; \
1045 : if( FD_UNLIKELY( !_fd_once_block_state ) ) { \
1046 : do
1047 :
1048 : #define FD_ONCE_END \
1049 : while(0); \
1050 : _fd_once_block_state = 1; \
1051 : } \
1052 : } while(0)
1053 :
1054 : #define FD_THREAD_ONCE_BEGIN FD_ONCE_BEGIN
1055 : #define FD_THREAD_ONCE_END FD_ONCE_END
1056 :
1057 : #define FD_TURNSTILE_BEGIN(blocking) do { \
1058 : (void)(blocking); \
1059 : FD_COMPILER_MFENCE(); \
1060 : if( 1 ) { \
1061 : do
1062 :
1063 : #define FD_TURNSTILE_BLOCKED \
1064 : while(0); \
1065 : } else { \
1066 : do
1067 :
1068 : #define FD_TURNSTILE_END \
1069 : while(0); \
1070 : } \
1071 : FD_COMPILER_MFENCE(); \
1072 : } while(0)
1073 :
1074 : #endif
1075 :
1076 : /* An ideal fd_clock_func_t is a function such that:
1077 :
1078 : long dx = clock( args );
1079 : ... stuff ...
1080 : dx = clock( args ) - dx;
1081 :
1082 : yields a strictly positive dx where dx approximates the amount of
1083 : wallclock time elapsed on the caller in some clock specific unit
1084 : (e.g. nanoseconds, CPU ticks, etc) for a reasonable amount of "stuff"
1085 : (including no "stuff"). args allows arbitrary clock specific context
1086 : to be passed to the clock implementation. (clocks that need a
1087 : non-const args can cast away the const in the implementation or cast
1088 : the function pointer as necessary.) */
1089 :
1090 : typedef long (*fd_clock_func_t)( void const * args );
1091 :
1092 : FD_PROTOTYPES_BEGIN
1093 :
1094 : /* fd_memcpy(d,s,sz): On modern x86 in some circumstances, rep mov will
1095 : be faster than memcpy under the hood (basically due to RFO /
1096 : read-for-ownership optimizations in the cache protocol under the hood
1097 : that aren't easily done from the ISA ... see Intel docs on enhanced
1098 : rep mov). Compile time configurable though as this is not always
1099 : true. So application can tune to taste. Hard to beat rep mov for
1100 : code density though (2 bytes) and pretty hard to beat in situations
1101 : needing a completely generic memcpy. But it can be beaten in
1102 : specialized situations for the usual reasons. */
1103 :
1104 : /* FIXME: CONSIDER MEMCMP TOO! */
1105 : /* FIXME: CONSIDER MEMCPY RELATED FUNC ATTRS */
1106 :
1107 : #ifndef FD_USE_ARCH_MEMCPY
1108 : #define FD_USE_ARCH_MEMCPY 0
1109 : #endif
1110 :
1111 : #if FD_HAS_X86 && FD_USE_ARCH_MEMCPY && !defined(CBMC) && !FD_HAS_DEEPASAN && !FD_HAS_MSAN
1112 :
1113 : static inline void *
1114 : fd_memcpy( void * FD_RESTRICT d,
1115 : void const * FD_RESTRICT s,
1116 37580450 : ulong sz ) {
1117 37580450 : void * p = d;
1118 37580450 : __asm__ __volatile__( "rep movsb" : "+D" (p), "+S" (s), "+c" (sz) :: "memory" );
1119 37580450 : return d;
1120 37580450 : }
1121 :
1122 : #elif FD_HAS_MSAN
1123 :
1124 : void * __msan_memcpy( void * dest, void const * src, ulong n );
1125 :
1126 : static inline void *
1127 : fd_memcpy( void * FD_RESTRICT d,
1128 : void const * FD_RESTRICT s,
1129 : ulong sz ) {
1130 : return __msan_memcpy( d, s, sz );
1131 : }
1132 :
1133 : #else
1134 :
1135 : static inline void *
1136 : fd_memcpy( void * FD_RESTRICT d,
1137 : void const * FD_RESTRICT s,
1138 : ulong sz ) {
1139 : #if defined(CBMC) || FD_HAS_ASAN || FD_HAS_UBSAN
1140 : if( FD_UNLIKELY( !sz ) ) return d; /* Standard says sz 0 is UB, uncomment if target is insane and doesn't treat sz 0 as a nop */
1141 : #endif
1142 : return memcpy( d, s, sz );
1143 : }
1144 :
1145 : #endif
1146 :
1147 : /* fd_memset(d,c,sz): architecturally optimized memset. See fd_memcpy
1148 : for considerations. */
1149 :
1150 : /* FIXME: CONSIDER MEMSET RELATED FUNC ATTRS */
1151 :
1152 : #ifndef FD_USE_ARCH_MEMSET
1153 : #define FD_USE_ARCH_MEMSET 0
1154 : #endif
1155 :
1156 : #if FD_HAS_X86 && FD_USE_ARCH_MEMSET && !defined(CBMC) && !FD_HAS_DEEPASAN && !FD_HAS_MSAN
1157 :
1158 : static inline void *
1159 : fd_memset( void * d,
1160 : int c,
1161 773721 : ulong sz ) {
1162 773721 : void * p = d;
1163 773721 : __asm__ __volatile__( "rep stosb" : "+D" (p), "+c" (sz) : "a" (c) : "memory" );
1164 773721 : return d;
1165 773721 : }
1166 :
1167 : #else
1168 :
1169 : static inline void *
1170 : fd_memset( void * d,
1171 : int c,
1172 : ulong sz ) {
1173 : # ifdef CBMC
1174 : if( FD_UNLIKELY( !sz ) ) return d; /* See fd_memcpy note */
1175 : # endif
1176 : return memset( d, c, sz );
1177 : }
1178 :
1179 : #endif
1180 :
1181 : /* Calling fd_memzero_explicit will fill the provided region with zeroes.
1182 : It is guaranteed to not be optimized away. */
1183 : FD_FN_UNUSED static inline void
1184 : fd_memzero_explicit( void * d,
1185 0 : ulong sz ) {
1186 : /* We don't want to depend on explicit_bzero or memset_s, so the simplest
1187 : way to ensure the memset is not optimized away is to use a compiler fence,
1188 : identical to how explicit_bzero is implemented.
1189 : https://elixir.bootlin.com/glibc/glibc-2.40/source/string/explicit_bzero.c#L33 */
1190 0 : memset( d, 0, sz );
1191 0 : __asm__ __volatile__( "" ::: "memory" );
1192 0 : }
1193 :
1194 : /* fd_memeq(s0,s1,sz): Compares two blocks of memory. Returns 1 if
1195 : equal or sz is zero and 0 otherwise. No memory accesses made if sz
1196 : is zero (pointers may be invalid). On x86, uses repe cmpsb which is
1197 : preferable to __builtin_memcmp in some cases. */
1198 :
1199 : #ifndef FD_USE_ARCH_MEMEQ
1200 : #define FD_USE_ARCH_MEMEQ 0
1201 : #endif
1202 :
1203 : #if FD_HAS_X86 && FD_USE_ARCH_MEMEQ && defined(__GCC_ASM_FLAG_OUTPUTS__) && __STDC_VERSION__>=199901L
1204 :
1205 : FD_FN_PURE static inline int
1206 : fd_memeq( void const * s0,
1207 : void const * s1,
1208 : ulong sz ) {
1209 : /* ZF flag is set and exported in two cases:
1210 : a) size is zero (via test)
1211 : b) buffer is equal (via repe cmpsb) */
1212 : int r;
1213 : __asm__( "test %3, %3;"
1214 : "repe cmpsb"
1215 : : "=@cce" (r), "+S" (s0), "+D" (s1), "+c" (sz)
1216 : : "m" (*(char const (*)[sz]) s0), "m" (*(char const (*)[sz]) s1)
1217 : : "cc" );
1218 : return r;
1219 : }
1220 :
1221 : #else
1222 :
1223 : FD_FN_PURE static inline int
1224 : fd_memeq( void const * s1,
1225 : void const * s2,
1226 565710 : ulong sz ) {
1227 565710 : return 0==memcmp( s1, s2, sz );
1228 565710 : }
1229 :
1230 : #endif
1231 :
1232 : /* Returns 1 if all sz bytes starting at s are zero, 0 otherwise. */
1233 : FD_FN_PURE static inline int
1234 : fd_mem_iszero( uchar const * s,
1235 8 : ulong sz ) {
1236 8 : for( ulong i=0UL; i<sz; i++ ) {
1237 8 : if( s[i]!=0 ) return 0;
1238 8 : }
1239 0 : return 1;
1240 8 : }
1241 :
1242 : /* fd_hash(seed,buf,sz), fd_hash_memcpy(seed,d,s,sz): High quality
1243 : (full avalanche) high speed variable length buffer -> 64-bit hash
1244 : function (memcpy_hash is often as fast as plain memcpy). Based on
1245 : the xxhash-r39 (open source BSD licensed) implementation. In-place
1246 : and out-of-place variants provided (out-of-place variant assumes dst
1247 : and src do not overlap). Caller promises valid input arguments,
1248 : cannot fail given valid input arguments. sz==0 is fine. */
1249 :
1250 : FD_FN_PURE ulong
1251 : fd_hash( ulong seed,
1252 : void const * buf,
1253 : ulong sz );
1254 :
1255 : ulong
1256 : fd_hash_memcpy( ulong seed,
1257 : void * FD_RESTRICT d,
1258 : void const * FD_RESTRICT s,
1259 : ulong sz );
1260 :
1261 : #ifndef FD_TICKCOUNT_STYLE
1262 : #if FD_HAS_X86 /* Use RDTSC */
1263 : #define FD_TICKCOUNT_STYLE 1
1264 : #elif FD_HAS_ARM /* Use CNTVCT_EL0 */
1265 : #define FD_TICKCOUNT_STYLE 2
1266 : #elif FD_HAS_RISCV /* Uses shared timebase instead of CPU cycles. */
1267 : #define FD_TICKCOUNT_STYLE 3
1268 : #else /* Use portable fallback */
1269 : #define FD_TICKCOUNT_STYLE 0
1270 : #endif
1271 : #endif
1272 :
1273 : #if FD_TICKCOUNT_STYLE==0 /* Portable fallback (slow). Ticks at 1 ns / tick */
1274 :
1275 : #define fd_tickcount() fd_log_wallclock() /* TODO: fix ugly pre-log usage */
1276 :
1277 : #elif FD_TICKCOUNT_STYLE==1 /* RDTSC (fast) */
1278 :
1279 : /* fd_tickcount: Reads the hardware invariant tickcounter ("RDTSC").
1280 : This monotonically increases at an approximately constant rate
1281 : relative to the system wallclock and is synchronous across all CPUs
1282 : on a host.
1283 :
1284 : The rate this ticks at is not precisely defined (see Intel docs for
1285 : more details) but it is typically in the ballpark of the CPU base
1286 : clock frequency. The relationship to the wallclock is very well
1287 : approximated as linear over short periods of time (i.e. less than a
1288 : fraction of a second) and this should not exhibit any sudden changes
1289 : in its rate relative to the wallclock. Notably, its rate is not
1290 : directly impacted by CPU clock frequency adaptation / Turbo mode (see
1291 : other Intel performance monitoring counters for various CPU cycle
1292 : counters). It can drift over longer periods of time for the usual
1293 : clock synchronization reasons.
1294 :
1295 : This is a reasonably fast O(1) cost (~6-8 ns on recent Intel).
1296 : Because of all compiler options and parallel execution going on in
1297 : modern CPUs cores, other instructions might be reordered around this
1298 : by the compiler and/or CPU. It is up to the user to do lower level
1299 : tricks as necessary when the precise location of this in the
1300 : execution stream and/or when executed by the CPU is needed. (This is
1301 : often unnecessary as such levels of precision are not frequently
1302 : required and often have self-defeating overheads.)
1303 :
1304 : It is worth noting that RDTSC and/or (even more frequently) lower
1305 : level performance counters are often restricted from use in user
1306 : space applications. It is recommended that applications use this
1307 : primarily for debugging / performance tuning on unrestricted hosts
1308 : and/or when the developer is confident that applications using this
1309 : will have appropriate permissions when deployed. */
1310 :
1311 58690 : #define fd_tickcount() ((long)__builtin_ia32_rdtsc())
1312 :
1313 : #elif FD_TICKCOUNT_STYLE==2 /* armv8 (fast) */
1314 :
1315 : /* fd_tickcount (ARM): https://developer.arm.com/documentation/ddi0601/2021-12/AArch64-Registers/CNTVCT-EL0--Counter-timer-Virtual-Count-register
1316 : Approx 24 MHz on Apple M1. */
1317 :
1318 : static inline long
1319 : fd_tickcount( void ) {
1320 : /* consider using 'isb' */
1321 : ulong value;
1322 : __asm__ __volatile__ (
1323 : "mrs %0, cntvct_el0\n"
1324 : "nop"
1325 : : "=r" (value) );
1326 : return (long)value;
1327 : }
1328 :
1329 : #elif FD_TICKCOUNT_STYLE==3 /* RV64 time CSR */
1330 :
1331 : /* The time CSR is a platform timebase shared across harts, which is
1332 : independent of core frequency. Unlike cycle, it is suitable for
1333 : migration between heterogeneous cores. The kernel must permit
1334 : user-mode reads. Select FD_TICKCOUNT_STYLE=0 on systems that do
1335 : not allow it (and trap on rdtime).
1336 :
1337 : We may not assume a frequency or nanoseconds per tick, tempo calibrate
1338 : it against wallclock time. */
1339 :
1340 : #if !defined(__riscv) || (__riscv_xlen!=64)
1341 : #error "FD_TICKCOUNT_STYLE=3 requires RV64"
1342 : #endif
1343 :
1344 : static inline long
1345 : fd_tickcount( void ) {
1346 : ulong value;
1347 : __asm__ __volatile__( "rdtime %0" : "=r" (value) );
1348 : return (long)value;
1349 : }
1350 :
1351 : #else
1352 : #error "Unknown FD_TICKCOUNT_STYLE"
1353 : #endif
1354 :
1355 : long _fd_tickcount( void const * _ ); /* fd_clock_func_t compat */
1356 :
1357 : #if FD_HAS_HOSTED
1358 :
1359 : /* fd_yield yields the calling thread to the operating system scheduler. */
1360 :
1361 : void
1362 : fd_yield( void );
1363 :
1364 : #endif
1365 :
1366 : #if FD_HAS_ARM
1367 :
1368 : /* fd_arm_stp16 stores two ulongs to a 16-byte memory location.
1369 : If LSE2 and p is aligned, is single-copy atomic. */
1370 :
1371 : static inline void
1372 : fd_arm_stp16( ulong * p,
1373 : ulong a,
1374 : ulong b ) {
1375 : __asm__(
1376 : "stp %x[a], %x[b], [%[p]]"
1377 : :
1378 : : [a] "r"(a), [b] "r"(b), [p] "r"(p)
1379 : : "memory"
1380 : );
1381 : }
1382 :
1383 : /* fd_arm_ldp16 loads two ulongs from a 16-byte memory location.
1384 : If LSE2 and p is aligned, is single-copy atomic. */
1385 :
1386 : #define fd_arm_ldp16(p_,a_,b_) \
1387 : __asm__( \
1388 : "ldp %x[a], %x[b], [%[p]]" \
1389 : : [a] "=r"(a_), [b] "=r"(b_) \
1390 : : [p] "r"(p_) \
1391 : : "memory" \
1392 : )
1393 :
1394 : #endif /* FD_HAS_ARM */
1395 :
1396 : FD_PROTOTYPES_END
1397 :
1398 : #endif /* HEADER_fd_src_util_fd_util_base_h */
|
