/src/simdutf/include/simdutf/scalar/atomic_util.h
Line | Count | Source |
1 | | #ifndef SIMDUTF_ATOMIC_UTIL_H |
2 | | #define SIMDUTF_ATOMIC_UTIL_H |
3 | | #if SIMDUTF_ATOMIC_REF |
4 | | #include <atomic> |
5 | | #include <cstring> |
6 | | namespace simdutf { |
7 | | namespace scalar { |
8 | | |
9 | | // This function is a memcpy that uses atomic operations to read from the |
10 | | // source. |
11 | 0 | inline void memcpy_atomic_read(char *dst, const char *src, size_t len) { |
12 | 0 | static_assert(std::atomic_ref<char>::required_alignment == sizeof(char), |
13 | 0 | "std::atomic_ref requires the same alignment as char_type"); |
14 | | // We expect all 64-bit systems to be able to read 64-bit words from an |
15 | | // aligned memory region atomically. You might be able to do better on |
16 | | // specific systems, e.g., x64 systems can read 128-bit words atomically. |
17 | 0 | constexpr size_t alignment = sizeof(uint64_t); |
18 | | |
19 | | // Lambda for atomic byte-by-byte copy |
20 | 0 | auto bbb_memcpy_atomic_read = [](char *bytedst, const char *bytesrc, |
21 | 0 | size_t bytelen) noexcept { |
22 | 0 | char *mutable_src = const_cast<char *>(bytesrc); |
23 | 0 | for (size_t j = 0; j < bytelen; ++j) { |
24 | 0 | bytedst[j] = |
25 | 0 | std::atomic_ref<char>(mutable_src[j]).load(std::memory_order_relaxed); |
26 | 0 | } |
27 | 0 | }; |
28 | | |
29 | | // Handle unaligned start |
30 | 0 | size_t offset = reinterpret_cast<std::uintptr_t>(src) % alignment; |
31 | 0 | if (offset) { |
32 | 0 | size_t to_align = detail::min(len, alignment - offset); |
33 | 0 | bbb_memcpy_atomic_read(dst, src, to_align); |
34 | 0 | src += to_align; |
35 | 0 | dst += to_align; |
36 | 0 | len -= to_align; |
37 | 0 | } |
38 | | |
39 | | // Process aligned 64-bit chunks |
40 | 0 | while (len >= alignment) { |
41 | 0 | auto *src_aligned = reinterpret_cast<uint64_t *>(const_cast<char *>(src)); |
42 | 0 | const auto dst_value = |
43 | 0 | std::atomic_ref<uint64_t>(*src_aligned).load(std::memory_order_relaxed); |
44 | 0 | std::memcpy(dst, &dst_value, sizeof(uint64_t)); |
45 | 0 | src += alignment; |
46 | 0 | dst += alignment; |
47 | 0 | len -= alignment; |
48 | 0 | } |
49 | | |
50 | | // Handle remaining bytes |
51 | 0 | if (len) { |
52 | 0 | bbb_memcpy_atomic_read(dst, src, len); |
53 | 0 | } |
54 | 0 | } |
55 | | |
56 | | // This function is a memcpy that uses atomic operations to write to the |
57 | | // destination. |
58 | 0 | inline void memcpy_atomic_write(char *dst, const char *src, size_t len) { |
59 | 0 | static_assert(std::atomic_ref<char>::required_alignment == sizeof(char), |
60 | 0 | "std::atomic_ref requires the same alignment as char"); |
61 | | // We expect all 64-bit systems to be able to write 64-bit words to an aligned |
62 | | // memory region atomically. |
63 | | // You might be able to do better on specific systems, e.g., x64 systems can |
64 | | // write 128-bit words atomically. |
65 | 0 | constexpr size_t alignment = sizeof(uint64_t); |
66 | | |
67 | | // Lambda for atomic byte-by-byte write |
68 | 0 | auto bbb_memcpy_atomic_write = [](char *bytedst, const char *bytesrc, |
69 | 0 | size_t bytelen) noexcept { |
70 | 0 | for (size_t j = 0; j < bytelen; ++j) { |
71 | 0 | std::atomic_ref<char>(bytedst[j]) |
72 | 0 | .store(bytesrc[j], std::memory_order_relaxed); |
73 | 0 | } |
74 | 0 | }; |
75 | | |
76 | | // Handle unaligned start |
77 | 0 | size_t offset = reinterpret_cast<std::uintptr_t>(dst) % alignment; |
78 | 0 | if (offset) { |
79 | 0 | size_t to_align = detail::min(len, alignment - offset); |
80 | 0 | bbb_memcpy_atomic_write(dst, src, to_align); |
81 | 0 | dst += to_align; |
82 | 0 | src += to_align; |
83 | 0 | len -= to_align; |
84 | 0 | } |
85 | | |
86 | | // Process aligned 64-bit chunks |
87 | 0 | while (len >= alignment) { |
88 | 0 | auto *dst_aligned = reinterpret_cast<uint64_t *>(dst); |
89 | 0 | uint64_t src_val; |
90 | 0 | std::memcpy(&src_val, src, sizeof(uint64_t)); // Non-atomic read from src |
91 | 0 | std::atomic_ref<uint64_t>(*dst_aligned) |
92 | 0 | .store(src_val, std::memory_order_relaxed); |
93 | 0 | dst += alignment; |
94 | 0 | src += alignment; |
95 | 0 | len -= alignment; |
96 | 0 | } |
97 | | |
98 | | // Handle remaining bytes |
99 | 0 | if (len) { |
100 | 0 | bbb_memcpy_atomic_write(dst, src, len); |
101 | 0 | } |
102 | 0 | } |
103 | | } // namespace scalar |
104 | | } // namespace simdutf |
105 | | #endif // SIMDUTF_ATOMIC_REF |
106 | | #endif // SIMDUTF_ATOMIC_UTIL_H |