/src/abseil-cpp/absl/hash/internal/low_level_hash.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright 2020 The Abseil Authors |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // https://www.apache.org/licenses/LICENSE-2.0 |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | #include "absl/hash/internal/low_level_hash.h" |
16 | | |
17 | | #include <cstddef> |
18 | | #include <cstdint> |
19 | | |
20 | | #include "absl/base/internal/unaligned_access.h" |
21 | | #include "absl/base/prefetch.h" |
22 | | #include "absl/numeric/int128.h" |
23 | | |
24 | | namespace absl { |
25 | | ABSL_NAMESPACE_BEGIN |
26 | | namespace hash_internal { |
27 | | |
28 | 76 | static uint64_t Mix(uint64_t v0, uint64_t v1) { |
29 | 76 | absl::uint128 p = v0; |
30 | 76 | p *= v1; |
31 | 76 | return absl::Uint128Low64(p) ^ absl::Uint128High64(p); |
32 | 76 | } |
33 | | |
34 | | uint64_t LowLevelHashLenGt16(const void* data, size_t len, uint64_t seed, |
35 | 28 | const uint64_t salt[5]) { |
36 | 28 | const uint8_t* ptr = static_cast<const uint8_t*>(data); |
37 | 28 | uint64_t starting_length = static_cast<uint64_t>(len); |
38 | 28 | const uint8_t* last_16_ptr = ptr + starting_length - 16; |
39 | 28 | uint64_t current_state = seed ^ salt[0]; |
40 | | |
41 | 28 | if (len > 64) { |
42 | | // If we have more than 64 bytes, we're going to handle chunks of 64 |
43 | | // bytes at a time. We're going to build up four separate hash states |
44 | | // which we will then hash together. This avoids short dependency chains. |
45 | 0 | uint64_t duplicated_state0 = current_state; |
46 | 0 | uint64_t duplicated_state1 = current_state; |
47 | 0 | uint64_t duplicated_state2 = current_state; |
48 | |
|
49 | 0 | do { |
50 | | // Always prefetch the next cacheline. |
51 | 0 | PrefetchToLocalCache(ptr + ABSL_CACHELINE_SIZE); |
52 | |
|
53 | 0 | uint64_t a = absl::base_internal::UnalignedLoad64(ptr); |
54 | 0 | uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); |
55 | 0 | uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16); |
56 | 0 | uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24); |
57 | 0 | uint64_t e = absl::base_internal::UnalignedLoad64(ptr + 32); |
58 | 0 | uint64_t f = absl::base_internal::UnalignedLoad64(ptr + 40); |
59 | 0 | uint64_t g = absl::base_internal::UnalignedLoad64(ptr + 48); |
60 | 0 | uint64_t h = absl::base_internal::UnalignedLoad64(ptr + 56); |
61 | |
|
62 | 0 | current_state = Mix(a ^ salt[1], b ^ current_state); |
63 | 0 | duplicated_state0 = Mix(c ^ salt[2], d ^ duplicated_state0); |
64 | |
|
65 | 0 | duplicated_state1 = Mix(e ^ salt[3], f ^ duplicated_state1); |
66 | 0 | duplicated_state2 = Mix(g ^ salt[4], h ^ duplicated_state2); |
67 | |
|
68 | 0 | ptr += 64; |
69 | 0 | len -= 64; |
70 | 0 | } while (len > 64); |
71 | |
|
72 | 0 | current_state = (current_state ^ duplicated_state0) ^ |
73 | 0 | (duplicated_state1 + duplicated_state2); |
74 | 0 | } |
75 | | |
76 | | // We now have a data `ptr` with at most 64 bytes and the current state |
77 | | // of the hashing state machine stored in current_state. |
78 | 28 | if (len > 32) { |
79 | 20 | uint64_t a = absl::base_internal::UnalignedLoad64(ptr); |
80 | 20 | uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); |
81 | 20 | uint64_t c = absl::base_internal::UnalignedLoad64(ptr + 16); |
82 | 20 | uint64_t d = absl::base_internal::UnalignedLoad64(ptr + 24); |
83 | | |
84 | 20 | uint64_t cs0 = Mix(a ^ salt[1], b ^ current_state); |
85 | 20 | uint64_t cs1 = Mix(c ^ salt[2], d ^ current_state); |
86 | 20 | current_state = cs0 ^ cs1; |
87 | | |
88 | 20 | ptr += 32; |
89 | 20 | len -= 32; |
90 | 20 | } |
91 | | |
92 | | // We now have a data `ptr` with at most 32 bytes and the current state |
93 | | // of the hashing state machine stored in current_state. |
94 | 28 | if (len > 16) { |
95 | 8 | uint64_t a = absl::base_internal::UnalignedLoad64(ptr); |
96 | 8 | uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); |
97 | | |
98 | 8 | current_state = Mix(a ^ salt[1], b ^ current_state); |
99 | 8 | } |
100 | | |
101 | | // We now have a data `ptr` with at least 1 and at most 16 bytes. But we can |
102 | | // safely read from `ptr + len - 16`. |
103 | 28 | uint64_t a = absl::base_internal::UnalignedLoad64(last_16_ptr); |
104 | 28 | uint64_t b = absl::base_internal::UnalignedLoad64(last_16_ptr + 8); |
105 | | |
106 | 28 | return Mix(a ^ salt[1] ^ starting_length, b ^ current_state); |
107 | 28 | } |
108 | | |
109 | | uint64_t LowLevelHash(const void* data, size_t len, uint64_t seed, |
110 | 0 | const uint64_t salt[5]) { |
111 | 0 | if (len > 16) return LowLevelHashLenGt16(data, len, seed, salt); |
112 | | |
113 | | // Prefetch the cacheline that data resides in. |
114 | 0 | PrefetchToLocalCache(data); |
115 | 0 | const uint8_t* ptr = static_cast<const uint8_t*>(data); |
116 | 0 | uint64_t starting_length = static_cast<uint64_t>(len); |
117 | 0 | uint64_t current_state = seed ^ salt[0]; |
118 | 0 | if (len == 0) return current_state; |
119 | | |
120 | 0 | uint64_t a = 0; |
121 | 0 | uint64_t b = 0; |
122 | | |
123 | | // We now have a data `ptr` with at least 1 and at most 16 bytes. |
124 | 0 | if (len > 8) { |
125 | | // When we have at least 9 and at most 16 bytes, set A to the first 64 |
126 | | // bits of the input and B to the last 64 bits of the input. Yes, they |
127 | | // will overlap in the middle if we are working with less than the full 16 |
128 | | // bytes. |
129 | 0 | a = absl::base_internal::UnalignedLoad64(ptr); |
130 | 0 | b = absl::base_internal::UnalignedLoad64(ptr + len - 8); |
131 | 0 | } else if (len > 3) { |
132 | | // If we have at least 4 and at most 8 bytes, set A to the first 32 |
133 | | // bits and B to the last 32 bits. |
134 | 0 | a = absl::base_internal::UnalignedLoad32(ptr); |
135 | 0 | b = absl::base_internal::UnalignedLoad32(ptr + len - 4); |
136 | 0 | } else { |
137 | | // If we have at least 1 and at most 3 bytes, read 2 bytes into A and the |
138 | | // other byte into B, with some adjustments. |
139 | 0 | a = static_cast<uint64_t>((ptr[0] << 8) | ptr[len - 1]); |
140 | 0 | b = static_cast<uint64_t>(ptr[len >> 1]); |
141 | 0 | } |
142 | |
|
143 | 0 | return Mix(a ^ salt[1] ^ starting_length, b ^ current_state); |
144 | 0 | } |
145 | | |
146 | | } // namespace hash_internal |
147 | | ABSL_NAMESPACE_END |
148 | | } // namespace absl |