/src/openssl34/crypto/hashtable/hashtable.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright 2024-2026 The OpenSSL Project Authors. All Rights Reserved. |
3 | | * |
4 | | * Licensed under the Apache License 2.0 (the "License"). You may not use |
5 | | * this file except in compliance with the License. You can obtain a copy |
6 | | * in the file LICENSE in the source distribution or at |
7 | | * https://www.openssl.org/source/license.html |
8 | | * |
9 | | * |
10 | | * |
11 | | * Notes On hash table design and layout |
12 | | * This hashtable uses a hopscotch algorithm to do indexing. The data structure |
13 | | * looks as follows: |
14 | | * |
15 | | * hash +--------------+ |
16 | | * value+------->+ HT_VALUE | |
17 | | * + +--------------+ |
18 | | * +-------+ |
19 | | * | | |
20 | | * +---------------------------------------------------------+ |
21 | | * | | | | | | |
22 | | * | entry | entry | entry | entry | | |
23 | | * | | | | | | |
24 | | * +---------------------------------------------------------+ |
25 | | * | | | |
26 | | * | | | |
27 | | * +---------------------------------------------------------+ |
28 | | * | + + + |
29 | | * | neighborhood[0] neighborhood[1] | |
30 | | * | | |
31 | | * | | |
32 | | * +---------------------------------------------------------+ |
33 | | * | |
34 | | * + |
35 | | * neighborhoods |
36 | | * |
37 | | * On lookup/insert/delete, the items key is hashed to a 64 bit value |
38 | | * and the result is masked to provide an index into the neighborhoods |
39 | | * table. Once a neighborhood is determined, an in-order search is done |
40 | | * of the elements in the neighborhood indexes entries for a matching hash |
41 | | * value, if found, the corresponding HT_VALUE is used for the respective |
42 | | * operation. The number of entries in a neighborhood is determined at build |
43 | | * time based on the cacheline size of the target CPU. The intent is for a |
44 | | * neighborhood to have all entries in the neighborhood fit into a single cache |
45 | | * line to speed up lookups. If all entries in a neighborhood are in use at the |
46 | | * time of an insert, the table is expanded and rehashed. |
47 | | * |
48 | | * Lockless reads hash table is based on the same design but does not |
49 | | * allow growing and deletion. Thus subsequent neighborhoods are always |
50 | | * searched for a match until an empty entry is found. |
51 | | */ |
52 | | |
53 | | #include <string.h> |
54 | | #include <internal/rcu.h> |
55 | | #include <internal/hashtable.h> |
56 | | #include <openssl/rand.h> |
57 | | |
58 | | /* |
59 | | * gcc defines __SANITIZE_THREAD__ |
60 | | * but clang uses the feature attributes api |
61 | | * map the latter to the former |
62 | | */ |
63 | | #if defined(__clang__) && defined(__has_feature) |
64 | | #if __has_feature(thread_sanitizer) |
65 | | #define __SANITIZE_THREADS__ |
66 | | #endif |
67 | | #endif |
68 | | |
69 | | #ifdef __SANITIZE_THREADS__ |
70 | | #include <sanitizer/tsan_interface.h> |
71 | | #endif |
72 | | |
73 | | #include "internal/numbers.h" |
74 | | /* |
75 | | * When we do a lookup/insert/delete, there is a high likelihood |
76 | | * that we will iterate over at least part of the neighborhood list |
77 | | * As such, because we design a neighborhood entry to fit into a single |
78 | | * cache line it is advantageous, when supported to fetch the entire |
79 | | * structure for faster lookups |
80 | | */ |
81 | | #if defined(__GNUC__) || defined(__CLANG__) |
82 | 25.9M | #define PREFETCH_NEIGHBORHOOD(x) __builtin_prefetch(x.entries) |
83 | 60.6M | #define PREFETCH(x) __builtin_prefetch(x) |
84 | | #define ALIGN __attribute__((aligned(8))) |
85 | | #else |
86 | | #define PREFETCH_NEIGHBORHOOD(x) |
87 | | #define PREFETCH(x) |
88 | | #define ALIGN |
89 | | #endif |
90 | | |
91 | | static ossl_unused uint64_t fnv1a_hash(uint8_t *key, size_t len) |
92 | 7.92M | { |
93 | 7.92M | uint64_t hash = 0xcbf29ce484222325ULL; |
94 | 7.92M | size_t i; |
95 | | |
96 | 514M | for (i = 0; i < len; i++) { |
97 | 506M | hash ^= key[i]; |
98 | 506M | hash *= 0x00000100000001B3ULL; |
99 | 506M | } |
100 | 7.92M | return hash; |
101 | 7.92M | } |
102 | | |
103 | | /* |
104 | | * Define our neighborhood list length |
105 | | * Note: It should always be a power of 2 |
106 | | */ |
107 | 477 | #define DEFAULT_NEIGH_LEN_LOG 4 |
108 | 477 | #define DEFAULT_NEIGH_LEN (1 << DEFAULT_NEIGH_LEN_LOG) |
109 | | |
110 | | /* |
111 | | * For now assume cache line size is 64 bytes |
112 | | */ |
113 | 34.5M | #define CACHE_LINE_BYTES 64 |
114 | | #define CACHE_LINE_ALIGNMENT CACHE_LINE_BYTES |
115 | | |
116 | 34.5M | #define NEIGHBORHOOD_LEN (CACHE_LINE_BYTES / sizeof(struct ht_neighborhood_entry_st)) |
117 | | /* |
118 | | * Defines our chains of values |
119 | | */ |
120 | | struct ht_internal_value_st { |
121 | | HT_VALUE value; |
122 | | HT *ht; |
123 | | }; |
124 | | |
125 | | struct ht_neighborhood_entry_st { |
126 | | uint64_t hash; |
127 | | struct ht_internal_value_st *value; |
128 | | } ALIGN; |
129 | | |
130 | | struct ht_neighborhood_st { |
131 | | struct ht_neighborhood_entry_st entries[NEIGHBORHOOD_LEN]; |
132 | | }; |
133 | | |
134 | | /* |
135 | | * Updates to data in this struct |
136 | | * require an rcu sync after modification |
137 | | * prior to free |
138 | | */ |
139 | | struct ht_mutable_data_st { |
140 | | struct ht_neighborhood_st *neighborhoods; |
141 | | void *neighborhood_ptr_to_free; |
142 | | uint64_t neighborhood_mask; |
143 | | }; |
144 | | |
145 | | /* |
146 | | * Private data may be updated on the write |
147 | | * side only, and so do not require rcu sync |
148 | | */ |
149 | | struct ht_write_private_data_st { |
150 | | size_t neighborhood_len; |
151 | | size_t value_count; |
152 | | int need_sync; |
153 | | }; |
154 | | |
155 | | struct ht_internal_st { |
156 | | HT_CONFIG config; |
157 | | CRYPTO_RCU_LOCK *lock; |
158 | | CRYPTO_RWLOCK *atomic_lock; |
159 | | struct ht_mutable_data_st *md; |
160 | | struct ht_write_private_data_st wpd; |
161 | | }; |
162 | | |
163 | | static void free_value(struct ht_internal_value_st *v); |
164 | | |
165 | | static struct ht_neighborhood_st *alloc_new_neighborhood_list(size_t len, |
166 | | void **freeptr) |
167 | 30.2k | { |
168 | 30.2k | struct ht_neighborhood_st *ret; |
169 | | |
170 | 30.2k | ret = OPENSSL_aligned_alloc(sizeof(struct ht_neighborhood_st) * len, |
171 | 30.2k | CACHE_LINE_BYTES, freeptr); |
172 | | |
173 | | /* fall back to regular malloc */ |
174 | 30.2k | if (ret == NULL) { |
175 | 0 | ret = *freeptr = OPENSSL_malloc(sizeof(struct ht_neighborhood_st) * len); |
176 | 0 | if (ret == NULL) |
177 | 0 | return NULL; |
178 | 0 | } |
179 | 30.2k | memset(ret, 0, sizeof(struct ht_neighborhood_st) * len); |
180 | 30.2k | return ret; |
181 | 30.2k | } |
182 | | |
183 | | static void internal_free_nop(HT_VALUE *v) |
184 | 24.8k | { |
185 | 24.8k | return; |
186 | 24.8k | } |
187 | | |
188 | | HT *ossl_ht_new(const HT_CONFIG *conf) |
189 | 242 | { |
190 | 242 | HT *new = OPENSSL_zalloc(sizeof(*new)); |
191 | | |
192 | 242 | if (new == NULL) |
193 | 0 | return NULL; |
194 | | |
195 | 242 | new->atomic_lock = CRYPTO_THREAD_lock_new(); |
196 | 242 | if (new->atomic_lock == NULL) |
197 | 0 | goto err; |
198 | | |
199 | 242 | memcpy(&new->config, conf, sizeof(*conf)); |
200 | | |
201 | 242 | if (new->config.init_neighborhoods != 0) { |
202 | 236 | new->wpd.neighborhood_len = new->config.init_neighborhoods; |
203 | | /* round up to the next power of 2 */ |
204 | 236 | new->wpd.neighborhood_len--; |
205 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 1; |
206 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 2; |
207 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 4; |
208 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 8; |
209 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 16; |
210 | 236 | new->wpd.neighborhood_len++; |
211 | 236 | } else { |
212 | 6 | new->wpd.neighborhood_len = DEFAULT_NEIGH_LEN; |
213 | 6 | } |
214 | | |
215 | 242 | if (new->config.ht_free_fn == NULL) |
216 | 236 | new->config.ht_free_fn = internal_free_nop; |
217 | | |
218 | 242 | new->md = OPENSSL_zalloc(sizeof(*new->md)); |
219 | 242 | if (new->md == NULL) |
220 | 0 | goto err; |
221 | | |
222 | 242 | new->md->neighborhoods = alloc_new_neighborhood_list(new->wpd.neighborhood_len, |
223 | 242 | &new->md->neighborhood_ptr_to_free); |
224 | 242 | if (new->md->neighborhoods == NULL) |
225 | 0 | goto err; |
226 | 242 | new->md->neighborhood_mask = new->wpd.neighborhood_len - 1; |
227 | | |
228 | 242 | new->lock = ossl_rcu_lock_new(1, conf->ctx); |
229 | 242 | if (new->lock == NULL) |
230 | 0 | goto err; |
231 | | |
232 | 242 | if (new->config.ht_hash_fn == NULL) |
233 | 242 | new->config.ht_hash_fn = fnv1a_hash; |
234 | | |
235 | 242 | return new; |
236 | | |
237 | 0 | err: |
238 | 0 | CRYPTO_THREAD_lock_free(new->atomic_lock); |
239 | 0 | ossl_rcu_lock_free(new->lock); |
240 | 0 | if (new->md != NULL) |
241 | 0 | OPENSSL_free(new->md->neighborhood_ptr_to_free); |
242 | 0 | OPENSSL_free(new->md); |
243 | 0 | OPENSSL_free(new); |
244 | 0 | return NULL; |
245 | 242 | } |
246 | | |
247 | | void ossl_ht_read_lock(HT *htable) |
248 | 29 | { |
249 | 29 | ossl_rcu_read_lock(htable->lock); |
250 | 29 | } |
251 | | |
252 | | void ossl_ht_read_unlock(HT *htable) |
253 | 50 | { |
254 | 50 | ossl_rcu_read_unlock(htable->lock); |
255 | 50 | } |
256 | | |
257 | | void ossl_ht_write_lock(HT *htable) |
258 | 270 | { |
259 | 270 | ossl_rcu_write_lock(htable->lock); |
260 | 270 | htable->wpd.need_sync = 0; |
261 | 270 | } |
262 | | |
263 | | void ossl_ht_write_unlock(HT *htable) |
264 | 270 | { |
265 | 270 | int need_sync = htable->wpd.need_sync; |
266 | | |
267 | 270 | htable->wpd.need_sync = 0; |
268 | 270 | ossl_rcu_write_unlock(htable->lock); |
269 | 270 | if (need_sync) |
270 | 175 | ossl_synchronize_rcu(htable->lock); |
271 | 270 | } |
272 | | |
273 | | static void free_oldmd(void *arg) |
274 | 29.9k | { |
275 | 29.9k | struct ht_mutable_data_st *oldmd = arg; |
276 | 29.9k | size_t i, j; |
277 | 29.9k | size_t neighborhood_len = (size_t)oldmd->neighborhood_mask + 1; |
278 | 29.9k | struct ht_internal_value_st *v; |
279 | | |
280 | 347k | for (i = 0; i < neighborhood_len; i++) { |
281 | 317k | PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[i + 1]); |
282 | 1.58M | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
283 | 1.26M | if (oldmd->neighborhoods[i].entries[j].value != NULL) { |
284 | 25.8k | v = oldmd->neighborhoods[i].entries[j].value; |
285 | 25.8k | v->ht->config.ht_free_fn((HT_VALUE *)v); |
286 | 25.8k | free_value(v); |
287 | 25.8k | } |
288 | 1.26M | } |
289 | 317k | } |
290 | | |
291 | 29.9k | OPENSSL_free(oldmd->neighborhood_ptr_to_free); |
292 | 29.9k | OPENSSL_free(oldmd); |
293 | 29.9k | } |
294 | | |
295 | | static int ossl_ht_flush_internal(HT *h) |
296 | 157 | { |
297 | 157 | struct ht_mutable_data_st *newmd = NULL; |
298 | 157 | struct ht_mutable_data_st *oldmd = NULL; |
299 | 157 | CRYPTO_RCU_CB_ITEM *cbi = NULL; |
300 | | |
301 | 157 | newmd = OPENSSL_zalloc(sizeof(*newmd)); |
302 | 157 | if (newmd == NULL) |
303 | 0 | return 0; |
304 | | |
305 | 157 | newmd->neighborhoods = alloc_new_neighborhood_list(DEFAULT_NEIGH_LEN, |
306 | 157 | &newmd->neighborhood_ptr_to_free); |
307 | 157 | if (newmd->neighborhoods == NULL) { |
308 | 0 | OPENSSL_free(newmd); |
309 | 0 | return 0; |
310 | 0 | } |
311 | | |
312 | 157 | newmd->neighborhood_mask = DEFAULT_NEIGH_LEN - 1; |
313 | | |
314 | 157 | cbi = ossl_rcu_cb_item_new(); |
315 | 157 | if (cbi == NULL) { |
316 | 0 | OPENSSL_free(newmd->neighborhood_ptr_to_free); |
317 | 0 | OPENSSL_free(newmd); |
318 | 0 | return 0; |
319 | 0 | } |
320 | | |
321 | | /* Swap the old and new mutable data sets */ |
322 | 157 | oldmd = ossl_rcu_deref(&h->md); |
323 | 157 | ossl_rcu_assign_ptr(&h->md, &newmd); |
324 | | |
325 | | /* Set the number of entries to 0 */ |
326 | 157 | h->wpd.value_count = 0; |
327 | 157 | h->wpd.neighborhood_len = DEFAULT_NEIGH_LEN; |
328 | | |
329 | 157 | ossl_rcu_call(h->lock, cbi, free_oldmd, oldmd); |
330 | 157 | h->wpd.need_sync = 1; |
331 | | |
332 | 157 | return 1; |
333 | 157 | } |
334 | | |
335 | | int ossl_ht_flush(HT *h) |
336 | 4 | { |
337 | 4 | return ossl_ht_flush_internal(h); |
338 | 4 | } |
339 | | |
340 | | void ossl_ht_free(HT *h) |
341 | 154 | { |
342 | 154 | int flush_ok; |
343 | | |
344 | 154 | if (h == NULL) |
345 | 0 | return; |
346 | | |
347 | 154 | ossl_ht_write_lock(h); |
348 | 154 | flush_ok = ossl_ht_flush_internal(h); |
349 | 154 | ossl_ht_write_unlock(h); |
350 | | /* Freeing the lock does a final sync for us */ |
351 | 154 | CRYPTO_THREAD_lock_free(h->atomic_lock); |
352 | 154 | ossl_rcu_lock_free(h->lock); |
353 | 154 | if (flush_ok) { |
354 | 154 | OPENSSL_free(h->md->neighborhood_ptr_to_free); |
355 | 154 | OPENSSL_free(h->md); |
356 | 154 | } else { |
357 | 0 | free_oldmd(h->md); |
358 | 0 | } |
359 | 154 | OPENSSL_free(h); |
360 | 154 | return; |
361 | 154 | } |
362 | | |
363 | | size_t ossl_ht_count(HT *h) |
364 | 0 | { |
365 | 0 | size_t count; |
366 | |
|
367 | 0 | count = h->wpd.value_count; |
368 | 0 | return count; |
369 | 0 | } |
370 | | |
371 | | void ossl_ht_foreach_until(HT *h, int (*cb)(HT_VALUE *obj, void *arg), |
372 | | void *arg) |
373 | 83 | { |
374 | 83 | size_t i, j; |
375 | 83 | struct ht_mutable_data_st *md; |
376 | | |
377 | 83 | md = ossl_rcu_deref(&h->md); |
378 | 13.3k | for (i = 0; i < md->neighborhood_mask + 1; i++) { |
379 | 13.3k | PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]); |
380 | 66.5k | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
381 | 53.2k | if (md->neighborhoods[i].entries[j].value != NULL) { |
382 | 488 | if (!cb((HT_VALUE *)md->neighborhoods[i].entries[j].value, arg)) |
383 | 3 | goto out; |
384 | 488 | } |
385 | 53.2k | } |
386 | 13.3k | } |
387 | 83 | out: |
388 | 83 | return; |
389 | 83 | } |
390 | | |
391 | | HT_VALUE_LIST *ossl_ht_filter(HT *h, size_t max_len, |
392 | | int (*filter)(HT_VALUE *obj, void *arg), |
393 | | void *arg) |
394 | 61 | { |
395 | 61 | struct ht_mutable_data_st *md; |
396 | 61 | HT_VALUE_LIST *list = OPENSSL_zalloc(sizeof(HT_VALUE_LIST) |
397 | 61 | + (sizeof(HT_VALUE *) * max_len)); |
398 | 61 | size_t i, j; |
399 | 61 | struct ht_internal_value_st *v; |
400 | | |
401 | 61 | if (list == NULL) |
402 | 0 | return NULL; |
403 | | |
404 | | /* |
405 | | * The list array lives just beyond the end of |
406 | | * the struct |
407 | | */ |
408 | 61 | list->list = (HT_VALUE **)(list + 1); |
409 | | |
410 | 61 | md = ossl_rcu_deref(&h->md); |
411 | 6.75k | for (i = 0; i < md->neighborhood_mask + 1; i++) { |
412 | 6.70k | PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]); |
413 | 33.4k | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
414 | 26.8k | v = md->neighborhoods[i].entries[j].value; |
415 | 26.8k | if (v != NULL && filter((HT_VALUE *)v, arg)) { |
416 | 5 | list->list[list->list_len++] = (HT_VALUE *)v; |
417 | 5 | if (list->list_len == max_len) |
418 | 5 | goto out; |
419 | 5 | } |
420 | 26.8k | } |
421 | 6.70k | } |
422 | 61 | out: |
423 | 61 | return list; |
424 | 61 | } |
425 | | |
426 | | void ossl_ht_value_list_free(HT_VALUE_LIST *list) |
427 | 61 | { |
428 | 61 | OPENSSL_free(list); |
429 | 61 | } |
430 | | |
431 | | static int compare_hash(uint64_t hash1, uint64_t hash2) |
432 | 41.5M | { |
433 | 41.5M | return (hash1 == hash2); |
434 | 41.5M | } |
435 | | |
436 | | static void free_old_neigh_table(void *arg) |
437 | 18 | { |
438 | 18 | struct ht_mutable_data_st *oldmd = arg; |
439 | | |
440 | 18 | OPENSSL_free(oldmd->neighborhood_ptr_to_free); |
441 | 18 | OPENSSL_free(oldmd); |
442 | 18 | } |
443 | | |
444 | | /* |
445 | | * Increase hash table bucket list |
446 | | * must be called with write_lock held |
447 | | */ |
448 | | static int grow_hashtable(HT *h, size_t oldsize) |
449 | 13 | { |
450 | 13 | struct ht_mutable_data_st *newmd; |
451 | 13 | struct ht_mutable_data_st *oldmd = ossl_rcu_deref(&h->md); |
452 | 13 | CRYPTO_RCU_CB_ITEM *cbi = NULL; |
453 | 13 | int rc = 0; |
454 | 13 | uint64_t oldi, oldj, newi, newj; |
455 | 13 | uint64_t oldhash; |
456 | 13 | struct ht_internal_value_st *oldv; |
457 | 13 | int rehashed; |
458 | 13 | size_t newsize = oldsize * 2; |
459 | | |
460 | 13 | if (h->config.lockless_reads) |
461 | 0 | goto out; |
462 | | |
463 | 13 | if ((newmd = OPENSSL_zalloc(sizeof(*newmd))) == NULL) |
464 | 0 | goto out; |
465 | | |
466 | | /* bucket list is always a power of 2 */ |
467 | 13 | newmd->neighborhoods = alloc_new_neighborhood_list(oldsize * 2, |
468 | 13 | &newmd->neighborhood_ptr_to_free); |
469 | 13 | if (newmd->neighborhoods == NULL) |
470 | 0 | goto out_free; |
471 | | |
472 | | /* being a power of 2 makes for easy mask computation */ |
473 | 13 | newmd->neighborhood_mask = (newsize - 1); |
474 | | |
475 | | /* |
476 | | * Now we need to start rehashing entries |
477 | | * Note we don't need to use atomics here as the new |
478 | | * mutable data hasn't been published |
479 | | */ |
480 | 877 | for (oldi = 0; oldi < h->wpd.neighborhood_len; oldi++) { |
481 | 864 | PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[oldi + 1]); |
482 | 4.32k | for (oldj = 0; oldj < NEIGHBORHOOD_LEN; oldj++) { |
483 | 3.45k | oldv = oldmd->neighborhoods[oldi].entries[oldj].value; |
484 | 3.45k | if (oldv == NULL) |
485 | 3.37k | continue; |
486 | 77 | oldhash = oldmd->neighborhoods[oldi].entries[oldj].hash; |
487 | 77 | newi = oldhash & newmd->neighborhood_mask; |
488 | 77 | rehashed = 0; |
489 | 144 | for (newj = 0; newj < NEIGHBORHOOD_LEN; newj++) { |
490 | 144 | if (newmd->neighborhoods[newi].entries[newj].value == NULL) { |
491 | 77 | newmd->neighborhoods[newi].entries[newj].value = oldv; |
492 | 77 | newmd->neighborhoods[newi].entries[newj].hash = oldhash; |
493 | 77 | rehashed = 1; |
494 | 77 | break; |
495 | 77 | } |
496 | 144 | } |
497 | 77 | if (rehashed == 0) { |
498 | | /* we ran out of space in a neighborhood, grow again */ |
499 | 0 | OPENSSL_free(newmd->neighborhood_ptr_to_free); |
500 | 0 | OPENSSL_free(newmd); |
501 | 0 | return grow_hashtable(h, newsize); |
502 | 0 | } |
503 | 77 | } |
504 | 864 | } |
505 | | |
506 | | /* |
507 | | * Pre allocate the rcu callback item before assigning the newmd. |
508 | | */ |
509 | 13 | cbi = ossl_rcu_cb_item_new(); |
510 | 13 | if (cbi == NULL) |
511 | 0 | goto out_free; |
512 | | |
513 | | /* |
514 | | * Now that our entries are all hashed into the new bucket list |
515 | | * update our bucket_len and target_max_load |
516 | | */ |
517 | 13 | h->wpd.neighborhood_len = newsize; |
518 | | |
519 | | /* |
520 | | * Now we replace the old mutable data with the new |
521 | | */ |
522 | 13 | ossl_rcu_assign_ptr(&h->md, &newmd); |
523 | 13 | ossl_rcu_call(h->lock, cbi, free_old_neigh_table, oldmd); |
524 | 13 | h->wpd.need_sync = 1; |
525 | | /* |
526 | | * And we're done |
527 | | */ |
528 | 13 | rc = 1; |
529 | | |
530 | 13 | out: |
531 | 13 | return rc; |
532 | 0 | out_free: |
533 | 0 | OPENSSL_free(newmd->neighborhood_ptr_to_free); |
534 | 0 | OPENSSL_free(newmd); |
535 | 0 | goto out; |
536 | 13 | } |
537 | | |
538 | | static void free_old_ht_value(void *arg) |
539 | 5 | { |
540 | 5 | HT_VALUE *h = (HT_VALUE *)arg; |
541 | | |
542 | | /* |
543 | | * Note, this is only called on replacement, |
544 | | * the caller is responsible for freeing the |
545 | | * held data, we just need to free the wrapping |
546 | | * struct here |
547 | | */ |
548 | 5 | OPENSSL_free(h); |
549 | 5 | } |
550 | | |
551 | | static ossl_inline int match_key(HT_KEY *a, HT_KEY *b) |
552 | 30.3M | { |
553 | | /* |
554 | | * keys match if they are both present, the same size |
555 | | * and compare equal in memory |
556 | | */ |
557 | 30.3M | PREFETCH(a->keybuf); |
558 | 30.3M | PREFETCH(b->keybuf); |
559 | 30.3M | if (a->keybuf != NULL && b->keybuf != NULL && a->keysize == b->keysize) |
560 | 30.3M | return !memcmp(a->keybuf, b->keybuf, a->keysize); |
561 | | |
562 | 1.92k | return 1; |
563 | 30.3M | } |
564 | | |
565 | | static int ossl_ht_insert_locked(HT *h, uint64_t hash, |
566 | | struct ht_internal_value_st *newval, |
567 | | HT_VALUE **olddata) |
568 | 28.4k | { |
569 | 28.4k | struct ht_mutable_data_st *md = h->md; |
570 | 28.4k | uint64_t neigh_idx_start = hash & md->neighborhood_mask; |
571 | 28.4k | uint64_t neigh_idx = neigh_idx_start; |
572 | 28.4k | size_t j; |
573 | 28.4k | uint64_t ihash; |
574 | 28.4k | HT_VALUE *ival; |
575 | 28.4k | size_t empty_idx = SIZE_MAX; |
576 | 28.4k | int lockless_reads = h->config.lockless_reads; |
577 | 28.4k | CRYPTO_RCU_CB_ITEM *cbi; |
578 | | |
579 | 28.7k | do { |
580 | 28.7k | PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]); |
581 | | |
582 | 43.4k | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
583 | 43.1k | ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value); |
584 | 43.1k | if (ival == NULL) { |
585 | 28.5k | empty_idx = j; |
586 | | /* lockless_reads implies no deletion, we can break out */ |
587 | 28.5k | if (lockless_reads) |
588 | 28.3k | goto not_found; |
589 | 168 | continue; |
590 | 28.5k | } |
591 | 14.5k | if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash, |
592 | 14.5k | &ihash, h->atomic_lock)) |
593 | 0 | return 0; |
594 | 14.5k | if (compare_hash(hash, ihash) && match_key(&newval->value.key, &ival->key)) { |
595 | 11 | if (olddata == NULL) { |
596 | | /* This would insert a duplicate -> fail */ |
597 | 7 | return 0; |
598 | 7 | } |
599 | | /* Do a replacement */ |
600 | 4 | cbi = ossl_rcu_cb_item_new(); |
601 | 4 | if (cbi == NULL) |
602 | 0 | return 0; |
603 | 4 | if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[j].hash, |
604 | 4 | hash, h->atomic_lock)) |
605 | 0 | return 0; |
606 | 4 | *olddata = (HT_VALUE *)md->neighborhoods[neigh_idx].entries[j].value; |
607 | 4 | ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[j].value, |
608 | 4 | &newval); |
609 | 4 | ossl_rcu_call(h->lock, cbi, free_old_ht_value, *olddata); |
610 | 4 | h->wpd.need_sync = 1; |
611 | 4 | return 1; |
612 | 4 | } |
613 | 14.5k | } |
614 | 333 | if (!lockless_reads) |
615 | 66 | break; |
616 | | /* Continue search in subsequent neighborhoods */ |
617 | 267 | neigh_idx = (neigh_idx + 1) & md->neighborhood_mask; |
618 | 267 | } while (neigh_idx != neigh_idx_start); |
619 | | |
620 | 28.4k | not_found: |
621 | | /* If we get to here, its just an insert */ |
622 | 28.4k | if (empty_idx == SIZE_MAX) |
623 | 13 | return -1; /* out of space */ |
624 | 28.4k | if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[empty_idx].hash, |
625 | 28.4k | hash, h->atomic_lock)) |
626 | 0 | return 0; |
627 | 28.4k | h->wpd.value_count++; |
628 | 28.4k | ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[empty_idx].value, |
629 | 28.4k | &newval); |
630 | 28.4k | return 1; |
631 | 28.4k | } |
632 | | |
633 | | static struct ht_internal_value_st *alloc_new_value(HT *h, HT_KEY *key, |
634 | | void *data, |
635 | | uintptr_t *type) |
636 | 40.0k | { |
637 | 40.0k | struct ht_internal_value_st *tmp; |
638 | 40.0k | size_t nvsize = sizeof(*tmp); |
639 | | |
640 | 40.0k | if (h->config.collision_check == 1) |
641 | 38.9k | nvsize += key->keysize; |
642 | | |
643 | 40.0k | tmp = OPENSSL_malloc(nvsize); |
644 | | |
645 | 40.0k | if (tmp == NULL) |
646 | 0 | return NULL; |
647 | | |
648 | 40.0k | tmp->ht = h; |
649 | 40.0k | tmp->value.value = data; |
650 | 40.0k | tmp->value.type_id = type; |
651 | 40.0k | tmp->value.key.keybuf = NULL; |
652 | 40.0k | if (h->config.collision_check) { |
653 | 38.9k | tmp->value.key.keybuf = (uint8_t *)(tmp + 1); |
654 | 38.9k | tmp->value.key.keysize = key->keysize; |
655 | 38.9k | memcpy(tmp->value.key.keybuf, key->keybuf, key->keysize); |
656 | 38.9k | } |
657 | | |
658 | 40.0k | return tmp; |
659 | 40.0k | } |
660 | | |
661 | | static void free_value(struct ht_internal_value_st *v) |
662 | 25.8k | { |
663 | 25.8k | OPENSSL_free(v); |
664 | 25.8k | } |
665 | | |
666 | | int ossl_ht_insert(HT *h, HT_KEY *key, HT_VALUE *data, HT_VALUE **olddata) |
667 | 28.4k | { |
668 | 28.4k | struct ht_internal_value_st *newval = NULL; |
669 | 28.4k | uint64_t hash; |
670 | 28.4k | int rc = 0; |
671 | 28.4k | int i; |
672 | | |
673 | 28.4k | if (data->value == NULL) |
674 | 0 | goto out; |
675 | | |
676 | 28.4k | newval = alloc_new_value(h, key, data->value, data->type_id); |
677 | 28.4k | if (newval == NULL) |
678 | 0 | goto out; |
679 | | |
680 | | /* |
681 | | * we have to take our lock here to prevent other changes |
682 | | * to the bucket list |
683 | | */ |
684 | 28.4k | hash = h->config.ht_hash_fn(key->keybuf, key->keysize); |
685 | | |
686 | 28.4k | for (i = 0; |
687 | 28.4k | (rc = ossl_ht_insert_locked(h, hash, newval, olddata)) == -1 |
688 | 13 | && i <= (int)NEIGHBORHOOD_LEN; |
689 | 28.4k | ++i) |
690 | 13 | if (!grow_hashtable(h, h->wpd.neighborhood_len)) { |
691 | 0 | rc = -1; |
692 | 0 | break; |
693 | 0 | } |
694 | | |
695 | 28.4k | if (rc <= 0) |
696 | 7 | free_value(newval); |
697 | | |
698 | 28.4k | out: |
699 | 28.4k | return rc; |
700 | 28.4k | } |
701 | | |
702 | | HT_VALUE *ossl_ht_get(HT *h, HT_KEY *key) |
703 | 25.6M | { |
704 | 25.6M | struct ht_mutable_data_st *md; |
705 | 25.6M | uint64_t hash; |
706 | 25.6M | uint64_t neigh_idx_start; |
707 | 25.6M | uint64_t neigh_idx; |
708 | 25.6M | struct ht_internal_value_st *ival = NULL; |
709 | 25.6M | size_t j; |
710 | 25.6M | uint64_t ehash; |
711 | 25.6M | int lockless_reads = h->config.lockless_reads; |
712 | | |
713 | 25.6M | hash = h->config.ht_hash_fn(key->keybuf, key->keysize); |
714 | | |
715 | 25.6M | md = ossl_rcu_deref(&h->md); |
716 | 25.6M | neigh_idx = neigh_idx_start = hash & md->neighborhood_mask; |
717 | 25.6M | do { |
718 | 25.6M | PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]); |
719 | 32.7M | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
720 | 32.7M | ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value); |
721 | 32.7M | if (ival == NULL) { |
722 | 2.33M | if (lockless_reads) |
723 | | /* lockless_reads implies no deletion, we can break out */ |
724 | 2.33M | return NULL; |
725 | 112 | continue; |
726 | 2.33M | } |
727 | 30.4M | if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash, |
728 | 30.4M | &ehash, h->atomic_lock)) |
729 | 0 | return NULL; |
730 | 30.4M | if (compare_hash(hash, ehash) && match_key(&ival->value.key, key)) |
731 | 23.2M | return (HT_VALUE *)ival; |
732 | 30.4M | } |
733 | 5.20k | if (!lockless_reads) |
734 | 43 | break; |
735 | | /* Continue search in subsequent neighborhoods */ |
736 | 5.16k | neigh_idx = (neigh_idx + 1) & md->neighborhood_mask; |
737 | 5.16k | } while (neigh_idx != neigh_idx_start); |
738 | | |
739 | 43 | return NULL; |
740 | 25.6M | } |
741 | | |
742 | | static void free_old_entry(void *arg) |
743 | 13 | { |
744 | 13 | struct ht_internal_value_st *v = arg; |
745 | | |
746 | 13 | v->ht->config.ht_free_fn((HT_VALUE *)v); |
747 | 13 | free_value(v); |
748 | 13 | } |
749 | | |
750 | | int ossl_ht_delete(HT *h, HT_KEY *key) |
751 | 49 | { |
752 | 49 | uint64_t hash; |
753 | 49 | uint64_t neigh_idx; |
754 | 49 | size_t j; |
755 | 49 | struct ht_internal_value_st *v = NULL; |
756 | 49 | HT_VALUE *nv = NULL; |
757 | 49 | int rc = 0; |
758 | | |
759 | 49 | if (h->config.lockless_reads) |
760 | 0 | return 0; |
761 | | |
762 | 49 | hash = h->config.ht_hash_fn(key->keybuf, key->keysize); |
763 | | |
764 | 49 | neigh_idx = hash & h->md->neighborhood_mask; |
765 | 49 | PREFETCH_NEIGHBORHOOD(h->md->neighborhoods[neigh_idx]); |
766 | 228 | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
767 | 187 | v = (struct ht_internal_value_st *)h->md->neighborhoods[neigh_idx].entries[j].value; |
768 | 187 | if (v == NULL) |
769 | 102 | continue; |
770 | 85 | if (compare_hash(hash, h->md->neighborhoods[neigh_idx].entries[j].hash) |
771 | 8 | && match_key(key, &v->value.key)) { |
772 | 8 | CRYPTO_RCU_CB_ITEM *cbi = ossl_rcu_cb_item_new(); |
773 | 8 | if (cbi == NULL) |
774 | 0 | break; |
775 | 8 | if (!CRYPTO_atomic_store(&h->md->neighborhoods[neigh_idx].entries[j].hash, |
776 | 8 | 0, h->atomic_lock)) |
777 | 0 | break; |
778 | 8 | h->wpd.value_count--; |
779 | 8 | ossl_rcu_assign_ptr(&h->md->neighborhoods[neigh_idx].entries[j].value, |
780 | 8 | &nv); |
781 | 8 | ossl_rcu_call(h->lock, cbi, free_old_entry, v); |
782 | 8 | h->wpd.need_sync = 1; |
783 | 8 | rc = 1; |
784 | 8 | break; |
785 | 8 | } |
786 | 85 | } |
787 | 49 | return rc; |
788 | 49 | } |