/src/openssl34/crypto/hashtable/hashtable.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright 2024-2026 The OpenSSL Project Authors. All Rights Reserved. |
3 | | * |
4 | | * Licensed under the Apache License 2.0 (the "License"). You may not use |
5 | | * this file except in compliance with the License. You can obtain a copy |
6 | | * in the file LICENSE in the source distribution or at |
7 | | * https://www.openssl.org/source/license.html |
8 | | * |
9 | | * |
10 | | * |
11 | | * Notes On hash table design and layout |
12 | | * This hashtable uses a hopscotch algorithm to do indexing. The data structure |
13 | | * looks as follows: |
14 | | * |
15 | | * hash +--------------+ |
16 | | * value+------->+ HT_VALUE | |
17 | | * + +--------------+ |
18 | | * +-------+ |
19 | | * | | |
20 | | * +---------------------------------------------------------+ |
21 | | * | | | | | | |
22 | | * | entry | entry | entry | entry | | |
23 | | * | | | | | | |
24 | | * +---------------------------------------------------------+ |
25 | | * | | | |
26 | | * | | | |
27 | | * +---------------------------------------------------------+ |
28 | | * | + + + |
29 | | * | neighborhood[0] neighborhood[1] | |
30 | | * | | |
31 | | * | | |
32 | | * +---------------------------------------------------------+ |
33 | | * | |
34 | | * + |
35 | | * neighborhoods |
36 | | * |
37 | | * On lookup/insert/delete, the items key is hashed to a 64 bit value |
38 | | * and the result is masked to provide an index into the neighborhoods |
39 | | * table. Once a neighborhood is determined, an in-order search is done |
40 | | * of the elements in the neighborhood indexes entries for a matching hash |
41 | | * value, if found, the corresponding HT_VALUE is used for the respective |
42 | | * operation. The number of entries in a neighborhood is determined at build |
43 | | * time based on the cacheline size of the target CPU. The intent is for a |
44 | | * neighborhood to have all entries in the neighborhood fit into a single cache |
45 | | * line to speed up lookups. If all entries in a neighborhood are in use at the |
46 | | * time of an insert, the table is expanded and rehashed. |
47 | | * |
48 | | * Lockless reads hash table is based on the same design but does not |
49 | | * allow growing and deletion. Thus subsequent neighborhoods are always |
50 | | * searched for a match until an empty entry is found. |
51 | | */ |
52 | | |
53 | | #include <string.h> |
54 | | #include <internal/rcu.h> |
55 | | #include <internal/hashtable.h> |
56 | | #include <openssl/rand.h> |
57 | | |
58 | | /* |
59 | | * gcc defines __SANITIZE_THREAD__ |
60 | | * but clang uses the feature attributes api |
61 | | * map the latter to the former |
62 | | */ |
63 | | #if defined(__clang__) && defined(__has_feature) |
64 | | #if __has_feature(thread_sanitizer) |
65 | | #define __SANITIZE_THREADS__ |
66 | | #endif |
67 | | #endif |
68 | | |
69 | | #ifdef __SANITIZE_THREADS__ |
70 | | #include <sanitizer/tsan_interface.h> |
71 | | #endif |
72 | | |
73 | | #include "internal/numbers.h" |
74 | | /* |
75 | | * When we do a lookup/insert/delete, there is a high likelihood |
76 | | * that we will iterate over at least part of the neighborhood list |
77 | | * As such, because we design a neighborhood entry to fit into a single |
78 | | * cache line it is advantageous, when supported to fetch the entire |
79 | | * structure for faster lookups |
80 | | */ |
81 | | #if defined(__GNUC__) || defined(__CLANG__) |
82 | 27.2M | #define PREFETCH_NEIGHBORHOOD(x) __builtin_prefetch(x.entries) |
83 | 72.4M | #define PREFETCH(x) __builtin_prefetch(x) |
84 | | #define ALIGN __attribute__((aligned(8))) |
85 | | #else |
86 | | #define PREFETCH_NEIGHBORHOOD(x) |
87 | | #define PREFETCH(x) |
88 | | #define ALIGN |
89 | | #endif |
90 | | |
91 | | static ossl_unused uint64_t fnv1a_hash(uint8_t *key, size_t len) |
92 | 8.31M | { |
93 | 8.31M | uint64_t hash = 0xcbf29ce484222325ULL; |
94 | 8.31M | size_t i; |
95 | | |
96 | 540M | for (i = 0; i < len; i++) { |
97 | 532M | hash ^= key[i]; |
98 | 532M | hash *= 0x00000100000001B3ULL; |
99 | 532M | } |
100 | 8.31M | return hash; |
101 | 8.31M | } |
102 | | |
103 | | /* |
104 | | * Define our neighborhood list length |
105 | | * Note: It should always be a power of 2 |
106 | | */ |
107 | 477 | #define DEFAULT_NEIGH_LEN_LOG 4 |
108 | 477 | #define DEFAULT_NEIGH_LEN (1 << DEFAULT_NEIGH_LEN_LOG) |
109 | | |
110 | | /* |
111 | | * For now assume cache line size is 64 bytes |
112 | | */ |
113 | 36.3M | #define CACHE_LINE_BYTES 64 |
114 | | #define CACHE_LINE_ALIGNMENT CACHE_LINE_BYTES |
115 | | |
116 | 36.3M | #define NEIGHBORHOOD_LEN (CACHE_LINE_BYTES / sizeof(struct ht_neighborhood_entry_st)) |
117 | | /* |
118 | | * Defines our chains of values |
119 | | */ |
120 | | struct ht_internal_value_st { |
121 | | HT_VALUE value; |
122 | | HT *ht; |
123 | | }; |
124 | | |
125 | | struct ht_neighborhood_entry_st { |
126 | | uint64_t hash; |
127 | | struct ht_internal_value_st *value; |
128 | | } ALIGN; |
129 | | |
130 | | struct ht_neighborhood_st { |
131 | | struct ht_neighborhood_entry_st entries[NEIGHBORHOOD_LEN]; |
132 | | }; |
133 | | |
134 | | /* |
135 | | * Updates to data in this struct |
136 | | * require an rcu sync after modification |
137 | | * prior to free |
138 | | */ |
139 | | struct ht_mutable_data_st { |
140 | | struct ht_neighborhood_st *neighborhoods; |
141 | | void *neighborhood_ptr_to_free; |
142 | | uint64_t neighborhood_mask; |
143 | | }; |
144 | | |
145 | | /* |
146 | | * Private data may be updated on the write |
147 | | * side only, and so do not require rcu sync |
148 | | */ |
149 | | struct ht_write_private_data_st { |
150 | | size_t neighborhood_len; |
151 | | size_t value_count; |
152 | | int need_sync; |
153 | | }; |
154 | | |
155 | | struct ht_internal_st { |
156 | | HT_CONFIG config; |
157 | | CRYPTO_RCU_LOCK *lock; |
158 | | CRYPTO_RWLOCK *atomic_lock; |
159 | | struct ht_mutable_data_st *md; |
160 | | struct ht_write_private_data_st wpd; |
161 | | }; |
162 | | |
163 | | static void free_value(struct ht_internal_value_st *v); |
164 | | |
165 | | static struct ht_neighborhood_st *alloc_new_neighborhood_list(size_t len, |
166 | | void **freeptr) |
167 | 34.2k | { |
168 | 34.2k | struct ht_neighborhood_st *ret; |
169 | | |
170 | 34.2k | ret = OPENSSL_aligned_alloc(sizeof(struct ht_neighborhood_st) * len, |
171 | 34.2k | CACHE_LINE_BYTES, freeptr); |
172 | | |
173 | | /* fall back to regular malloc */ |
174 | 34.2k | if (ret == NULL) { |
175 | 0 | ret = *freeptr = OPENSSL_malloc(sizeof(struct ht_neighborhood_st) * len); |
176 | 0 | if (ret == NULL) |
177 | 0 | return NULL; |
178 | 0 | } |
179 | 34.2k | memset(ret, 0, sizeof(struct ht_neighborhood_st) * len); |
180 | 34.2k | return ret; |
181 | 34.2k | } |
182 | | |
183 | | static void internal_free_nop(HT_VALUE *v) |
184 | 24.8k | { |
185 | 24.8k | return; |
186 | 24.8k | } |
187 | | |
188 | | HT *ossl_ht_new(const HT_CONFIG *conf) |
189 | 242 | { |
190 | 242 | HT *new = OPENSSL_zalloc(sizeof(*new)); |
191 | | |
192 | 242 | if (new == NULL) |
193 | 0 | return NULL; |
194 | | |
195 | 242 | new->atomic_lock = CRYPTO_THREAD_lock_new(); |
196 | 242 | if (new->atomic_lock == NULL) |
197 | 0 | goto err; |
198 | | |
199 | 242 | memcpy(&new->config, conf, sizeof(*conf)); |
200 | | |
201 | 242 | if (new->config.init_neighborhoods != 0) { |
202 | 236 | new->wpd.neighborhood_len = new->config.init_neighborhoods; |
203 | | /* round up to the next power of 2 */ |
204 | 236 | new->wpd.neighborhood_len--; |
205 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 1; |
206 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 2; |
207 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 4; |
208 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 8; |
209 | 236 | new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 16; |
210 | 236 | new->wpd.neighborhood_len++; |
211 | 236 | } else { |
212 | 6 | new->wpd.neighborhood_len = DEFAULT_NEIGH_LEN; |
213 | 6 | } |
214 | | |
215 | 242 | if (new->config.ht_free_fn == NULL) |
216 | 236 | new->config.ht_free_fn = internal_free_nop; |
217 | | |
218 | 242 | new->md = OPENSSL_zalloc(sizeof(*new->md)); |
219 | 242 | if (new->md == NULL) |
220 | 0 | goto err; |
221 | | |
222 | 242 | new->md->neighborhoods = alloc_new_neighborhood_list(new->wpd.neighborhood_len, |
223 | 242 | &new->md->neighborhood_ptr_to_free); |
224 | 242 | if (new->md->neighborhoods == NULL) |
225 | 0 | goto err; |
226 | 242 | new->md->neighborhood_mask = new->wpd.neighborhood_len - 1; |
227 | | |
228 | 242 | new->lock = ossl_rcu_lock_new(1, conf->ctx); |
229 | 242 | if (new->lock == NULL) |
230 | 0 | goto err; |
231 | | |
232 | 242 | if (new->config.ht_hash_fn == NULL) |
233 | 242 | new->config.ht_hash_fn = fnv1a_hash; |
234 | | |
235 | 242 | return new; |
236 | | |
237 | 0 | err: |
238 | 0 | CRYPTO_THREAD_lock_free(new->atomic_lock); |
239 | 0 | ossl_rcu_lock_free(new->lock); |
240 | 0 | if (new->md != NULL) |
241 | 0 | OPENSSL_free(new->md->neighborhood_ptr_to_free); |
242 | 0 | OPENSSL_free(new->md); |
243 | 0 | OPENSSL_free(new); |
244 | 0 | return NULL; |
245 | 242 | } |
246 | | |
247 | | void ossl_ht_read_lock(HT *htable) |
248 | 32 | { |
249 | 32 | ossl_rcu_read_lock(htable->lock); |
250 | 32 | } |
251 | | |
252 | | void ossl_ht_read_unlock(HT *htable) |
253 | 49 | { |
254 | 49 | ossl_rcu_read_unlock(htable->lock); |
255 | 49 | } |
256 | | |
257 | | void ossl_ht_write_lock(HT *htable) |
258 | 245 | { |
259 | 245 | ossl_rcu_write_lock(htable->lock); |
260 | 245 | htable->wpd.need_sync = 0; |
261 | 245 | } |
262 | | |
263 | | void ossl_ht_write_unlock(HT *htable) |
264 | 245 | { |
265 | 245 | int need_sync = htable->wpd.need_sync; |
266 | | |
267 | 245 | htable->wpd.need_sync = 0; |
268 | 245 | ossl_rcu_write_unlock(htable->lock); |
269 | 245 | if (need_sync) |
270 | 172 | ossl_synchronize_rcu(htable->lock); |
271 | 245 | } |
272 | | |
273 | | static void free_oldmd(void *arg) |
274 | 32.5k | { |
275 | 32.5k | struct ht_mutable_data_st *oldmd = arg; |
276 | 32.5k | size_t i, j; |
277 | 32.5k | size_t neighborhood_len = (size_t)oldmd->neighborhood_mask + 1; |
278 | 32.5k | struct ht_internal_value_st *v; |
279 | | |
280 | 368k | for (i = 0; i < neighborhood_len; i++) { |
281 | 335k | PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[i + 1]); |
282 | 1.67M | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
283 | 1.34M | if (oldmd->neighborhoods[i].entries[j].value != NULL) { |
284 | 26.1k | v = oldmd->neighborhoods[i].entries[j].value; |
285 | 26.1k | v->ht->config.ht_free_fn((HT_VALUE *)v); |
286 | 26.1k | free_value(v); |
287 | 26.1k | } |
288 | 1.34M | } |
289 | 335k | } |
290 | | |
291 | 32.5k | OPENSSL_free(oldmd->neighborhood_ptr_to_free); |
292 | 32.5k | OPENSSL_free(oldmd); |
293 | 32.5k | } |
294 | | |
295 | | static int ossl_ht_flush_internal(HT *h) |
296 | 157 | { |
297 | 157 | struct ht_mutable_data_st *newmd = NULL; |
298 | 157 | struct ht_mutable_data_st *oldmd = NULL; |
299 | 157 | CRYPTO_RCU_CB_ITEM *cbi = NULL; |
300 | | |
301 | 157 | newmd = OPENSSL_zalloc(sizeof(*newmd)); |
302 | 157 | if (newmd == NULL) |
303 | 0 | return 0; |
304 | | |
305 | 157 | newmd->neighborhoods = alloc_new_neighborhood_list(DEFAULT_NEIGH_LEN, |
306 | 157 | &newmd->neighborhood_ptr_to_free); |
307 | 157 | if (newmd->neighborhoods == NULL) { |
308 | 0 | OPENSSL_free(newmd); |
309 | 0 | return 0; |
310 | 0 | } |
311 | | |
312 | 157 | newmd->neighborhood_mask = DEFAULT_NEIGH_LEN - 1; |
313 | | |
314 | 157 | cbi = ossl_rcu_cb_item_new(); |
315 | 157 | if (cbi == NULL) { |
316 | 0 | OPENSSL_free(newmd->neighborhood_ptr_to_free); |
317 | 0 | OPENSSL_free(newmd); |
318 | 0 | return 0; |
319 | 0 | } |
320 | | |
321 | | /* Swap the old and new mutable data sets */ |
322 | 157 | oldmd = ossl_rcu_deref(&h->md); |
323 | 157 | ossl_rcu_assign_ptr(&h->md, &newmd); |
324 | | |
325 | | /* Set the number of entries to 0 */ |
326 | 157 | h->wpd.value_count = 0; |
327 | 157 | h->wpd.neighborhood_len = DEFAULT_NEIGH_LEN; |
328 | | |
329 | 157 | ossl_rcu_call(h->lock, cbi, free_oldmd, oldmd); |
330 | 157 | h->wpd.need_sync = 1; |
331 | | |
332 | 157 | return 1; |
333 | 157 | } |
334 | | |
335 | | int ossl_ht_flush(HT *h) |
336 | 420 | { |
337 | 420 | return ossl_ht_flush_internal(h); |
338 | 420 | } |
339 | | |
340 | | void ossl_ht_free(HT *h) |
341 | 154 | { |
342 | 154 | int flush_ok; |
343 | | |
344 | 154 | if (h == NULL) |
345 | 0 | return; |
346 | | |
347 | 154 | ossl_ht_write_lock(h); |
348 | 154 | flush_ok = ossl_ht_flush_internal(h); |
349 | 154 | ossl_ht_write_unlock(h); |
350 | | /* Freeing the lock does a final sync for us */ |
351 | 154 | CRYPTO_THREAD_lock_free(h->atomic_lock); |
352 | 154 | ossl_rcu_lock_free(h->lock); |
353 | 154 | if (flush_ok) { |
354 | 154 | OPENSSL_free(h->md->neighborhood_ptr_to_free); |
355 | 154 | OPENSSL_free(h->md); |
356 | 154 | } else { |
357 | 0 | free_oldmd(h->md); |
358 | 0 | } |
359 | 154 | OPENSSL_free(h); |
360 | 154 | return; |
361 | 154 | } |
362 | | |
363 | | size_t ossl_ht_count(HT *h) |
364 | 1.97k | { |
365 | 1.97k | size_t count; |
366 | | |
367 | 1.97k | count = h->wpd.value_count; |
368 | 1.97k | return count; |
369 | 1.97k | } |
370 | | |
371 | | void ossl_ht_foreach_until(HT *h, int (*cb)(HT_VALUE *obj, void *arg), |
372 | | void *arg) |
373 | 100 | { |
374 | 100 | size_t i, j; |
375 | 100 | struct ht_mutable_data_st *md; |
376 | | |
377 | 100 | md = ossl_rcu_deref(&h->md); |
378 | 37.9k | for (i = 0; i < md->neighborhood_mask + 1; i++) { |
379 | 37.8k | PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]); |
380 | 189k | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
381 | 151k | if (md->neighborhoods[i].entries[j].value != NULL) { |
382 | 650 | if (!cb((HT_VALUE *)md->neighborhoods[i].entries[j].value, arg)) |
383 | 9 | goto out; |
384 | 650 | } |
385 | 151k | } |
386 | 37.8k | } |
387 | 100 | out: |
388 | 100 | return; |
389 | 100 | } |
390 | | |
391 | | HT_VALUE_LIST *ossl_ht_filter(HT *h, size_t max_len, |
392 | | int (*filter)(HT_VALUE *obj, void *arg), |
393 | | void *arg) |
394 | 78 | { |
395 | 78 | struct ht_mutable_data_st *md; |
396 | 78 | HT_VALUE_LIST *list = OPENSSL_zalloc(sizeof(HT_VALUE_LIST) |
397 | 78 | + (sizeof(HT_VALUE *) * max_len)); |
398 | 78 | size_t i, j; |
399 | 78 | struct ht_internal_value_st *v; |
400 | | |
401 | 78 | if (list == NULL) |
402 | 0 | return NULL; |
403 | | |
404 | | /* |
405 | | * The list array lives just beyond the end of |
406 | | * the struct |
407 | | */ |
408 | 78 | list->list = (HT_VALUE **)(list + 1); |
409 | | |
410 | 78 | md = ossl_rcu_deref(&h->md); |
411 | 21.8k | for (i = 0; i < md->neighborhood_mask + 1; i++) { |
412 | 21.7k | PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]); |
413 | 108k | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
414 | 86.9k | v = md->neighborhoods[i].entries[j].value; |
415 | 86.9k | if (v != NULL && filter((HT_VALUE *)v, arg)) { |
416 | 5 | list->list[list->list_len++] = (HT_VALUE *)v; |
417 | 5 | if (list->list_len == max_len) |
418 | 5 | goto out; |
419 | 5 | } |
420 | 86.9k | } |
421 | 21.7k | } |
422 | 78 | out: |
423 | 78 | return list; |
424 | 78 | } |
425 | | |
426 | | void ossl_ht_value_list_free(HT_VALUE_LIST *list) |
427 | 78 | { |
428 | 78 | OPENSSL_free(list); |
429 | 78 | } |
430 | | |
431 | | static int compare_hash(uint64_t hash1, uint64_t hash2) |
432 | 49.9M | { |
433 | 49.9M | return (hash1 == hash2); |
434 | 49.9M | } |
435 | | |
436 | | static void free_old_neigh_table(void *arg) |
437 | 100 | { |
438 | 100 | struct ht_mutable_data_st *oldmd = arg; |
439 | | |
440 | 100 | OPENSSL_free(oldmd->neighborhood_ptr_to_free); |
441 | 100 | OPENSSL_free(oldmd); |
442 | 100 | } |
443 | | |
444 | | /* |
445 | | * Increase hash table bucket list |
446 | | * must be called with write_lock held |
447 | | */ |
448 | | static int grow_hashtable(HT *h, size_t oldsize) |
449 | 18 | { |
450 | 18 | struct ht_mutable_data_st *newmd; |
451 | 18 | struct ht_mutable_data_st *oldmd = ossl_rcu_deref(&h->md); |
452 | 18 | CRYPTO_RCU_CB_ITEM *cbi = NULL; |
453 | 18 | int rc = 0; |
454 | 18 | uint64_t oldi, oldj, newi, newj; |
455 | 18 | uint64_t oldhash; |
456 | 18 | struct ht_internal_value_st *oldv; |
457 | 18 | int rehashed; |
458 | 18 | size_t newsize = oldsize * 2; |
459 | | |
460 | 18 | if (h->config.lockless_reads) |
461 | 0 | goto out; |
462 | | |
463 | 18 | if ((newmd = OPENSSL_zalloc(sizeof(*newmd))) == NULL) |
464 | 0 | goto out; |
465 | | |
466 | | /* bucket list is always a power of 2 */ |
467 | 18 | newmd->neighborhoods = alloc_new_neighborhood_list(oldsize * 2, |
468 | 18 | &newmd->neighborhood_ptr_to_free); |
469 | 18 | if (newmd->neighborhoods == NULL) |
470 | 0 | goto out_free; |
471 | | |
472 | | /* being a power of 2 makes for easy mask computation */ |
473 | 18 | newmd->neighborhood_mask = (newsize - 1); |
474 | | |
475 | | /* |
476 | | * Now we need to start rehashing entries |
477 | | * Note we don't need to use atomics here as the new |
478 | | * mutable data hasn't been published |
479 | | */ |
480 | 1.87k | for (oldi = 0; oldi < h->wpd.neighborhood_len; oldi++) { |
481 | 1.85k | PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[oldi + 1]); |
482 | 9.28k | for (oldj = 0; oldj < NEIGHBORHOOD_LEN; oldj++) { |
483 | 7.42k | oldv = oldmd->neighborhoods[oldi].entries[oldj].value; |
484 | 7.42k | if (oldv == NULL) |
485 | 7.33k | continue; |
486 | 89 | oldhash = oldmd->neighborhoods[oldi].entries[oldj].hash; |
487 | 89 | newi = oldhash & newmd->neighborhood_mask; |
488 | 89 | rehashed = 0; |
489 | 178 | for (newj = 0; newj < NEIGHBORHOOD_LEN; newj++) { |
490 | 178 | if (newmd->neighborhoods[newi].entries[newj].value == NULL) { |
491 | 89 | newmd->neighborhoods[newi].entries[newj].value = oldv; |
492 | 89 | newmd->neighborhoods[newi].entries[newj].hash = oldhash; |
493 | 89 | rehashed = 1; |
494 | 89 | break; |
495 | 89 | } |
496 | 178 | } |
497 | 89 | if (rehashed == 0) { |
498 | | /* we ran out of space in a neighborhood, grow again */ |
499 | 0 | OPENSSL_free(newmd->neighborhood_ptr_to_free); |
500 | 0 | OPENSSL_free(newmd); |
501 | 0 | return grow_hashtable(h, newsize); |
502 | 0 | } |
503 | 89 | } |
504 | 1.85k | } |
505 | | |
506 | | /* |
507 | | * Pre allocate the rcu callback item before assigning the newmd. |
508 | | */ |
509 | 18 | cbi = ossl_rcu_cb_item_new(); |
510 | 18 | if (cbi == NULL) |
511 | 0 | goto out_free; |
512 | | |
513 | | /* |
514 | | * Now that our entries are all hashed into the new bucket list |
515 | | * update our bucket_len and target_max_load |
516 | | */ |
517 | 18 | h->wpd.neighborhood_len = newsize; |
518 | | |
519 | | /* |
520 | | * Now we replace the old mutable data with the new |
521 | | */ |
522 | 18 | ossl_rcu_assign_ptr(&h->md, &newmd); |
523 | 18 | ossl_rcu_call(h->lock, cbi, free_old_neigh_table, oldmd); |
524 | 18 | h->wpd.need_sync = 1; |
525 | | /* |
526 | | * And we're done |
527 | | */ |
528 | 18 | rc = 1; |
529 | | |
530 | 18 | out: |
531 | 18 | return rc; |
532 | 0 | out_free: |
533 | 0 | OPENSSL_free(newmd->neighborhood_ptr_to_free); |
534 | 0 | OPENSSL_free(newmd); |
535 | 0 | goto out; |
536 | 18 | } |
537 | | |
538 | | static void free_old_ht_value(void *arg) |
539 | 3 | { |
540 | 3 | HT_VALUE *h = (HT_VALUE *)arg; |
541 | | |
542 | | /* |
543 | | * Note, this is only called on replacement, |
544 | | * the caller is responsible for freeing the |
545 | | * held data, we just need to free the wrapping |
546 | | * struct here |
547 | | */ |
548 | 3 | OPENSSL_free(h); |
549 | 3 | } |
550 | | |
551 | | static ossl_inline int match_key(HT_KEY *a, HT_KEY *b) |
552 | 36.2M | { |
553 | | /* |
554 | | * keys match if they are both present, the same size |
555 | | * and compare equal in memory |
556 | | */ |
557 | 36.2M | PREFETCH(a->keybuf); |
558 | 36.2M | PREFETCH(b->keybuf); |
559 | 36.2M | if (a->keybuf != NULL && b->keybuf != NULL && a->keysize == b->keysize) |
560 | 36.2M | return !memcmp(a->keybuf, b->keybuf, a->keysize); |
561 | | |
562 | 4.27k | return 1; |
563 | 36.2M | } |
564 | | |
565 | | static int ossl_ht_insert_locked(HT *h, uint64_t hash, |
566 | | struct ht_internal_value_st *newval, |
567 | | HT_VALUE **olddata) |
568 | 28.4k | { |
569 | 28.4k | struct ht_mutable_data_st *md = h->md; |
570 | 28.4k | uint64_t neigh_idx_start = hash & md->neighborhood_mask; |
571 | 28.4k | uint64_t neigh_idx = neigh_idx_start; |
572 | 28.4k | size_t j; |
573 | 28.4k | uint64_t ihash; |
574 | 28.4k | HT_VALUE *ival; |
575 | 28.4k | size_t empty_idx = SIZE_MAX; |
576 | 28.4k | int lockless_reads = h->config.lockless_reads; |
577 | 28.4k | CRYPTO_RCU_CB_ITEM *cbi; |
578 | | |
579 | 28.7k | do { |
580 | 28.7k | PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]); |
581 | | |
582 | 43.4k | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
583 | 43.1k | ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value); |
584 | 43.1k | if (ival == NULL) { |
585 | 28.5k | empty_idx = j; |
586 | | /* lockless_reads implies no deletion, we can break out */ |
587 | 28.5k | if (lockless_reads) |
588 | 28.3k | goto not_found; |
589 | 141 | continue; |
590 | 28.5k | } |
591 | 14.5k | if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash, |
592 | 14.5k | &ihash, h->atomic_lock)) |
593 | 0 | return 0; |
594 | 14.5k | if (compare_hash(hash, ihash) && match_key(&newval->value.key, &ival->key)) { |
595 | 7 | if (olddata == NULL) { |
596 | | /* This would insert a duplicate -> fail */ |
597 | 5 | return 0; |
598 | 5 | } |
599 | | /* Do a replacement */ |
600 | 2 | cbi = ossl_rcu_cb_item_new(); |
601 | 2 | if (cbi == NULL) |
602 | 0 | return 0; |
603 | 2 | if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[j].hash, |
604 | 2 | hash, h->atomic_lock)) |
605 | 0 | return 0; |
606 | 2 | *olddata = (HT_VALUE *)md->neighborhoods[neigh_idx].entries[j].value; |
607 | 2 | ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[j].value, |
608 | 2 | &newval); |
609 | 2 | ossl_rcu_call(h->lock, cbi, free_old_ht_value, *olddata); |
610 | 2 | h->wpd.need_sync = 1; |
611 | 2 | return 1; |
612 | 2 | } |
613 | 14.5k | } |
614 | 331 | if (!lockless_reads) |
615 | 64 | break; |
616 | | /* Continue search in subsequent neighborhoods */ |
617 | 267 | neigh_idx = (neigh_idx + 1) & md->neighborhood_mask; |
618 | 267 | } while (neigh_idx != neigh_idx_start); |
619 | | |
620 | 28.4k | not_found: |
621 | | /* If we get to here, its just an insert */ |
622 | 28.4k | if (empty_idx == SIZE_MAX) |
623 | 18 | return -1; /* out of space */ |
624 | 28.4k | if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[empty_idx].hash, |
625 | 28.4k | hash, h->atomic_lock)) |
626 | 0 | return 0; |
627 | 28.4k | h->wpd.value_count++; |
628 | 28.4k | ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[empty_idx].value, |
629 | 28.4k | &newval); |
630 | 28.4k | return 1; |
631 | 28.4k | } |
632 | | |
633 | | static struct ht_internal_value_st *alloc_new_value(HT *h, HT_KEY *key, |
634 | | void *data, |
635 | | uintptr_t *type) |
636 | 44.5k | { |
637 | 44.5k | struct ht_internal_value_st *tmp; |
638 | 44.5k | size_t nvsize = sizeof(*tmp); |
639 | | |
640 | 44.5k | if (h->config.collision_check == 1) |
641 | 43.1k | nvsize += key->keysize; |
642 | | |
643 | 44.5k | tmp = OPENSSL_malloc(nvsize); |
644 | | |
645 | 44.5k | if (tmp == NULL) |
646 | 0 | return NULL; |
647 | | |
648 | 44.5k | tmp->ht = h; |
649 | 44.5k | tmp->value.value = data; |
650 | 44.5k | tmp->value.type_id = type; |
651 | 44.5k | tmp->value.key.keybuf = NULL; |
652 | 44.5k | if (h->config.collision_check) { |
653 | 43.1k | tmp->value.key.keybuf = (uint8_t *)(tmp + 1); |
654 | 43.1k | tmp->value.key.keysize = key->keysize; |
655 | 43.1k | memcpy(tmp->value.key.keybuf, key->keybuf, key->keysize); |
656 | 43.1k | } |
657 | | |
658 | 44.5k | return tmp; |
659 | 44.5k | } |
660 | | |
661 | | static void free_value(struct ht_internal_value_st *v) |
662 | 28.3k | { |
663 | 28.3k | OPENSSL_free(v); |
664 | 28.3k | } |
665 | | |
666 | | int ossl_ht_insert(HT *h, HT_KEY *key, HT_VALUE *data, HT_VALUE **olddata) |
667 | 28.4k | { |
668 | 28.4k | struct ht_internal_value_st *newval = NULL; |
669 | 28.4k | uint64_t hash; |
670 | 28.4k | int rc = 0; |
671 | 28.4k | int i; |
672 | | |
673 | 28.4k | if (data->value == NULL) |
674 | 0 | goto out; |
675 | | |
676 | 28.4k | newval = alloc_new_value(h, key, data->value, data->type_id); |
677 | 28.4k | if (newval == NULL) |
678 | 0 | goto out; |
679 | | |
680 | | /* |
681 | | * we have to take our lock here to prevent other changes |
682 | | * to the bucket list |
683 | | */ |
684 | 28.4k | hash = h->config.ht_hash_fn(key->keybuf, key->keysize); |
685 | | |
686 | 28.4k | for (i = 0; |
687 | 28.4k | (rc = ossl_ht_insert_locked(h, hash, newval, olddata)) == -1 |
688 | 18 | && i <= (int)NEIGHBORHOOD_LEN; |
689 | 28.4k | ++i) |
690 | 18 | if (!grow_hashtable(h, h->wpd.neighborhood_len)) { |
691 | 0 | rc = -1; |
692 | 0 | break; |
693 | 0 | } |
694 | | |
695 | 28.4k | if (rc <= 0) |
696 | 5 | free_value(newval); |
697 | | |
698 | 28.4k | out: |
699 | 28.4k | return rc; |
700 | 28.4k | } |
701 | | |
702 | | HT_VALUE *ossl_ht_get(HT *h, HT_KEY *key) |
703 | 26.8M | { |
704 | 26.8M | struct ht_mutable_data_st *md; |
705 | 26.8M | uint64_t hash; |
706 | 26.8M | uint64_t neigh_idx_start; |
707 | 26.8M | uint64_t neigh_idx; |
708 | 26.8M | struct ht_internal_value_st *ival = NULL; |
709 | 26.8M | size_t j; |
710 | 26.8M | uint64_t ehash; |
711 | 26.8M | int lockless_reads = h->config.lockless_reads; |
712 | | |
713 | 26.8M | hash = h->config.ht_hash_fn(key->keybuf, key->keysize); |
714 | | |
715 | 26.8M | md = ossl_rcu_deref(&h->md); |
716 | 26.8M | neigh_idx = neigh_idx_start = hash & md->neighborhood_mask; |
717 | 26.8M | do { |
718 | 26.8M | PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]); |
719 | 34.2M | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
720 | 34.2M | ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value); |
721 | 34.2M | if (ival == NULL) { |
722 | 2.43M | if (lockless_reads) |
723 | | /* lockless_reads implies no deletion, we can break out */ |
724 | 2.43M | return NULL; |
725 | 114 | continue; |
726 | 2.43M | } |
727 | 31.8M | if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash, |
728 | 31.8M | &ehash, h->atomic_lock)) |
729 | 0 | return NULL; |
730 | 31.8M | if (compare_hash(hash, ehash) && match_key(&ival->value.key, key)) |
731 | 24.3M | return (HT_VALUE *)ival; |
732 | 31.8M | } |
733 | 4.84k | if (!lockless_reads) |
734 | 43 | break; |
735 | | /* Continue search in subsequent neighborhoods */ |
736 | 4.79k | neigh_idx = (neigh_idx + 1) & md->neighborhood_mask; |
737 | 4.79k | } while (neigh_idx != neigh_idx_start); |
738 | | |
739 | 43 | return NULL; |
740 | 26.8M | } |
741 | | |
742 | | static void free_old_entry(void *arg) |
743 | 2.13k | { |
744 | 2.13k | struct ht_internal_value_st *v = arg; |
745 | | |
746 | 2.13k | v->ht->config.ht_free_fn((HT_VALUE *)v); |
747 | 2.13k | free_value(v); |
748 | 2.13k | } |
749 | | |
750 | | int ossl_ht_delete(HT *h, HT_KEY *key) |
751 | 35 | { |
752 | 35 | uint64_t hash; |
753 | 35 | uint64_t neigh_idx; |
754 | 35 | size_t j; |
755 | 35 | struct ht_internal_value_st *v = NULL; |
756 | 35 | HT_VALUE *nv = NULL; |
757 | 35 | int rc = 0; |
758 | | |
759 | 35 | if (h->config.lockless_reads) |
760 | 0 | return 0; |
761 | | |
762 | 35 | hash = h->config.ht_hash_fn(key->keybuf, key->keysize); |
763 | | |
764 | 35 | neigh_idx = hash & h->md->neighborhood_mask; |
765 | 35 | PREFETCH_NEIGHBORHOOD(h->md->neighborhoods[neigh_idx]); |
766 | 161 | for (j = 0; j < NEIGHBORHOOD_LEN; j++) { |
767 | 131 | v = (struct ht_internal_value_st *)h->md->neighborhoods[neigh_idx].entries[j].value; |
768 | 131 | if (v == NULL) |
769 | 68 | continue; |
770 | 63 | if (compare_hash(hash, h->md->neighborhoods[neigh_idx].entries[j].hash) |
771 | 5 | && match_key(key, &v->value.key)) { |
772 | 5 | CRYPTO_RCU_CB_ITEM *cbi = ossl_rcu_cb_item_new(); |
773 | 5 | if (cbi == NULL) |
774 | 0 | break; |
775 | 5 | if (!CRYPTO_atomic_store(&h->md->neighborhoods[neigh_idx].entries[j].hash, |
776 | 5 | 0, h->atomic_lock)) |
777 | 0 | break; |
778 | 5 | h->wpd.value_count--; |
779 | 5 | ossl_rcu_assign_ptr(&h->md->neighborhoods[neigh_idx].entries[j].value, |
780 | 5 | &nv); |
781 | 5 | ossl_rcu_call(h->lock, cbi, free_old_entry, v); |
782 | 5 | h->wpd.need_sync = 1; |
783 | 5 | rc = 1; |
784 | 5 | break; |
785 | 5 | } |
786 | 63 | } |
787 | 35 | return rc; |
788 | 35 | } |