Coverage Report

Created: 2026-05-24 07:14

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/openssl34/crypto/hashtable/hashtable.c
Line
Count
Source
1
/*
2
 * Copyright 2024-2026 The OpenSSL Project Authors. All Rights Reserved.
3
 *
4
 * Licensed under the Apache License 2.0 (the "License").  You may not use
5
 * this file except in compliance with the License.  You can obtain a copy
6
 * in the file LICENSE in the source distribution or at
7
 * https://www.openssl.org/source/license.html
8
 *
9
 *
10
 *
11
 * Notes On hash table design and layout
12
 * This hashtable uses a hopscotch algorithm to do indexing.  The data structure
13
 * looks as follows:
14
 *
15
 *   hash          +--------------+
16
 *   value+------->+ HT_VALUE     |
17
 *      +          +--------------+
18
 *  +-------+
19
 *  |       |
20
 *  +---------------------------------------------------------+
21
 *  |       |       |       |       |                         |
22
 *  | entry | entry | entry | entry |                         |
23
 *  |       |       |       |       |                         |
24
 *  +---------------------------------------------------------+
25
 *  |                               |                         |
26
 *  |                               |                         |
27
 *  +---------------------------------------------------------+
28
 *  |              +                             +            +
29
 *  |        neighborhood[0]               neighborhood[1]    |
30
 *  |                                                         |
31
 *  |                                                         |
32
 *  +---------------------------------------------------------+
33
 *                              |
34
 *                              +
35
 *                         neighborhoods
36
 *
37
 * On lookup/insert/delete, the items key is hashed to a 64 bit value
38
 * and the result is masked to provide an index into the neighborhoods
39
 * table.  Once a neighborhood is determined, an in-order search is done
40
 * of the elements in the neighborhood indexes entries for a matching hash
41
 * value, if found, the corresponding HT_VALUE is used for the respective
42
 * operation.  The number of entries in a neighborhood is determined at build
43
 * time based on the cacheline size of the target CPU.  The intent is for a
44
 * neighborhood to have all entries in the neighborhood fit into a single cache
45
 * line to speed up lookups.  If all entries in a neighborhood are in use at the
46
 * time of an insert, the table is expanded and rehashed.
47
 *
48
 * Lockless reads hash table is based on the same design but does not
49
 * allow growing and deletion. Thus subsequent neighborhoods are always
50
 * searched for a match until an empty entry is found.
51
 */
52
53
#include <string.h>
54
#include <internal/rcu.h>
55
#include <internal/hashtable.h>
56
#include <openssl/rand.h>
57
58
/*
59
 * gcc defines __SANITIZE_THREAD__
60
 * but clang uses the feature attributes api
61
 * map the latter to the former
62
 */
63
#if defined(__clang__) && defined(__has_feature)
64
#if __has_feature(thread_sanitizer)
65
#define __SANITIZE_THREADS__
66
#endif
67
#endif
68
69
#ifdef __SANITIZE_THREADS__
70
#include <sanitizer/tsan_interface.h>
71
#endif
72
73
#include "internal/numbers.h"
74
/*
75
 * When we do a lookup/insert/delete, there is a high likelihood
76
 * that we will iterate over at least part of the neighborhood list
77
 * As such, because we design a neighborhood entry to fit into a single
78
 * cache line it is advantageous, when supported to fetch the entire
79
 * structure for faster lookups
80
 */
81
#if defined(__GNUC__) || defined(__CLANG__)
82
27.2M
#define PREFETCH_NEIGHBORHOOD(x) __builtin_prefetch(x.entries)
83
72.4M
#define PREFETCH(x) __builtin_prefetch(x)
84
#define ALIGN __attribute__((aligned(8)))
85
#else
86
#define PREFETCH_NEIGHBORHOOD(x)
87
#define PREFETCH(x)
88
#define ALIGN
89
#endif
90
91
static ossl_unused uint64_t fnv1a_hash(uint8_t *key, size_t len)
92
8.31M
{
93
8.31M
    uint64_t hash = 0xcbf29ce484222325ULL;
94
8.31M
    size_t i;
95
96
540M
    for (i = 0; i < len; i++) {
97
532M
        hash ^= key[i];
98
532M
        hash *= 0x00000100000001B3ULL;
99
532M
    }
100
8.31M
    return hash;
101
8.31M
}
102
103
/*
104
 * Define our neighborhood list length
105
 * Note: It should always be a power of 2
106
 */
107
477
#define DEFAULT_NEIGH_LEN_LOG 4
108
477
#define DEFAULT_NEIGH_LEN (1 << DEFAULT_NEIGH_LEN_LOG)
109
110
/*
111
 * For now assume cache line size is 64 bytes
112
 */
113
36.3M
#define CACHE_LINE_BYTES 64
114
#define CACHE_LINE_ALIGNMENT CACHE_LINE_BYTES
115
116
36.3M
#define NEIGHBORHOOD_LEN (CACHE_LINE_BYTES / sizeof(struct ht_neighborhood_entry_st))
117
/*
118
 * Defines our chains of values
119
 */
120
struct ht_internal_value_st {
121
    HT_VALUE value;
122
    HT *ht;
123
};
124
125
struct ht_neighborhood_entry_st {
126
    uint64_t hash;
127
    struct ht_internal_value_st *value;
128
} ALIGN;
129
130
struct ht_neighborhood_st {
131
    struct ht_neighborhood_entry_st entries[NEIGHBORHOOD_LEN];
132
};
133
134
/*
135
 * Updates to data in this struct
136
 * require an rcu sync after modification
137
 * prior to free
138
 */
139
struct ht_mutable_data_st {
140
    struct ht_neighborhood_st *neighborhoods;
141
    void *neighborhood_ptr_to_free;
142
    uint64_t neighborhood_mask;
143
};
144
145
/*
146
 * Private data may be updated on the write
147
 * side only, and so do not require rcu sync
148
 */
149
struct ht_write_private_data_st {
150
    size_t neighborhood_len;
151
    size_t value_count;
152
    int need_sync;
153
};
154
155
struct ht_internal_st {
156
    HT_CONFIG config;
157
    CRYPTO_RCU_LOCK *lock;
158
    CRYPTO_RWLOCK *atomic_lock;
159
    struct ht_mutable_data_st *md;
160
    struct ht_write_private_data_st wpd;
161
};
162
163
static void free_value(struct ht_internal_value_st *v);
164
165
static struct ht_neighborhood_st *alloc_new_neighborhood_list(size_t len,
166
    void **freeptr)
167
34.2k
{
168
34.2k
    struct ht_neighborhood_st *ret;
169
170
34.2k
    ret = OPENSSL_aligned_alloc(sizeof(struct ht_neighborhood_st) * len,
171
34.2k
        CACHE_LINE_BYTES, freeptr);
172
173
    /* fall back to regular malloc */
174
34.2k
    if (ret == NULL) {
175
0
        ret = *freeptr = OPENSSL_malloc(sizeof(struct ht_neighborhood_st) * len);
176
0
        if (ret == NULL)
177
0
            return NULL;
178
0
    }
179
34.2k
    memset(ret, 0, sizeof(struct ht_neighborhood_st) * len);
180
34.2k
    return ret;
181
34.2k
}
182
183
static void internal_free_nop(HT_VALUE *v)
184
24.8k
{
185
24.8k
    return;
186
24.8k
}
187
188
HT *ossl_ht_new(const HT_CONFIG *conf)
189
242
{
190
242
    HT *new = OPENSSL_zalloc(sizeof(*new));
191
192
242
    if (new == NULL)
193
0
        return NULL;
194
195
242
    new->atomic_lock = CRYPTO_THREAD_lock_new();
196
242
    if (new->atomic_lock == NULL)
197
0
        goto err;
198
199
242
    memcpy(&new->config, conf, sizeof(*conf));
200
201
242
    if (new->config.init_neighborhoods != 0) {
202
236
        new->wpd.neighborhood_len = new->config.init_neighborhoods;
203
        /* round up to the next power of 2 */
204
236
        new->wpd.neighborhood_len--;
205
236
        new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 1;
206
236
        new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 2;
207
236
        new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 4;
208
236
        new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 8;
209
236
        new->wpd.neighborhood_len |= new->wpd.neighborhood_len >> 16;
210
236
        new->wpd.neighborhood_len++;
211
236
    } else {
212
6
        new->wpd.neighborhood_len = DEFAULT_NEIGH_LEN;
213
6
    }
214
215
242
    if (new->config.ht_free_fn == NULL)
216
236
        new->config.ht_free_fn = internal_free_nop;
217
218
242
    new->md = OPENSSL_zalloc(sizeof(*new->md));
219
242
    if (new->md == NULL)
220
0
        goto err;
221
222
242
    new->md->neighborhoods = alloc_new_neighborhood_list(new->wpd.neighborhood_len,
223
242
        &new->md->neighborhood_ptr_to_free);
224
242
    if (new->md->neighborhoods == NULL)
225
0
        goto err;
226
242
    new->md->neighborhood_mask = new->wpd.neighborhood_len - 1;
227
228
242
    new->lock = ossl_rcu_lock_new(1, conf->ctx);
229
242
    if (new->lock == NULL)
230
0
        goto err;
231
232
242
    if (new->config.ht_hash_fn == NULL)
233
242
        new->config.ht_hash_fn = fnv1a_hash;
234
235
242
    return new;
236
237
0
err:
238
0
    CRYPTO_THREAD_lock_free(new->atomic_lock);
239
0
    ossl_rcu_lock_free(new->lock);
240
0
    if (new->md != NULL)
241
0
        OPENSSL_free(new->md->neighborhood_ptr_to_free);
242
0
    OPENSSL_free(new->md);
243
0
    OPENSSL_free(new);
244
0
    return NULL;
245
242
}
246
247
void ossl_ht_read_lock(HT *htable)
248
32
{
249
32
    ossl_rcu_read_lock(htable->lock);
250
32
}
251
252
void ossl_ht_read_unlock(HT *htable)
253
49
{
254
49
    ossl_rcu_read_unlock(htable->lock);
255
49
}
256
257
void ossl_ht_write_lock(HT *htable)
258
245
{
259
245
    ossl_rcu_write_lock(htable->lock);
260
245
    htable->wpd.need_sync = 0;
261
245
}
262
263
void ossl_ht_write_unlock(HT *htable)
264
245
{
265
245
    int need_sync = htable->wpd.need_sync;
266
267
245
    htable->wpd.need_sync = 0;
268
245
    ossl_rcu_write_unlock(htable->lock);
269
245
    if (need_sync)
270
172
        ossl_synchronize_rcu(htable->lock);
271
245
}
272
273
static void free_oldmd(void *arg)
274
32.5k
{
275
32.5k
    struct ht_mutable_data_st *oldmd = arg;
276
32.5k
    size_t i, j;
277
32.5k
    size_t neighborhood_len = (size_t)oldmd->neighborhood_mask + 1;
278
32.5k
    struct ht_internal_value_st *v;
279
280
368k
    for (i = 0; i < neighborhood_len; i++) {
281
335k
        PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[i + 1]);
282
1.67M
        for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
283
1.34M
            if (oldmd->neighborhoods[i].entries[j].value != NULL) {
284
26.1k
                v = oldmd->neighborhoods[i].entries[j].value;
285
26.1k
                v->ht->config.ht_free_fn((HT_VALUE *)v);
286
26.1k
                free_value(v);
287
26.1k
            }
288
1.34M
        }
289
335k
    }
290
291
32.5k
    OPENSSL_free(oldmd->neighborhood_ptr_to_free);
292
32.5k
    OPENSSL_free(oldmd);
293
32.5k
}
294
295
static int ossl_ht_flush_internal(HT *h)
296
157
{
297
157
    struct ht_mutable_data_st *newmd = NULL;
298
157
    struct ht_mutable_data_st *oldmd = NULL;
299
157
    CRYPTO_RCU_CB_ITEM *cbi = NULL;
300
301
157
    newmd = OPENSSL_zalloc(sizeof(*newmd));
302
157
    if (newmd == NULL)
303
0
        return 0;
304
305
157
    newmd->neighborhoods = alloc_new_neighborhood_list(DEFAULT_NEIGH_LEN,
306
157
        &newmd->neighborhood_ptr_to_free);
307
157
    if (newmd->neighborhoods == NULL) {
308
0
        OPENSSL_free(newmd);
309
0
        return 0;
310
0
    }
311
312
157
    newmd->neighborhood_mask = DEFAULT_NEIGH_LEN - 1;
313
314
157
    cbi = ossl_rcu_cb_item_new();
315
157
    if (cbi == NULL) {
316
0
        OPENSSL_free(newmd->neighborhood_ptr_to_free);
317
0
        OPENSSL_free(newmd);
318
0
        return 0;
319
0
    }
320
321
    /* Swap the old and new mutable data sets */
322
157
    oldmd = ossl_rcu_deref(&h->md);
323
157
    ossl_rcu_assign_ptr(&h->md, &newmd);
324
325
    /* Set the number of entries to 0 */
326
157
    h->wpd.value_count = 0;
327
157
    h->wpd.neighborhood_len = DEFAULT_NEIGH_LEN;
328
329
157
    ossl_rcu_call(h->lock, cbi, free_oldmd, oldmd);
330
157
    h->wpd.need_sync = 1;
331
332
157
    return 1;
333
157
}
334
335
int ossl_ht_flush(HT *h)
336
420
{
337
420
    return ossl_ht_flush_internal(h);
338
420
}
339
340
void ossl_ht_free(HT *h)
341
154
{
342
154
    int flush_ok;
343
344
154
    if (h == NULL)
345
0
        return;
346
347
154
    ossl_ht_write_lock(h);
348
154
    flush_ok = ossl_ht_flush_internal(h);
349
154
    ossl_ht_write_unlock(h);
350
    /* Freeing the lock does a final sync for us */
351
154
    CRYPTO_THREAD_lock_free(h->atomic_lock);
352
154
    ossl_rcu_lock_free(h->lock);
353
154
    if (flush_ok) {
354
154
        OPENSSL_free(h->md->neighborhood_ptr_to_free);
355
154
        OPENSSL_free(h->md);
356
154
    } else {
357
0
        free_oldmd(h->md);
358
0
    }
359
154
    OPENSSL_free(h);
360
154
    return;
361
154
}
362
363
size_t ossl_ht_count(HT *h)
364
1.97k
{
365
1.97k
    size_t count;
366
367
1.97k
    count = h->wpd.value_count;
368
1.97k
    return count;
369
1.97k
}
370
371
void ossl_ht_foreach_until(HT *h, int (*cb)(HT_VALUE *obj, void *arg),
372
    void *arg)
373
100
{
374
100
    size_t i, j;
375
100
    struct ht_mutable_data_st *md;
376
377
100
    md = ossl_rcu_deref(&h->md);
378
37.9k
    for (i = 0; i < md->neighborhood_mask + 1; i++) {
379
37.8k
        PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]);
380
189k
        for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
381
151k
            if (md->neighborhoods[i].entries[j].value != NULL) {
382
650
                if (!cb((HT_VALUE *)md->neighborhoods[i].entries[j].value, arg))
383
9
                    goto out;
384
650
            }
385
151k
        }
386
37.8k
    }
387
100
out:
388
100
    return;
389
100
}
390
391
HT_VALUE_LIST *ossl_ht_filter(HT *h, size_t max_len,
392
    int (*filter)(HT_VALUE *obj, void *arg),
393
    void *arg)
394
78
{
395
78
    struct ht_mutable_data_st *md;
396
78
    HT_VALUE_LIST *list = OPENSSL_zalloc(sizeof(HT_VALUE_LIST)
397
78
        + (sizeof(HT_VALUE *) * max_len));
398
78
    size_t i, j;
399
78
    struct ht_internal_value_st *v;
400
401
78
    if (list == NULL)
402
0
        return NULL;
403
404
    /*
405
     * The list array lives just beyond the end of
406
     * the struct
407
     */
408
78
    list->list = (HT_VALUE **)(list + 1);
409
410
78
    md = ossl_rcu_deref(&h->md);
411
21.8k
    for (i = 0; i < md->neighborhood_mask + 1; i++) {
412
21.7k
        PREFETCH_NEIGHBORHOOD(md->neighborhoods[i + 1]);
413
108k
        for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
414
86.9k
            v = md->neighborhoods[i].entries[j].value;
415
86.9k
            if (v != NULL && filter((HT_VALUE *)v, arg)) {
416
5
                list->list[list->list_len++] = (HT_VALUE *)v;
417
5
                if (list->list_len == max_len)
418
5
                    goto out;
419
5
            }
420
86.9k
        }
421
21.7k
    }
422
78
out:
423
78
    return list;
424
78
}
425
426
void ossl_ht_value_list_free(HT_VALUE_LIST *list)
427
78
{
428
78
    OPENSSL_free(list);
429
78
}
430
431
static int compare_hash(uint64_t hash1, uint64_t hash2)
432
49.9M
{
433
49.9M
    return (hash1 == hash2);
434
49.9M
}
435
436
static void free_old_neigh_table(void *arg)
437
100
{
438
100
    struct ht_mutable_data_st *oldmd = arg;
439
440
100
    OPENSSL_free(oldmd->neighborhood_ptr_to_free);
441
100
    OPENSSL_free(oldmd);
442
100
}
443
444
/*
445
 * Increase hash table bucket list
446
 * must be called with write_lock held
447
 */
448
static int grow_hashtable(HT *h, size_t oldsize)
449
18
{
450
18
    struct ht_mutable_data_st *newmd;
451
18
    struct ht_mutable_data_st *oldmd = ossl_rcu_deref(&h->md);
452
18
    CRYPTO_RCU_CB_ITEM *cbi = NULL;
453
18
    int rc = 0;
454
18
    uint64_t oldi, oldj, newi, newj;
455
18
    uint64_t oldhash;
456
18
    struct ht_internal_value_st *oldv;
457
18
    int rehashed;
458
18
    size_t newsize = oldsize * 2;
459
460
18
    if (h->config.lockless_reads)
461
0
        goto out;
462
463
18
    if ((newmd = OPENSSL_zalloc(sizeof(*newmd))) == NULL)
464
0
        goto out;
465
466
    /* bucket list is always a power of 2 */
467
18
    newmd->neighborhoods = alloc_new_neighborhood_list(oldsize * 2,
468
18
        &newmd->neighborhood_ptr_to_free);
469
18
    if (newmd->neighborhoods == NULL)
470
0
        goto out_free;
471
472
    /* being a power of 2 makes for easy mask computation */
473
18
    newmd->neighborhood_mask = (newsize - 1);
474
475
    /*
476
     * Now we need to start rehashing entries
477
     * Note we don't need to use atomics here as the new
478
     * mutable data hasn't been published
479
     */
480
1.87k
    for (oldi = 0; oldi < h->wpd.neighborhood_len; oldi++) {
481
1.85k
        PREFETCH_NEIGHBORHOOD(oldmd->neighborhoods[oldi + 1]);
482
9.28k
        for (oldj = 0; oldj < NEIGHBORHOOD_LEN; oldj++) {
483
7.42k
            oldv = oldmd->neighborhoods[oldi].entries[oldj].value;
484
7.42k
            if (oldv == NULL)
485
7.33k
                continue;
486
89
            oldhash = oldmd->neighborhoods[oldi].entries[oldj].hash;
487
89
            newi = oldhash & newmd->neighborhood_mask;
488
89
            rehashed = 0;
489
178
            for (newj = 0; newj < NEIGHBORHOOD_LEN; newj++) {
490
178
                if (newmd->neighborhoods[newi].entries[newj].value == NULL) {
491
89
                    newmd->neighborhoods[newi].entries[newj].value = oldv;
492
89
                    newmd->neighborhoods[newi].entries[newj].hash = oldhash;
493
89
                    rehashed = 1;
494
89
                    break;
495
89
                }
496
178
            }
497
89
            if (rehashed == 0) {
498
                /* we ran out of space in a neighborhood, grow again */
499
0
                OPENSSL_free(newmd->neighborhood_ptr_to_free);
500
0
                OPENSSL_free(newmd);
501
0
                return grow_hashtable(h, newsize);
502
0
            }
503
89
        }
504
1.85k
    }
505
506
    /*
507
     * Pre allocate the rcu callback item before assigning the newmd.
508
     */
509
18
    cbi = ossl_rcu_cb_item_new();
510
18
    if (cbi == NULL)
511
0
        goto out_free;
512
513
    /*
514
     * Now that our entries are all hashed into the new bucket list
515
     * update our bucket_len and target_max_load
516
     */
517
18
    h->wpd.neighborhood_len = newsize;
518
519
    /*
520
     * Now we replace the old mutable data with the new
521
     */
522
18
    ossl_rcu_assign_ptr(&h->md, &newmd);
523
18
    ossl_rcu_call(h->lock, cbi, free_old_neigh_table, oldmd);
524
18
    h->wpd.need_sync = 1;
525
    /*
526
     * And we're done
527
     */
528
18
    rc = 1;
529
530
18
out:
531
18
    return rc;
532
0
out_free:
533
0
    OPENSSL_free(newmd->neighborhood_ptr_to_free);
534
0
    OPENSSL_free(newmd);
535
0
    goto out;
536
18
}
537
538
static void free_old_ht_value(void *arg)
539
3
{
540
3
    HT_VALUE *h = (HT_VALUE *)arg;
541
542
    /*
543
     * Note, this is only called on replacement,
544
     * the caller is responsible for freeing the
545
     * held data, we just need to free the wrapping
546
     * struct here
547
     */
548
3
    OPENSSL_free(h);
549
3
}
550
551
static ossl_inline int match_key(HT_KEY *a, HT_KEY *b)
552
36.2M
{
553
    /*
554
     * keys match if they are both present, the same size
555
     * and compare equal in memory
556
     */
557
36.2M
    PREFETCH(a->keybuf);
558
36.2M
    PREFETCH(b->keybuf);
559
36.2M
    if (a->keybuf != NULL && b->keybuf != NULL && a->keysize == b->keysize)
560
36.2M
        return !memcmp(a->keybuf, b->keybuf, a->keysize);
561
562
4.27k
    return 1;
563
36.2M
}
564
565
static int ossl_ht_insert_locked(HT *h, uint64_t hash,
566
    struct ht_internal_value_st *newval,
567
    HT_VALUE **olddata)
568
28.4k
{
569
28.4k
    struct ht_mutable_data_st *md = h->md;
570
28.4k
    uint64_t neigh_idx_start = hash & md->neighborhood_mask;
571
28.4k
    uint64_t neigh_idx = neigh_idx_start;
572
28.4k
    size_t j;
573
28.4k
    uint64_t ihash;
574
28.4k
    HT_VALUE *ival;
575
28.4k
    size_t empty_idx = SIZE_MAX;
576
28.4k
    int lockless_reads = h->config.lockless_reads;
577
28.4k
    CRYPTO_RCU_CB_ITEM *cbi;
578
579
28.7k
    do {
580
28.7k
        PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]);
581
582
43.4k
        for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
583
43.1k
            ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value);
584
43.1k
            if (ival == NULL) {
585
28.5k
                empty_idx = j;
586
                /* lockless_reads implies no deletion, we can break out */
587
28.5k
                if (lockless_reads)
588
28.3k
                    goto not_found;
589
141
                continue;
590
28.5k
            }
591
14.5k
            if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash,
592
14.5k
                    &ihash, h->atomic_lock))
593
0
                return 0;
594
14.5k
            if (compare_hash(hash, ihash) && match_key(&newval->value.key, &ival->key)) {
595
7
                if (olddata == NULL) {
596
                    /* This would insert a duplicate -> fail */
597
5
                    return 0;
598
5
                }
599
                /* Do a replacement */
600
2
                cbi = ossl_rcu_cb_item_new();
601
2
                if (cbi == NULL)
602
0
                    return 0;
603
2
                if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[j].hash,
604
2
                        hash, h->atomic_lock))
605
0
                    return 0;
606
2
                *olddata = (HT_VALUE *)md->neighborhoods[neigh_idx].entries[j].value;
607
2
                ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[j].value,
608
2
                    &newval);
609
2
                ossl_rcu_call(h->lock, cbi, free_old_ht_value, *olddata);
610
2
                h->wpd.need_sync = 1;
611
2
                return 1;
612
2
            }
613
14.5k
        }
614
331
        if (!lockless_reads)
615
64
            break;
616
        /* Continue search in subsequent neighborhoods */
617
267
        neigh_idx = (neigh_idx + 1) & md->neighborhood_mask;
618
267
    } while (neigh_idx != neigh_idx_start);
619
620
28.4k
not_found:
621
    /* If we get to here, its just an insert */
622
28.4k
    if (empty_idx == SIZE_MAX)
623
18
        return -1; /* out of space */
624
28.4k
    if (!CRYPTO_atomic_store(&md->neighborhoods[neigh_idx].entries[empty_idx].hash,
625
28.4k
            hash, h->atomic_lock))
626
0
        return 0;
627
28.4k
    h->wpd.value_count++;
628
28.4k
    ossl_rcu_assign_ptr(&md->neighborhoods[neigh_idx].entries[empty_idx].value,
629
28.4k
        &newval);
630
28.4k
    return 1;
631
28.4k
}
632
633
static struct ht_internal_value_st *alloc_new_value(HT *h, HT_KEY *key,
634
    void *data,
635
    uintptr_t *type)
636
44.5k
{
637
44.5k
    struct ht_internal_value_st *tmp;
638
44.5k
    size_t nvsize = sizeof(*tmp);
639
640
44.5k
    if (h->config.collision_check == 1)
641
43.1k
        nvsize += key->keysize;
642
643
44.5k
    tmp = OPENSSL_malloc(nvsize);
644
645
44.5k
    if (tmp == NULL)
646
0
        return NULL;
647
648
44.5k
    tmp->ht = h;
649
44.5k
    tmp->value.value = data;
650
44.5k
    tmp->value.type_id = type;
651
44.5k
    tmp->value.key.keybuf = NULL;
652
44.5k
    if (h->config.collision_check) {
653
43.1k
        tmp->value.key.keybuf = (uint8_t *)(tmp + 1);
654
43.1k
        tmp->value.key.keysize = key->keysize;
655
43.1k
        memcpy(tmp->value.key.keybuf, key->keybuf, key->keysize);
656
43.1k
    }
657
658
44.5k
    return tmp;
659
44.5k
}
660
661
static void free_value(struct ht_internal_value_st *v)
662
28.3k
{
663
28.3k
    OPENSSL_free(v);
664
28.3k
}
665
666
int ossl_ht_insert(HT *h, HT_KEY *key, HT_VALUE *data, HT_VALUE **olddata)
667
28.4k
{
668
28.4k
    struct ht_internal_value_st *newval = NULL;
669
28.4k
    uint64_t hash;
670
28.4k
    int rc = 0;
671
28.4k
    int i;
672
673
28.4k
    if (data->value == NULL)
674
0
        goto out;
675
676
28.4k
    newval = alloc_new_value(h, key, data->value, data->type_id);
677
28.4k
    if (newval == NULL)
678
0
        goto out;
679
680
    /*
681
     * we have to take our lock here to prevent other changes
682
     * to the bucket list
683
     */
684
28.4k
    hash = h->config.ht_hash_fn(key->keybuf, key->keysize);
685
686
28.4k
    for (i = 0;
687
28.4k
        (rc = ossl_ht_insert_locked(h, hash, newval, olddata)) == -1
688
18
        && i <= (int)NEIGHBORHOOD_LEN;
689
28.4k
        ++i)
690
18
        if (!grow_hashtable(h, h->wpd.neighborhood_len)) {
691
0
            rc = -1;
692
0
            break;
693
0
        }
694
695
28.4k
    if (rc <= 0)
696
5
        free_value(newval);
697
698
28.4k
out:
699
28.4k
    return rc;
700
28.4k
}
701
702
HT_VALUE *ossl_ht_get(HT *h, HT_KEY *key)
703
26.8M
{
704
26.8M
    struct ht_mutable_data_st *md;
705
26.8M
    uint64_t hash;
706
26.8M
    uint64_t neigh_idx_start;
707
26.8M
    uint64_t neigh_idx;
708
26.8M
    struct ht_internal_value_st *ival = NULL;
709
26.8M
    size_t j;
710
26.8M
    uint64_t ehash;
711
26.8M
    int lockless_reads = h->config.lockless_reads;
712
713
26.8M
    hash = h->config.ht_hash_fn(key->keybuf, key->keysize);
714
715
26.8M
    md = ossl_rcu_deref(&h->md);
716
26.8M
    neigh_idx = neigh_idx_start = hash & md->neighborhood_mask;
717
26.8M
    do {
718
26.8M
        PREFETCH_NEIGHBORHOOD(md->neighborhoods[neigh_idx]);
719
34.2M
        for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
720
34.2M
            ival = ossl_rcu_deref(&md->neighborhoods[neigh_idx].entries[j].value);
721
34.2M
            if (ival == NULL) {
722
2.43M
                if (lockless_reads)
723
                    /* lockless_reads implies no deletion, we can break out */
724
2.43M
                    return NULL;
725
114
                continue;
726
2.43M
            }
727
31.8M
            if (!CRYPTO_atomic_load(&md->neighborhoods[neigh_idx].entries[j].hash,
728
31.8M
                    &ehash, h->atomic_lock))
729
0
                return NULL;
730
31.8M
            if (compare_hash(hash, ehash) && match_key(&ival->value.key, key))
731
24.3M
                return (HT_VALUE *)ival;
732
31.8M
        }
733
4.84k
        if (!lockless_reads)
734
43
            break;
735
        /* Continue search in subsequent neighborhoods */
736
4.79k
        neigh_idx = (neigh_idx + 1) & md->neighborhood_mask;
737
4.79k
    } while (neigh_idx != neigh_idx_start);
738
739
43
    return NULL;
740
26.8M
}
741
742
static void free_old_entry(void *arg)
743
2.13k
{
744
2.13k
    struct ht_internal_value_st *v = arg;
745
746
2.13k
    v->ht->config.ht_free_fn((HT_VALUE *)v);
747
2.13k
    free_value(v);
748
2.13k
}
749
750
int ossl_ht_delete(HT *h, HT_KEY *key)
751
35
{
752
35
    uint64_t hash;
753
35
    uint64_t neigh_idx;
754
35
    size_t j;
755
35
    struct ht_internal_value_st *v = NULL;
756
35
    HT_VALUE *nv = NULL;
757
35
    int rc = 0;
758
759
35
    if (h->config.lockless_reads)
760
0
        return 0;
761
762
35
    hash = h->config.ht_hash_fn(key->keybuf, key->keysize);
763
764
35
    neigh_idx = hash & h->md->neighborhood_mask;
765
35
    PREFETCH_NEIGHBORHOOD(h->md->neighborhoods[neigh_idx]);
766
161
    for (j = 0; j < NEIGHBORHOOD_LEN; j++) {
767
131
        v = (struct ht_internal_value_st *)h->md->neighborhoods[neigh_idx].entries[j].value;
768
131
        if (v == NULL)
769
68
            continue;
770
63
        if (compare_hash(hash, h->md->neighborhoods[neigh_idx].entries[j].hash)
771
5
            && match_key(key, &v->value.key)) {
772
5
            CRYPTO_RCU_CB_ITEM *cbi = ossl_rcu_cb_item_new();
773
5
            if (cbi == NULL)
774
0
                break;
775
5
            if (!CRYPTO_atomic_store(&h->md->neighborhoods[neigh_idx].entries[j].hash,
776
5
                    0, h->atomic_lock))
777
0
                break;
778
5
            h->wpd.value_count--;
779
5
            ossl_rcu_assign_ptr(&h->md->neighborhoods[neigh_idx].entries[j].value,
780
5
                &nv);
781
5
            ossl_rcu_call(h->lock, cbi, free_old_entry, v);
782
5
            h->wpd.need_sync = 1;
783
5
            rc = 1;
784
5
            break;
785
5
        }
786
63
    }
787
35
    return rc;
788
35
}