Coverage Report

Created: 2024-07-27 06:36

/src/openssl/crypto/threads_pthread.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved.
3
 *
4
 * Licensed under the Apache License 2.0 (the "License").  You may not use
5
 * this file except in compliance with the License.  You can obtain a copy
6
 * in the file LICENSE in the source distribution or at
7
 * https://www.openssl.org/source/license.html
8
 */
9
10
/* We need to use the OPENSSL_fork_*() deprecated APIs */
11
#define OPENSSL_SUPPRESS_DEPRECATED
12
13
#include <openssl/crypto.h>
14
#include <crypto/cryptlib.h>
15
#include "internal/cryptlib.h"
16
#include "internal/rcu.h"
17
#include "rcu_internal.h"
18
19
#if defined(__clang__) && defined(__has_feature)
20
# if __has_feature(thread_sanitizer)
21
#  define __SANITIZE_THREAD__
22
# endif
23
#endif
24
25
#if defined(__SANITIZE_THREAD__)
26
# include <sanitizer/tsan_interface.h>
27
# define TSAN_FAKE_UNLOCK(x)   __tsan_mutex_pre_unlock((x), 0); \
28
__tsan_mutex_post_unlock((x), 0)
29
30
# define TSAN_FAKE_LOCK(x)  __tsan_mutex_pre_lock((x), 0); \
31
__tsan_mutex_post_lock((x), 0, 0)
32
#else
33
# define TSAN_FAKE_UNLOCK(x)
34
# define TSAN_FAKE_LOCK(x)
35
#endif
36
37
#if defined(__sun)
38
# include <atomic.h>
39
#endif
40
41
#if defined(__apple_build_version__) && __apple_build_version__ < 6000000
42
/*
43
 * OS/X 10.7 and 10.8 had a weird version of clang which has __ATOMIC_ACQUIRE and
44
 * __ATOMIC_ACQ_REL but which expects only one parameter for __atomic_is_lock_free()
45
 * rather than two which has signature __atomic_is_lock_free(sizeof(_Atomic(T))).
46
 * All of this makes impossible to use __atomic_is_lock_free here.
47
 *
48
 * See: https://github.com/llvm/llvm-project/commit/a4c2602b714e6c6edb98164550a5ae829b2de760
49
 */
50
# define BROKEN_CLANG_ATOMICS
51
#endif
52
53
#if defined(OPENSSL_THREADS) && !defined(CRYPTO_TDEBUG) && !defined(OPENSSL_SYS_WINDOWS)
54
55
# if defined(OPENSSL_SYS_UNIX)
56
#  include <sys/types.h>
57
#  include <unistd.h>
58
# endif
59
60
# include <assert.h>
61
62
# ifdef PTHREAD_RWLOCK_INITIALIZER
63
#  define USE_RWLOCK
64
# endif
65
66
/*
67
 * For all GNU/clang atomic builtins, we also need fallbacks, to cover all
68
 * other compilers.
69
70
 * Unfortunately, we can't do that with some "generic type", because there's no
71
 * guarantee that the chosen generic type is large enough to cover all cases.
72
 * Therefore, we implement fallbacks for each applicable type, with composed
73
 * names that include the type they handle.
74
 *
75
 * (an anecdote: we previously tried to use |void *| as the generic type, with
76
 * the thought that the pointer itself is the largest type.  However, this is
77
 * not true on 32-bit pointer platforms, as a |uint64_t| is twice as large)
78
 *
79
 * All applicable ATOMIC_ macros take the intended type as first parameter, so
80
 * they can map to the correct fallback function.  In the GNU/clang case, that
81
 * parameter is simply ignored.
82
 */
83
84
/*
85
 * Internal types used with the ATOMIC_ macros, to make it possible to compose
86
 * fallback function names.
87
 */
88
typedef void *pvoid;
89
typedef struct rcu_cb_item *prcu_cb_item;
90
91
# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) \
92
    && !defined(USE_ATOMIC_FALLBACKS)
93
#  if defined(__APPLE__) && defined(__clang__) && defined(__aarch64__)
94
/*
95
 * For pointers, Apple M1 virtualized cpu seems to have some problem using the
96
 * ldapr instruction (see https://github.com/openssl/openssl/pull/23974)
97
 * When using the native apple clang compiler, this instruction is emitted for
98
 * atomic loads, which is bad.  So, if
99
 * 1) We are building on a target that defines __APPLE__ AND
100
 * 2) We are building on a target using clang (__clang__) AND
101
 * 3) We are building for an M1 processor (__aarch64__)
102
 * Then we should not use __atomic_load_n and instead implement our own
103
 * function to issue the ldar instruction instead, which produces the proper
104
 * sequencing guarantees
105
 */
106
static inline void *apple_atomic_load_n_pvoid(void **p,
107
                                              ossl_unused int memorder)
108
{
109
    void *ret;
110
111
    __asm volatile("ldar %0, [%1]" : "=r" (ret): "r" (p):);
112
113
    return ret;
114
}
115
116
/* For uint64_t, we should be fine, though */
117
#   define apple_atomic_load_n_uint32_t(p, o) __atomic_load_n(p, o)
118
#   define apple_atomic_load_n_uint64_t(p, o) __atomic_load_n(p, o)
119
120
#   define ATOMIC_LOAD_N(t, p, o) apple_atomic_load_n_##t(p, o)
121
#  else
122
5
#   define ATOMIC_LOAD_N(t, p, o) __atomic_load_n(p, o)
123
#  endif
124
3
#  define ATOMIC_STORE_N(t, p, v, o) __atomic_store_n(p, v, o)
125
2
#  define ATOMIC_STORE(t, p, v, o) __atomic_store(p, v, o)
126
0
#  define ATOMIC_EXCHANGE_N(t, p, v, o) __atomic_exchange_n(p, v, o)
127
0
#  define ATOMIC_ADD_FETCH(p, v, o) __atomic_add_fetch(p, v, o)
128
#  define ATOMIC_FETCH_ADD(p, v, o) __atomic_fetch_add(p, v, o)
129
0
#  define ATOMIC_SUB_FETCH(p, v, o) __atomic_sub_fetch(p, v, o)
130
3
#  define ATOMIC_AND_FETCH(p, m, o) __atomic_and_fetch(p, m, o)
131
3
#  define ATOMIC_OR_FETCH(p, m, o) __atomic_or_fetch(p, m, o)
132
# else
133
static pthread_mutex_t atomic_sim_lock = PTHREAD_MUTEX_INITIALIZER;
134
135
#  define IMPL_fallback_atomic_load_n(t)                        \
136
    static ossl_inline t fallback_atomic_load_n_##t(t *p)            \
137
    {                                                           \
138
        t ret;                                                  \
139
                                                                \
140
        pthread_mutex_lock(&atomic_sim_lock);                   \
141
        ret = *p;                                               \
142
        pthread_mutex_unlock(&atomic_sim_lock);                 \
143
        return ret;                                             \
144
    }
145
IMPL_fallback_atomic_load_n(uint32_t)
146
IMPL_fallback_atomic_load_n(uint64_t)
147
IMPL_fallback_atomic_load_n(pvoid)
148
149
#  define ATOMIC_LOAD_N(t, p, o) fallback_atomic_load_n_##t(p)
150
151
#  define IMPL_fallback_atomic_store_n(t)                       \
152
    static ossl_inline t fallback_atomic_store_n_##t(t *p, t v)      \
153
    {                                                           \
154
        t ret;                                                  \
155
                                                                \
156
        pthread_mutex_lock(&atomic_sim_lock);                   \
157
        ret = *p;                                               \
158
        *p = v;                                                 \
159
        pthread_mutex_unlock(&atomic_sim_lock);                 \
160
        return ret;                                             \
161
    }
162
IMPL_fallback_atomic_store_n(uint32_t)
163
IMPL_fallback_atomic_store_n(uint64_t)
164
165
#  define ATOMIC_STORE_N(t, p, v, o) fallback_atomic_store_n_##t(p, v)
166
167
#  define IMPL_fallback_atomic_store(t)                         \
168
    static ossl_inline void fallback_atomic_store_##t(t *p, t *v)    \
169
    {                                                           \
170
        pthread_mutex_lock(&atomic_sim_lock);                   \
171
        *p = *v;                                                \
172
        pthread_mutex_unlock(&atomic_sim_lock);                 \
173
    }
174
IMPL_fallback_atomic_store(uint64_t)
175
IMPL_fallback_atomic_store(pvoid)
176
177
#  define ATOMIC_STORE(t, p, v, o) fallback_atomic_store_##t(p, v)
178
179
#  define IMPL_fallback_atomic_exchange_n(t)                            \
180
    static ossl_inline t fallback_atomic_exchange_n_##t(t *p, t v)           \
181
    {                                                                   \
182
        t ret;                                                          \
183
                                                                        \
184
        pthread_mutex_lock(&atomic_sim_lock);                           \
185
        ret = *p;                                                       \
186
        *p = v;                                                         \
187
        pthread_mutex_unlock(&atomic_sim_lock);                         \
188
        return ret;                                                     \
189
    }
190
IMPL_fallback_atomic_exchange_n(uint64_t)
191
IMPL_fallback_atomic_exchange_n(prcu_cb_item)
192
193
#  define ATOMIC_EXCHANGE_N(t, p, v, o) fallback_atomic_exchange_n_##t(p, v)
194
195
/*
196
 * The fallbacks that follow don't need any per type implementation, as
197
 * they are designed for uint64_t only.  If there comes a time when multiple
198
 * types need to be covered, it's relatively easy to refactor them the same
199
 * way as the fallbacks above.
200
 */
201
202
static ossl_inline uint64_t fallback_atomic_add_fetch(uint64_t *p, uint64_t v)
203
{
204
    uint64_t ret;
205
206
    pthread_mutex_lock(&atomic_sim_lock);
207
    *p += v;
208
    ret = *p;
209
    pthread_mutex_unlock(&atomic_sim_lock);
210
    return ret;
211
}
212
213
#  define ATOMIC_ADD_FETCH(p, v, o) fallback_atomic_add_fetch(p, v)
214
215
static ossl_inline uint64_t fallback_atomic_fetch_add(uint64_t *p, uint64_t v)
216
{
217
    uint64_t ret;
218
219
    pthread_mutex_lock(&atomic_sim_lock);
220
    ret = *p;
221
    *p += v;
222
    pthread_mutex_unlock(&atomic_sim_lock);
223
    return ret;
224
}
225
226
#  define ATOMIC_FETCH_ADD(p, v, o) fallback_atomic_fetch_add(p, v)
227
228
static ossl_inline uint64_t fallback_atomic_sub_fetch(uint64_t *p, uint64_t v)
229
{
230
    uint64_t ret;
231
232
    pthread_mutex_lock(&atomic_sim_lock);
233
    *p -= v;
234
    ret = *p;
235
    pthread_mutex_unlock(&atomic_sim_lock);
236
    return ret;
237
}
238
239
#  define ATOMIC_SUB_FETCH(p, v, o) fallback_atomic_sub_fetch(p, v)
240
241
static ossl_inline uint64_t fallback_atomic_and_fetch(uint64_t *p, uint64_t m)
242
{
243
    uint64_t ret;
244
245
    pthread_mutex_lock(&atomic_sim_lock);
246
    *p &= m;
247
    ret = *p;
248
    pthread_mutex_unlock(&atomic_sim_lock);
249
    return ret;
250
}
251
252
#  define ATOMIC_AND_FETCH(p, v, o) fallback_atomic_and_fetch(p, v)
253
254
static ossl_inline uint64_t fallback_atomic_or_fetch(uint64_t *p, uint64_t m)
255
{
256
    uint64_t ret;
257
258
    pthread_mutex_lock(&atomic_sim_lock);
259
    *p |= m;
260
    ret = *p;
261
    pthread_mutex_unlock(&atomic_sim_lock);
262
    return ret;
263
}
264
265
#  define ATOMIC_OR_FETCH(p, v, o) fallback_atomic_or_fetch(p, v)
266
# endif
267
268
/*
269
 * users is broken up into 2 parts
270
 * bits 0-15 current readers
271
 * bit 32-63 ID
272
 */
273
3
# define READER_SHIFT 0
274
6
# define ID_SHIFT 32
275
/* TODO: READER_SIZE 32 in threads_win.c */
276
3
# define READER_SIZE 16
277
3
# define ID_SIZE 32
278
279
3
# define READER_MASK     (((uint64_t)1 << READER_SIZE) - 1)
280
3
# define ID_MASK         (((uint64_t)1 << ID_SIZE) - 1)
281
3
# define READER_COUNT(x) ((uint32_t)(((uint64_t)(x) >> READER_SHIFT) & \
282
3
                                     READER_MASK))
283
3
# define ID_VAL(x)       ((uint32_t)(((uint64_t)(x) >> ID_SHIFT) & ID_MASK))
284
# define VAL_READER      ((uint64_t)1 << READER_SHIFT)
285
3
# define VAL_ID(x)       ((uint64_t)x << ID_SHIFT)
286
287
/*
288
 * This is the core of an rcu lock. It tracks the readers and writers for the
289
 * current quiescence point for a given lock. Users is the 64 bit value that
290
 * stores the READERS/ID as defined above
291
 *
292
 */
293
struct rcu_qp {
294
    uint64_t users;
295
};
296
297
struct thread_qp {
298
    struct rcu_qp *qp;
299
    unsigned int depth;
300
    CRYPTO_RCU_LOCK *lock;
301
};
302
303
0
# define MAX_QPS 10
304
/*
305
 * This is the per thread tracking data
306
 * that is assigned to each thread participating
307
 * in an rcu qp
308
 *
309
 * qp points to the qp that it last acquired
310
 *
311
 */
312
struct rcu_thr_data {
313
    struct thread_qp thread_qps[MAX_QPS];
314
};
315
316
/*
317
 * This is the internal version of a CRYPTO_RCU_LOCK
318
 * it is cast from CRYPTO_RCU_LOCK
319
 */
320
struct rcu_lock_st {
321
    /* Callbacks to call for next ossl_synchronize_rcu */
322
    struct rcu_cb_item *cb_items;
323
324
    /* The context we are being created against */
325
    OSSL_LIB_CTX *ctx;
326
327
    /* rcu generation counter for in-order retirement */
328
    uint32_t id_ctr;
329
330
    /* TODO: can be moved before id_ctr for better alignment */
331
    /* Array of quiescent points for synchronization */
332
    struct rcu_qp *qp_group;
333
334
    /* Number of elements in qp_group array */
335
    uint32_t group_count;
336
337
    /* Index of the current qp in the qp_group array */
338
    uint32_t reader_idx;
339
340
    /* value of the next id_ctr value to be retired */
341
    uint32_t next_to_retire;
342
343
    /* index of the next free rcu_qp in the qp_group */
344
    uint32_t current_alloc_idx;
345
346
    /* number of qp's in qp_group array currently being retired */
347
    uint32_t writers_alloced;
348
349
    /* lock protecting write side operations */
350
    pthread_mutex_t write_lock;
351
352
    /* lock protecting updates to writers_alloced/current_alloc_idx */
353
    pthread_mutex_t alloc_lock;
354
355
    /* signal to wake threads waiting on alloc_lock */
356
    pthread_cond_t alloc_signal;
357
358
    /* lock to enforce in-order retirement */
359
    pthread_mutex_t prior_lock;
360
361
    /* signal to wake threads waiting on prior_lock */
362
    pthread_cond_t prior_signal;
363
};
364
365
/* Read side acquisition of the current qp */
366
static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock)
367
0
{
368
0
    uint32_t qp_idx;
369
370
    /* get the current qp index */
371
0
    for (;;) {
372
        /*
373
         * Notes on use of __ATOMIC_ACQUIRE
374
         * We need to ensure the following:
375
         * 1) That subsequent operations aren't optimized by hoisting them above
376
         * this operation.  Specifically, we don't want the below re-load of
377
         * qp_idx to get optimized away
378
         * 2) We want to ensure that any updating of reader_idx on the write side
379
         * of the lock is flushed from a local cpu cache so that we see any
380
         * updates prior to the load.  This is a non-issue on cache coherent
381
         * systems like x86, but is relevant on other arches
382
         * Note: This applies to the reload below as well
383
         */
384
0
        qp_idx = ATOMIC_LOAD_N(uint32_t, &lock->reader_idx, __ATOMIC_ACQUIRE);
385
386
        /*
387
         * Notes of use of __ATOMIC_RELEASE
388
         * This counter is only read by the write side of the lock, and so we
389
         * specify __ATOMIC_RELEASE here to ensure that the write side of the
390
         * lock see this during the spin loop read of users, as it waits for the
391
         * reader count to approach zero
392
         */
393
0
        ATOMIC_ADD_FETCH(&lock->qp_group[qp_idx].users, VAL_READER,
394
0
                         __ATOMIC_RELEASE);
395
396
        /* if the idx hasn't changed, we're good, else try again */
397
0
        if (qp_idx == ATOMIC_LOAD_N(uint32_t, &lock->reader_idx, __ATOMIC_ACQUIRE))
398
0
            break;
399
400
        /*
401
         * Notes on use of __ATOMIC_RELEASE
402
         * As with the add above, we want to ensure that this decrement is
403
         * seen by the write side of the lock as soon as it happens to prevent
404
         * undue spinning waiting for write side completion
405
         */
406
0
        ATOMIC_SUB_FETCH(&lock->qp_group[qp_idx].users, VAL_READER,
407
0
                         __ATOMIC_RELEASE);
408
0
    }
409
410
0
    return &lock->qp_group[qp_idx];
411
0
}
412
413
static void ossl_rcu_free_local_data(void *arg)
414
0
{
415
0
    OSSL_LIB_CTX *ctx = arg;
416
0
    CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(ctx);
417
0
    struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey);
418
419
0
    OPENSSL_free(data);
420
0
    CRYPTO_THREAD_set_local(lkey, NULL);
421
0
}
422
423
void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock)
424
0
{
425
0
    struct rcu_thr_data *data;
426
0
    int i, available_qp = -1;
427
0
    CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx);
428
429
    /*
430
     * we're going to access current_qp here so ask the
431
     * processor to fetch it
432
     */
433
0
    data = CRYPTO_THREAD_get_local(lkey);
434
435
0
    if (data == NULL) {
436
0
        data = OPENSSL_zalloc(sizeof(*data));
437
0
        OPENSSL_assert(data != NULL);
438
0
        CRYPTO_THREAD_set_local(lkey, data);
439
0
        ossl_init_thread_start(NULL, lock->ctx, ossl_rcu_free_local_data);
440
0
    }
441
442
0
    for (i = 0; i < MAX_QPS; i++) {
443
0
        if (data->thread_qps[i].qp == NULL && available_qp == -1)
444
0
            available_qp = i;
445
        /* If we have a hold on this lock already, we're good */
446
0
        if (data->thread_qps[i].lock == lock) {
447
0
            data->thread_qps[i].depth++;
448
0
            return;
449
0
        }
450
0
    }
451
452
    /*
453
     * if we get here, then we don't have a hold on this lock yet
454
     */
455
0
    assert(available_qp != -1);
456
457
0
    data->thread_qps[available_qp].qp = get_hold_current_qp(lock);
458
0
    data->thread_qps[available_qp].depth = 1;
459
0
    data->thread_qps[available_qp].lock = lock;
460
0
}
461
462
void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock)
463
0
{
464
0
    int i;
465
0
    CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx);
466
0
    struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey);
467
0
    uint64_t ret;
468
469
0
    assert(data != NULL);
470
471
0
    for (i = 0; i < MAX_QPS; i++) {
472
0
        if (data->thread_qps[i].lock == lock) {
473
            /*
474
             * As with read side acquisition, we use __ATOMIC_RELEASE here
475
             * to ensure that the decrement is published immediately
476
             * to any write side waiters
477
             */
478
0
            data->thread_qps[i].depth--;
479
0
            if (data->thread_qps[i].depth == 0) {
480
0
                ret = ATOMIC_SUB_FETCH(&data->thread_qps[i].qp->users, VAL_READER,
481
0
                                       __ATOMIC_RELEASE);
482
0
                OPENSSL_assert(ret != UINT64_MAX);
483
0
                data->thread_qps[i].qp = NULL;
484
0
                data->thread_qps[i].lock = NULL;
485
0
            }
486
0
            return;
487
0
        }
488
0
    }
489
    /*
490
     * If we get here, we're trying to unlock a lock that we never acquired -
491
     * that's fatal.
492
     */
493
0
    assert(0);
494
0
}
495
496
/*
497
 * Write side allocation routine to get the current qp
498
 * and replace it with a new one
499
 */
500
static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock)
501
3
{
502
3
    uint64_t new_id;
503
3
    uint32_t current_idx;
504
505
3
    pthread_mutex_lock(&lock->alloc_lock);
506
507
    /*
508
     * we need at least one qp to be available with one
509
     * left over, so that readers can start working on
510
     * one that isn't yet being waited on
511
     */
512
3
    while (lock->group_count - lock->writers_alloced < 2)
513
        /* we have to wait for one to be free */
514
0
        pthread_cond_wait(&lock->alloc_signal, &lock->alloc_lock);
515
516
3
    current_idx = lock->current_alloc_idx;
517
518
    /* Allocate the qp */
519
3
    lock->writers_alloced++;
520
521
    /* increment the allocation index */
522
3
    lock->current_alloc_idx =
523
3
        (lock->current_alloc_idx + 1) % lock->group_count;
524
525
    /* get and insert a new id */
526
3
    new_id = VAL_ID(lock->id_ctr);
527
3
    lock->id_ctr++;
528
529
    /*
530
     * Even though we are under a write side lock here
531
     * We need to use atomic instructions to ensure that the results
532
     * of this update are published to the read side prior to updating the
533
     * reader idx below
534
     */
535
3
    ATOMIC_AND_FETCH(&lock->qp_group[current_idx].users, ID_MASK,
536
3
                     __ATOMIC_RELEASE);
537
3
    ATOMIC_OR_FETCH(&lock->qp_group[current_idx].users, new_id,
538
3
                    __ATOMIC_RELEASE);
539
540
    /*
541
     * Update the reader index to be the prior qp.
542
     * Note the use of __ATOMIC_RELEASE here is based on the corresponding use
543
     * of __ATOMIC_ACQUIRE in get_hold_current_qp, as we want any publication
544
     * of this value to be seen on the read side immediately after it happens
545
     */
546
3
    ATOMIC_STORE_N(uint32_t, &lock->reader_idx, lock->current_alloc_idx,
547
3
                   __ATOMIC_RELEASE);
548
549
    /* wake up any waiters */
550
3
    pthread_cond_signal(&lock->alloc_signal);
551
3
    pthread_mutex_unlock(&lock->alloc_lock);
552
3
    return &lock->qp_group[current_idx];
553
3
}
554
555
static void retire_qp(CRYPTO_RCU_LOCK *lock, struct rcu_qp *qp)
556
3
{
557
3
    pthread_mutex_lock(&lock->alloc_lock);
558
3
    lock->writers_alloced--;
559
3
    pthread_cond_signal(&lock->alloc_signal);
560
3
    pthread_mutex_unlock(&lock->alloc_lock);
561
3
}
562
563
/* TODO: count should be unsigned, e.g uint32_t */
564
/* a negative value could result in unexpected behaviour */
565
static struct rcu_qp *allocate_new_qp_group(CRYPTO_RCU_LOCK *lock,
566
                                            int count)
567
1
{
568
1
    struct rcu_qp *new =
569
1
        OPENSSL_zalloc(sizeof(*new) * count);
570
571
1
    lock->group_count = count;
572
1
    return new;
573
1
}
574
575
void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock)
576
2
{
577
2
    pthread_mutex_lock(&lock->write_lock);
578
2
    TSAN_FAKE_UNLOCK(&lock->write_lock);
579
2
}
580
581
void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock)
582
2
{
583
2
    TSAN_FAKE_LOCK(&lock->write_lock);
584
2
    pthread_mutex_unlock(&lock->write_lock);
585
2
}
586
587
void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock)
588
3
{
589
3
    struct rcu_qp *qp;
590
3
    uint64_t count;
591
3
    struct rcu_cb_item *cb_items, *tmpcb;
592
593
3
    pthread_mutex_lock(&lock->write_lock);
594
3
    cb_items = lock->cb_items;
595
3
    lock->cb_items = NULL;
596
3
    pthread_mutex_unlock(&lock->write_lock);
597
598
3
    qp = update_qp(lock);
599
600
    /*
601
     * wait for the reader count to reach zero
602
     * Note the use of __ATOMIC_ACQUIRE here to ensure that any
603
     * prior __ATOMIC_RELEASE write operation in get_hold_current_qp
604
     * is visible prior to our read
605
     */
606
3
    do {
607
3
        count = ATOMIC_LOAD_N(uint64_t, &qp->users, __ATOMIC_ACQUIRE);
608
3
    } while (READER_COUNT(count) != 0);
609
610
    /* retire in order */
611
3
    pthread_mutex_lock(&lock->prior_lock);
612
3
    while (lock->next_to_retire != ID_VAL(count))
613
0
        pthread_cond_wait(&lock->prior_signal, &lock->prior_lock);
614
3
    lock->next_to_retire++;
615
3
    pthread_cond_broadcast(&lock->prior_signal);
616
3
    pthread_mutex_unlock(&lock->prior_lock);
617
618
3
    retire_qp(lock, qp);
619
620
    /* handle any callbacks that we have */
621
3
    while (cb_items != NULL) {
622
0
        tmpcb = cb_items;
623
0
        cb_items = cb_items->next;
624
0
        tmpcb->fn(tmpcb->data);
625
0
        OPENSSL_free(tmpcb);
626
0
    }
627
3
}
628
629
int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data)
630
0
{
631
0
    struct rcu_cb_item *new =
632
0
        OPENSSL_zalloc(sizeof(*new));
633
634
0
    if (new == NULL)
635
0
        return 0;
636
637
0
    new->data = data;
638
0
    new->fn = cb;
639
    /*
640
     * Use __ATOMIC_ACQ_REL here to indicate that any prior writes to this
641
     * list are visible to us prior to reading, and publish the new value
642
     * immediately
643
     */
644
0
    new->next = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, new,
645
0
                                  __ATOMIC_ACQ_REL);
646
647
0
    return 1;
648
0
}
649
650
void *ossl_rcu_uptr_deref(void **p)
651
2
{
652
2
    return ATOMIC_LOAD_N(pvoid, p, __ATOMIC_ACQUIRE);
653
2
}
654
655
void ossl_rcu_assign_uptr(void **p, void **v)
656
2
{
657
2
    ATOMIC_STORE(pvoid, p, v, __ATOMIC_RELEASE);
658
2
}
659
660
CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers, OSSL_LIB_CTX *ctx)
661
1
{
662
1
    struct rcu_lock_st *new;
663
664
1
    if (num_writers < 1)
665
0
        num_writers = 1;
666
667
1
    ctx = ossl_lib_ctx_get_concrete(ctx);
668
1
    if (ctx == NULL)
669
0
        return 0;
670
671
1
    new = OPENSSL_zalloc(sizeof(*new));
672
1
    if (new == NULL)
673
0
        return NULL;
674
675
1
    new->ctx = ctx;
676
1
    pthread_mutex_init(&new->write_lock, NULL);
677
1
    pthread_mutex_init(&new->prior_lock, NULL);
678
1
    pthread_mutex_init(&new->alloc_lock, NULL);
679
1
    pthread_cond_init(&new->prior_signal, NULL);
680
1
    pthread_cond_init(&new->alloc_signal, NULL);
681
1
    new->qp_group = allocate_new_qp_group(new, num_writers + 1);
682
1
    if (new->qp_group == NULL) {
683
0
        OPENSSL_free(new);
684
0
        new = NULL;
685
0
    }
686
1
    return new;
687
1
}
688
689
void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock)
690
1
{
691
1
    struct rcu_lock_st *rlock = (struct rcu_lock_st *)lock;
692
693
1
    if (lock == NULL)
694
0
        return;
695
696
    /* make sure we're synchronized */
697
1
    ossl_synchronize_rcu(rlock);
698
699
1
    OPENSSL_free(rlock->qp_group);
700
    /* There should only be a single qp left now */
701
1
    OPENSSL_free(rlock);
702
1
}
703
704
CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void)
705
24
{
706
24
# ifdef USE_RWLOCK
707
24
    CRYPTO_RWLOCK *lock;
708
709
24
    if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL)
710
        /* Don't set error, to avoid recursion blowup. */
711
0
        return NULL;
712
713
24
    if (pthread_rwlock_init(lock, NULL) != 0) {
714
0
        OPENSSL_free(lock);
715
0
        return NULL;
716
0
    }
717
# else
718
    pthread_mutexattr_t attr;
719
    CRYPTO_RWLOCK *lock;
720
721
    if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL)
722
        /* Don't set error, to avoid recursion blowup. */
723
        return NULL;
724
725
    /*
726
     * We don't use recursive mutexes, but try to catch errors if we do.
727
     */
728
    pthread_mutexattr_init(&attr);
729
#  if !defined (__TANDEM) && !defined (_SPT_MODEL_)
730
#   if !defined(NDEBUG) && !defined(OPENSSL_NO_MUTEX_ERRORCHECK)
731
    pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
732
#   endif
733
#  else
734
    /* The SPT Thread Library does not define MUTEX attributes. */
735
#  endif
736
737
    if (pthread_mutex_init(lock, &attr) != 0) {
738
        pthread_mutexattr_destroy(&attr);
739
        OPENSSL_free(lock);
740
        return NULL;
741
    }
742
743
    pthread_mutexattr_destroy(&attr);
744
# endif
745
746
24
    return lock;
747
24
}
748
749
__owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock)
750
290k
{
751
290k
# ifdef USE_RWLOCK
752
290k
    if (pthread_rwlock_rdlock(lock) != 0)
753
0
        return 0;
754
# else
755
    if (pthread_mutex_lock(lock) != 0) {
756
        assert(errno != EDEADLK && errno != EBUSY);
757
        return 0;
758
    }
759
# endif
760
761
290k
    return 1;
762
290k
}
763
764
__owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock)
765
115
{
766
115
# ifdef USE_RWLOCK
767
115
    if (pthread_rwlock_wrlock(lock) != 0)
768
0
        return 0;
769
# else
770
    if (pthread_mutex_lock(lock) != 0) {
771
        assert(errno != EDEADLK && errno != EBUSY);
772
        return 0;
773
    }
774
# endif
775
776
115
    return 1;
777
115
}
778
779
int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock)
780
290k
{
781
290k
# ifdef USE_RWLOCK
782
290k
    if (pthread_rwlock_unlock(lock) != 0)
783
0
        return 0;
784
# else
785
    if (pthread_mutex_unlock(lock) != 0) {
786
        assert(errno != EPERM);
787
        return 0;
788
    }
789
# endif
790
791
290k
    return 1;
792
290k
}
793
794
void CRYPTO_THREAD_lock_free(CRYPTO_RWLOCK *lock)
795
30
{
796
30
    if (lock == NULL)
797
6
        return;
798
799
24
# ifdef USE_RWLOCK
800
24
    pthread_rwlock_destroy(lock);
801
# else
802
    pthread_mutex_destroy(lock);
803
# endif
804
24
    OPENSSL_free(lock);
805
806
24
    return;
807
30
}
808
809
int CRYPTO_THREAD_run_once(CRYPTO_ONCE *once, void (*init)(void))
810
421k
{
811
421k
    if (pthread_once(once, init) != 0)
812
0
        return 0;
813
814
421k
    return 1;
815
421k
}
816
817
int CRYPTO_THREAD_init_local(CRYPTO_THREAD_LOCAL *key, void (*cleanup)(void *))
818
7
{
819
7
    if (pthread_key_create(key, cleanup) != 0)
820
0
        return 0;
821
822
7
    return 1;
823
7
}
824
825
void *CRYPTO_THREAD_get_local(CRYPTO_THREAD_LOCAL *key)
826
65.5k
{
827
65.5k
    return pthread_getspecific(*key);
828
65.5k
}
829
830
int CRYPTO_THREAD_set_local(CRYPTO_THREAD_LOCAL *key, void *val)
831
8
{
832
8
    if (pthread_setspecific(*key, val) != 0)
833
0
        return 0;
834
835
8
    return 1;
836
8
}
837
838
int CRYPTO_THREAD_cleanup_local(CRYPTO_THREAD_LOCAL *key)
839
7
{
840
7
    if (pthread_key_delete(*key) != 0)
841
0
        return 0;
842
843
7
    return 1;
844
7
}
845
846
CRYPTO_THREAD_ID CRYPTO_THREAD_get_current_id(void)
847
0
{
848
0
    return pthread_self();
849
0
}
850
851
int CRYPTO_THREAD_compare_id(CRYPTO_THREAD_ID a, CRYPTO_THREAD_ID b)
852
0
{
853
0
    return pthread_equal(a, b);
854
0
}
855
856
int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock)
857
0
{
858
0
# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS)
859
0
    if (__atomic_is_lock_free(sizeof(*val), val)) {
860
0
        *ret = __atomic_add_fetch(val, amount, __ATOMIC_ACQ_REL);
861
0
        return 1;
862
0
    }
863
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
864
    /* This will work for all future Solaris versions. */
865
    if (ret != NULL) {
866
        *ret = atomic_add_int_nv((volatile unsigned int *)val, amount);
867
        return 1;
868
    }
869
# endif
870
0
    if (lock == NULL || !CRYPTO_THREAD_write_lock(lock))
871
0
        return 0;
872
873
0
    *val += amount;
874
0
    *ret  = *val;
875
876
0
    if (!CRYPTO_THREAD_unlock(lock))
877
0
        return 0;
878
879
0
    return 1;
880
0
}
881
882
int CRYPTO_atomic_add64(uint64_t *val, uint64_t op, uint64_t *ret,
883
                        CRYPTO_RWLOCK *lock)
884
0
{
885
0
# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS)
886
0
    if (__atomic_is_lock_free(sizeof(*val), val)) {
887
0
        *ret = __atomic_add_fetch(val, op, __ATOMIC_ACQ_REL);
888
0
        return 1;
889
0
    }
890
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
891
    /* This will work for all future Solaris versions. */
892
    if (ret != NULL) {
893
        *ret = atomic_add_64_nv(val, op);
894
        return 1;
895
    }
896
# endif
897
0
    if (lock == NULL || !CRYPTO_THREAD_write_lock(lock))
898
0
        return 0;
899
0
    *val += op;
900
0
    *ret  = *val;
901
902
0
    if (!CRYPTO_THREAD_unlock(lock))
903
0
        return 0;
904
905
0
    return 1;
906
0
}
907
908
int CRYPTO_atomic_and(uint64_t *val, uint64_t op, uint64_t *ret,
909
                      CRYPTO_RWLOCK *lock)
910
0
{
911
0
# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS)
912
0
    if (__atomic_is_lock_free(sizeof(*val), val)) {
913
0
        *ret = __atomic_and_fetch(val, op, __ATOMIC_ACQ_REL);
914
0
        return 1;
915
0
    }
916
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
917
    /* This will work for all future Solaris versions. */
918
    if (ret != NULL) {
919
        *ret = atomic_and_64_nv(val, op);
920
        return 1;
921
    }
922
# endif
923
0
    if (lock == NULL || !CRYPTO_THREAD_write_lock(lock))
924
0
        return 0;
925
0
    *val &= op;
926
0
    *ret  = *val;
927
928
0
    if (!CRYPTO_THREAD_unlock(lock))
929
0
        return 0;
930
931
0
    return 1;
932
0
}
933
934
int CRYPTO_atomic_or(uint64_t *val, uint64_t op, uint64_t *ret,
935
                     CRYPTO_RWLOCK *lock)
936
2
{
937
2
# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS)
938
2
    if (__atomic_is_lock_free(sizeof(*val), val)) {
939
2
        *ret = __atomic_or_fetch(val, op, __ATOMIC_ACQ_REL);
940
2
        return 1;
941
2
    }
942
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
943
    /* This will work for all future Solaris versions. */
944
    if (ret != NULL) {
945
        *ret = atomic_or_64_nv(val, op);
946
        return 1;
947
    }
948
# endif
949
0
    if (lock == NULL || !CRYPTO_THREAD_write_lock(lock))
950
0
        return 0;
951
0
    *val |= op;
952
0
    *ret  = *val;
953
954
0
    if (!CRYPTO_THREAD_unlock(lock))
955
0
        return 0;
956
957
0
    return 1;
958
0
}
959
960
int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock)
961
355k
{
962
355k
# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS)
963
355k
    if (__atomic_is_lock_free(sizeof(*val), val)) {
964
355k
        __atomic_load(val, ret, __ATOMIC_ACQUIRE);
965
355k
        return 1;
966
355k
    }
967
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
968
    /* This will work for all future Solaris versions. */
969
    if (ret != NULL) {
970
        *ret = atomic_or_64_nv(val, 0);
971
        return 1;
972
    }
973
# endif
974
0
    if (lock == NULL || !CRYPTO_THREAD_read_lock(lock))
975
0
        return 0;
976
0
    *ret  = *val;
977
0
    if (!CRYPTO_THREAD_unlock(lock))
978
0
        return 0;
979
980
0
    return 1;
981
0
}
982
983
int CRYPTO_atomic_store(uint64_t *dst, uint64_t val, CRYPTO_RWLOCK *lock)
984
0
{
985
0
# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS)
986
0
    if (__atomic_is_lock_free(sizeof(*dst), dst)) {
987
0
        __atomic_store(dst, &val, __ATOMIC_RELEASE);
988
0
        return 1;
989
0
    }
990
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
991
    /* This will work for all future Solaris versions. */
992
    if (ret != NULL) {
993
        atomic_swap_64(dst, val);
994
        return 1;
995
    }
996
# endif
997
0
    if (lock == NULL || !CRYPTO_THREAD_read_lock(lock))
998
0
        return 0;
999
0
    *dst  = val;
1000
0
    if (!CRYPTO_THREAD_unlock(lock))
1001
0
        return 0;
1002
1003
0
    return 1;
1004
0
}
1005
1006
int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock)
1007
0
{
1008
0
# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS)
1009
0
    if (__atomic_is_lock_free(sizeof(*val), val)) {
1010
0
        __atomic_load(val, ret, __ATOMIC_ACQUIRE);
1011
0
        return 1;
1012
0
    }
1013
# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
1014
    /* This will work for all future Solaris versions. */
1015
    if (ret != NULL) {
1016
        *ret = (int)atomic_or_uint_nv((unsigned int *)val, 0);
1017
        return 1;
1018
    }
1019
# endif
1020
0
    if (lock == NULL || !CRYPTO_THREAD_read_lock(lock))
1021
0
        return 0;
1022
0
    *ret  = *val;
1023
0
    if (!CRYPTO_THREAD_unlock(lock))
1024
0
        return 0;
1025
1026
0
    return 1;
1027
0
}
1028
1029
# ifndef FIPS_MODULE
1030
int openssl_init_fork_handlers(void)
1031
0
{
1032
0
    return 1;
1033
0
}
1034
# endif /* FIPS_MODULE */
1035
1036
int openssl_get_fork_id(void)
1037
0
{
1038
0
    return getpid();
1039
0
}
1040
#endif