/src/openssl32/include/internal/ring_buf.h
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright 2022-2023 The OpenSSL Project Authors. All Rights Reserved. |
3 | | * |
4 | | * Licensed under the Apache License 2.0 (the "License"). You may not use |
5 | | * this file except in compliance with the License. You can obtain a copy |
6 | | * in the file LICENSE in the source distribution or at |
7 | | * https://www.openssl.org/source/license.html |
8 | | */ |
9 | | |
10 | | #ifndef OSSL_INTERNAL_RING_BUF_H |
11 | | # define OSSL_INTERNAL_RING_BUF_H |
12 | | # pragma once |
13 | | |
14 | | # include <openssl/e_os2.h> /* For 'ossl_inline' */ |
15 | | # include "internal/safe_math.h" |
16 | | |
17 | | /* |
18 | | * ================================================================== |
19 | | * Byte-wise ring buffer which supports pushing and popping blocks of multiple |
20 | | * bytes at a time. The logical offset of each byte for the purposes of a QUIC |
21 | | * stream is tracked. Bytes can be popped from the ring buffer in two stages; |
22 | | * first they are popped, and then they are culled. Bytes which have been popped |
23 | | * but not yet culled will not be overwritten, and can be restored. |
24 | | */ |
25 | | struct ring_buf { |
26 | | void *start; |
27 | | size_t alloc; /* size of buffer allocation in bytes */ |
28 | | |
29 | | /* |
30 | | * Logical offset of the head (where we append to). This is the current size |
31 | | * of the QUIC stream. This increases monotonically. |
32 | | */ |
33 | | uint64_t head_offset; |
34 | | |
35 | | /* |
36 | | * Logical offset of the cull tail. Data is no longer needed and is |
37 | | * deallocated as the cull tail advances, which occurs as data is |
38 | | * acknowledged. This increases monotonically. |
39 | | */ |
40 | | uint64_t ctail_offset; |
41 | | }; |
42 | | |
43 | | OSSL_SAFE_MATH_UNSIGNED(u64, uint64_t) |
44 | | |
45 | 104k | #define MAX_OFFSET (((uint64_t)1) << 62) /* QUIC-imposed limit */ |
46 | | |
47 | | static ossl_inline int ring_buf_init(struct ring_buf *r) |
48 | 212k | { |
49 | 212k | r->start = NULL; |
50 | 212k | r->alloc = 0; |
51 | 212k | r->head_offset = r->ctail_offset = 0; |
52 | 212k | return 1; |
53 | 212k | } quic_rstream.c:ring_buf_init Line | Count | Source | 48 | 127k | { | 49 | 127k | r->start = NULL; | 50 | 127k | r->alloc = 0; | 51 | 127k | r->head_offset = r->ctail_offset = 0; | 52 | 127k | return 1; | 53 | 127k | } |
quic_sstream.c:ring_buf_init Line | Count | Source | 48 | 84.7k | { | 49 | 84.7k | r->start = NULL; | 50 | 84.7k | r->alloc = 0; | 51 | 84.7k | r->head_offset = r->ctail_offset = 0; | 52 | 84.7k | return 1; | 53 | 84.7k | } |
|
54 | | |
55 | | static ossl_inline void ring_buf_destroy(struct ring_buf *r, int cleanse) |
56 | 296k | { |
57 | 296k | if (cleanse) |
58 | 0 | OPENSSL_clear_free(r->start, r->alloc); |
59 | 296k | else |
60 | 296k | OPENSSL_free(r->start); |
61 | 296k | r->start = NULL; |
62 | 296k | r->alloc = 0; |
63 | 296k | } quic_rstream.c:ring_buf_destroy Line | Count | Source | 56 | 127k | { | 57 | 127k | if (cleanse) | 58 | 0 | OPENSSL_clear_free(r->start, r->alloc); | 59 | 127k | else | 60 | 127k | OPENSSL_free(r->start); | 61 | 127k | r->start = NULL; | 62 | 127k | r->alloc = 0; | 63 | 127k | } |
quic_sstream.c:ring_buf_destroy Line | Count | Source | 56 | 169k | { | 57 | 169k | if (cleanse) | 58 | 0 | OPENSSL_clear_free(r->start, r->alloc); | 59 | 169k | else | 60 | 169k | OPENSSL_free(r->start); | 61 | 169k | r->start = NULL; | 62 | 169k | r->alloc = 0; | 63 | 169k | } |
|
64 | | |
65 | | static ossl_inline size_t ring_buf_used(struct ring_buf *r) |
66 | 235k | { |
67 | 235k | return (size_t)(r->head_offset - r->ctail_offset); |
68 | 235k | } Unexecuted instantiation: quic_rstream.c:ring_buf_used quic_sstream.c:ring_buf_used Line | Count | Source | 66 | 235k | { | 67 | 235k | return (size_t)(r->head_offset - r->ctail_offset); | 68 | 235k | } |
|
69 | | |
70 | | static ossl_inline size_t ring_buf_avail(struct ring_buf *r) |
71 | 65.7k | { |
72 | 65.7k | return r->alloc - ring_buf_used(r); |
73 | 65.7k | } Unexecuted instantiation: quic_rstream.c:ring_buf_avail quic_sstream.c:ring_buf_avail Line | Count | Source | 71 | 65.7k | { | 72 | 65.7k | return r->alloc - ring_buf_used(r); | 73 | 65.7k | } |
|
74 | | |
75 | | static ossl_inline int ring_buf_write_at(struct ring_buf *r, |
76 | | uint64_t logical_offset, |
77 | | const unsigned char *buf, |
78 | | size_t buf_len) |
79 | 0 | { |
80 | 0 | size_t avail, idx, l; |
81 | 0 | unsigned char *start = r->start; |
82 | 0 | int i, err = 0; |
83 | |
|
84 | 0 | avail = ring_buf_avail(r); |
85 | 0 | if (logical_offset < r->ctail_offset |
86 | 0 | || safe_add_u64(logical_offset, buf_len, &err) |
87 | 0 | > safe_add_u64(r->head_offset, avail, &err) |
88 | 0 | || safe_add_u64(r->head_offset, buf_len, &err) |
89 | 0 | > MAX_OFFSET |
90 | 0 | || err) |
91 | 0 | return 0; |
92 | | |
93 | 0 | for (i = 0; buf_len > 0 && i < 2; ++i) { |
94 | 0 | idx = logical_offset % r->alloc; |
95 | 0 | l = r->alloc - idx; |
96 | 0 | if (buf_len < l) |
97 | 0 | l = buf_len; |
98 | |
|
99 | 0 | memcpy(start + idx, buf, l); |
100 | 0 | if (r->head_offset < logical_offset + l) |
101 | 0 | r->head_offset = logical_offset + l; |
102 | |
|
103 | 0 | logical_offset += l; |
104 | 0 | buf += l; |
105 | 0 | buf_len -= l; |
106 | 0 | } |
107 | |
|
108 | 0 | assert(buf_len == 0); |
109 | | |
110 | 0 | return 1; |
111 | 0 | } Unexecuted instantiation: quic_rstream.c:ring_buf_write_at Unexecuted instantiation: quic_sstream.c:ring_buf_write_at |
112 | | |
113 | | static ossl_inline size_t ring_buf_push(struct ring_buf *r, |
114 | | const unsigned char *buf, |
115 | | size_t buf_len) |
116 | 29.8k | { |
117 | 29.8k | size_t pushed = 0, avail, idx, l; |
118 | 29.8k | unsigned char *start = r->start; |
119 | | |
120 | 59.6k | for (;;) { |
121 | 59.6k | avail = ring_buf_avail(r); |
122 | 59.6k | if (buf_len > avail) |
123 | 0 | buf_len = avail; |
124 | | |
125 | 59.6k | if (buf_len > MAX_OFFSET - r->head_offset) |
126 | 0 | buf_len = (size_t)(MAX_OFFSET - r->head_offset); |
127 | | |
128 | 59.6k | if (buf_len == 0) |
129 | 29.8k | break; |
130 | | |
131 | 29.8k | idx = r->head_offset % r->alloc; |
132 | 29.8k | l = r->alloc - idx; |
133 | 29.8k | if (buf_len < l) |
134 | 29.8k | l = buf_len; |
135 | | |
136 | 29.8k | memcpy(start + idx, buf, l); |
137 | 29.8k | r->head_offset += l; |
138 | 29.8k | buf += l; |
139 | 29.8k | buf_len -= l; |
140 | 29.8k | pushed += l; |
141 | 29.8k | } |
142 | | |
143 | 29.8k | return pushed; |
144 | 29.8k | } Unexecuted instantiation: quic_rstream.c:ring_buf_push quic_sstream.c:ring_buf_push Line | Count | Source | 116 | 29.8k | { | 117 | 29.8k | size_t pushed = 0, avail, idx, l; | 118 | 29.8k | unsigned char *start = r->start; | 119 | | | 120 | 59.6k | for (;;) { | 121 | 59.6k | avail = ring_buf_avail(r); | 122 | 59.6k | if (buf_len > avail) | 123 | 0 | buf_len = avail; | 124 | | | 125 | 59.6k | if (buf_len > MAX_OFFSET - r->head_offset) | 126 | 0 | buf_len = (size_t)(MAX_OFFSET - r->head_offset); | 127 | | | 128 | 59.6k | if (buf_len == 0) | 129 | 29.8k | break; | 130 | | | 131 | 29.8k | idx = r->head_offset % r->alloc; | 132 | 29.8k | l = r->alloc - idx; | 133 | 29.8k | if (buf_len < l) | 134 | 29.8k | l = buf_len; | 135 | | | 136 | 29.8k | memcpy(start + idx, buf, l); | 137 | 29.8k | r->head_offset += l; | 138 | 29.8k | buf += l; | 139 | 29.8k | buf_len -= l; | 140 | 29.8k | pushed += l; | 141 | 29.8k | } | 142 | | | 143 | 29.8k | return pushed; | 144 | 29.8k | } |
|
145 | | |
146 | | static ossl_inline const unsigned char *ring_buf_get_ptr(const struct ring_buf *r, |
147 | | uint64_t logical_offset, |
148 | | size_t *max_len) |
149 | 0 | { |
150 | 0 | unsigned char *start = r->start; |
151 | 0 | size_t idx; |
152 | |
|
153 | 0 | if (logical_offset >= r->head_offset || logical_offset < r->ctail_offset) |
154 | 0 | return NULL; |
155 | 0 | idx = logical_offset % r->alloc; |
156 | 0 | *max_len = r->alloc - idx; |
157 | 0 | return start + idx; |
158 | 0 | } Unexecuted instantiation: quic_rstream.c:ring_buf_get_ptr Unexecuted instantiation: quic_sstream.c:ring_buf_get_ptr |
159 | | |
160 | | /* |
161 | | * Retrieves data out of the read side of the ring buffer starting at the given |
162 | | * logical offset. *buf is set to point to a contiguous span of bytes and |
163 | | * *buf_len is set to the number of contiguous bytes. After this function |
164 | | * returns, there may or may not be more bytes available at the logical offset |
165 | | * of (logical_offset + *buf_len) by calling this function again. If the logical |
166 | | * offset is out of the range retained by the ring buffer, returns 0, else |
167 | | * returns 1. A logical offset at the end of the range retained by the ring |
168 | | * buffer is not considered an error and is returned with a *buf_len of 0. |
169 | | * |
170 | | * The ring buffer state is not changed. |
171 | | */ |
172 | | static ossl_inline int ring_buf_get_buf_at(const struct ring_buf *r, |
173 | | uint64_t logical_offset, |
174 | | const unsigned char **buf, |
175 | | size_t *buf_len) |
176 | 186k | { |
177 | 186k | const unsigned char *start = r->start; |
178 | 186k | size_t idx, l; |
179 | | |
180 | 186k | if (logical_offset > r->head_offset || logical_offset < r->ctail_offset) |
181 | 0 | return 0; |
182 | | |
183 | 186k | if (r->alloc == 0) { |
184 | 84.7k | *buf = NULL; |
185 | 84.7k | *buf_len = 0; |
186 | 84.7k | return 1; |
187 | 84.7k | } |
188 | | |
189 | 101k | idx = logical_offset % r->alloc; |
190 | 101k | l = (size_t)(r->head_offset - logical_offset); |
191 | 101k | if (l > r->alloc - idx) |
192 | 0 | l = r->alloc - idx; |
193 | | |
194 | 101k | *buf = start + idx; |
195 | 101k | *buf_len = l; |
196 | 101k | return 1; |
197 | 186k | } Unexecuted instantiation: quic_rstream.c:ring_buf_get_buf_at quic_sstream.c:ring_buf_get_buf_at Line | Count | Source | 176 | 186k | { | 177 | 186k | const unsigned char *start = r->start; | 178 | 186k | size_t idx, l; | 179 | | | 180 | 186k | if (logical_offset > r->head_offset || logical_offset < r->ctail_offset) | 181 | 0 | return 0; | 182 | | | 183 | 186k | if (r->alloc == 0) { | 184 | 84.7k | *buf = NULL; | 185 | 84.7k | *buf_len = 0; | 186 | 84.7k | return 1; | 187 | 84.7k | } | 188 | | | 189 | 101k | idx = logical_offset % r->alloc; | 190 | 101k | l = (size_t)(r->head_offset - logical_offset); | 191 | 101k | if (l > r->alloc - idx) | 192 | 0 | l = r->alloc - idx; | 193 | | | 194 | 101k | *buf = start + idx; | 195 | 101k | *buf_len = l; | 196 | 101k | return 1; | 197 | 186k | } |
|
198 | | |
199 | | static ossl_inline void ring_buf_cpop_range(struct ring_buf *r, |
200 | | uint64_t start, uint64_t end, |
201 | | int cleanse) |
202 | 46.7k | { |
203 | 46.7k | assert(end >= start); |
204 | | |
205 | 46.7k | if (start > r->ctail_offset || end >= MAX_OFFSET) |
206 | 1.40k | return; |
207 | | |
208 | 45.3k | if (cleanse && r->alloc > 0 && end > r->ctail_offset) { |
209 | 0 | size_t idx = r->ctail_offset % r->alloc; |
210 | 0 | uint64_t cleanse_end = end + 1; |
211 | 0 | size_t l; |
212 | |
|
213 | 0 | if (cleanse_end > r->head_offset) |
214 | 0 | cleanse_end = r->head_offset; |
215 | 0 | l = (size_t)(cleanse_end - r->ctail_offset); |
216 | 0 | if (l > r->alloc - idx) { |
217 | 0 | OPENSSL_cleanse((unsigned char *)r->start + idx, r->alloc - idx); |
218 | 0 | l -= r->alloc - idx; |
219 | 0 | idx = 0; |
220 | 0 | } |
221 | 0 | if (l > 0) |
222 | 0 | OPENSSL_cleanse((unsigned char *)r->start + idx, l); |
223 | 0 | } |
224 | | |
225 | 45.3k | r->ctail_offset = end + 1; |
226 | | /* Allow culling unpushed data */ |
227 | 45.3k | if (r->head_offset < r->ctail_offset) |
228 | 32.4k | r->head_offset = r->ctail_offset; |
229 | 45.3k | } quic_rstream.c:ring_buf_cpop_range Line | Count | Source | 202 | 32.4k | { | 203 | 32.4k | assert(end >= start); | 204 | | | 205 | 32.4k | if (start > r->ctail_offset || end >= MAX_OFFSET) | 206 | 0 | return; | 207 | | | 208 | 32.4k | if (cleanse && r->alloc > 0 && end > r->ctail_offset) { | 209 | 0 | size_t idx = r->ctail_offset % r->alloc; | 210 | 0 | uint64_t cleanse_end = end + 1; | 211 | 0 | size_t l; | 212 | |
| 213 | 0 | if (cleanse_end > r->head_offset) | 214 | 0 | cleanse_end = r->head_offset; | 215 | 0 | l = (size_t)(cleanse_end - r->ctail_offset); | 216 | 0 | if (l > r->alloc - idx) { | 217 | 0 | OPENSSL_cleanse((unsigned char *)r->start + idx, r->alloc - idx); | 218 | 0 | l -= r->alloc - idx; | 219 | 0 | idx = 0; | 220 | 0 | } | 221 | 0 | if (l > 0) | 222 | 0 | OPENSSL_cleanse((unsigned char *)r->start + idx, l); | 223 | 0 | } | 224 | | | 225 | 32.4k | r->ctail_offset = end + 1; | 226 | | /* Allow culling unpushed data */ | 227 | 32.4k | if (r->head_offset < r->ctail_offset) | 228 | 32.4k | r->head_offset = r->ctail_offset; | 229 | 32.4k | } |
quic_sstream.c:ring_buf_cpop_range Line | Count | Source | 202 | 14.3k | { | 203 | 14.3k | assert(end >= start); | 204 | | | 205 | 14.3k | if (start > r->ctail_offset || end >= MAX_OFFSET) | 206 | 1.40k | return; | 207 | | | 208 | 12.9k | if (cleanse && r->alloc > 0 && end > r->ctail_offset) { | 209 | 0 | size_t idx = r->ctail_offset % r->alloc; | 210 | 0 | uint64_t cleanse_end = end + 1; | 211 | 0 | size_t l; | 212 | |
| 213 | 0 | if (cleanse_end > r->head_offset) | 214 | 0 | cleanse_end = r->head_offset; | 215 | 0 | l = (size_t)(cleanse_end - r->ctail_offset); | 216 | 0 | if (l > r->alloc - idx) { | 217 | 0 | OPENSSL_cleanse((unsigned char *)r->start + idx, r->alloc - idx); | 218 | 0 | l -= r->alloc - idx; | 219 | 0 | idx = 0; | 220 | 0 | } | 221 | 0 | if (l > 0) | 222 | 0 | OPENSSL_cleanse((unsigned char *)r->start + idx, l); | 223 | 0 | } | 224 | | | 225 | 12.9k | r->ctail_offset = end + 1; | 226 | | /* Allow culling unpushed data */ | 227 | 12.9k | if (r->head_offset < r->ctail_offset) | 228 | 0 | r->head_offset = r->ctail_offset; | 229 | 12.9k | } |
|
230 | | |
231 | | static ossl_inline int ring_buf_resize(struct ring_buf *r, size_t num_bytes, |
232 | | int cleanse) |
233 | 212k | { |
234 | 212k | struct ring_buf rnew = {0}; |
235 | 212k | const unsigned char *src = NULL; |
236 | 212k | size_t src_len = 0, copied = 0; |
237 | | |
238 | 212k | if (num_bytes == r->alloc) |
239 | 127k | return 1; |
240 | | |
241 | 84.7k | if (num_bytes < ring_buf_used(r)) |
242 | 0 | return 0; |
243 | | |
244 | 84.7k | rnew.start = OPENSSL_malloc(num_bytes); |
245 | 84.7k | if (rnew.start == NULL) |
246 | 0 | return 0; |
247 | | |
248 | 84.7k | rnew.alloc = num_bytes; |
249 | 84.7k | rnew.head_offset = r->head_offset - ring_buf_used(r); |
250 | 84.7k | rnew.ctail_offset = rnew.head_offset; |
251 | | |
252 | 84.7k | for (;;) { |
253 | 84.7k | if (!ring_buf_get_buf_at(r, r->ctail_offset + copied, &src, &src_len)) { |
254 | 0 | OPENSSL_free(rnew.start); |
255 | 0 | return 0; |
256 | 0 | } |
257 | | |
258 | 84.7k | if (src_len == 0) |
259 | 84.7k | break; |
260 | | |
261 | 0 | if (ring_buf_push(&rnew, src, src_len) != src_len) { |
262 | 0 | OPENSSL_free(rnew.start); |
263 | 0 | return 0; |
264 | 0 | } |
265 | | |
266 | 0 | copied += src_len; |
267 | 0 | } |
268 | | |
269 | 84.7k | assert(rnew.head_offset == r->head_offset); |
270 | 84.7k | rnew.ctail_offset = r->ctail_offset; |
271 | | |
272 | 84.7k | ring_buf_destroy(r, cleanse); |
273 | 84.7k | memcpy(r, &rnew, sizeof(*r)); |
274 | 84.7k | return 1; |
275 | 84.7k | } quic_rstream.c:ring_buf_resize Line | Count | Source | 233 | 127k | { | 234 | 127k | struct ring_buf rnew = {0}; | 235 | 127k | const unsigned char *src = NULL; | 236 | 127k | size_t src_len = 0, copied = 0; | 237 | | | 238 | 127k | if (num_bytes == r->alloc) | 239 | 127k | return 1; | 240 | | | 241 | 0 | if (num_bytes < ring_buf_used(r)) | 242 | 0 | return 0; | 243 | | | 244 | 0 | rnew.start = OPENSSL_malloc(num_bytes); | 245 | 0 | if (rnew.start == NULL) | 246 | 0 | return 0; | 247 | | | 248 | 0 | rnew.alloc = num_bytes; | 249 | 0 | rnew.head_offset = r->head_offset - ring_buf_used(r); | 250 | 0 | rnew.ctail_offset = rnew.head_offset; | 251 | |
| 252 | 0 | for (;;) { | 253 | 0 | if (!ring_buf_get_buf_at(r, r->ctail_offset + copied, &src, &src_len)) { | 254 | 0 | OPENSSL_free(rnew.start); | 255 | 0 | return 0; | 256 | 0 | } | 257 | | | 258 | 0 | if (src_len == 0) | 259 | 0 | break; | 260 | | | 261 | 0 | if (ring_buf_push(&rnew, src, src_len) != src_len) { | 262 | 0 | OPENSSL_free(rnew.start); | 263 | 0 | return 0; | 264 | 0 | } | 265 | | | 266 | 0 | copied += src_len; | 267 | 0 | } | 268 | | | 269 | 0 | assert(rnew.head_offset == r->head_offset); | 270 | 0 | rnew.ctail_offset = r->ctail_offset; | 271 | |
| 272 | 0 | ring_buf_destroy(r, cleanse); | 273 | 0 | memcpy(r, &rnew, sizeof(*r)); | 274 | 0 | return 1; | 275 | 0 | } |
quic_sstream.c:ring_buf_resize Line | Count | Source | 233 | 84.7k | { | 234 | 84.7k | struct ring_buf rnew = {0}; | 235 | 84.7k | const unsigned char *src = NULL; | 236 | 84.7k | size_t src_len = 0, copied = 0; | 237 | | | 238 | 84.7k | if (num_bytes == r->alloc) | 239 | 0 | return 1; | 240 | | | 241 | 84.7k | if (num_bytes < ring_buf_used(r)) | 242 | 0 | return 0; | 243 | | | 244 | 84.7k | rnew.start = OPENSSL_malloc(num_bytes); | 245 | 84.7k | if (rnew.start == NULL) | 246 | 0 | return 0; | 247 | | | 248 | 84.7k | rnew.alloc = num_bytes; | 249 | 84.7k | rnew.head_offset = r->head_offset - ring_buf_used(r); | 250 | 84.7k | rnew.ctail_offset = rnew.head_offset; | 251 | | | 252 | 84.7k | for (;;) { | 253 | 84.7k | if (!ring_buf_get_buf_at(r, r->ctail_offset + copied, &src, &src_len)) { | 254 | 0 | OPENSSL_free(rnew.start); | 255 | 0 | return 0; | 256 | 0 | } | 257 | | | 258 | 84.7k | if (src_len == 0) | 259 | 84.7k | break; | 260 | | | 261 | 0 | if (ring_buf_push(&rnew, src, src_len) != src_len) { | 262 | 0 | OPENSSL_free(rnew.start); | 263 | 0 | return 0; | 264 | 0 | } | 265 | | | 266 | 0 | copied += src_len; | 267 | 0 | } | 268 | | | 269 | 84.7k | assert(rnew.head_offset == r->head_offset); | 270 | 84.7k | rnew.ctail_offset = r->ctail_offset; | 271 | | | 272 | 84.7k | ring_buf_destroy(r, cleanse); | 273 | 84.7k | memcpy(r, &rnew, sizeof(*r)); | 274 | 84.7k | return 1; | 275 | 84.7k | } |
|
276 | | |
277 | | #endif /* OSSL_INTERNAL_RING_BUF_H */ |