/src/php-src/Zend/zend_gc.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | +----------------------------------------------------------------------+ |
3 | | | Zend Engine | |
4 | | +----------------------------------------------------------------------+ |
5 | | | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) | |
6 | | +----------------------------------------------------------------------+ |
7 | | | This source file is subject to version 2.00 of the Zend license, | |
8 | | | that is bundled with this package in the file LICENSE, and is | |
9 | | | available through the world-wide-web at the following url: | |
10 | | | http://www.zend.com/license/2_00.txt. | |
11 | | | If you did not receive a copy of the Zend license and are unable to | |
12 | | | obtain it through the world-wide-web, please send a note to | |
13 | | | license@zend.com so we can mail you a copy immediately. | |
14 | | +----------------------------------------------------------------------+ |
15 | | | Authors: David Wang <planetbeing@gmail.com> | |
16 | | | Dmitry Stogov <dmitry@php.net> | |
17 | | +----------------------------------------------------------------------+ |
18 | | */ |
19 | | |
20 | | /** |
21 | | * zend_gc_collect_cycles |
22 | | * ====================== |
23 | | * |
24 | | * Colors and its meaning |
25 | | * ---------------------- |
26 | | * |
27 | | * BLACK (GC_BLACK) - In use or free. |
28 | | * GREY (GC_GREY) - Possible member of cycle. |
29 | | * WHITE (GC_WHITE) - Member of garbage cycle. |
30 | | * PURPLE (GC_PURPLE) - Possible root of cycle. |
31 | | * |
32 | | * Colors described in the paper but not used |
33 | | * ------------------------------------------ |
34 | | * |
35 | | * GREEN - Acyclic |
36 | | * RED - Candidate cycle undergoing |
37 | | * ORANGE - Candidate cycle awaiting epoch boundary. |
38 | | * |
39 | | * |
40 | | * Flow |
41 | | * ===== |
42 | | * |
43 | | * The garbage collect cycle starts from 'gc_mark_roots', which traverses the |
44 | | * possible roots, and calls mark_grey for roots are marked purple with |
45 | | * depth-first traverse. |
46 | | * |
47 | | * After all possible roots are traversed and marked, |
48 | | * gc_scan_roots will be called, and each root will be called with |
49 | | * gc_scan(root->ref) |
50 | | * |
51 | | * gc_scan checks the colors of possible members. |
52 | | * |
53 | | * If the node is marked as grey and the refcount > 0 |
54 | | * gc_scan_black will be called on that node to scan it's subgraph. |
55 | | * otherwise (refcount == 0), it marks the node white. |
56 | | * |
57 | | * A node MAY be added to possible roots when ZEND_UNSET_VAR happens or |
58 | | * zend_assign_to_variable is called only when possible garbage node is |
59 | | * produced. |
60 | | * gc_possible_root() will be called to add the nodes to possible roots. |
61 | | * |
62 | | * |
63 | | * For objects, we call their get_gc handler (by default 'zend_std_get_gc') to |
64 | | * get the object properties to scan. |
65 | | * |
66 | | * |
67 | | * @see http://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf |
68 | | */ |
69 | | #include "zend.h" |
70 | | #include "zend_API.h" |
71 | | #include "zend_compile.h" |
72 | | #include "zend_errors.h" |
73 | | #include "zend_fibers.h" |
74 | | #include "zend_hrtime.h" |
75 | | #include "zend_portability.h" |
76 | | #include "zend_types.h" |
77 | | #include "zend_weakrefs.h" |
78 | | #include "zend_string.h" |
79 | | |
80 | | #ifndef GC_BENCH |
81 | | # define GC_BENCH 0 |
82 | | #endif |
83 | | |
84 | | #ifndef ZEND_GC_DEBUG |
85 | | # define ZEND_GC_DEBUG 0 |
86 | | #endif |
87 | | |
88 | | /* GC_INFO layout */ |
89 | 3.53M | #define GC_ADDRESS 0x0fffffu |
90 | 21.2M | #define GC_COLOR 0x300000u |
91 | | |
92 | | #define GC_BLACK 0x000000u /* must be zero */ |
93 | | #define GC_WHITE 0x100000u |
94 | | #define GC_GREY 0x200000u |
95 | | #define GC_PURPLE 0x300000u |
96 | | |
97 | | /* Debug tracing */ |
98 | | #if ZEND_GC_DEBUG > 1 |
99 | | # define GC_TRACE(format, ...) fprintf(stderr, format "\n", ##__VA_ARGS__); |
100 | | # define GC_TRACE_REF(ref, format, ...) \ |
101 | | do { \ |
102 | | gc_trace_ref((zend_refcounted *) ref); \ |
103 | | fprintf(stderr, format "\n", ##__VA_ARGS__); \ |
104 | | } while (0) |
105 | | # define GC_TRACE_SET_COLOR(ref, color) \ |
106 | | GC_TRACE_REF(ref, "->%s", gc_color_name(color)) |
107 | | #else |
108 | | # define GC_TRACE_REF(ref, format, ...) |
109 | | # define GC_TRACE_SET_COLOR(ref, new_color) |
110 | | # define GC_TRACE(str) |
111 | | #endif |
112 | | |
113 | | /* GC_INFO access */ |
114 | | #define GC_REF_ADDRESS(ref) \ |
115 | 3.53M | (((GC_TYPE_INFO(ref)) & (GC_ADDRESS << GC_INFO_SHIFT)) >> GC_INFO_SHIFT) |
116 | | |
117 | | #define GC_REF_COLOR(ref) \ |
118 | | (((GC_TYPE_INFO(ref)) & (GC_COLOR << GC_INFO_SHIFT)) >> GC_INFO_SHIFT) |
119 | | |
120 | | #define GC_REF_CHECK_COLOR(ref, color) \ |
121 | 15.1M | ((GC_TYPE_INFO(ref) & (GC_COLOR << GC_INFO_SHIFT)) == ((color) << GC_INFO_SHIFT)) |
122 | | |
123 | 7.21M | #define GC_REF_SET_INFO(ref, info) do { \ |
124 | 7.21M | GC_TYPE_INFO(ref) = \ |
125 | 7.21M | (GC_TYPE_INFO(ref) & (GC_TYPE_MASK | GC_FLAGS_MASK)) | \ |
126 | 7.21M | ((info) << GC_INFO_SHIFT); \ |
127 | 7.21M | } while (0) |
128 | | |
129 | 3.98M | #define GC_REF_SET_COLOR(ref, c) do { \ |
130 | 3.98M | GC_TRACE_SET_COLOR(ref, c); \ |
131 | 3.98M | GC_TYPE_INFO(ref) = \ |
132 | 3.98M | (GC_TYPE_INFO(ref) & ~(GC_COLOR << GC_INFO_SHIFT)) | \ |
133 | 3.98M | ((c) << GC_INFO_SHIFT); \ |
134 | 3.98M | } while (0) |
135 | | |
136 | 2.09M | #define GC_REF_SET_BLACK(ref) do { \ |
137 | 2.09M | GC_TRACE_SET_COLOR(ref, GC_BLACK); \ |
138 | 2.09M | GC_TYPE_INFO(ref) &= ~(GC_COLOR << GC_INFO_SHIFT); \ |
139 | 2.09M | } while (0) |
140 | | |
141 | | #define GC_REF_SET_PURPLE(ref) do { \ |
142 | | GC_TRACE_SET_COLOR(ref, GC_PURPLE); \ |
143 | | GC_TYPE_INFO(ref) |= (GC_COLOR << GC_INFO_SHIFT); \ |
144 | | } while (0) |
145 | | |
146 | | /* bit stealing tags for gc_root_buffer.ref */ |
147 | 3.33M | #define GC_BITS 0x3 |
148 | | |
149 | 701k | #define GC_ROOT 0x0 /* possible root of circular garbage */ |
150 | 3.78M | #define GC_UNUSED 0x1 /* part of linked list of unused buffers */ |
151 | 3.54M | #define GC_GARBAGE 0x2 /* garbage to delete */ |
152 | 12.9k | #define GC_DTOR_GARBAGE 0x3 /* garbage on which only the dtor should be invoked */ |
153 | | |
154 | | #define GC_GET_PTR(ptr) \ |
155 | 127k | ((void*)(((uintptr_t)(ptr)) & ~GC_BITS)) |
156 | | |
157 | | #define GC_IS_ROOT(ptr) \ |
158 | 701k | ((((uintptr_t)(ptr)) & GC_BITS) == GC_ROOT) |
159 | | #define GC_IS_UNUSED(ptr) \ |
160 | 157k | ((((uintptr_t)(ptr)) & GC_BITS) == GC_UNUSED) |
161 | | #define GC_IS_GARBAGE(ptr) \ |
162 | 2.33M | ((((uintptr_t)(ptr)) & GC_BITS) == GC_GARBAGE) |
163 | | #define GC_IS_DTOR_GARBAGE(ptr) \ |
164 | 10.0k | ((((uintptr_t)(ptr)) & GC_BITS) == GC_DTOR_GARBAGE) |
165 | | |
166 | | #define GC_MAKE_GARBAGE(ptr) \ |
167 | 1.21M | ((void*)(((uintptr_t)(ptr)) | GC_GARBAGE)) |
168 | | #define GC_MAKE_DTOR_GARBAGE(ptr) \ |
169 | 2.94k | ((void*)(((uintptr_t)(ptr)) | GC_DTOR_GARBAGE)) |
170 | | |
171 | | /* GC address conversion */ |
172 | 9.83M | #define GC_IDX2PTR(idx) (GC_G(buf) + (idx)) |
173 | 3.63M | #define GC_PTR2IDX(ptr) ((ptr) - GC_G(buf)) |
174 | | |
175 | 3.62M | #define GC_IDX2LIST(idx) ((void*)(uintptr_t)(((idx) * sizeof(void*)) | GC_UNUSED)) |
176 | 954k | #define GC_LIST2IDX(list) (((uint32_t)(uintptr_t)(list)) / sizeof(void*)) |
177 | | |
178 | | /* GC buffers */ |
179 | 1.74M | #define GC_INVALID 0 |
180 | 1.99M | #define GC_FIRST_ROOT 1 |
181 | | |
182 | 16 | #define GC_DEFAULT_BUF_SIZE (16 * 1024) |
183 | 1 | #define GC_BUF_GROW_STEP (128 * 1024) |
184 | | |
185 | 0 | #define GC_MAX_UNCOMPRESSED (512 * 1024) |
186 | 2 | #define GC_MAX_BUF_SIZE 0x40000000 |
187 | | |
188 | 16 | #define GC_THRESHOLD_DEFAULT (10000 + GC_FIRST_ROOT) |
189 | 0 | #define GC_THRESHOLD_STEP 10000 |
190 | 0 | #define GC_THRESHOLD_MAX 1000000000 |
191 | 0 | #define GC_THRESHOLD_TRIGGER 100 |
192 | | |
193 | | /* GC flags */ |
194 | 7.81k | #define GC_HAS_DESTRUCTORS (1<<0) |
195 | | |
196 | | /* Weak maps */ |
197 | 1.76k | #define Z_FROM_WEAKMAP_KEY (1<<0) |
198 | 1.69k | #define Z_FROM_WEAKMAP (1<<1) |
199 | | |
200 | | /* The WeakMap entry zv is reachable from roots by following the virtual |
201 | | * reference from the a WeakMap key to the entry */ |
202 | | #define GC_FROM_WEAKMAP_KEY(zv) \ |
203 | 337 | (Z_TYPE_INFO_P((zv)) & (Z_FROM_WEAKMAP_KEY << Z_TYPE_INFO_EXTRA_SHIFT)) |
204 | | |
205 | 707 | #define GC_SET_FROM_WEAKMAP_KEY(zv) do { \ |
206 | 707 | zval *_z = (zv); \ |
207 | 707 | Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) | (Z_FROM_WEAKMAP_KEY << Z_TYPE_INFO_EXTRA_SHIFT); \ |
208 | 707 | } while (0) |
209 | | |
210 | 722 | #define GC_UNSET_FROM_WEAKMAP_KEY(zv) do { \ |
211 | 722 | zval *_z = (zv); \ |
212 | 722 | Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) & ~(Z_FROM_WEAKMAP_KEY << Z_TYPE_INFO_EXTRA_SHIFT); \ |
213 | 722 | } while (0) |
214 | | |
215 | | /* The WeakMap entry zv is reachable from roots by following the reference from |
216 | | * the WeakMap */ |
217 | | #define GC_FROM_WEAKMAP(zv) \ |
218 | 1.31k | (Z_TYPE_INFO_P((zv)) & (Z_FROM_WEAKMAP << Z_TYPE_INFO_EXTRA_SHIFT)) |
219 | | |
220 | 166 | #define GC_SET_FROM_WEAKMAP(zv) do { \ |
221 | 166 | zval *_z = (zv); \ |
222 | 166 | Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) | (Z_FROM_WEAKMAP << Z_TYPE_INFO_EXTRA_SHIFT); \ |
223 | 166 | } while (0) |
224 | | |
225 | 211 | #define GC_UNSET_FROM_WEAKMAP(zv) do { \ |
226 | 211 | zval *_z = (zv); \ |
227 | 211 | Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) & ~(Z_FROM_WEAKMAP << Z_TYPE_INFO_EXTRA_SHIFT); \ |
228 | 211 | } while (0) |
229 | | |
230 | | /* unused buffers */ |
231 | | #define GC_HAS_UNUSED() \ |
232 | 982k | (GC_G(unused) != GC_INVALID) |
233 | | #define GC_FETCH_UNUSED() \ |
234 | 954k | gc_fetch_unused() |
235 | | #define GC_LINK_UNUSED(root) \ |
236 | 3.62M | gc_link_unused(root) |
237 | | |
238 | | #define GC_HAS_NEXT_UNUSED_UNDER_THRESHOLD() \ |
239 | | (GC_G(first_unused) < GC_G(gc_threshold)) |
240 | | #define GC_HAS_NEXT_UNUSED() \ |
241 | 982k | (GC_G(first_unused) != GC_G(buf_size)) |
242 | | #define GC_FETCH_NEXT_UNUSED() \ |
243 | 2.67M | gc_fetch_next_unused() |
244 | | |
245 | | ZEND_API int (*gc_collect_cycles)(void); |
246 | | |
247 | | typedef struct _gc_root_buffer { |
248 | | zend_refcounted *ref; |
249 | | } gc_root_buffer; |
250 | | |
251 | | typedef struct _zend_gc_globals { |
252 | | gc_root_buffer *buf; /* preallocated arrays of buffers */ |
253 | | |
254 | | bool gc_enabled; |
255 | | bool gc_active; /* GC currently running, forbid nested GC */ |
256 | | bool gc_protected; /* GC protected, forbid root additions */ |
257 | | bool gc_full; |
258 | | |
259 | | uint32_t unused; /* linked list of unused buffers */ |
260 | | uint32_t first_unused; /* first unused buffer */ |
261 | | uint32_t gc_threshold; /* GC collection threshold */ |
262 | | uint32_t buf_size; /* size of the GC buffer */ |
263 | | uint32_t num_roots; /* number of roots in GC buffer */ |
264 | | |
265 | | uint32_t gc_runs; |
266 | | uint32_t collected; |
267 | | |
268 | | zend_hrtime_t activated_at; |
269 | | zend_hrtime_t collector_time; |
270 | | zend_hrtime_t dtor_time; |
271 | | zend_hrtime_t free_time; |
272 | | |
273 | | uint32_t dtor_idx; /* root buffer index */ |
274 | | uint32_t dtor_end; |
275 | | zend_fiber *dtor_fiber; |
276 | | bool dtor_fiber_running; |
277 | | |
278 | | #if GC_BENCH |
279 | | uint32_t root_buf_length; |
280 | | uint32_t root_buf_peak; |
281 | | uint32_t zval_possible_root; |
282 | | uint32_t zval_buffered; |
283 | | uint32_t zval_remove_from_buffer; |
284 | | uint32_t zval_marked_grey; |
285 | | #endif |
286 | | } zend_gc_globals; |
287 | | |
288 | | #ifdef ZTS |
289 | | static int gc_globals_id; |
290 | | static size_t gc_globals_offset; |
291 | | #define GC_G(v) ZEND_TSRMG_FAST(gc_globals_offset, zend_gc_globals *, v) |
292 | | #else |
293 | 51.6M | #define GC_G(v) (gc_globals.v) |
294 | | static zend_gc_globals gc_globals; |
295 | | #endif |
296 | | |
297 | | #if GC_BENCH |
298 | | # define GC_BENCH_INC(counter) GC_G(counter)++ |
299 | | # define GC_BENCH_DEC(counter) GC_G(counter)-- |
300 | | # define GC_BENCH_PEAK(peak, counter) do { \ |
301 | | if (GC_G(counter) > GC_G(peak)) { \ |
302 | | GC_G(peak) = GC_G(counter); \ |
303 | | } \ |
304 | | } while (0) |
305 | | #else |
306 | | # define GC_BENCH_INC(counter) |
307 | | # define GC_BENCH_DEC(counter) |
308 | | # define GC_BENCH_PEAK(peak, counter) |
309 | | #endif |
310 | | |
311 | | |
312 | 5.11k | #define GC_STACK_SEGMENT_SIZE (((4096 - ZEND_MM_OVERHEAD) / sizeof(void*)) - 2) |
313 | | |
314 | | typedef struct _gc_stack gc_stack; |
315 | | |
316 | | struct _gc_stack { |
317 | | gc_stack *prev; |
318 | | gc_stack *next; |
319 | | zend_refcounted *data[GC_STACK_SEGMENT_SIZE]; |
320 | | }; |
321 | | |
322 | | #define GC_STACK_DCL(init) \ |
323 | 198k | gc_stack *_stack = init; \ |
324 | 198k | size_t _top = 0; |
325 | | |
326 | | #define GC_STACK_PUSH(ref) \ |
327 | 3.12M | gc_stack_push(&_stack, &_top, ref); |
328 | | |
329 | | #define GC_STACK_POP() \ |
330 | 3.32M | gc_stack_pop(&_stack, &_top) |
331 | | |
332 | | static zend_never_inline gc_stack* gc_stack_next(gc_stack *stack) |
333 | 21.9k | { |
334 | 21.9k | if (UNEXPECTED(!stack->next)) { |
335 | 20.1k | gc_stack *segment = emalloc(sizeof(gc_stack)); |
336 | 20.1k | segment->prev = stack; |
337 | 20.1k | segment->next = NULL; |
338 | 20.1k | stack->next = segment; |
339 | 20.1k | } |
340 | 21.9k | return stack->next; |
341 | 21.9k | } |
342 | | |
343 | | static zend_always_inline void gc_stack_push(gc_stack **stack, size_t *top, zend_refcounted *ref) |
344 | 3.12M | { |
345 | 3.12M | if (UNEXPECTED(*top == GC_STACK_SEGMENT_SIZE)) { |
346 | 2.55k | (*stack) = gc_stack_next(*stack); |
347 | 2.55k | (*top) = 0; |
348 | 2.55k | } |
349 | 3.12M | (*stack)->data[(*top)++] = ref; |
350 | 3.12M | } |
351 | | |
352 | | static zend_always_inline zend_refcounted* gc_stack_pop(gc_stack **stack, size_t *top) |
353 | 3.32M | { |
354 | 3.32M | if (UNEXPECTED((*top) == 0)) { |
355 | 201k | if (!(*stack)->prev) { |
356 | 198k | return NULL; |
357 | 198k | } else { |
358 | 2.55k | (*stack) = (*stack)->prev; |
359 | 2.55k | (*top) = GC_STACK_SEGMENT_SIZE - 1; |
360 | 2.55k | return (*stack)->data[GC_STACK_SEGMENT_SIZE - 1]; |
361 | 2.55k | } |
362 | 3.12M | } else { |
363 | 3.12M | return (*stack)->data[--(*top)]; |
364 | 3.12M | } |
365 | 3.32M | } |
366 | | |
367 | | static void gc_stack_free(gc_stack *stack) |
368 | 23.5k | { |
369 | 23.5k | gc_stack *p = stack->next; |
370 | | |
371 | 43.7k | while (p) { |
372 | 20.1k | stack = p->next; |
373 | 20.1k | efree(p); |
374 | 20.1k | p = stack; |
375 | 20.1k | } |
376 | 23.5k | } |
377 | | |
378 | | static zend_always_inline uint32_t gc_compress(uint32_t idx) |
379 | 3.63M | { |
380 | 3.63M | if (EXPECTED(idx < GC_MAX_UNCOMPRESSED)) { |
381 | 3.63M | return idx; |
382 | 3.63M | } |
383 | 0 | return (idx % GC_MAX_UNCOMPRESSED) | GC_MAX_UNCOMPRESSED; |
384 | 3.63M | } |
385 | | |
386 | | static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, uint32_t idx) |
387 | 0 | { |
388 | 0 | gc_root_buffer *root = GC_IDX2PTR(idx); |
389 | |
|
390 | 0 | if (EXPECTED(GC_GET_PTR(root->ref) == ref)) { |
391 | 0 | return root; |
392 | 0 | } |
393 | | |
394 | 0 | while (1) { |
395 | 0 | idx += GC_MAX_UNCOMPRESSED; |
396 | 0 | ZEND_ASSERT(idx < GC_G(first_unused)); |
397 | 0 | root = GC_IDX2PTR(idx); |
398 | 0 | if (GC_GET_PTR(root->ref) == ref) { |
399 | 0 | return root; |
400 | 0 | } |
401 | 0 | } |
402 | 0 | } |
403 | | |
404 | | static zend_always_inline uint32_t gc_fetch_unused(void) |
405 | 954k | { |
406 | 954k | uint32_t idx; |
407 | 954k | gc_root_buffer *root; |
408 | | |
409 | 954k | ZEND_ASSERT(GC_HAS_UNUSED()); |
410 | 954k | idx = GC_G(unused); |
411 | 954k | root = GC_IDX2PTR(idx); |
412 | 954k | ZEND_ASSERT(GC_IS_UNUSED(root->ref)); |
413 | 954k | GC_G(unused) = GC_LIST2IDX(root->ref); |
414 | 954k | return idx; |
415 | 954k | } |
416 | | |
417 | | static zend_always_inline void gc_link_unused(gc_root_buffer *root) |
418 | 3.62M | { |
419 | 3.62M | root->ref = GC_IDX2LIST(GC_G(unused)); |
420 | 3.62M | GC_G(unused) = GC_PTR2IDX(root); |
421 | 3.62M | } |
422 | | |
423 | | static zend_always_inline uint32_t gc_fetch_next_unused(void) |
424 | 2.67M | { |
425 | 2.67M | uint32_t idx; |
426 | | |
427 | 2.67M | ZEND_ASSERT(GC_HAS_NEXT_UNUSED()); |
428 | 2.67M | idx = GC_G(first_unused); |
429 | 2.67M | GC_G(first_unused) = GC_G(first_unused) + 1; |
430 | 2.67M | return idx; |
431 | 2.67M | } |
432 | | |
433 | | #if ZEND_GC_DEBUG > 1 |
434 | | static const char *gc_color_name(uint32_t color) { |
435 | | switch (color) { |
436 | | case GC_BLACK: return "black"; |
437 | | case GC_WHITE: return "white"; |
438 | | case GC_GREY: return "grey"; |
439 | | case GC_PURPLE: return "purple"; |
440 | | default: return "unknown"; |
441 | | } |
442 | | } |
443 | | static void gc_trace_ref(zend_refcounted *ref) { |
444 | | if (GC_TYPE(ref) == IS_OBJECT) { |
445 | | zend_object *obj = (zend_object *) ref; |
446 | | fprintf(stderr, "[%p] rc=%d addr=%d %s object(%s)#%d ", |
447 | | ref, GC_REFCOUNT(ref), GC_REF_ADDRESS(ref), |
448 | | gc_color_name(GC_REF_COLOR(ref)), |
449 | | obj->ce->name->val, obj->handle); |
450 | | } else if (GC_TYPE(ref) == IS_ARRAY) { |
451 | | zend_array *arr = (zend_array *) ref; |
452 | | fprintf(stderr, "[%p] rc=%d addr=%d %s array(%d) ", |
453 | | ref, GC_REFCOUNT(ref), GC_REF_ADDRESS(ref), |
454 | | gc_color_name(GC_REF_COLOR(ref)), |
455 | | zend_hash_num_elements(arr)); |
456 | | } else { |
457 | | fprintf(stderr, "[%p] rc=%d addr=%d %s %s ", |
458 | | ref, GC_REFCOUNT(ref), GC_REF_ADDRESS(ref), |
459 | | gc_color_name(GC_REF_COLOR(ref)), |
460 | | GC_TYPE(ref) == IS_REFERENCE |
461 | | ? "reference" : zend_get_type_by_const(GC_TYPE(ref))); |
462 | | } |
463 | | } |
464 | | #endif |
465 | | |
466 | | static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root) |
467 | 3.57M | { |
468 | 3.57M | GC_LINK_UNUSED(root); |
469 | 3.57M | GC_G(num_roots)--; |
470 | 3.57M | GC_BENCH_DEC(root_buf_length); |
471 | 3.57M | } |
472 | | |
473 | | static void root_buffer_dtor(zend_gc_globals *gc_globals) |
474 | 0 | { |
475 | 0 | if (gc_globals->buf) { |
476 | 0 | free(gc_globals->buf); |
477 | 0 | gc_globals->buf = NULL; |
478 | 0 | } |
479 | 0 | } |
480 | | |
481 | | static void gc_globals_ctor_ex(zend_gc_globals *gc_globals) |
482 | 16 | { |
483 | 16 | gc_globals->gc_enabled = 0; |
484 | 16 | gc_globals->gc_active = 0; |
485 | 16 | gc_globals->gc_protected = 1; |
486 | 16 | gc_globals->gc_full = 0; |
487 | | |
488 | 16 | gc_globals->buf = NULL; |
489 | 16 | gc_globals->unused = GC_INVALID; |
490 | 16 | gc_globals->first_unused = GC_INVALID; |
491 | 16 | gc_globals->gc_threshold = GC_INVALID; |
492 | 16 | gc_globals->buf_size = GC_INVALID; |
493 | 16 | gc_globals->num_roots = 0; |
494 | | |
495 | 16 | gc_globals->gc_runs = 0; |
496 | 16 | gc_globals->collected = 0; |
497 | 16 | gc_globals->collector_time = 0; |
498 | 16 | gc_globals->dtor_time = 0; |
499 | 16 | gc_globals->free_time = 0; |
500 | 16 | gc_globals->activated_at = 0; |
501 | | |
502 | 16 | gc_globals->dtor_idx = GC_FIRST_ROOT; |
503 | 16 | gc_globals->dtor_end = 0; |
504 | 16 | gc_globals->dtor_fiber = NULL; |
505 | 16 | gc_globals->dtor_fiber_running = false; |
506 | | |
507 | | #if GC_BENCH |
508 | | gc_globals->root_buf_length = 0; |
509 | | gc_globals->root_buf_peak = 0; |
510 | | gc_globals->zval_possible_root = 0; |
511 | | gc_globals->zval_buffered = 0; |
512 | | gc_globals->zval_remove_from_buffer = 0; |
513 | | gc_globals->zval_marked_grey = 0; |
514 | | #endif |
515 | 16 | } |
516 | | |
517 | | void gc_globals_ctor(void) |
518 | 16 | { |
519 | | #ifdef ZTS |
520 | | ts_allocate_fast_id(&gc_globals_id, &gc_globals_offset, sizeof(zend_gc_globals), (ts_allocate_ctor) gc_globals_ctor_ex, (ts_allocate_dtor) root_buffer_dtor); |
521 | | #else |
522 | 16 | gc_globals_ctor_ex(&gc_globals); |
523 | 16 | #endif |
524 | 16 | } |
525 | | |
526 | | void gc_globals_dtor(void) |
527 | 0 | { |
528 | 0 | #ifndef ZTS |
529 | 0 | root_buffer_dtor(&gc_globals); |
530 | 0 | #endif |
531 | 0 | } |
532 | | |
533 | | void gc_reset(void) |
534 | 300k | { |
535 | 300k | if (GC_G(buf)) { |
536 | 300k | GC_G(gc_active) = 0; |
537 | 300k | GC_G(gc_protected) = 0; |
538 | 300k | GC_G(gc_full) = 0; |
539 | 300k | GC_G(unused) = GC_INVALID; |
540 | 300k | GC_G(first_unused) = GC_FIRST_ROOT; |
541 | 300k | GC_G(num_roots) = 0; |
542 | | |
543 | 300k | GC_G(gc_runs) = 0; |
544 | 300k | GC_G(collected) = 0; |
545 | | |
546 | 300k | GC_G(collector_time) = 0; |
547 | 300k | GC_G(dtor_time) = 0; |
548 | 300k | GC_G(free_time) = 0; |
549 | | |
550 | 300k | GC_G(dtor_idx) = GC_FIRST_ROOT; |
551 | 300k | GC_G(dtor_end) = 0; |
552 | 300k | GC_G(dtor_fiber) = NULL; |
553 | 300k | GC_G(dtor_fiber_running) = false; |
554 | | |
555 | | #if GC_BENCH |
556 | | GC_G(root_buf_length) = 0; |
557 | | GC_G(root_buf_peak) = 0; |
558 | | GC_G(zval_possible_root) = 0; |
559 | | GC_G(zval_buffered) = 0; |
560 | | GC_G(zval_remove_from_buffer) = 0; |
561 | | GC_G(zval_marked_grey) = 0; |
562 | | #endif |
563 | 300k | } |
564 | | |
565 | 300k | GC_G(activated_at) = zend_hrtime(); |
566 | 300k | } |
567 | | |
568 | | ZEND_API bool gc_enable(bool enable) |
569 | 134k | { |
570 | 134k | bool old_enabled = GC_G(gc_enabled); |
571 | 134k | GC_G(gc_enabled) = enable; |
572 | 134k | if (enable && !old_enabled && GC_G(buf) == NULL) { |
573 | 16 | GC_G(buf) = (gc_root_buffer*) pemalloc(sizeof(gc_root_buffer) * GC_DEFAULT_BUF_SIZE, 1); |
574 | 16 | GC_G(buf)[0].ref = NULL; |
575 | 16 | GC_G(buf_size) = GC_DEFAULT_BUF_SIZE; |
576 | 16 | GC_G(gc_threshold) = GC_THRESHOLD_DEFAULT; |
577 | 16 | gc_reset(); |
578 | 16 | } |
579 | 134k | return old_enabled; |
580 | 134k | } |
581 | | |
582 | | ZEND_API bool gc_enabled(void) |
583 | 91 | { |
584 | 91 | return GC_G(gc_enabled); |
585 | 91 | } |
586 | | |
587 | | ZEND_API bool gc_protect(bool protect) |
588 | 32.8k | { |
589 | 32.8k | bool old_protected = GC_G(gc_protected); |
590 | 32.8k | GC_G(gc_protected) = protect; |
591 | 32.8k | return old_protected; |
592 | 32.8k | } |
593 | | |
594 | | ZEND_API bool gc_protected(void) |
595 | 0 | { |
596 | 0 | return GC_G(gc_protected); |
597 | 0 | } |
598 | | |
599 | | static void gc_grow_root_buffer(void) |
600 | 1 | { |
601 | 1 | size_t new_size; |
602 | | |
603 | 1 | if (GC_G(buf_size) >= GC_MAX_BUF_SIZE) { |
604 | 0 | if (!GC_G(gc_full)) { |
605 | 0 | zend_error(E_WARNING, "GC buffer overflow (GC disabled)\n"); |
606 | 0 | GC_G(gc_active) = 1; |
607 | 0 | GC_G(gc_protected) = 1; |
608 | 0 | GC_G(gc_full) = 1; |
609 | 0 | return; |
610 | 0 | } |
611 | 0 | } |
612 | 1 | if (GC_G(buf_size) < GC_BUF_GROW_STEP) { |
613 | 1 | new_size = GC_G(buf_size) * 2; |
614 | 1 | } else { |
615 | 0 | new_size = GC_G(buf_size) + GC_BUF_GROW_STEP; |
616 | 0 | } |
617 | 1 | if (new_size > GC_MAX_BUF_SIZE) { |
618 | 0 | new_size = GC_MAX_BUF_SIZE; |
619 | 0 | } |
620 | 1 | GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1); |
621 | 1 | GC_G(buf_size) = new_size; |
622 | 1 | } |
623 | | |
624 | | static void gc_adjust_threshold(int count) |
625 | 0 | { |
626 | 0 | uint32_t new_threshold; |
627 | | |
628 | | /* TODO Very simple heuristic for dynamic GC buffer resizing: |
629 | | * If there are "too few" collections, increase the collection threshold |
630 | | * by a fixed step */ |
631 | 0 | if (count < GC_THRESHOLD_TRIGGER || GC_G(num_roots) >= GC_G(gc_threshold)) { |
632 | | /* increase */ |
633 | 0 | if (GC_G(gc_threshold) < GC_THRESHOLD_MAX) { |
634 | 0 | new_threshold = GC_G(gc_threshold) + GC_THRESHOLD_STEP; |
635 | 0 | if (new_threshold > GC_THRESHOLD_MAX) { |
636 | 0 | new_threshold = GC_THRESHOLD_MAX; |
637 | 0 | } |
638 | 0 | if (new_threshold > GC_G(buf_size)) { |
639 | 0 | gc_grow_root_buffer(); |
640 | 0 | } |
641 | 0 | if (new_threshold <= GC_G(buf_size)) { |
642 | 0 | GC_G(gc_threshold) = new_threshold; |
643 | 0 | } |
644 | 0 | } |
645 | 0 | } else if (GC_G(gc_threshold) > GC_THRESHOLD_DEFAULT) { |
646 | 0 | new_threshold = GC_G(gc_threshold) - GC_THRESHOLD_STEP; |
647 | 0 | if (new_threshold < GC_THRESHOLD_DEFAULT) { |
648 | 0 | new_threshold = GC_THRESHOLD_DEFAULT; |
649 | 0 | } |
650 | 0 | GC_G(gc_threshold) = new_threshold; |
651 | 0 | } |
652 | 0 | } |
653 | | |
654 | | static zend_never_inline void ZEND_FASTCALL gc_possible_root_when_full(zend_refcounted *ref) |
655 | 0 | { |
656 | 0 | uint32_t idx; |
657 | 0 | gc_root_buffer *newRoot; |
658 | |
|
659 | 0 | ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT); |
660 | 0 | ZEND_ASSERT(GC_INFO(ref) == 0); |
661 | | |
662 | 0 | if (GC_G(gc_enabled) && !GC_G(gc_active)) { |
663 | 0 | GC_ADDREF(ref); |
664 | 0 | gc_adjust_threshold(gc_collect_cycles()); |
665 | 0 | if (UNEXPECTED(GC_DELREF(ref) == 0)) { |
666 | 0 | rc_dtor_func(ref); |
667 | 0 | return; |
668 | 0 | } else if (UNEXPECTED(GC_INFO(ref))) { |
669 | 0 | return; |
670 | 0 | } |
671 | 0 | } |
672 | | |
673 | 0 | if (GC_HAS_UNUSED()) { |
674 | 0 | idx = GC_FETCH_UNUSED(); |
675 | 0 | } else if (EXPECTED(GC_HAS_NEXT_UNUSED())) { |
676 | 0 | idx = GC_FETCH_NEXT_UNUSED(); |
677 | 0 | } else { |
678 | 0 | gc_grow_root_buffer(); |
679 | 0 | if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) { |
680 | 0 | return; |
681 | 0 | } |
682 | 0 | idx = GC_FETCH_NEXT_UNUSED(); |
683 | 0 | } |
684 | | |
685 | 0 | newRoot = GC_IDX2PTR(idx); |
686 | 0 | newRoot->ref = ref; /* GC_ROOT tag is 0 */ |
687 | 0 | GC_TRACE_SET_COLOR(ref, GC_PURPLE); |
688 | |
|
689 | 0 | idx = gc_compress(idx); |
690 | 0 | GC_REF_SET_INFO(ref, idx | GC_PURPLE); |
691 | 0 | GC_G(num_roots)++; |
692 | |
|
693 | 0 | GC_BENCH_INC(zval_buffered); |
694 | 0 | GC_BENCH_INC(root_buf_length); |
695 | 0 | GC_BENCH_PEAK(root_buf_peak, root_buf_length); |
696 | 0 | } |
697 | | |
698 | | ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref) |
699 | 2.87M | { |
700 | 2.87M | uint32_t idx; |
701 | 2.87M | gc_root_buffer *newRoot; |
702 | | |
703 | 2.87M | if (UNEXPECTED(GC_G(gc_protected))) { |
704 | 226k | return; |
705 | 226k | } |
706 | | |
707 | 2.64M | GC_BENCH_INC(zval_possible_root); |
708 | | |
709 | 2.64M | if (EXPECTED(GC_HAS_UNUSED())) { |
710 | 954k | idx = GC_FETCH_UNUSED(); |
711 | 1.69M | } else if (EXPECTED(GC_HAS_NEXT_UNUSED_UNDER_THRESHOLD())) { |
712 | 1.69M | idx = GC_FETCH_NEXT_UNUSED(); |
713 | 1.69M | } else { |
714 | 0 | gc_possible_root_when_full(ref); |
715 | 0 | return; |
716 | 0 | } |
717 | | |
718 | 2.64M | ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT); |
719 | 2.64M | ZEND_ASSERT(GC_INFO(ref) == 0); |
720 | | |
721 | 2.64M | newRoot = GC_IDX2PTR(idx); |
722 | 2.64M | newRoot->ref = ref; /* GC_ROOT tag is 0 */ |
723 | 2.64M | GC_TRACE_SET_COLOR(ref, GC_PURPLE); |
724 | | |
725 | 2.64M | idx = gc_compress(idx); |
726 | 2.64M | GC_REF_SET_INFO(ref, idx | GC_PURPLE); |
727 | 2.64M | GC_G(num_roots)++; |
728 | | |
729 | 2.64M | GC_BENCH_INC(zval_buffered); |
730 | 2.64M | GC_BENCH_INC(root_buf_length); |
731 | 2.64M | GC_BENCH_PEAK(root_buf_peak, root_buf_length); |
732 | 2.64M | } |
733 | | |
734 | | static void ZEND_FASTCALL gc_extra_root(zend_refcounted *ref) |
735 | 50 | { |
736 | 50 | uint32_t idx; |
737 | 50 | gc_root_buffer *newRoot; |
738 | | |
739 | 50 | if (EXPECTED(GC_HAS_UNUSED())) { |
740 | 0 | idx = GC_FETCH_UNUSED(); |
741 | 50 | } else if (EXPECTED(GC_HAS_NEXT_UNUSED())) { |
742 | 50 | idx = GC_FETCH_NEXT_UNUSED(); |
743 | 50 | } else { |
744 | 0 | gc_grow_root_buffer(); |
745 | 0 | if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) { |
746 | | /* TODO: can this really happen? */ |
747 | 0 | return; |
748 | 0 | } |
749 | 0 | idx = GC_FETCH_NEXT_UNUSED(); |
750 | 0 | } |
751 | | |
752 | 50 | ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT); |
753 | 50 | ZEND_ASSERT(GC_REF_ADDRESS(ref) == 0); |
754 | | |
755 | 50 | newRoot = GC_IDX2PTR(idx); |
756 | 50 | newRoot->ref = ref; /* GC_ROOT tag is 0 */ |
757 | | |
758 | 50 | idx = gc_compress(idx); |
759 | 50 | GC_REF_SET_INFO(ref, idx | GC_REF_COLOR(ref)); |
760 | 50 | GC_G(num_roots)++; |
761 | | |
762 | 50 | GC_BENCH_INC(zval_buffered); |
763 | 50 | GC_BENCH_INC(root_buf_length); |
764 | 50 | GC_BENCH_PEAK(root_buf_peak, root_buf_length); |
765 | 50 | } |
766 | | |
767 | | static zend_never_inline void ZEND_FASTCALL gc_remove_compressed(zend_refcounted *ref, uint32_t idx) |
768 | 0 | { |
769 | 0 | gc_root_buffer *root = gc_decompress(ref, idx); |
770 | 0 | gc_remove_from_roots(root); |
771 | 0 | } |
772 | | |
773 | | ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref) |
774 | 3.52M | { |
775 | 3.52M | gc_root_buffer *root; |
776 | 3.52M | uint32_t idx = GC_REF_ADDRESS(ref); |
777 | | |
778 | 3.52M | GC_BENCH_INC(zval_remove_from_buffer); |
779 | | |
780 | 3.52M | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
781 | 2.41M | GC_TRACE_SET_COLOR(ref, GC_BLACK); |
782 | 2.41M | } |
783 | 3.52M | GC_REF_SET_INFO(ref, 0); |
784 | | |
785 | | /* Perform decompression only in case of large buffers */ |
786 | 3.52M | if (UNEXPECTED(GC_G(first_unused) >= GC_MAX_UNCOMPRESSED)) { |
787 | 0 | gc_remove_compressed(ref, idx); |
788 | 0 | return; |
789 | 0 | } |
790 | | |
791 | 3.52M | ZEND_ASSERT(idx); |
792 | 3.52M | root = GC_IDX2PTR(idx); |
793 | 3.52M | gc_remove_from_roots(root); |
794 | 3.52M | } |
795 | | |
796 | | static void gc_scan_black(zend_refcounted *ref, gc_stack *stack) |
797 | 36.2k | { |
798 | 36.2k | HashTable *ht; |
799 | 36.2k | Bucket *p; |
800 | 36.2k | zval *zv; |
801 | 36.2k | uint32_t n; |
802 | 36.2k | GC_STACK_DCL(stack); |
803 | | |
804 | 237k | tail_call: |
805 | 237k | if (GC_TYPE(ref) == IS_OBJECT) { |
806 | 38.3k | zend_object *obj = (zend_object*)ref; |
807 | | |
808 | 38.3k | if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { |
809 | 38.3k | zval *table; |
810 | 38.3k | int len; |
811 | | |
812 | 38.3k | if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) { |
813 | 810 | zend_weakmap_get_object_key_entry_gc(obj, &table, &len); |
814 | 810 | n = len; |
815 | 810 | zv = table; |
816 | 1.53k | for (; n != 0; n-=2) { |
817 | 723 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
818 | 723 | zval *entry = (zval*) Z_PTR_P(zv); |
819 | 723 | zval *weakmap = zv+1; |
820 | 723 | ZEND_ASSERT(Z_REFCOUNTED_P(weakmap)); |
821 | 723 | if (Z_OPT_COLLECTABLE_P(entry)) { |
822 | 613 | GC_UNSET_FROM_WEAKMAP_KEY(entry); |
823 | 613 | if (GC_REF_CHECK_COLOR(Z_COUNTED_P(weakmap), GC_GREY)) { |
824 | | /* Weakmap was scanned in gc_mark_roots, we must |
825 | | * ensure that it's eventually scanned in |
826 | | * gc_scan_roots as well. */ |
827 | 66 | if (!GC_REF_ADDRESS(Z_COUNTED_P(weakmap))) { |
828 | 45 | gc_extra_root(Z_COUNTED_P(weakmap)); |
829 | 45 | } |
830 | 547 | } else if (/* GC_REF_CHECK_COLOR(Z_COUNTED_P(weakmap), GC_BLACK) && */ !GC_FROM_WEAKMAP(entry)) { |
831 | | /* Both the entry weakmap and key are BLACK, so we |
832 | | * can mark the entry BLACK as well. |
833 | | * !GC_FROM_WEAKMAP(entry) means that the weakmap |
834 | | * was already scanned black (or will not be |
835 | | * scanned), so it's our responsibility to mark the |
836 | | * entry */ |
837 | 544 | ZEND_ASSERT(GC_REF_CHECK_COLOR(Z_COUNTED_P(weakmap), GC_BLACK)); |
838 | 544 | ref = Z_COUNTED_P(entry); |
839 | 544 | GC_ADDREF(ref); |
840 | 544 | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
841 | 27 | GC_REF_SET_BLACK(ref); |
842 | 27 | GC_STACK_PUSH(ref); |
843 | 27 | } |
844 | 544 | } |
845 | 613 | } |
846 | 723 | zv+=2; |
847 | 723 | } |
848 | 810 | } |
849 | | |
850 | 38.3k | if (UNEXPECTED(obj->handlers->get_gc == zend_weakmap_get_gc)) { |
851 | 109 | zend_weakmap_get_key_entry_gc(obj, &table, &len); |
852 | 109 | n = len; |
853 | 109 | zv = table; |
854 | 305 | for (; n != 0; n-=2) { |
855 | 196 | ZEND_ASSERT(Z_TYPE_P(zv+1) == IS_PTR); |
856 | 196 | zval *key = zv; |
857 | 196 | zval *entry = (zval*) Z_PTR_P(zv+1); |
858 | 196 | if (Z_OPT_COLLECTABLE_P(entry)) { |
859 | 102 | GC_UNSET_FROM_WEAKMAP(entry); |
860 | 102 | if (GC_REF_CHECK_COLOR(Z_COUNTED_P(key), GC_GREY)) { |
861 | | /* Key was scanned in gc_mark_roots, we must |
862 | | * ensure that it's eventually scanned in |
863 | | * gc_scan_roots as well. */ |
864 | 25 | if (!GC_REF_ADDRESS(Z_COUNTED_P(key))) { |
865 | 5 | gc_extra_root(Z_COUNTED_P(key)); |
866 | 5 | } |
867 | 77 | } else if (/* GC_REF_CHECK_COLOR(Z_COUNTED_P(key), GC_BLACK) && */ !GC_FROM_WEAKMAP_KEY(entry)) { |
868 | | /* Both the entry weakmap and key are BLACK, so we |
869 | | * can mark the entry BLACK as well. |
870 | | * !GC_FROM_WEAKMAP_KEY(entry) means that the key |
871 | | * was already scanned black (or will not be |
872 | | * scanned), so it's our responsibility to mark the |
873 | | * entry */ |
874 | 67 | ZEND_ASSERT(GC_REF_CHECK_COLOR(Z_COUNTED_P(key), GC_BLACK)); |
875 | 67 | ref = Z_COUNTED_P(entry); |
876 | 67 | GC_ADDREF(ref); |
877 | 67 | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
878 | 62 | GC_REF_SET_BLACK(ref); |
879 | 62 | GC_STACK_PUSH(ref); |
880 | 62 | } |
881 | 67 | } |
882 | 102 | } |
883 | 196 | zv += 2; |
884 | 196 | } |
885 | 109 | goto next; |
886 | 109 | } |
887 | | |
888 | 38.2k | ht = obj->handlers->get_gc(obj, &table, &len); |
889 | 38.2k | n = len; |
890 | 38.2k | zv = table; |
891 | 38.2k | if (UNEXPECTED(ht)) { |
892 | 13.2k | GC_ADDREF(ht); |
893 | 13.2k | if (!GC_REF_CHECK_COLOR(ht, GC_BLACK)) { |
894 | 13.2k | GC_REF_SET_BLACK(ht); |
895 | 16.8k | for (; n != 0; n--) { |
896 | 3.62k | if (Z_COLLECTABLE_P(zv)) { |
897 | 608 | ref = Z_COUNTED_P(zv); |
898 | 608 | GC_ADDREF(ref); |
899 | 608 | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
900 | 479 | GC_REF_SET_BLACK(ref); |
901 | 479 | GC_STACK_PUSH(ref); |
902 | 479 | } |
903 | 608 | } |
904 | 3.62k | zv++; |
905 | 3.62k | } |
906 | 13.2k | goto handle_ht; |
907 | 13.2k | } |
908 | 13.2k | } |
909 | | |
910 | 133k | handle_zvals: |
911 | 409k | for (; n != 0; n--) { |
912 | 300k | if (Z_COLLECTABLE_P(zv)) { |
913 | 27.6k | ref = Z_COUNTED_P(zv); |
914 | 27.6k | GC_ADDREF(ref); |
915 | 27.6k | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
916 | 23.8k | GC_REF_SET_BLACK(ref); |
917 | 23.8k | zv++; |
918 | 124k | while (--n) { |
919 | 101k | if (Z_COLLECTABLE_P(zv)) { |
920 | 90.3k | zend_refcounted *ref = Z_COUNTED_P(zv); |
921 | 90.3k | GC_ADDREF(ref); |
922 | 90.3k | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
923 | 89.0k | GC_REF_SET_BLACK(ref); |
924 | 89.0k | GC_STACK_PUSH(ref); |
925 | 89.0k | } |
926 | 90.3k | } |
927 | 101k | zv++; |
928 | 101k | } |
929 | 23.8k | goto tail_call; |
930 | 23.8k | } |
931 | 27.6k | } |
932 | 276k | zv++; |
933 | 276k | } |
934 | 133k | } |
935 | 199k | } else if (GC_TYPE(ref) == IS_ARRAY) { |
936 | 196k | ZEND_ASSERT((zend_array*)ref != &EG(symbol_table)); |
937 | 196k | ht = (zend_array*)ref; |
938 | 209k | handle_ht: |
939 | 209k | n = ht->nNumUsed; |
940 | 209k | zv = ht->arPacked; |
941 | 209k | if (HT_IS_PACKED(ht)) { |
942 | 108k | goto handle_zvals; |
943 | 108k | } |
944 | | |
945 | 101k | p = (Bucket*)zv; |
946 | 301k | for (; n != 0; n--) { |
947 | 283k | zv = &p->val; |
948 | 283k | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
949 | 4.94k | zv = Z_INDIRECT_P(zv); |
950 | 4.94k | } |
951 | 283k | if (Z_COLLECTABLE_P(zv)) { |
952 | 85.0k | ref = Z_COUNTED_P(zv); |
953 | 85.0k | GC_ADDREF(ref); |
954 | 85.0k | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
955 | 83.5k | GC_REF_SET_BLACK(ref); |
956 | 83.5k | p++; |
957 | 168k | while (--n) { |
958 | 84.5k | zv = &p->val; |
959 | 84.5k | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
960 | 369 | zv = Z_INDIRECT_P(zv); |
961 | 369 | } |
962 | 84.5k | if (Z_COLLECTABLE_P(zv)) { |
963 | 4.43k | zend_refcounted *ref = Z_COUNTED_P(zv); |
964 | 4.43k | GC_ADDREF(ref); |
965 | 4.43k | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
966 | 3.01k | GC_REF_SET_BLACK(ref); |
967 | 3.01k | GC_STACK_PUSH(ref); |
968 | 3.01k | } |
969 | 4.43k | } |
970 | 84.5k | p++; |
971 | 84.5k | } |
972 | 83.5k | goto tail_call; |
973 | 83.5k | } |
974 | 85.0k | } |
975 | 200k | p++; |
976 | 200k | } |
977 | 101k | } else if (GC_TYPE(ref) == IS_REFERENCE) { |
978 | 3.28k | if (Z_COLLECTABLE(((zend_reference*)ref)->val)) { |
979 | 1.65k | ref = Z_COUNTED(((zend_reference*)ref)->val); |
980 | 1.65k | GC_ADDREF(ref); |
981 | 1.65k | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
982 | 1.57k | GC_REF_SET_BLACK(ref); |
983 | 1.57k | goto tail_call; |
984 | 1.57k | } |
985 | 1.65k | } |
986 | 3.28k | } |
987 | | |
988 | 128k | next: |
989 | 128k | ref = GC_STACK_POP(); |
990 | 128k | if (ref) { |
991 | 92.6k | goto tail_call; |
992 | 92.6k | } |
993 | 128k | } |
994 | | |
995 | | static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) |
996 | 68.0k | { |
997 | 68.0k | HashTable *ht; |
998 | 68.0k | Bucket *p; |
999 | 68.0k | zval *zv; |
1000 | 68.0k | uint32_t n; |
1001 | 68.0k | GC_STACK_DCL(stack); |
1002 | | |
1003 | 1.42M | tail_call: |
1004 | 1.42M | GC_BENCH_INC(zval_marked_grey); |
1005 | | |
1006 | 1.42M | if (GC_TYPE(ref) == IS_OBJECT) { |
1007 | 707k | zend_object *obj = (zend_object*)ref; |
1008 | | |
1009 | 707k | if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { |
1010 | 707k | zval *table; |
1011 | 707k | int len; |
1012 | | |
1013 | 707k | if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) { |
1014 | 1.03k | zend_weakmap_get_object_key_entry_gc(obj, &table, &len); |
1015 | 1.03k | n = len; |
1016 | 1.03k | zv = table; |
1017 | 1.86k | for (; n != 0; n-=2) { |
1018 | 822 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
1019 | 822 | zval *entry = (zval*) Z_PTR_P(zv); |
1020 | 822 | zval *weakmap = zv+1; |
1021 | 822 | ZEND_ASSERT(Z_REFCOUNTED_P(weakmap)); |
1022 | 822 | if (Z_COLLECTABLE_P(entry)) { |
1023 | 707 | GC_SET_FROM_WEAKMAP_KEY(entry); |
1024 | 707 | ref = Z_COUNTED_P(entry); |
1025 | | /* Only DELREF if the contribution from the weakmap has |
1026 | | * not been cancelled yet */ |
1027 | 707 | if (!GC_FROM_WEAKMAP(entry)) { |
1028 | 656 | GC_DELREF(ref); |
1029 | 656 | } |
1030 | 707 | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1031 | 126 | GC_REF_SET_COLOR(ref, GC_GREY); |
1032 | 126 | GC_STACK_PUSH(ref); |
1033 | 126 | } |
1034 | 707 | } |
1035 | 822 | zv+=2; |
1036 | 822 | } |
1037 | 1.03k | } |
1038 | | |
1039 | 707k | if (UNEXPECTED(obj->handlers->get_gc == zend_weakmap_get_gc)) { |
1040 | 158 | zend_weakmap_get_entry_gc(obj, &table, &len); |
1041 | 158 | n = len; |
1042 | 158 | zv = table; |
1043 | 418 | for (; n != 0; n--) { |
1044 | 260 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
1045 | 260 | zval *entry = (zval*) Z_PTR_P(zv); |
1046 | 260 | if (Z_COLLECTABLE_P(entry)) { |
1047 | 166 | GC_SET_FROM_WEAKMAP(entry); |
1048 | 166 | ref = Z_COUNTED_P(entry); |
1049 | | /* Only DELREF if the contribution from the weakmap key |
1050 | | * has not been cancelled yet */ |
1051 | 166 | if (!GC_FROM_WEAKMAP_KEY(entry)) { |
1052 | 64 | GC_DELREF(ref); |
1053 | 64 | } |
1054 | 166 | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1055 | 51 | GC_REF_SET_COLOR(ref, GC_GREY); |
1056 | 51 | GC_STACK_PUSH(ref); |
1057 | 51 | } |
1058 | 166 | } |
1059 | 260 | zv++; |
1060 | 260 | } |
1061 | 158 | goto next; |
1062 | 158 | } |
1063 | | |
1064 | 707k | ht = obj->handlers->get_gc(obj, &table, &len); |
1065 | 707k | n = len; |
1066 | 707k | zv = table; |
1067 | 707k | if (UNEXPECTED(ht)) { |
1068 | 666k | GC_DELREF(ht); |
1069 | 666k | if (!GC_REF_CHECK_COLOR(ht, GC_GREY)) { |
1070 | 665k | GC_REF_SET_COLOR(ht, GC_GREY); |
1071 | 891k | for (; n != 0; n--) { |
1072 | 225k | if (Z_COLLECTABLE_P(zv)) { |
1073 | 222k | ref = Z_COUNTED_P(zv); |
1074 | 222k | GC_DELREF(ref); |
1075 | 222k | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1076 | 218k | GC_REF_SET_COLOR(ref, GC_GREY); |
1077 | 218k | GC_STACK_PUSH(ref); |
1078 | 218k | } |
1079 | 222k | } |
1080 | 225k | zv++; |
1081 | 225k | } |
1082 | 665k | goto handle_ht; |
1083 | 665k | } |
1084 | 666k | } |
1085 | 168k | handle_zvals: |
1086 | 596k | for (; n != 0; n--) { |
1087 | 464k | if (Z_COLLECTABLE_P(zv)) { |
1088 | 77.4k | ref = Z_COUNTED_P(zv); |
1089 | 77.4k | GC_DELREF(ref); |
1090 | 77.4k | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1091 | 36.8k | GC_REF_SET_COLOR(ref, GC_GREY); |
1092 | 36.8k | zv++; |
1093 | 144k | while (--n) { |
1094 | 107k | if (Z_COLLECTABLE_P(zv)) { |
1095 | 89.0k | zend_refcounted *ref = Z_COUNTED_P(zv); |
1096 | 89.0k | GC_DELREF(ref); |
1097 | 89.0k | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1098 | 83.9k | GC_REF_SET_COLOR(ref, GC_GREY); |
1099 | 83.9k | GC_STACK_PUSH(ref); |
1100 | 83.9k | } |
1101 | 89.0k | } |
1102 | 107k | zv++; |
1103 | 107k | } |
1104 | 36.8k | goto tail_call; |
1105 | 36.8k | } |
1106 | 77.4k | } |
1107 | 427k | zv++; |
1108 | 427k | } |
1109 | 168k | } |
1110 | 717k | } else if (GC_TYPE(ref) == IS_ARRAY) { |
1111 | 694k | ZEND_ASSERT(((zend_array*)ref) != &EG(symbol_table)); |
1112 | 694k | ht = (zend_array*)ref; |
1113 | 1.36M | handle_ht: |
1114 | 1.36M | n = ht->nNumUsed; |
1115 | 1.36M | if (HT_IS_PACKED(ht)) { |
1116 | 127k | zv = ht->arPacked; |
1117 | 127k | goto handle_zvals; |
1118 | 127k | } |
1119 | | |
1120 | 1.23M | p = ht->arData; |
1121 | 2.55M | for (; n != 0; n--) { |
1122 | 1.79M | zv = &p->val; |
1123 | 1.79M | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1124 | 815k | zv = Z_INDIRECT_P(zv); |
1125 | 815k | } |
1126 | 1.79M | if (Z_COLLECTABLE_P(zv)) { |
1127 | 612k | ref = Z_COUNTED_P(zv); |
1128 | 612k | GC_DELREF(ref); |
1129 | 612k | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1130 | 465k | GC_REF_SET_COLOR(ref, GC_GREY); |
1131 | 465k | p++; |
1132 | 1.72M | while (--n) { |
1133 | 1.25M | zv = &p->val; |
1134 | 1.25M | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1135 | 139k | zv = Z_INDIRECT_P(zv); |
1136 | 139k | } |
1137 | 1.25M | if (Z_COLLECTABLE_P(zv)) { |
1138 | 971k | zend_refcounted *ref = Z_COUNTED_P(zv); |
1139 | 971k | GC_DELREF(ref); |
1140 | 971k | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1141 | 544k | GC_REF_SET_COLOR(ref, GC_GREY); |
1142 | 544k | GC_STACK_PUSH(ref); |
1143 | 544k | } |
1144 | 971k | } |
1145 | 1.25M | p++; |
1146 | 1.25M | } |
1147 | 465k | goto tail_call; |
1148 | 465k | } |
1149 | 612k | } |
1150 | 1.32M | p++; |
1151 | 1.32M | } |
1152 | 1.23M | } else if (GC_TYPE(ref) == IS_REFERENCE) { |
1153 | 22.7k | if (Z_COLLECTABLE(((zend_reference*)ref)->val)) { |
1154 | 18.0k | ref = Z_COUNTED(((zend_reference*)ref)->val); |
1155 | 18.0k | GC_DELREF(ref); |
1156 | 18.0k | if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1157 | 6.84k | GC_REF_SET_COLOR(ref, GC_GREY); |
1158 | 6.84k | goto tail_call; |
1159 | 6.84k | } |
1160 | 18.0k | } |
1161 | 22.7k | } |
1162 | | |
1163 | 915k | next: |
1164 | 915k | ref = GC_STACK_POP(); |
1165 | 915k | if (ref) { |
1166 | 847k | goto tail_call; |
1167 | 847k | } |
1168 | 915k | } |
1169 | | |
1170 | | /* Two-Finger compaction algorithm */ |
1171 | | static void gc_compact(void) |
1172 | 880k | { |
1173 | 880k | if (GC_G(num_roots) + GC_FIRST_ROOT != GC_G(first_unused)) { |
1174 | 459k | if (GC_G(num_roots)) { |
1175 | 11.2k | gc_root_buffer *free = GC_IDX2PTR(GC_FIRST_ROOT); |
1176 | 11.2k | gc_root_buffer *scan = GC_IDX2PTR(GC_G(first_unused) - 1); |
1177 | 11.2k | gc_root_buffer *end = GC_IDX2PTR(GC_G(num_roots)); |
1178 | 11.2k | uint32_t idx; |
1179 | 11.2k | zend_refcounted *p; |
1180 | | |
1181 | 20.9k | while (free < scan) { |
1182 | 113k | while (!GC_IS_UNUSED(free->ref)) { |
1183 | 99.5k | free++; |
1184 | 99.5k | } |
1185 | 43.6k | while (GC_IS_UNUSED(scan->ref)) { |
1186 | 29.5k | scan--; |
1187 | 29.5k | } |
1188 | 14.1k | if (scan > free) { |
1189 | 7.47k | p = scan->ref; |
1190 | 7.47k | free->ref = p; |
1191 | 7.47k | p = GC_GET_PTR(p); |
1192 | 7.47k | idx = gc_compress(GC_PTR2IDX(free)); |
1193 | 7.47k | GC_REF_SET_INFO(p, idx | GC_REF_COLOR(p)); |
1194 | 7.47k | free++; |
1195 | 7.47k | scan--; |
1196 | 7.47k | if (scan <= end) { |
1197 | 4.38k | break; |
1198 | 4.38k | } |
1199 | 7.47k | } |
1200 | 14.1k | } |
1201 | 11.2k | } |
1202 | 459k | GC_G(unused) = GC_INVALID; |
1203 | 459k | GC_G(first_unused) = GC_G(num_roots) + GC_FIRST_ROOT; |
1204 | 459k | } |
1205 | 880k | } |
1206 | | |
1207 | | static void gc_mark_roots(gc_stack *stack) |
1208 | 23.6k | { |
1209 | 23.6k | gc_root_buffer *current, *last; |
1210 | | |
1211 | 23.6k | gc_compact(); |
1212 | | |
1213 | 23.6k | current = GC_IDX2PTR(GC_FIRST_ROOT); |
1214 | 23.6k | last = GC_IDX2PTR(GC_G(first_unused)); |
1215 | 257k | while (current != last) { |
1216 | 233k | if (GC_IS_ROOT(current->ref)) { |
1217 | 233k | if (GC_REF_CHECK_COLOR(current->ref, GC_PURPLE)) { |
1218 | 68.0k | GC_REF_SET_COLOR(current->ref, GC_GREY); |
1219 | 68.0k | gc_mark_grey(current->ref, stack); |
1220 | 68.0k | } |
1221 | 233k | } |
1222 | 233k | current++; |
1223 | 233k | } |
1224 | 23.6k | } |
1225 | | |
1226 | | static void gc_scan(zend_refcounted *ref, gc_stack *stack) |
1227 | 68.0k | { |
1228 | 68.0k | HashTable *ht; |
1229 | 68.0k | Bucket *p; |
1230 | 68.0k | zval *zv; |
1231 | 68.0k | uint32_t n; |
1232 | 68.0k | GC_STACK_DCL(stack); |
1233 | | |
1234 | 1.89M | tail_call: |
1235 | 1.89M | if (!GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1236 | 333 | goto next; |
1237 | 333 | } |
1238 | | |
1239 | 1.89M | if (GC_REFCOUNT(ref) > 0) { |
1240 | 36.2k | if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
1241 | 36.2k | GC_REF_SET_BLACK(ref); |
1242 | 36.2k | if (UNEXPECTED(!_stack->next)) { |
1243 | 19.3k | gc_stack_next(_stack); |
1244 | 19.3k | } |
1245 | | /* Split stack and reuse the tail */ |
1246 | 36.2k | _stack->next->prev = NULL; |
1247 | 36.2k | gc_scan_black(ref, _stack->next); |
1248 | 36.2k | _stack->next->prev = _stack; |
1249 | 36.2k | } |
1250 | 36.2k | goto next; |
1251 | 36.2k | } |
1252 | | |
1253 | 1.85M | if (GC_TYPE(ref) == IS_OBJECT) { |
1254 | 671k | zend_object *obj = (zend_object*)ref; |
1255 | 671k | if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { |
1256 | 671k | zval *table; |
1257 | 671k | int len; |
1258 | | |
1259 | 671k | if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) { |
1260 | 313 | zend_weakmap_get_object_entry_gc(obj, &table, &len); |
1261 | 313 | n = len; |
1262 | 313 | zv = table; |
1263 | 494 | for (; n != 0; n--) { |
1264 | 181 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
1265 | 181 | zval *entry = (zval*) Z_PTR_P(zv); |
1266 | 181 | if (Z_OPT_COLLECTABLE_P(entry)) { |
1267 | 113 | ref = Z_COUNTED_P(entry); |
1268 | 113 | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1269 | 50 | GC_REF_SET_COLOR(ref, GC_WHITE); |
1270 | 50 | GC_STACK_PUSH(ref); |
1271 | 50 | } |
1272 | 113 | } |
1273 | 181 | zv++; |
1274 | 181 | } |
1275 | 313 | } |
1276 | | |
1277 | 671k | ht = obj->handlers->get_gc(obj, &table, &len); |
1278 | 671k | n = len; |
1279 | 671k | zv = table; |
1280 | 671k | if (UNEXPECTED(ht)) { |
1281 | 654k | if (GC_REF_CHECK_COLOR(ht, GC_GREY)) { |
1282 | 654k | GC_REF_SET_COLOR(ht, GC_WHITE); |
1283 | 654k | GC_STACK_PUSH((zend_refcounted *) ht); |
1284 | 876k | for (; n != 0; n--) { |
1285 | 222k | if (Z_COLLECTABLE_P(zv)) { |
1286 | 221k | ref = Z_COUNTED_P(zv); |
1287 | 221k | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1288 | 218k | GC_REF_SET_COLOR(ref, GC_WHITE); |
1289 | 218k | GC_STACK_PUSH(ref); |
1290 | 218k | } |
1291 | 221k | } |
1292 | 222k | zv++; |
1293 | 222k | } |
1294 | 654k | goto handle_ht; |
1295 | 654k | } |
1296 | 654k | } |
1297 | | |
1298 | 64.1k | handle_zvals: |
1299 | 333k | for (; n != 0; n--) { |
1300 | 285k | if (Z_COLLECTABLE_P(zv)) { |
1301 | 79.0k | ref = Z_COUNTED_P(zv); |
1302 | 79.0k | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1303 | 15.8k | GC_REF_SET_COLOR(ref, GC_WHITE); |
1304 | 15.8k | zv++; |
1305 | 33.4k | while (--n) { |
1306 | 17.5k | if (Z_COLLECTABLE_P(zv)) { |
1307 | 8.76k | zend_refcounted *ref = Z_COUNTED_P(zv); |
1308 | 8.76k | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1309 | 5.03k | GC_REF_SET_COLOR(ref, GC_WHITE); |
1310 | 5.03k | GC_STACK_PUSH(ref); |
1311 | 5.03k | } |
1312 | 8.76k | } |
1313 | 17.5k | zv++; |
1314 | 17.5k | } |
1315 | 15.8k | goto tail_call; |
1316 | 15.8k | } |
1317 | 79.0k | } |
1318 | 269k | zv++; |
1319 | 269k | } |
1320 | 64.1k | } |
1321 | 1.18M | } else if (GC_TYPE(ref) == IS_ARRAY) { |
1322 | 1.16M | ht = (HashTable *)ref; |
1323 | 1.16M | ZEND_ASSERT(ht != &EG(symbol_table)); |
1324 | | |
1325 | 1.82M | handle_ht: |
1326 | 1.82M | n = ht->nNumUsed; |
1327 | 1.82M | if (HT_IS_PACKED(ht)) { |
1328 | 46.7k | zv = ht->arPacked; |
1329 | 46.7k | goto handle_zvals; |
1330 | 46.7k | } |
1331 | | |
1332 | 1.77M | p = ht->arData; |
1333 | 4.97M | for (; n != 0; n--) { |
1334 | 3.57M | zv = &p->val; |
1335 | 3.57M | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1336 | 1.76M | zv = Z_INDIRECT_P(zv); |
1337 | 1.76M | } |
1338 | 3.57M | if (Z_COLLECTABLE_P(zv)) { |
1339 | 1.50M | ref = Z_COUNTED_P(zv); |
1340 | 1.50M | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1341 | 383k | GC_REF_SET_COLOR(ref, GC_WHITE); |
1342 | 383k | p++; |
1343 | 1.56M | while (--n) { |
1344 | 1.17M | zv = &p->val; |
1345 | 1.17M | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1346 | 139k | zv = Z_INDIRECT_P(zv); |
1347 | 139k | } |
1348 | 1.17M | if (Z_COLLECTABLE_P(zv)) { |
1349 | 969k | zend_refcounted *ref = Z_COUNTED_P(zv); |
1350 | 969k | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1351 | 543k | GC_REF_SET_COLOR(ref, GC_WHITE); |
1352 | 543k | GC_STACK_PUSH(ref); |
1353 | 543k | } |
1354 | 969k | } |
1355 | 1.17M | p++; |
1356 | 1.17M | } |
1357 | 383k | goto tail_call; |
1358 | 383k | } |
1359 | 1.50M | } |
1360 | 3.19M | p++; |
1361 | 3.19M | } |
1362 | 1.77M | } else if (GC_TYPE(ref) == IS_REFERENCE) { |
1363 | 19.9k | if (Z_COLLECTABLE(((zend_reference*)ref)->val)) { |
1364 | 16.7k | ref = Z_COUNTED(((zend_reference*)ref)->val); |
1365 | 16.7k | if (GC_REF_CHECK_COLOR(ref, GC_GREY)) { |
1366 | 5.92k | GC_REF_SET_COLOR(ref, GC_WHITE); |
1367 | 5.92k | goto tail_call; |
1368 | 5.92k | } |
1369 | 16.7k | } |
1370 | 19.9k | } |
1371 | | |
1372 | 1.48M | next: |
1373 | 1.48M | ref = GC_STACK_POP(); |
1374 | 1.48M | if (ref) { |
1375 | 1.42M | goto tail_call; |
1376 | 1.42M | } |
1377 | 1.48M | } |
1378 | | |
1379 | | static void gc_scan_roots(gc_stack *stack) |
1380 | 23.6k | { |
1381 | 23.6k | uint32_t idx, end; |
1382 | 23.6k | gc_root_buffer *current; |
1383 | | |
1384 | | /* Root buffer might be reallocated during gc_scan, |
1385 | | * make sure to reload pointers. */ |
1386 | 23.6k | idx = GC_FIRST_ROOT; |
1387 | 23.6k | end = GC_G(first_unused); |
1388 | 257k | while (idx != end) { |
1389 | 233k | current = GC_IDX2PTR(idx); |
1390 | 233k | if (GC_IS_ROOT(current->ref)) { |
1391 | 233k | if (GC_REF_CHECK_COLOR(current->ref, GC_GREY)) { |
1392 | 68.0k | GC_REF_SET_COLOR(current->ref, GC_WHITE); |
1393 | 68.0k | gc_scan(current->ref, stack); |
1394 | 68.0k | } |
1395 | 233k | } |
1396 | 233k | idx++; |
1397 | 233k | } |
1398 | | |
1399 | | /* Scan extra roots added during gc_scan */ |
1400 | 23.6k | while (idx != GC_G(first_unused)) { |
1401 | 50 | current = GC_IDX2PTR(idx); |
1402 | 50 | if (GC_IS_ROOT(current->ref)) { |
1403 | 50 | if (GC_REF_CHECK_COLOR(current->ref, GC_GREY)) { |
1404 | 27 | GC_REF_SET_COLOR(current->ref, GC_WHITE); |
1405 | 27 | gc_scan(current->ref, stack); |
1406 | 27 | } |
1407 | 50 | } |
1408 | 50 | idx++; |
1409 | 50 | } |
1410 | 23.6k | } |
1411 | | |
1412 | | static void gc_add_garbage(zend_refcounted *ref) |
1413 | 982k | { |
1414 | 982k | uint32_t idx; |
1415 | 982k | gc_root_buffer *buf; |
1416 | | |
1417 | 982k | if (GC_HAS_UNUSED()) { |
1418 | 0 | idx = GC_FETCH_UNUSED(); |
1419 | 982k | } else if (GC_HAS_NEXT_UNUSED()) { |
1420 | 982k | idx = GC_FETCH_NEXT_UNUSED(); |
1421 | 982k | } else { |
1422 | 1 | gc_grow_root_buffer(); |
1423 | 1 | if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) { |
1424 | 0 | return; |
1425 | 0 | } |
1426 | 1 | idx = GC_FETCH_NEXT_UNUSED(); |
1427 | 1 | } |
1428 | | |
1429 | 982k | buf = GC_IDX2PTR(idx); |
1430 | 982k | buf->ref = GC_MAKE_GARBAGE(ref); |
1431 | | |
1432 | 982k | idx = gc_compress(idx); |
1433 | 982k | GC_REF_SET_INFO(ref, idx | GC_BLACK); |
1434 | 982k | GC_G(num_roots)++; |
1435 | 982k | } |
1436 | | |
1437 | | static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *stack) |
1438 | 23.5k | { |
1439 | 23.5k | int count = 0; |
1440 | 23.5k | HashTable *ht; |
1441 | 23.5k | Bucket *p; |
1442 | 23.5k | zval *zv; |
1443 | 23.5k | uint32_t n; |
1444 | 23.5k | GC_STACK_DCL(stack); |
1445 | | |
1446 | 1.18M | tail_call: |
1447 | | /* don't count references for compatibility ??? */ |
1448 | 1.18M | if (GC_TYPE(ref) != IS_REFERENCE) { |
1449 | 1.16M | count++; |
1450 | 1.16M | } |
1451 | | |
1452 | 1.18M | if (GC_TYPE(ref) == IS_OBJECT) { |
1453 | 669k | zend_object *obj = (zend_object*)ref; |
1454 | | |
1455 | 669k | if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { |
1456 | 669k | int len; |
1457 | 669k | zval *table; |
1458 | | |
1459 | | /* optimization: color is GC_BLACK (0) */ |
1460 | 669k | if (!GC_INFO(ref)) { |
1461 | 493k | gc_add_garbage(ref); |
1462 | 493k | } |
1463 | 669k | if (!(OBJ_FLAGS(obj) & IS_OBJ_DESTRUCTOR_CALLED) |
1464 | 669k | && (obj->handlers->dtor_obj != zend_objects_destroy_object |
1465 | 525k | || obj->ce->destructor != NULL)) { |
1466 | 2.94k | *flags |= GC_HAS_DESTRUCTORS; |
1467 | 2.94k | } |
1468 | | |
1469 | 669k | if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) { |
1470 | 229 | zend_weakmap_get_object_entry_gc(obj, &table, &len); |
1471 | 229 | n = len; |
1472 | 229 | zv = table; |
1473 | 328 | for (; n != 0; n--) { |
1474 | 99 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
1475 | 99 | zval *entry = (zval*) Z_PTR_P(zv); |
1476 | 99 | if (Z_COLLECTABLE_P(entry) && GC_FROM_WEAKMAP_KEY(entry)) { |
1477 | 73 | GC_UNSET_FROM_WEAKMAP_KEY(entry); |
1478 | 73 | GC_UNSET_FROM_WEAKMAP(entry); |
1479 | 73 | ref = Z_COUNTED_P(entry); |
1480 | 73 | GC_ADDREF(ref); |
1481 | 73 | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1482 | 26 | GC_REF_SET_BLACK(ref); |
1483 | 26 | GC_STACK_PUSH(ref); |
1484 | 26 | } |
1485 | 73 | } |
1486 | 99 | zv++; |
1487 | 99 | } |
1488 | 229 | } |
1489 | | |
1490 | 669k | if (UNEXPECTED(obj->handlers->get_gc == zend_weakmap_get_gc)) { |
1491 | 49 | zend_weakmap_get_entry_gc(obj, &table, &len); |
1492 | 49 | n = len; |
1493 | 49 | zv = table; |
1494 | 113 | for (; n != 0; n--) { |
1495 | 64 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
1496 | 64 | zval *entry = (zval*) Z_PTR_P(zv); |
1497 | 64 | if (Z_COLLECTABLE_P(entry) && GC_FROM_WEAKMAP(entry)) { |
1498 | 36 | GC_UNSET_FROM_WEAKMAP_KEY(entry); |
1499 | 36 | GC_UNSET_FROM_WEAKMAP(entry); |
1500 | 36 | ref = Z_COUNTED_P(entry); |
1501 | 36 | GC_ADDREF(ref); |
1502 | 36 | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1503 | 26 | GC_REF_SET_BLACK(ref); |
1504 | 26 | GC_STACK_PUSH(ref); |
1505 | 26 | } |
1506 | 36 | } |
1507 | 64 | zv++; |
1508 | 64 | } |
1509 | 49 | goto next; |
1510 | 49 | } |
1511 | | |
1512 | 669k | ht = obj->handlers->get_gc(obj, &table, &len); |
1513 | 669k | n = len; |
1514 | 669k | zv = table; |
1515 | 669k | if (UNEXPECTED(ht)) { |
1516 | 652k | GC_ADDREF(ht); |
1517 | 652k | if (GC_REF_CHECK_COLOR(ht, GC_WHITE)) { |
1518 | 652k | GC_REF_SET_BLACK(ht); |
1519 | 875k | for (; n != 0; n--) { |
1520 | 222k | if (Z_COLLECTABLE_P(zv)) { |
1521 | 221k | ref = Z_COUNTED_P(zv); |
1522 | 221k | GC_ADDREF(ref); |
1523 | 221k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1524 | 218k | GC_REF_SET_BLACK(ref); |
1525 | 218k | GC_STACK_PUSH(ref); |
1526 | 218k | } |
1527 | 221k | } |
1528 | 222k | zv++; |
1529 | 222k | } |
1530 | 652k | goto handle_ht; |
1531 | 652k | } |
1532 | 652k | } |
1533 | | |
1534 | 35.5k | handle_zvals: |
1535 | 180k | for (; n != 0; n--) { |
1536 | 159k | if (Z_COLLECTABLE_P(zv)) { |
1537 | 45.2k | ref = Z_COUNTED_P(zv); |
1538 | 45.2k | GC_ADDREF(ref); |
1539 | 45.2k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1540 | 14.5k | GC_REF_SET_BLACK(ref); |
1541 | 14.5k | zv++; |
1542 | 25.3k | while (--n) { |
1543 | 10.8k | if (Z_COLLECTABLE_P(zv)) { |
1544 | 3.19k | zend_refcounted *ref = Z_COUNTED_P(zv); |
1545 | 3.19k | GC_ADDREF(ref); |
1546 | 3.19k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1547 | 966 | GC_REF_SET_BLACK(ref); |
1548 | 966 | GC_STACK_PUSH(ref); |
1549 | 966 | } |
1550 | 3.19k | } |
1551 | 10.8k | zv++; |
1552 | 10.8k | } |
1553 | 14.5k | goto tail_call; |
1554 | 14.5k | } |
1555 | 45.2k | } |
1556 | 144k | zv++; |
1557 | 144k | } |
1558 | 35.5k | } |
1559 | 669k | } else if (GC_TYPE(ref) == IS_ARRAY) { |
1560 | | /* optimization: color is GC_BLACK (0) */ |
1561 | 498k | if (!GC_INFO(ref)) { |
1562 | 488k | gc_add_garbage(ref); |
1563 | 488k | } |
1564 | 498k | ht = (zend_array*)ref; |
1565 | | |
1566 | 1.15M | handle_ht: |
1567 | 1.15M | n = ht->nNumUsed; |
1568 | 1.15M | if (HT_IS_PACKED(ht)) { |
1569 | 19.0k | zv = ht->arPacked; |
1570 | 19.0k | goto handle_zvals; |
1571 | 19.0k | } |
1572 | | |
1573 | 1.13M | p = ht->arData; |
1574 | 2.25M | for (; n != 0; n--) { |
1575 | 1.50M | zv = &p->val; |
1576 | 1.50M | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1577 | 810k | zv = Z_INDIRECT_P(zv); |
1578 | 810k | } |
1579 | 1.50M | if (Z_COLLECTABLE_P(zv)) { |
1580 | 527k | ref = Z_COUNTED_P(zv); |
1581 | 527k | GC_ADDREF(ref); |
1582 | 527k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1583 | 382k | GC_REF_SET_BLACK(ref); |
1584 | 382k | p++; |
1585 | 1.55M | while (--n) { |
1586 | 1.17M | zv = &p->val; |
1587 | 1.17M | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1588 | 139k | zv = Z_INDIRECT_P(zv); |
1589 | 139k | } |
1590 | 1.17M | if (Z_COLLECTABLE_P(zv)) { |
1591 | 967k | zend_refcounted *ref = Z_COUNTED_P(zv); |
1592 | 967k | GC_ADDREF(ref); |
1593 | 967k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1594 | 542k | GC_REF_SET_BLACK(ref); |
1595 | 542k | GC_STACK_PUSH(ref); |
1596 | 542k | } |
1597 | 967k | } |
1598 | 1.17M | p++; |
1599 | 1.17M | } |
1600 | 382k | goto tail_call; |
1601 | 382k | } |
1602 | 527k | } |
1603 | 1.12M | p++; |
1604 | 1.12M | } |
1605 | 1.13M | } else if (GC_TYPE(ref) == IS_REFERENCE) { |
1606 | 19.4k | if (Z_COLLECTABLE(((zend_reference*)ref)->val)) { |
1607 | 16.3k | ref = Z_COUNTED(((zend_reference*)ref)->val); |
1608 | 16.3k | GC_ADDREF(ref); |
1609 | 16.3k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1610 | 5.43k | GC_REF_SET_BLACK(ref); |
1611 | 5.43k | goto tail_call; |
1612 | 5.43k | } |
1613 | 16.3k | } |
1614 | 19.4k | } |
1615 | | |
1616 | 785k | next: |
1617 | 785k | ref = GC_STACK_POP(); |
1618 | 785k | if (ref) { |
1619 | 761k | goto tail_call; |
1620 | 761k | } |
1621 | | |
1622 | 23.5k | return count; |
1623 | 785k | } |
1624 | | |
1625 | | static int gc_collect_roots(uint32_t *flags, gc_stack *stack) |
1626 | 23.6k | { |
1627 | 23.6k | uint32_t idx, end; |
1628 | 23.6k | zend_refcounted *ref; |
1629 | 23.6k | int count = 0; |
1630 | 23.6k | gc_root_buffer *current = GC_IDX2PTR(GC_FIRST_ROOT); |
1631 | 23.6k | gc_root_buffer *last = GC_IDX2PTR(GC_G(first_unused)); |
1632 | | |
1633 | | /* remove non-garbage from the list */ |
1634 | 257k | while (current != last) { |
1635 | 234k | if (GC_IS_ROOT(current->ref)) { |
1636 | 234k | if (GC_REF_CHECK_COLOR(current->ref, GC_BLACK)) { |
1637 | 48.3k | GC_REF_SET_INFO(current->ref, 0); /* reset GC_ADDRESS() and keep GC_BLACK */ |
1638 | 48.3k | gc_remove_from_roots(current); |
1639 | 48.3k | } |
1640 | 234k | } |
1641 | 234k | current++; |
1642 | 234k | } |
1643 | | |
1644 | 23.6k | gc_compact(); |
1645 | | |
1646 | | /* Root buffer might be reallocated during gc_collect_white, |
1647 | | * make sure to reload pointers. */ |
1648 | 23.6k | idx = GC_FIRST_ROOT; |
1649 | 23.6k | end = GC_G(first_unused); |
1650 | 209k | while (idx != end) { |
1651 | 185k | current = GC_IDX2PTR(idx); |
1652 | 185k | ref = current->ref; |
1653 | 185k | ZEND_ASSERT(GC_IS_ROOT(ref)); |
1654 | 185k | current->ref = GC_MAKE_GARBAGE(ref); |
1655 | 185k | if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) { |
1656 | 23.5k | GC_REF_SET_BLACK(ref); |
1657 | 23.5k | count += gc_collect_white(ref, flags, stack); |
1658 | 23.5k | } |
1659 | 185k | idx++; |
1660 | 185k | } |
1661 | | |
1662 | 23.6k | return count; |
1663 | 23.6k | } |
1664 | | |
1665 | | static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffer *root, gc_stack *stack) |
1666 | 2.94k | { |
1667 | 2.94k | HashTable *ht; |
1668 | 2.94k | Bucket *p; |
1669 | 2.94k | zval *zv; |
1670 | 2.94k | uint32_t n; |
1671 | 2.94k | int count = 0; |
1672 | 2.94k | GC_STACK_DCL(stack); |
1673 | | |
1674 | 8.96k | tail_call: |
1675 | 8.96k | if (root) { |
1676 | 2.94k | root = NULL; |
1677 | 2.94k | count++; |
1678 | 6.01k | } else if (GC_REF_ADDRESS(ref) != 0 |
1679 | 6.01k | && GC_REF_CHECK_COLOR(ref, GC_BLACK)) { |
1680 | 1.84k | GC_TRACE_REF(ref, "removing from buffer"); |
1681 | 1.84k | GC_REMOVE_FROM_BUFFER(ref); |
1682 | 1.84k | count++; |
1683 | 4.17k | } else if (GC_TYPE(ref) == IS_REFERENCE) { |
1684 | 464 | if (Z_COLLECTABLE(((zend_reference*)ref)->val)) { |
1685 | 245 | ref = Z_COUNTED(((zend_reference*)ref)->val); |
1686 | 245 | goto tail_call; |
1687 | 245 | } |
1688 | 219 | goto next; |
1689 | 3.71k | } else { |
1690 | 3.71k | goto next; |
1691 | 3.71k | } |
1692 | | |
1693 | 4.79k | if (GC_TYPE(ref) == IS_OBJECT) { |
1694 | 4.48k | zend_object *obj = (zend_object*)ref; |
1695 | | |
1696 | 4.48k | if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) { |
1697 | 4.48k | int len; |
1698 | 4.48k | zval *table; |
1699 | | |
1700 | 4.48k | if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) { |
1701 | 72 | zend_weakmap_get_object_entry_gc(obj, &table, &len); |
1702 | 72 | n = len; |
1703 | 72 | zv = table; |
1704 | 77 | for (; n != 0; n--) { |
1705 | 5 | ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR); |
1706 | 5 | zval *entry = (zval*) Z_PTR_P(zv); |
1707 | 5 | if (Z_OPT_COLLECTABLE_P(entry)) { |
1708 | 5 | ref = Z_COUNTED_P(entry); |
1709 | 5 | GC_STACK_PUSH(ref); |
1710 | 5 | } |
1711 | 5 | zv++; |
1712 | 5 | } |
1713 | 72 | } |
1714 | | |
1715 | 4.48k | ht = obj->handlers->get_gc(obj, &table, &len); |
1716 | 4.48k | n = len; |
1717 | 4.48k | zv = table; |
1718 | 4.48k | if (UNEXPECTED(ht)) { |
1719 | 2.29k | for (; n != 0; n--) { |
1720 | 998 | if (Z_COLLECTABLE_P(zv)) { |
1721 | 801 | ref = Z_COUNTED_P(zv); |
1722 | 801 | GC_STACK_PUSH(ref); |
1723 | 801 | } |
1724 | 998 | zv++; |
1725 | 998 | } |
1726 | 1.29k | if (GC_REF_ADDRESS(ht) != 0 && GC_REF_CHECK_COLOR(ht, GC_BLACK)) { |
1727 | 5 | GC_TRACE_REF(ht, "removing from buffer"); |
1728 | 5 | GC_REMOVE_FROM_BUFFER(ht); |
1729 | 5 | } |
1730 | 1.29k | goto handle_ht; |
1731 | 1.29k | } |
1732 | | |
1733 | 3.46k | handle_zvals: |
1734 | 3.57k | for (; n != 0; n--) { |
1735 | 3.23k | if (Z_COLLECTABLE_P(zv)) { |
1736 | 3.12k | ref = Z_COUNTED_P(zv); |
1737 | 3.12k | zv++; |
1738 | 3.72k | while (--n) { |
1739 | 599 | if (Z_COLLECTABLE_P(zv)) { |
1740 | 553 | zend_refcounted *ref = Z_COUNTED_P(zv); |
1741 | 553 | GC_STACK_PUSH(ref); |
1742 | 553 | } |
1743 | 599 | zv++; |
1744 | 599 | } |
1745 | 3.12k | goto tail_call; |
1746 | 3.12k | } |
1747 | 110 | zv++; |
1748 | 110 | } |
1749 | 3.46k | } |
1750 | 4.48k | } else if (GC_TYPE(ref) == IS_ARRAY) { |
1751 | 305 | ht = (zend_array*)ref; |
1752 | | |
1753 | 1.59k | handle_ht: |
1754 | 1.59k | n = ht->nNumUsed; |
1755 | 1.59k | if (HT_IS_PACKED(ht)) { |
1756 | 273 | zv = ht->arPacked; |
1757 | 273 | goto handle_zvals; |
1758 | 273 | } |
1759 | | |
1760 | 1.32k | p = ht->arData; |
1761 | 1.72k | for (; n != 0; n--) { |
1762 | 1.48k | zv = &p->val; |
1763 | 1.48k | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1764 | 254 | zv = Z_INDIRECT_P(zv); |
1765 | 254 | } |
1766 | 1.48k | if (Z_COLLECTABLE_P(zv)) { |
1767 | 1.09k | ref = Z_COUNTED_P(zv); |
1768 | 1.09k | p++; |
1769 | 1.29k | while (--n) { |
1770 | 206 | zv = &p->val; |
1771 | 206 | if (Z_TYPE_P(zv) == IS_INDIRECT) { |
1772 | 20 | zv = Z_INDIRECT_P(zv); |
1773 | 20 | } |
1774 | 206 | if (Z_COLLECTABLE_P(zv)) { |
1775 | 196 | zend_refcounted *ref = Z_COUNTED_P(zv); |
1776 | 196 | GC_STACK_PUSH(ref); |
1777 | 196 | } |
1778 | 206 | p++; |
1779 | 206 | } |
1780 | 1.09k | goto tail_call; |
1781 | 1.09k | } |
1782 | 396 | p++; |
1783 | 396 | } |
1784 | 1.32k | } |
1785 | | |
1786 | 4.50k | next: |
1787 | 4.50k | ref = GC_STACK_POP(); |
1788 | 4.50k | if (ref) { |
1789 | 1.55k | goto tail_call; |
1790 | 1.55k | } |
1791 | | |
1792 | 2.94k | return count; |
1793 | 4.50k | } |
1794 | | |
1795 | | static void zend_get_gc_buffer_release(void); |
1796 | | static void zend_gc_check_root_tmpvars(void); |
1797 | | static void zend_gc_remove_root_tmpvars(void); |
1798 | | |
1799 | | static zend_internal_function gc_destructor_fiber; |
1800 | | |
1801 | | static ZEND_COLD ZEND_NORETURN void gc_create_destructor_fiber_error(void) |
1802 | 0 | { |
1803 | 0 | zend_error_noreturn(E_ERROR, "Unable to create destructor fiber"); |
1804 | 0 | } |
1805 | | |
1806 | | static ZEND_COLD ZEND_NORETURN void gc_start_destructor_fiber_error(void) |
1807 | 0 | { |
1808 | 0 | zend_error_noreturn(E_ERROR, "Unable to start destructor fiber"); |
1809 | 0 | } |
1810 | | |
1811 | | static zend_always_inline zend_result gc_call_destructors(uint32_t idx, uint32_t end, zend_fiber *fiber) |
1812 | 619 | { |
1813 | 619 | gc_root_buffer *current; |
1814 | 619 | zend_refcounted *p; |
1815 | | |
1816 | | /* The root buffer might be reallocated during destructors calls, |
1817 | | * make sure to reload pointers as necessary. */ |
1818 | 5.19k | while (idx != end) { |
1819 | 4.63k | current = GC_IDX2PTR(idx); |
1820 | 4.63k | if (GC_IS_DTOR_GARBAGE(current->ref)) { |
1821 | 2.37k | p = GC_GET_PTR(current->ref); |
1822 | | /* Mark this is as a normal root for the next GC run */ |
1823 | 2.37k | current->ref = p; |
1824 | | /* Double check that the destructor hasn't been called yet. It |
1825 | | * could have already been invoked indirectly by some other |
1826 | | * destructor. */ |
1827 | 2.37k | if (!(OBJ_FLAGS(p) & IS_OBJ_DESTRUCTOR_CALLED)) { |
1828 | 2.37k | if (fiber != NULL) { |
1829 | 185 | GC_G(dtor_idx) = idx; |
1830 | 185 | } |
1831 | 2.37k | zend_object *obj = (zend_object*)p; |
1832 | 2.37k | GC_TRACE_REF(obj, "calling destructor"); |
1833 | 2.37k | GC_ADD_FLAGS(obj, IS_OBJ_DESTRUCTOR_CALLED); |
1834 | 2.37k | GC_ADDREF(obj); |
1835 | 2.37k | obj->handlers->dtor_obj(obj); |
1836 | 2.37k | GC_TRACE_REF(obj, "returned from destructor"); |
1837 | 2.37k | GC_DELREF(obj); |
1838 | 2.37k | if (UNEXPECTED(fiber != NULL && GC_G(dtor_fiber) != fiber)) { |
1839 | | /* We resumed after suspension */ |
1840 | 53 | gc_check_possible_root((zend_refcounted*)&obj->gc); |
1841 | 53 | return FAILURE; |
1842 | 53 | } |
1843 | 2.37k | } |
1844 | 2.37k | } |
1845 | 4.58k | idx++; |
1846 | 4.58k | } |
1847 | | |
1848 | 566 | return SUCCESS; |
1849 | 619 | } |
1850 | | |
1851 | | static zend_fiber *gc_create_destructor_fiber(void) |
1852 | 125 | { |
1853 | 125 | zval zobj; |
1854 | 125 | zend_fiber *fiber; |
1855 | | |
1856 | 125 | GC_TRACE("starting destructor fiber"); |
1857 | | |
1858 | 125 | if (UNEXPECTED(object_init_ex(&zobj, zend_ce_fiber) == FAILURE)) { |
1859 | 0 | gc_create_destructor_fiber_error(); |
1860 | 0 | } |
1861 | | |
1862 | 125 | fiber = (zend_fiber *)Z_OBJ(zobj); |
1863 | 125 | fiber->fci.size = sizeof(fiber->fci); |
1864 | 125 | fiber->fci_cache.function_handler = (zend_function*) &gc_destructor_fiber; |
1865 | | |
1866 | 125 | GC_G(dtor_fiber) = fiber; |
1867 | | |
1868 | 125 | if (UNEXPECTED(zend_fiber_start(fiber, NULL) == FAILURE)) { |
1869 | 0 | gc_start_destructor_fiber_error(); |
1870 | 0 | } |
1871 | | |
1872 | 125 | return fiber; |
1873 | 125 | } |
1874 | | |
1875 | | static zend_never_inline void gc_call_destructors_in_fiber(uint32_t end) |
1876 | 76 | { |
1877 | 76 | ZEND_ASSERT(!GC_G(dtor_fiber_running)); |
1878 | | |
1879 | 76 | zend_fiber *fiber = GC_G(dtor_fiber); |
1880 | | |
1881 | 76 | GC_G(dtor_idx) = GC_FIRST_ROOT; |
1882 | 76 | GC_G(dtor_end) = GC_G(first_unused); |
1883 | | |
1884 | 76 | if (UNEXPECTED(!fiber)) { |
1885 | 72 | fiber = gc_create_destructor_fiber(); |
1886 | 72 | } else { |
1887 | 4 | zend_fiber_resume(fiber, NULL, NULL); |
1888 | 4 | } |
1889 | | |
1890 | 129 | for (;;) { |
1891 | | /* At this point, fiber has executed until suspension */ |
1892 | 129 | GC_TRACE("resumed from destructor fiber"); |
1893 | | |
1894 | 129 | if (UNEXPECTED(GC_G(dtor_fiber_running))) { |
1895 | | /* Fiber was suspended by a destructor. Start a new one for the |
1896 | | * remaining destructors. */ |
1897 | 53 | GC_TRACE("destructor fiber suspended by destructor"); |
1898 | 53 | GC_G(dtor_fiber) = NULL; |
1899 | 53 | GC_G(dtor_idx)++; |
1900 | | /* We do not own the fiber anymore. It may be collected if the |
1901 | | * application does not reference it. */ |
1902 | 53 | zend_object_release(&fiber->std); |
1903 | 53 | fiber = gc_create_destructor_fiber(); |
1904 | 53 | continue; |
1905 | 76 | } else { |
1906 | | /* Fiber suspended itself after calling all destructors */ |
1907 | 76 | GC_TRACE("destructor fiber suspended itself"); |
1908 | 76 | break; |
1909 | 76 | } |
1910 | 129 | } |
1911 | 76 | } |
1912 | | |
1913 | | ZEND_API int zend_gc_collect_cycles(void) |
1914 | 851k | { |
1915 | 851k | int total_count = 0; |
1916 | 851k | bool should_rerun_gc = 0; |
1917 | 851k | bool did_rerun_gc = 0; |
1918 | | |
1919 | 851k | zend_hrtime_t start_time = zend_hrtime(); |
1920 | 851k | if (GC_G(num_roots) && !GC_G(gc_active)) { |
1921 | 23.2k | zend_gc_remove_root_tmpvars(); |
1922 | 23.2k | } |
1923 | | |
1924 | 851k | rerun_gc: |
1925 | 851k | if (GC_G(num_roots)) { |
1926 | 23.6k | int count; |
1927 | 23.6k | gc_root_buffer *current, *last; |
1928 | 23.6k | zend_refcounted *p; |
1929 | 23.6k | uint32_t gc_flags = 0; |
1930 | 23.6k | uint32_t idx, end; |
1931 | 23.6k | gc_stack stack; |
1932 | | |
1933 | 23.6k | stack.prev = NULL; |
1934 | 23.6k | stack.next = NULL; |
1935 | | |
1936 | 23.6k | if (GC_G(gc_active)) { |
1937 | 33 | GC_G(collector_time) += zend_hrtime() - start_time; |
1938 | 33 | return 0; |
1939 | 33 | } |
1940 | | |
1941 | 23.6k | GC_TRACE("Collecting cycles"); |
1942 | 23.6k | GC_G(gc_runs)++; |
1943 | 23.6k | GC_G(gc_active) = 1; |
1944 | | |
1945 | 23.6k | GC_TRACE("Marking roots"); |
1946 | 23.6k | gc_mark_roots(&stack); |
1947 | 23.6k | GC_TRACE("Scanning roots"); |
1948 | 23.6k | gc_scan_roots(&stack); |
1949 | | |
1950 | 23.6k | GC_TRACE("Collecting roots"); |
1951 | 23.6k | count = gc_collect_roots(&gc_flags, &stack); |
1952 | | |
1953 | 23.6k | if (!GC_G(num_roots)) { |
1954 | | /* nothing to free */ |
1955 | 18.7k | GC_TRACE("Nothing to free"); |
1956 | 18.7k | gc_stack_free(&stack); |
1957 | 18.7k | GC_G(gc_active) = 0; |
1958 | 18.7k | goto finish; |
1959 | 18.7k | } |
1960 | | |
1961 | 4.86k | end = GC_G(first_unused); |
1962 | | |
1963 | 4.86k | if (gc_flags & GC_HAS_DESTRUCTORS) { |
1964 | 566 | GC_TRACE("Calling destructors"); |
1965 | | |
1966 | | /* During a destructor call, new externally visible references to nested data may |
1967 | | * be introduced. These references can be introduced in a way that does not |
1968 | | * modify any refcounts, so we have no real way to detect this situation |
1969 | | * short of rerunning full GC tracing. What we do instead is to only run |
1970 | | * destructors at this point and automatically re-run GC afterwards. */ |
1971 | 566 | should_rerun_gc = 1; |
1972 | | |
1973 | | /* Mark all roots for which a dtor will be invoked as DTOR_GARBAGE. Additionally |
1974 | | * color them purple. This serves a double purpose: First, they should be |
1975 | | * considered new potential roots for the next GC run. Second, it will prevent |
1976 | | * their removal from the root buffer by nested data removal. */ |
1977 | 566 | idx = GC_FIRST_ROOT; |
1978 | 566 | current = GC_IDX2PTR(GC_FIRST_ROOT); |
1979 | 5.94k | while (idx != end) { |
1980 | 5.38k | if (GC_IS_GARBAGE(current->ref)) { |
1981 | 5.38k | p = GC_GET_PTR(current->ref); |
1982 | 5.38k | if (GC_TYPE(p) == IS_OBJECT && !(OBJ_FLAGS(p) & IS_OBJ_DESTRUCTOR_CALLED)) { |
1983 | 4.69k | zend_object *obj = (zend_object *) p; |
1984 | 4.69k | if (obj->handlers->dtor_obj != zend_objects_destroy_object |
1985 | 4.69k | || obj->ce->destructor) { |
1986 | 2.94k | current->ref = GC_MAKE_DTOR_GARBAGE(obj); |
1987 | 2.94k | GC_REF_SET_COLOR(obj, GC_PURPLE); |
1988 | 2.94k | } else { |
1989 | 1.74k | GC_ADD_FLAGS(obj, IS_OBJ_DESTRUCTOR_CALLED); |
1990 | 1.74k | } |
1991 | 4.69k | } |
1992 | 5.38k | } |
1993 | 5.38k | current++; |
1994 | 5.38k | idx++; |
1995 | 5.38k | } |
1996 | | |
1997 | | /* Remove nested data for objects on which a destructor will be called. |
1998 | | * This will not remove the objects themselves, as they have been colored |
1999 | | * purple. */ |
2000 | 566 | idx = GC_FIRST_ROOT; |
2001 | 566 | current = GC_IDX2PTR(GC_FIRST_ROOT); |
2002 | 5.94k | while (idx != end) { |
2003 | 5.38k | if (GC_IS_DTOR_GARBAGE(current->ref)) { |
2004 | 2.94k | p = GC_GET_PTR(current->ref); |
2005 | 2.94k | count -= gc_remove_nested_data_from_buffer(p, current, &stack); |
2006 | 2.94k | } |
2007 | 5.38k | current++; |
2008 | 5.38k | idx++; |
2009 | 5.38k | } |
2010 | | |
2011 | | /* Actually call destructors. */ |
2012 | 566 | zend_hrtime_t dtor_start_time = zend_hrtime(); |
2013 | 566 | if (EXPECTED(!EG(active_fiber))) { |
2014 | 490 | gc_call_destructors(GC_FIRST_ROOT, end, NULL); |
2015 | 490 | } else { |
2016 | 76 | gc_call_destructors_in_fiber(end); |
2017 | 76 | } |
2018 | 566 | GC_G(dtor_time) += zend_hrtime() - dtor_start_time; |
2019 | | |
2020 | 566 | if (GC_G(gc_protected)) { |
2021 | | /* something went wrong */ |
2022 | 15 | zend_get_gc_buffer_release(); |
2023 | 15 | GC_G(collector_time) += zend_hrtime() - start_time; |
2024 | 15 | return 0; |
2025 | 15 | } |
2026 | 566 | } |
2027 | | |
2028 | 4.84k | gc_stack_free(&stack); |
2029 | | |
2030 | | /* Destroy zvals. The root buffer may be reallocated. */ |
2031 | 4.84k | GC_TRACE("Destroying zvals"); |
2032 | 4.84k | zend_hrtime_t free_start_time = zend_hrtime(); |
2033 | 4.84k | idx = GC_FIRST_ROOT; |
2034 | 1.16M | while (idx != end) { |
2035 | 1.16M | current = GC_IDX2PTR(idx); |
2036 | 1.16M | if (GC_IS_GARBAGE(current->ref)) { |
2037 | 54.8k | p = GC_GET_PTR(current->ref); |
2038 | 54.8k | GC_TRACE_REF(p, "destroying"); |
2039 | 54.8k | if (GC_TYPE(p) == IS_OBJECT) { |
2040 | 45.2k | zend_object *obj = (zend_object*)p; |
2041 | | |
2042 | 45.2k | EG(objects_store).object_buckets[obj->handle] = SET_OBJ_INVALID(obj); |
2043 | 45.2k | GC_TYPE_INFO(obj) = GC_NULL | |
2044 | 45.2k | (GC_TYPE_INFO(obj) & ~GC_TYPE_MASK); |
2045 | | /* Modify current before calling free_obj (bug #78811: free_obj() can cause the root buffer (with current) to be reallocated.) */ |
2046 | 45.2k | current->ref = GC_MAKE_GARBAGE(((char*)obj) - obj->handlers->offset); |
2047 | 45.2k | if (!(OBJ_FLAGS(obj) & IS_OBJ_FREE_CALLED)) { |
2048 | 45.2k | GC_ADD_FLAGS(obj, IS_OBJ_FREE_CALLED); |
2049 | 45.2k | GC_ADDREF(obj); |
2050 | 45.2k | obj->handlers->free_obj(obj); |
2051 | 45.2k | GC_DELREF(obj); |
2052 | 45.2k | } |
2053 | | |
2054 | 45.2k | ZEND_OBJECTS_STORE_ADD_TO_FREE_LIST(obj->handle); |
2055 | 45.2k | } else if (GC_TYPE(p) == IS_ARRAY) { |
2056 | 9.65k | zend_array *arr = (zend_array*)p; |
2057 | | |
2058 | 9.65k | GC_TYPE_INFO(arr) = GC_NULL | |
2059 | 9.65k | (GC_TYPE_INFO(arr) & ~GC_TYPE_MASK); |
2060 | | |
2061 | | /* GC may destroy arrays with rc>1. This is valid and safe. */ |
2062 | 9.65k | HT_ALLOW_COW_VIOLATION(arr); |
2063 | | |
2064 | 9.65k | zend_hash_destroy(arr); |
2065 | 9.65k | } |
2066 | 54.8k | } |
2067 | 1.16M | idx++; |
2068 | 1.16M | } |
2069 | | |
2070 | | /* Free objects */ |
2071 | 4.84k | current = GC_IDX2PTR(GC_FIRST_ROOT); |
2072 | 4.84k | last = GC_IDX2PTR(end); |
2073 | 1.16M | while (current != last) { |
2074 | 1.16M | if (GC_IS_GARBAGE(current->ref)) { |
2075 | 54.8k | p = GC_GET_PTR(current->ref); |
2076 | 54.8k | GC_LINK_UNUSED(current); |
2077 | 54.8k | GC_G(num_roots)--; |
2078 | 54.8k | efree(p); |
2079 | 54.8k | } |
2080 | 1.16M | current++; |
2081 | 1.16M | } |
2082 | | |
2083 | 4.84k | GC_G(free_time) += zend_hrtime() - free_start_time; |
2084 | | |
2085 | 4.84k | GC_TRACE("Collection finished"); |
2086 | 4.84k | GC_G(collected) += count; |
2087 | 4.84k | total_count += count; |
2088 | 4.84k | GC_G(gc_active) = 0; |
2089 | 4.84k | } |
2090 | | |
2091 | 833k | gc_compact(); |
2092 | | |
2093 | | /* Objects with destructors were removed from this GC run. Rerun GC right away to clean them |
2094 | | * up. We do this only once: If we encounter more destructors on the second run, we'll not |
2095 | | * run GC another time. */ |
2096 | 833k | if (should_rerun_gc && !did_rerun_gc) { |
2097 | 441 | did_rerun_gc = 1; |
2098 | 441 | goto rerun_gc; |
2099 | 441 | } |
2100 | | |
2101 | 851k | finish: |
2102 | 851k | zend_get_gc_buffer_release(); |
2103 | | |
2104 | | /* Prevent GC from running during zend_gc_check_root_tmpvars, before |
2105 | | * gc_threshold is adjusted, as this may result in unbounded recursion */ |
2106 | 851k | GC_G(gc_active) = 1; |
2107 | 851k | zend_gc_check_root_tmpvars(); |
2108 | 851k | GC_G(gc_active) = 0; |
2109 | | |
2110 | 851k | GC_G(collector_time) += zend_hrtime() - start_time; |
2111 | 851k | return total_count; |
2112 | 833k | } |
2113 | | |
2114 | | ZEND_API void zend_gc_get_status(zend_gc_status *status) |
2115 | 213 | { |
2116 | 213 | status->active = GC_G(gc_active); |
2117 | 213 | status->gc_protected = GC_G(gc_protected); |
2118 | 213 | status->full = GC_G(gc_full); |
2119 | 213 | status->runs = GC_G(gc_runs); |
2120 | 213 | status->collected = GC_G(collected); |
2121 | 213 | status->threshold = GC_G(gc_threshold); |
2122 | 213 | status->buf_size = GC_G(buf_size); |
2123 | 213 | status->num_roots = GC_G(num_roots); |
2124 | 213 | status->application_time = zend_hrtime() - GC_G(activated_at); |
2125 | 213 | status->collector_time = GC_G(collector_time); |
2126 | 213 | status->dtor_time = GC_G(dtor_time); |
2127 | 213 | status->free_time = GC_G(free_time); |
2128 | 213 | } |
2129 | | |
2130 | 73.4k | ZEND_API zend_get_gc_buffer *zend_get_gc_buffer_create(void) { |
2131 | | /* There can only be one get_gc() call active at a time, |
2132 | | * so there only needs to be one buffer. */ |
2133 | 73.4k | zend_get_gc_buffer *gc_buffer = &EG(get_gc_buffer); |
2134 | 73.4k | gc_buffer->cur = gc_buffer->start; |
2135 | 73.4k | return gc_buffer; |
2136 | 73.4k | } |
2137 | | |
2138 | 2.07k | ZEND_API void zend_get_gc_buffer_grow(zend_get_gc_buffer *gc_buffer) { |
2139 | 2.07k | size_t old_capacity = gc_buffer->end - gc_buffer->start; |
2140 | 2.07k | size_t new_capacity = old_capacity == 0 ? 64 : old_capacity * 2; |
2141 | 2.07k | gc_buffer->start = erealloc(gc_buffer->start, new_capacity * sizeof(zval)); |
2142 | 2.07k | gc_buffer->end = gc_buffer->start + new_capacity; |
2143 | 2.07k | gc_buffer->cur = gc_buffer->start + old_capacity; |
2144 | 2.07k | } |
2145 | | |
2146 | 851k | static void zend_get_gc_buffer_release(void) { |
2147 | 851k | zend_get_gc_buffer *gc_buffer = &EG(get_gc_buffer); |
2148 | 851k | efree(gc_buffer->start); |
2149 | 851k | gc_buffer->start = gc_buffer->end = gc_buffer->cur = NULL; |
2150 | 851k | } |
2151 | | |
2152 | | /* TMPVAR operands are destroyed using zval_ptr_dtor_nogc(), because they usually cannot contain |
2153 | | * cycles. However, there are some rare exceptions where this is possible, in which case we rely |
2154 | | * on the producing code to root the value. If a GC run occurs between the rooting and consumption |
2155 | | * of the value, we would end up leaking it. To avoid this, root all live TMPVAR values here. */ |
2156 | 851k | static void zend_gc_check_root_tmpvars(void) { |
2157 | 851k | zend_execute_data *ex = EG(current_execute_data); |
2158 | 1.11M | for (; ex; ex = ex->prev_execute_data) { |
2159 | 265k | zend_function *func = ex->func; |
2160 | 265k | if (!func || !ZEND_USER_CODE(func->type)) { |
2161 | 262k | continue; |
2162 | 262k | } |
2163 | | |
2164 | 2.96k | uint32_t op_num = ex->opline - ex->func->op_array.opcodes; |
2165 | 12.0k | for (uint32_t i = 0; i < func->op_array.last_live_range; i++) { |
2166 | 9.39k | const zend_live_range *range = &func->op_array.live_range[i]; |
2167 | 9.39k | if (range->start > op_num) { |
2168 | 261 | break; |
2169 | 261 | } |
2170 | 9.12k | if (range->end <= op_num) { |
2171 | 8.75k | continue; |
2172 | 8.75k | } |
2173 | | |
2174 | 374 | uint32_t kind = range->var & ZEND_LIVE_MASK; |
2175 | 374 | if (kind == ZEND_LIVE_TMPVAR || kind == ZEND_LIVE_LOOP) { |
2176 | 274 | uint32_t var_num = range->var & ~ZEND_LIVE_MASK; |
2177 | 274 | zval *var = ZEND_CALL_VAR(ex, var_num); |
2178 | 274 | if (Z_COLLECTABLE_P(var)) { |
2179 | 263 | gc_check_possible_root(Z_COUNTED_P(var)); |
2180 | 263 | } |
2181 | 274 | } |
2182 | 374 | } |
2183 | 2.96k | } |
2184 | 851k | } |
2185 | | |
2186 | 23.2k | static void zend_gc_remove_root_tmpvars(void) { |
2187 | 23.2k | zend_execute_data *ex = EG(current_execute_data); |
2188 | 29.8k | for (; ex; ex = ex->prev_execute_data) { |
2189 | 6.63k | zend_function *func = ex->func; |
2190 | 6.63k | if (!func || !ZEND_USER_CODE(func->type)) { |
2191 | 4.11k | continue; |
2192 | 4.11k | } |
2193 | | |
2194 | 2.52k | uint32_t op_num = ex->opline - ex->func->op_array.opcodes; |
2195 | 11.1k | for (uint32_t i = 0; i < func->op_array.last_live_range; i++) { |
2196 | 8.82k | const zend_live_range *range = &func->op_array.live_range[i]; |
2197 | 8.82k | if (range->start > op_num) { |
2198 | 196 | break; |
2199 | 196 | } |
2200 | 8.63k | if (range->end <= op_num) { |
2201 | 8.36k | continue; |
2202 | 8.36k | } |
2203 | | |
2204 | 266 | uint32_t kind = range->var & ZEND_LIVE_MASK; |
2205 | 266 | if (kind == ZEND_LIVE_TMPVAR || kind == ZEND_LIVE_LOOP) { |
2206 | 266 | uint32_t var_num = range->var & ~ZEND_LIVE_MASK; |
2207 | 266 | zval *var = ZEND_CALL_VAR(ex, var_num); |
2208 | 266 | if (Z_COLLECTABLE_P(var)) { |
2209 | 263 | GC_REMOVE_FROM_BUFFER(Z_COUNTED_P(var)); |
2210 | 263 | } |
2211 | 266 | } |
2212 | 266 | } |
2213 | 2.52k | } |
2214 | 23.2k | } |
2215 | | |
2216 | | #if GC_BENCH |
2217 | | void gc_bench_print(void) |
2218 | | { |
2219 | | fprintf(stderr, "GC Statistics\n"); |
2220 | | fprintf(stderr, "-------------\n"); |
2221 | | fprintf(stderr, "Runs: %d\n", GC_G(gc_runs)); |
2222 | | fprintf(stderr, "Collected: %d\n", GC_G(collected)); |
2223 | | fprintf(stderr, "Root buffer length: %d\n", GC_G(root_buf_length)); |
2224 | | fprintf(stderr, "Root buffer peak: %d\n\n", GC_G(root_buf_peak)); |
2225 | | fprintf(stderr, " Possible Remove from Marked\n"); |
2226 | | fprintf(stderr, " Root Buffered buffer grey\n"); |
2227 | | fprintf(stderr, " -------- -------- ----------- ------\n"); |
2228 | | fprintf(stderr, "ZVAL %8d %8d %9d %8d\n", GC_G(zval_possible_root), GC_G(zval_buffered), GC_G(zval_remove_from_buffer), GC_G(zval_marked_grey)); |
2229 | | } |
2230 | | #endif |
2231 | | |
2232 | | #ifdef ZTS |
2233 | | size_t zend_gc_globals_size(void) |
2234 | | { |
2235 | | return sizeof(zend_gc_globals); |
2236 | | } |
2237 | | #endif |
2238 | | |
2239 | | static ZEND_FUNCTION(gc_destructor_fiber) |
2240 | 125 | { |
2241 | 125 | uint32_t idx, end; |
2242 | | |
2243 | 125 | zend_fiber *fiber = GC_G(dtor_fiber); |
2244 | 125 | ZEND_ASSERT(fiber != NULL); |
2245 | 125 | ZEND_ASSERT(fiber == EG(active_fiber)); |
2246 | | |
2247 | 129 | for (;;) { |
2248 | 129 | GC_G(dtor_fiber_running) = true; |
2249 | | |
2250 | 129 | idx = GC_G(dtor_idx); |
2251 | 129 | end = GC_G(dtor_end); |
2252 | 129 | if (UNEXPECTED(gc_call_destructors(idx, end, fiber) == FAILURE)) { |
2253 | | /* We resumed after being suspended by a destructor */ |
2254 | 53 | return; |
2255 | 53 | } |
2256 | | |
2257 | | /* We have called all destructors. Suspend fiber until the next GC run |
2258 | | */ |
2259 | 76 | GC_G(dtor_fiber_running) = false; |
2260 | 76 | zend_fiber_suspend(fiber, NULL, NULL); |
2261 | | |
2262 | 76 | if (UNEXPECTED(fiber->flags & ZEND_FIBER_FLAG_DESTROYED)) { |
2263 | | /* Fiber is being destroyed by shutdown sequence */ |
2264 | 72 | if (GC_G(dtor_fiber) == fiber) { |
2265 | 72 | GC_G(dtor_fiber) = NULL; |
2266 | 72 | } |
2267 | 72 | GC_DELREF(&fiber->std); |
2268 | 72 | gc_check_possible_root((zend_refcounted*)&fiber->std.gc); |
2269 | 72 | return; |
2270 | 72 | } |
2271 | 76 | } |
2272 | 125 | } |
2273 | | |
2274 | | static zend_internal_function gc_destructor_fiber = { |
2275 | | .type = ZEND_INTERNAL_FUNCTION, |
2276 | | .fn_flags = ZEND_ACC_PUBLIC, |
2277 | | .handler = ZEND_FN(gc_destructor_fiber), |
2278 | | }; |
2279 | | |
2280 | | void gc_init(void) |
2281 | 16 | { |
2282 | 16 | gc_destructor_fiber.function_name = zend_string_init_interned( |
2283 | 16 | "gc_destructor_fiber", |
2284 | 16 | strlen("gc_destructor_fiber"), |
2285 | 16 | true); |
2286 | 16 | } |