Coverage Report

Created: 2025-06-13 06:43

/src/php-src/Zend/zend_gc.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
   +----------------------------------------------------------------------+
3
   | Zend Engine                                                          |
4
   +----------------------------------------------------------------------+
5
   | Copyright (c) Zend Technologies Ltd. (http://www.zend.com)           |
6
   +----------------------------------------------------------------------+
7
   | This source file is subject to version 2.00 of the Zend license,     |
8
   | that is bundled with this package in the file LICENSE, and is        |
9
   | available through the world-wide-web at the following url:           |
10
   | http://www.zend.com/license/2_00.txt.                                |
11
   | If you did not receive a copy of the Zend license and are unable to  |
12
   | obtain it through the world-wide-web, please send a note to          |
13
   | license@zend.com so we can mail you a copy immediately.              |
14
   +----------------------------------------------------------------------+
15
   | Authors: David Wang <planetbeing@gmail.com>                          |
16
   |          Dmitry Stogov <dmitry@php.net>                              |
17
   +----------------------------------------------------------------------+
18
*/
19
20
/**
21
 * zend_gc_collect_cycles
22
 * ======================
23
 *
24
 * Colors and its meaning
25
 * ----------------------
26
 *
27
 * BLACK  (GC_BLACK)   - In use or free.
28
 * GREY   (GC_GREY)    - Possible member of cycle.
29
 * WHITE  (GC_WHITE)   - Member of garbage cycle.
30
 * PURPLE (GC_PURPLE)  - Possible root of cycle.
31
 *
32
 * Colors described in the paper but not used
33
 * ------------------------------------------
34
 *
35
 * GREEN - Acyclic
36
 * RED   - Candidate cycle undergoing
37
 * ORANGE - Candidate cycle awaiting epoch boundary.
38
 *
39
 *
40
 * Flow
41
 * =====
42
 *
43
 * The garbage collect cycle starts from 'gc_mark_roots', which traverses the
44
 * possible roots, and calls mark_grey for roots are marked purple with
45
 * depth-first traverse.
46
 *
47
 * After all possible roots are traversed and marked,
48
 * gc_scan_roots will be called, and each root will be called with
49
 * gc_scan(root->ref)
50
 *
51
 * gc_scan checks the colors of possible members.
52
 *
53
 * If the node is marked as grey and the refcount > 0
54
 *    gc_scan_black will be called on that node to scan it's subgraph.
55
 * otherwise (refcount == 0), it marks the node white.
56
 *
57
 * A node MAY be added to possible roots when ZEND_UNSET_VAR happens or
58
 * zend_assign_to_variable is called only when possible garbage node is
59
 * produced.
60
 * gc_possible_root() will be called to add the nodes to possible roots.
61
 *
62
 *
63
 * For objects, we call their get_gc handler (by default 'zend_std_get_gc') to
64
 * get the object properties to scan.
65
 *
66
 *
67
 * @see http://researcher.watson.ibm.com/researcher/files/us-bacon/Bacon01Concurrent.pdf
68
 */
69
#include "zend.h"
70
#include "zend_API.h"
71
#include "zend_compile.h"
72
#include "zend_errors.h"
73
#include "zend_fibers.h"
74
#include "zend_hrtime.h"
75
#include "zend_portability.h"
76
#include "zend_types.h"
77
#include "zend_weakrefs.h"
78
#include "zend_string.h"
79
80
#ifndef GC_BENCH
81
# define GC_BENCH 0
82
#endif
83
84
#ifndef ZEND_GC_DEBUG
85
# define ZEND_GC_DEBUG 0
86
#endif
87
88
/* GC_INFO layout */
89
3.53M
#define GC_ADDRESS  0x0fffffu
90
21.2M
#define GC_COLOR    0x300000u
91
92
#define GC_BLACK    0x000000u /* must be zero */
93
#define GC_WHITE    0x100000u
94
#define GC_GREY     0x200000u
95
#define GC_PURPLE   0x300000u
96
97
/* Debug tracing */
98
#if ZEND_GC_DEBUG > 1
99
# define GC_TRACE(format, ...) fprintf(stderr, format "\n", ##__VA_ARGS__);
100
# define GC_TRACE_REF(ref, format, ...) \
101
  do { \
102
    gc_trace_ref((zend_refcounted *) ref); \
103
    fprintf(stderr, format "\n", ##__VA_ARGS__); \
104
  } while (0)
105
# define GC_TRACE_SET_COLOR(ref, color) \
106
  GC_TRACE_REF(ref, "->%s", gc_color_name(color))
107
#else
108
# define GC_TRACE_REF(ref, format, ...)
109
# define GC_TRACE_SET_COLOR(ref, new_color)
110
# define GC_TRACE(str)
111
#endif
112
113
/* GC_INFO access */
114
#define GC_REF_ADDRESS(ref) \
115
3.53M
  (((GC_TYPE_INFO(ref)) & (GC_ADDRESS << GC_INFO_SHIFT)) >> GC_INFO_SHIFT)
116
117
#define GC_REF_COLOR(ref) \
118
  (((GC_TYPE_INFO(ref)) & (GC_COLOR << GC_INFO_SHIFT)) >> GC_INFO_SHIFT)
119
120
#define GC_REF_CHECK_COLOR(ref, color) \
121
15.1M
  ((GC_TYPE_INFO(ref) & (GC_COLOR << GC_INFO_SHIFT)) == ((color) << GC_INFO_SHIFT))
122
123
7.21M
#define GC_REF_SET_INFO(ref, info) do { \
124
7.21M
    GC_TYPE_INFO(ref) = \
125
7.21M
      (GC_TYPE_INFO(ref) & (GC_TYPE_MASK | GC_FLAGS_MASK)) | \
126
7.21M
      ((info) << GC_INFO_SHIFT); \
127
7.21M
  } while (0)
128
129
3.98M
#define GC_REF_SET_COLOR(ref, c) do { \
130
3.98M
    GC_TRACE_SET_COLOR(ref, c); \
131
3.98M
    GC_TYPE_INFO(ref) = \
132
3.98M
      (GC_TYPE_INFO(ref) & ~(GC_COLOR << GC_INFO_SHIFT)) | \
133
3.98M
      ((c) << GC_INFO_SHIFT); \
134
3.98M
  } while (0)
135
136
2.09M
#define GC_REF_SET_BLACK(ref) do { \
137
2.09M
    GC_TRACE_SET_COLOR(ref, GC_BLACK); \
138
2.09M
    GC_TYPE_INFO(ref) &= ~(GC_COLOR << GC_INFO_SHIFT); \
139
2.09M
  } while (0)
140
141
#define GC_REF_SET_PURPLE(ref) do { \
142
    GC_TRACE_SET_COLOR(ref, GC_PURPLE); \
143
    GC_TYPE_INFO(ref) |= (GC_COLOR << GC_INFO_SHIFT); \
144
  } while (0)
145
146
/* bit stealing tags for gc_root_buffer.ref */
147
3.33M
#define GC_BITS    0x3
148
149
701k
#define GC_ROOT    0x0 /* possible root of circular garbage     */
150
3.78M
#define GC_UNUSED  0x1 /* part of linked list of unused buffers */
151
3.54M
#define GC_GARBAGE 0x2 /* garbage to delete                     */
152
12.9k
#define GC_DTOR_GARBAGE 0x3 /* garbage on which only the dtor should be invoked */
153
154
#define GC_GET_PTR(ptr) \
155
127k
  ((void*)(((uintptr_t)(ptr)) & ~GC_BITS))
156
157
#define GC_IS_ROOT(ptr) \
158
701k
  ((((uintptr_t)(ptr)) & GC_BITS) == GC_ROOT)
159
#define GC_IS_UNUSED(ptr) \
160
157k
  ((((uintptr_t)(ptr)) & GC_BITS) == GC_UNUSED)
161
#define GC_IS_GARBAGE(ptr) \
162
2.33M
  ((((uintptr_t)(ptr)) & GC_BITS) == GC_GARBAGE)
163
#define GC_IS_DTOR_GARBAGE(ptr) \
164
10.0k
  ((((uintptr_t)(ptr)) & GC_BITS) == GC_DTOR_GARBAGE)
165
166
#define GC_MAKE_GARBAGE(ptr) \
167
1.21M
  ((void*)(((uintptr_t)(ptr)) | GC_GARBAGE))
168
#define GC_MAKE_DTOR_GARBAGE(ptr) \
169
2.94k
  ((void*)(((uintptr_t)(ptr)) | GC_DTOR_GARBAGE))
170
171
/* GC address conversion */
172
9.83M
#define GC_IDX2PTR(idx)      (GC_G(buf) + (idx))
173
3.63M
#define GC_PTR2IDX(ptr)      ((ptr) - GC_G(buf))
174
175
3.62M
#define GC_IDX2LIST(idx)     ((void*)(uintptr_t)(((idx) * sizeof(void*)) | GC_UNUSED))
176
954k
#define GC_LIST2IDX(list)    (((uint32_t)(uintptr_t)(list)) / sizeof(void*))
177
178
/* GC buffers */
179
1.74M
#define GC_INVALID           0
180
1.99M
#define GC_FIRST_ROOT        1
181
182
16
#define GC_DEFAULT_BUF_SIZE  (16 * 1024)
183
1
#define GC_BUF_GROW_STEP     (128 * 1024)
184
185
0
#define GC_MAX_UNCOMPRESSED  (512 * 1024)
186
2
#define GC_MAX_BUF_SIZE      0x40000000
187
188
16
#define GC_THRESHOLD_DEFAULT (10000 + GC_FIRST_ROOT)
189
0
#define GC_THRESHOLD_STEP    10000
190
0
#define GC_THRESHOLD_MAX     1000000000
191
0
#define GC_THRESHOLD_TRIGGER 100
192
193
/* GC flags */
194
7.81k
#define GC_HAS_DESTRUCTORS  (1<<0)
195
196
/* Weak maps */
197
1.76k
#define Z_FROM_WEAKMAP_KEY    (1<<0)
198
1.69k
#define Z_FROM_WEAKMAP      (1<<1)
199
200
/* The WeakMap entry zv is reachable from roots by following the virtual
201
 * reference from the a WeakMap key to the entry */
202
#define GC_FROM_WEAKMAP_KEY(zv) \
203
337
  (Z_TYPE_INFO_P((zv)) & (Z_FROM_WEAKMAP_KEY << Z_TYPE_INFO_EXTRA_SHIFT))
204
205
707
#define GC_SET_FROM_WEAKMAP_KEY(zv) do {                    \
206
707
  zval *_z = (zv);                               \
207
707
  Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) | (Z_FROM_WEAKMAP_KEY << Z_TYPE_INFO_EXTRA_SHIFT); \
208
707
} while (0)
209
210
722
#define GC_UNSET_FROM_WEAKMAP_KEY(zv) do {                    \
211
722
  zval *_z = (zv);                               \
212
722
  Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) & ~(Z_FROM_WEAKMAP_KEY << Z_TYPE_INFO_EXTRA_SHIFT); \
213
722
} while (0)
214
215
/* The WeakMap entry zv is reachable from roots by following the reference from
216
 * the WeakMap */
217
#define GC_FROM_WEAKMAP(zv) \
218
1.31k
  (Z_TYPE_INFO_P((zv)) & (Z_FROM_WEAKMAP << Z_TYPE_INFO_EXTRA_SHIFT))
219
220
166
#define GC_SET_FROM_WEAKMAP(zv) do {                        \
221
166
  zval *_z = (zv);                               \
222
166
  Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) | (Z_FROM_WEAKMAP << Z_TYPE_INFO_EXTRA_SHIFT); \
223
166
} while (0)
224
225
211
#define GC_UNSET_FROM_WEAKMAP(zv) do {                      \
226
211
  zval *_z = (zv);                               \
227
211
  Z_TYPE_INFO_P(_z) = Z_TYPE_INFO_P(_z) & ~(Z_FROM_WEAKMAP << Z_TYPE_INFO_EXTRA_SHIFT); \
228
211
} while (0)
229
230
/* unused buffers */
231
#define GC_HAS_UNUSED() \
232
982k
  (GC_G(unused) != GC_INVALID)
233
#define GC_FETCH_UNUSED() \
234
954k
  gc_fetch_unused()
235
#define GC_LINK_UNUSED(root) \
236
3.62M
  gc_link_unused(root)
237
238
#define GC_HAS_NEXT_UNUSED_UNDER_THRESHOLD() \
239
  (GC_G(first_unused) < GC_G(gc_threshold))
240
#define GC_HAS_NEXT_UNUSED() \
241
982k
  (GC_G(first_unused) != GC_G(buf_size))
242
#define GC_FETCH_NEXT_UNUSED() \
243
2.67M
  gc_fetch_next_unused()
244
245
ZEND_API int (*gc_collect_cycles)(void);
246
247
typedef struct _gc_root_buffer {
248
  zend_refcounted  *ref;
249
} gc_root_buffer;
250
251
typedef struct _zend_gc_globals {
252
  gc_root_buffer   *buf;        /* preallocated arrays of buffers   */
253
254
  bool         gc_enabled;
255
  bool         gc_active;        /* GC currently running, forbid nested GC */
256
  bool         gc_protected;     /* GC protected, forbid root additions */
257
  bool         gc_full;
258
259
  uint32_t          unused;     /* linked list of unused buffers    */
260
  uint32_t          first_unused;   /* first unused buffer              */
261
  uint32_t          gc_threshold;     /* GC collection threshold          */
262
  uint32_t          buf_size;     /* size of the GC buffer            */
263
  uint32_t          num_roots;    /* number of roots in GC buffer     */
264
265
  uint32_t gc_runs;
266
  uint32_t collected;
267
268
  zend_hrtime_t activated_at;
269
  zend_hrtime_t collector_time;
270
  zend_hrtime_t dtor_time;
271
  zend_hrtime_t free_time;
272
273
  uint32_t dtor_idx;      /* root buffer index */
274
  uint32_t dtor_end;
275
  zend_fiber *dtor_fiber;
276
  bool dtor_fiber_running;
277
278
#if GC_BENCH
279
  uint32_t root_buf_length;
280
  uint32_t root_buf_peak;
281
  uint32_t zval_possible_root;
282
  uint32_t zval_buffered;
283
  uint32_t zval_remove_from_buffer;
284
  uint32_t zval_marked_grey;
285
#endif
286
} zend_gc_globals;
287
288
#ifdef ZTS
289
static int gc_globals_id;
290
static size_t gc_globals_offset;
291
#define GC_G(v) ZEND_TSRMG_FAST(gc_globals_offset, zend_gc_globals *, v)
292
#else
293
51.6M
#define GC_G(v) (gc_globals.v)
294
static zend_gc_globals gc_globals;
295
#endif
296
297
#if GC_BENCH
298
# define GC_BENCH_INC(counter) GC_G(counter)++
299
# define GC_BENCH_DEC(counter) GC_G(counter)--
300
# define GC_BENCH_PEAK(peak, counter) do {    \
301
    if (GC_G(counter) > GC_G(peak)) {   \
302
      GC_G(peak) = GC_G(counter);     \
303
    }                   \
304
  } while (0)
305
#else
306
# define GC_BENCH_INC(counter)
307
# define GC_BENCH_DEC(counter)
308
# define GC_BENCH_PEAK(peak, counter)
309
#endif
310
311
312
5.11k
#define GC_STACK_SEGMENT_SIZE (((4096 - ZEND_MM_OVERHEAD) / sizeof(void*)) - 2)
313
314
typedef struct _gc_stack gc_stack;
315
316
struct _gc_stack {
317
  gc_stack        *prev;
318
  gc_stack        *next;
319
  zend_refcounted *data[GC_STACK_SEGMENT_SIZE];
320
};
321
322
#define GC_STACK_DCL(init) \
323
198k
  gc_stack *_stack = init; \
324
198k
  size_t    _top = 0;
325
326
#define GC_STACK_PUSH(ref) \
327
3.12M
  gc_stack_push(&_stack, &_top, ref);
328
329
#define GC_STACK_POP() \
330
3.32M
  gc_stack_pop(&_stack, &_top)
331
332
static zend_never_inline gc_stack* gc_stack_next(gc_stack *stack)
333
21.9k
{
334
21.9k
  if (UNEXPECTED(!stack->next)) {
335
20.1k
    gc_stack *segment = emalloc(sizeof(gc_stack));
336
20.1k
    segment->prev = stack;
337
20.1k
    segment->next = NULL;
338
20.1k
    stack->next = segment;
339
20.1k
  }
340
21.9k
  return stack->next;
341
21.9k
}
342
343
static zend_always_inline void gc_stack_push(gc_stack **stack, size_t *top, zend_refcounted *ref)
344
3.12M
{
345
3.12M
  if (UNEXPECTED(*top == GC_STACK_SEGMENT_SIZE)) {
346
2.55k
    (*stack) = gc_stack_next(*stack);
347
2.55k
    (*top) = 0;
348
2.55k
  }
349
3.12M
  (*stack)->data[(*top)++] = ref;
350
3.12M
}
351
352
static zend_always_inline zend_refcounted* gc_stack_pop(gc_stack **stack, size_t *top)
353
3.32M
{
354
3.32M
  if (UNEXPECTED((*top) == 0)) {
355
201k
    if (!(*stack)->prev) {
356
198k
      return NULL;
357
198k
    } else {
358
2.55k
      (*stack) = (*stack)->prev;
359
2.55k
      (*top) = GC_STACK_SEGMENT_SIZE - 1;
360
2.55k
      return (*stack)->data[GC_STACK_SEGMENT_SIZE - 1];
361
2.55k
    }
362
3.12M
  } else {
363
3.12M
    return (*stack)->data[--(*top)];
364
3.12M
  }
365
3.32M
}
366
367
static void gc_stack_free(gc_stack *stack)
368
23.5k
{
369
23.5k
  gc_stack *p = stack->next;
370
371
43.7k
  while (p) {
372
20.1k
    stack = p->next;
373
20.1k
    efree(p);
374
20.1k
    p = stack;
375
20.1k
  }
376
23.5k
}
377
378
static zend_always_inline uint32_t gc_compress(uint32_t idx)
379
3.63M
{
380
3.63M
  if (EXPECTED(idx < GC_MAX_UNCOMPRESSED)) {
381
3.63M
    return idx;
382
3.63M
  }
383
0
  return (idx % GC_MAX_UNCOMPRESSED) | GC_MAX_UNCOMPRESSED;
384
3.63M
}
385
386
static zend_always_inline gc_root_buffer* gc_decompress(zend_refcounted *ref, uint32_t idx)
387
0
{
388
0
  gc_root_buffer *root = GC_IDX2PTR(idx);
389
390
0
  if (EXPECTED(GC_GET_PTR(root->ref) == ref)) {
391
0
    return root;
392
0
  }
393
394
0
  while (1) {
395
0
    idx += GC_MAX_UNCOMPRESSED;
396
0
    ZEND_ASSERT(idx < GC_G(first_unused));
397
0
    root = GC_IDX2PTR(idx);
398
0
    if (GC_GET_PTR(root->ref) == ref) {
399
0
      return root;
400
0
    }
401
0
  }
402
0
}
403
404
static zend_always_inline uint32_t gc_fetch_unused(void)
405
954k
{
406
954k
  uint32_t idx;
407
954k
  gc_root_buffer *root;
408
409
954k
  ZEND_ASSERT(GC_HAS_UNUSED());
410
954k
  idx = GC_G(unused);
411
954k
  root = GC_IDX2PTR(idx);
412
954k
  ZEND_ASSERT(GC_IS_UNUSED(root->ref));
413
954k
  GC_G(unused) = GC_LIST2IDX(root->ref);
414
954k
  return idx;
415
954k
}
416
417
static zend_always_inline void gc_link_unused(gc_root_buffer *root)
418
3.62M
{
419
3.62M
  root->ref = GC_IDX2LIST(GC_G(unused));
420
3.62M
  GC_G(unused) = GC_PTR2IDX(root);
421
3.62M
}
422
423
static zend_always_inline uint32_t gc_fetch_next_unused(void)
424
2.67M
{
425
2.67M
  uint32_t idx;
426
427
2.67M
  ZEND_ASSERT(GC_HAS_NEXT_UNUSED());
428
2.67M
  idx = GC_G(first_unused);
429
2.67M
  GC_G(first_unused) = GC_G(first_unused) + 1;
430
2.67M
  return idx;
431
2.67M
}
432
433
#if ZEND_GC_DEBUG > 1
434
static const char *gc_color_name(uint32_t color) {
435
  switch (color) {
436
    case GC_BLACK: return "black";
437
    case GC_WHITE: return "white";
438
    case GC_GREY: return "grey";
439
    case GC_PURPLE: return "purple";
440
    default: return "unknown";
441
  }
442
}
443
static void gc_trace_ref(zend_refcounted *ref) {
444
  if (GC_TYPE(ref) == IS_OBJECT) {
445
    zend_object *obj = (zend_object *) ref;
446
    fprintf(stderr, "[%p] rc=%d addr=%d %s object(%s)#%d ",
447
      ref, GC_REFCOUNT(ref), GC_REF_ADDRESS(ref),
448
      gc_color_name(GC_REF_COLOR(ref)),
449
      obj->ce->name->val, obj->handle);
450
  } else if (GC_TYPE(ref) == IS_ARRAY) {
451
    zend_array *arr = (zend_array *) ref;
452
    fprintf(stderr, "[%p] rc=%d addr=%d %s array(%d) ",
453
      ref, GC_REFCOUNT(ref), GC_REF_ADDRESS(ref),
454
      gc_color_name(GC_REF_COLOR(ref)),
455
      zend_hash_num_elements(arr));
456
  } else {
457
    fprintf(stderr, "[%p] rc=%d addr=%d %s %s ",
458
      ref, GC_REFCOUNT(ref), GC_REF_ADDRESS(ref),
459
      gc_color_name(GC_REF_COLOR(ref)),
460
      GC_TYPE(ref) == IS_REFERENCE
461
        ? "reference" : zend_get_type_by_const(GC_TYPE(ref)));
462
  }
463
}
464
#endif
465
466
static zend_always_inline void gc_remove_from_roots(gc_root_buffer *root)
467
3.57M
{
468
3.57M
  GC_LINK_UNUSED(root);
469
3.57M
  GC_G(num_roots)--;
470
3.57M
  GC_BENCH_DEC(root_buf_length);
471
3.57M
}
472
473
static void root_buffer_dtor(zend_gc_globals *gc_globals)
474
0
{
475
0
  if (gc_globals->buf) {
476
0
    free(gc_globals->buf);
477
0
    gc_globals->buf = NULL;
478
0
  }
479
0
}
480
481
static void gc_globals_ctor_ex(zend_gc_globals *gc_globals)
482
16
{
483
16
  gc_globals->gc_enabled = 0;
484
16
  gc_globals->gc_active = 0;
485
16
  gc_globals->gc_protected = 1;
486
16
  gc_globals->gc_full = 0;
487
488
16
  gc_globals->buf = NULL;
489
16
  gc_globals->unused = GC_INVALID;
490
16
  gc_globals->first_unused = GC_INVALID;
491
16
  gc_globals->gc_threshold = GC_INVALID;
492
16
  gc_globals->buf_size = GC_INVALID;
493
16
  gc_globals->num_roots = 0;
494
495
16
  gc_globals->gc_runs = 0;
496
16
  gc_globals->collected = 0;
497
16
  gc_globals->collector_time = 0;
498
16
  gc_globals->dtor_time = 0;
499
16
  gc_globals->free_time = 0;
500
16
  gc_globals->activated_at = 0;
501
502
16
  gc_globals->dtor_idx = GC_FIRST_ROOT;
503
16
  gc_globals->dtor_end = 0;
504
16
  gc_globals->dtor_fiber = NULL;
505
16
  gc_globals->dtor_fiber_running = false;
506
507
#if GC_BENCH
508
  gc_globals->root_buf_length = 0;
509
  gc_globals->root_buf_peak = 0;
510
  gc_globals->zval_possible_root = 0;
511
  gc_globals->zval_buffered = 0;
512
  gc_globals->zval_remove_from_buffer = 0;
513
  gc_globals->zval_marked_grey = 0;
514
#endif
515
16
}
516
517
void gc_globals_ctor(void)
518
16
{
519
#ifdef ZTS
520
  ts_allocate_fast_id(&gc_globals_id, &gc_globals_offset, sizeof(zend_gc_globals), (ts_allocate_ctor) gc_globals_ctor_ex, (ts_allocate_dtor) root_buffer_dtor);
521
#else
522
16
  gc_globals_ctor_ex(&gc_globals);
523
16
#endif
524
16
}
525
526
void gc_globals_dtor(void)
527
0
{
528
0
#ifndef ZTS
529
0
  root_buffer_dtor(&gc_globals);
530
0
#endif
531
0
}
532
533
void gc_reset(void)
534
300k
{
535
300k
  if (GC_G(buf)) {
536
300k
    GC_G(gc_active) = 0;
537
300k
    GC_G(gc_protected) = 0;
538
300k
    GC_G(gc_full) = 0;
539
300k
    GC_G(unused) = GC_INVALID;
540
300k
    GC_G(first_unused) = GC_FIRST_ROOT;
541
300k
    GC_G(num_roots) = 0;
542
543
300k
    GC_G(gc_runs) = 0;
544
300k
    GC_G(collected) = 0;
545
546
300k
    GC_G(collector_time) = 0;
547
300k
    GC_G(dtor_time) = 0;
548
300k
    GC_G(free_time) = 0;
549
550
300k
    GC_G(dtor_idx) = GC_FIRST_ROOT;
551
300k
    GC_G(dtor_end) = 0;
552
300k
    GC_G(dtor_fiber) = NULL;
553
300k
    GC_G(dtor_fiber_running) = false;
554
555
#if GC_BENCH
556
    GC_G(root_buf_length) = 0;
557
    GC_G(root_buf_peak) = 0;
558
    GC_G(zval_possible_root) = 0;
559
    GC_G(zval_buffered) = 0;
560
    GC_G(zval_remove_from_buffer) = 0;
561
    GC_G(zval_marked_grey) = 0;
562
#endif
563
300k
  }
564
565
300k
  GC_G(activated_at) = zend_hrtime();
566
300k
}
567
568
ZEND_API bool gc_enable(bool enable)
569
134k
{
570
134k
  bool old_enabled = GC_G(gc_enabled);
571
134k
  GC_G(gc_enabled) = enable;
572
134k
  if (enable && !old_enabled && GC_G(buf) == NULL) {
573
16
    GC_G(buf) = (gc_root_buffer*) pemalloc(sizeof(gc_root_buffer) * GC_DEFAULT_BUF_SIZE, 1);
574
16
    GC_G(buf)[0].ref = NULL;
575
16
    GC_G(buf_size) = GC_DEFAULT_BUF_SIZE;
576
16
    GC_G(gc_threshold) = GC_THRESHOLD_DEFAULT;
577
16
    gc_reset();
578
16
  }
579
134k
  return old_enabled;
580
134k
}
581
582
ZEND_API bool gc_enabled(void)
583
91
{
584
91
  return GC_G(gc_enabled);
585
91
}
586
587
ZEND_API bool gc_protect(bool protect)
588
32.8k
{
589
32.8k
  bool old_protected = GC_G(gc_protected);
590
32.8k
  GC_G(gc_protected) = protect;
591
32.8k
  return old_protected;
592
32.8k
}
593
594
ZEND_API bool gc_protected(void)
595
0
{
596
0
  return GC_G(gc_protected);
597
0
}
598
599
static void gc_grow_root_buffer(void)
600
1
{
601
1
  size_t new_size;
602
603
1
  if (GC_G(buf_size) >= GC_MAX_BUF_SIZE) {
604
0
    if (!GC_G(gc_full)) {
605
0
      zend_error(E_WARNING, "GC buffer overflow (GC disabled)\n");
606
0
      GC_G(gc_active) = 1;
607
0
      GC_G(gc_protected) = 1;
608
0
      GC_G(gc_full) = 1;
609
0
      return;
610
0
    }
611
0
  }
612
1
  if (GC_G(buf_size) < GC_BUF_GROW_STEP) {
613
1
    new_size = GC_G(buf_size) * 2;
614
1
  } else {
615
0
    new_size = GC_G(buf_size) + GC_BUF_GROW_STEP;
616
0
  }
617
1
  if (new_size > GC_MAX_BUF_SIZE) {
618
0
    new_size = GC_MAX_BUF_SIZE;
619
0
  }
620
1
  GC_G(buf) = perealloc(GC_G(buf), sizeof(gc_root_buffer) * new_size, 1);
621
1
  GC_G(buf_size) = new_size;
622
1
}
623
624
static void gc_adjust_threshold(int count)
625
0
{
626
0
  uint32_t new_threshold;
627
628
  /* TODO Very simple heuristic for dynamic GC buffer resizing:
629
   * If there are "too few" collections, increase the collection threshold
630
   * by a fixed step */
631
0
  if (count < GC_THRESHOLD_TRIGGER || GC_G(num_roots) >= GC_G(gc_threshold)) {
632
    /* increase */
633
0
    if (GC_G(gc_threshold) < GC_THRESHOLD_MAX) {
634
0
      new_threshold = GC_G(gc_threshold) + GC_THRESHOLD_STEP;
635
0
      if (new_threshold > GC_THRESHOLD_MAX) {
636
0
        new_threshold = GC_THRESHOLD_MAX;
637
0
      }
638
0
      if (new_threshold > GC_G(buf_size)) {
639
0
        gc_grow_root_buffer();
640
0
      }
641
0
      if (new_threshold <= GC_G(buf_size)) {
642
0
        GC_G(gc_threshold) = new_threshold;
643
0
      }
644
0
    }
645
0
  } else if (GC_G(gc_threshold) > GC_THRESHOLD_DEFAULT) {
646
0
    new_threshold = GC_G(gc_threshold) - GC_THRESHOLD_STEP;
647
0
    if (new_threshold < GC_THRESHOLD_DEFAULT) {
648
0
      new_threshold = GC_THRESHOLD_DEFAULT;
649
0
    }
650
0
    GC_G(gc_threshold) = new_threshold;
651
0
  }
652
0
}
653
654
static zend_never_inline void ZEND_FASTCALL gc_possible_root_when_full(zend_refcounted *ref)
655
0
{
656
0
  uint32_t idx;
657
0
  gc_root_buffer *newRoot;
658
659
0
  ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
660
0
  ZEND_ASSERT(GC_INFO(ref) == 0);
661
662
0
  if (GC_G(gc_enabled) && !GC_G(gc_active)) {
663
0
    GC_ADDREF(ref);
664
0
    gc_adjust_threshold(gc_collect_cycles());
665
0
    if (UNEXPECTED(GC_DELREF(ref) == 0)) {
666
0
      rc_dtor_func(ref);
667
0
      return;
668
0
    } else if (UNEXPECTED(GC_INFO(ref))) {
669
0
      return;
670
0
    }
671
0
  }
672
673
0
  if (GC_HAS_UNUSED()) {
674
0
    idx = GC_FETCH_UNUSED();
675
0
  } else if (EXPECTED(GC_HAS_NEXT_UNUSED())) {
676
0
    idx = GC_FETCH_NEXT_UNUSED();
677
0
  } else {
678
0
    gc_grow_root_buffer();
679
0
    if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
680
0
      return;
681
0
    }
682
0
    idx = GC_FETCH_NEXT_UNUSED();
683
0
  }
684
685
0
  newRoot = GC_IDX2PTR(idx);
686
0
  newRoot->ref = ref; /* GC_ROOT tag is 0 */
687
0
  GC_TRACE_SET_COLOR(ref, GC_PURPLE);
688
689
0
  idx = gc_compress(idx);
690
0
  GC_REF_SET_INFO(ref, idx | GC_PURPLE);
691
0
  GC_G(num_roots)++;
692
693
0
  GC_BENCH_INC(zval_buffered);
694
0
  GC_BENCH_INC(root_buf_length);
695
0
  GC_BENCH_PEAK(root_buf_peak, root_buf_length);
696
0
}
697
698
ZEND_API void ZEND_FASTCALL gc_possible_root(zend_refcounted *ref)
699
2.87M
{
700
2.87M
  uint32_t idx;
701
2.87M
  gc_root_buffer *newRoot;
702
703
2.87M
  if (UNEXPECTED(GC_G(gc_protected))) {
704
226k
    return;
705
226k
  }
706
707
2.64M
  GC_BENCH_INC(zval_possible_root);
708
709
2.64M
  if (EXPECTED(GC_HAS_UNUSED())) {
710
954k
    idx = GC_FETCH_UNUSED();
711
1.69M
  } else if (EXPECTED(GC_HAS_NEXT_UNUSED_UNDER_THRESHOLD())) {
712
1.69M
    idx = GC_FETCH_NEXT_UNUSED();
713
1.69M
  } else {
714
0
    gc_possible_root_when_full(ref);
715
0
    return;
716
0
  }
717
718
2.64M
  ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
719
2.64M
  ZEND_ASSERT(GC_INFO(ref) == 0);
720
721
2.64M
  newRoot = GC_IDX2PTR(idx);
722
2.64M
  newRoot->ref = ref; /* GC_ROOT tag is 0 */
723
2.64M
  GC_TRACE_SET_COLOR(ref, GC_PURPLE);
724
725
2.64M
  idx = gc_compress(idx);
726
2.64M
  GC_REF_SET_INFO(ref, idx | GC_PURPLE);
727
2.64M
  GC_G(num_roots)++;
728
729
2.64M
  GC_BENCH_INC(zval_buffered);
730
2.64M
  GC_BENCH_INC(root_buf_length);
731
2.64M
  GC_BENCH_PEAK(root_buf_peak, root_buf_length);
732
2.64M
}
733
734
static void ZEND_FASTCALL gc_extra_root(zend_refcounted *ref)
735
50
{
736
50
  uint32_t idx;
737
50
  gc_root_buffer *newRoot;
738
739
50
  if (EXPECTED(GC_HAS_UNUSED())) {
740
0
    idx = GC_FETCH_UNUSED();
741
50
  } else if (EXPECTED(GC_HAS_NEXT_UNUSED())) {
742
50
    idx = GC_FETCH_NEXT_UNUSED();
743
50
  } else {
744
0
    gc_grow_root_buffer();
745
0
    if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
746
      /* TODO: can this really happen? */
747
0
      return;
748
0
    }
749
0
    idx = GC_FETCH_NEXT_UNUSED();
750
0
  }
751
752
50
  ZEND_ASSERT(GC_TYPE(ref) == IS_ARRAY || GC_TYPE(ref) == IS_OBJECT);
753
50
  ZEND_ASSERT(GC_REF_ADDRESS(ref) == 0);
754
755
50
  newRoot = GC_IDX2PTR(idx);
756
50
  newRoot->ref = ref; /* GC_ROOT tag is 0 */
757
758
50
  idx = gc_compress(idx);
759
50
  GC_REF_SET_INFO(ref, idx | GC_REF_COLOR(ref));
760
50
  GC_G(num_roots)++;
761
762
50
  GC_BENCH_INC(zval_buffered);
763
50
  GC_BENCH_INC(root_buf_length);
764
50
  GC_BENCH_PEAK(root_buf_peak, root_buf_length);
765
50
}
766
767
static zend_never_inline void ZEND_FASTCALL gc_remove_compressed(zend_refcounted *ref, uint32_t idx)
768
0
{
769
0
  gc_root_buffer *root = gc_decompress(ref, idx);
770
0
  gc_remove_from_roots(root);
771
0
}
772
773
ZEND_API void ZEND_FASTCALL gc_remove_from_buffer(zend_refcounted *ref)
774
3.52M
{
775
3.52M
  gc_root_buffer *root;
776
3.52M
  uint32_t idx = GC_REF_ADDRESS(ref);
777
778
3.52M
  GC_BENCH_INC(zval_remove_from_buffer);
779
780
3.52M
  if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
781
2.41M
    GC_TRACE_SET_COLOR(ref, GC_BLACK);
782
2.41M
  }
783
3.52M
  GC_REF_SET_INFO(ref, 0);
784
785
  /* Perform decompression only in case of large buffers */
786
3.52M
  if (UNEXPECTED(GC_G(first_unused) >= GC_MAX_UNCOMPRESSED)) {
787
0
    gc_remove_compressed(ref, idx);
788
0
    return;
789
0
  }
790
791
3.52M
  ZEND_ASSERT(idx);
792
3.52M
  root = GC_IDX2PTR(idx);
793
3.52M
  gc_remove_from_roots(root);
794
3.52M
}
795
796
static void gc_scan_black(zend_refcounted *ref, gc_stack *stack)
797
36.2k
{
798
36.2k
  HashTable *ht;
799
36.2k
  Bucket *p;
800
36.2k
  zval *zv;
801
36.2k
  uint32_t n;
802
36.2k
  GC_STACK_DCL(stack);
803
804
237k
tail_call:
805
237k
  if (GC_TYPE(ref) == IS_OBJECT) {
806
38.3k
    zend_object *obj = (zend_object*)ref;
807
808
38.3k
    if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
809
38.3k
      zval *table;
810
38.3k
      int len;
811
812
38.3k
      if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) {
813
810
        zend_weakmap_get_object_key_entry_gc(obj, &table, &len);
814
810
        n = len;
815
810
        zv = table;
816
1.53k
        for (; n != 0; n-=2) {
817
723
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
818
723
          zval *entry = (zval*) Z_PTR_P(zv);
819
723
          zval *weakmap = zv+1;
820
723
          ZEND_ASSERT(Z_REFCOUNTED_P(weakmap));
821
723
          if (Z_OPT_COLLECTABLE_P(entry)) {
822
613
            GC_UNSET_FROM_WEAKMAP_KEY(entry);
823
613
            if (GC_REF_CHECK_COLOR(Z_COUNTED_P(weakmap), GC_GREY)) {
824
              /* Weakmap was scanned in gc_mark_roots, we must
825
               * ensure that it's eventually scanned in
826
               * gc_scan_roots as well. */
827
66
              if (!GC_REF_ADDRESS(Z_COUNTED_P(weakmap))) {
828
45
                gc_extra_root(Z_COUNTED_P(weakmap));
829
45
              }
830
547
            } else if (/* GC_REF_CHECK_COLOR(Z_COUNTED_P(weakmap), GC_BLACK) && */ !GC_FROM_WEAKMAP(entry)) {
831
              /* Both the entry weakmap and key are BLACK, so we
832
               * can mark the entry BLACK as well.
833
               * !GC_FROM_WEAKMAP(entry) means that the weakmap
834
               * was already scanned black (or will not be
835
               * scanned), so it's our responsibility to mark the
836
               * entry */
837
544
              ZEND_ASSERT(GC_REF_CHECK_COLOR(Z_COUNTED_P(weakmap), GC_BLACK));
838
544
              ref = Z_COUNTED_P(entry);
839
544
              GC_ADDREF(ref);
840
544
              if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
841
27
                GC_REF_SET_BLACK(ref);
842
27
                GC_STACK_PUSH(ref);
843
27
              }
844
544
            }
845
613
          }
846
723
          zv+=2;
847
723
        }
848
810
      }
849
850
38.3k
      if (UNEXPECTED(obj->handlers->get_gc == zend_weakmap_get_gc)) {
851
109
        zend_weakmap_get_key_entry_gc(obj, &table, &len);
852
109
        n = len;
853
109
        zv = table;
854
305
        for (; n != 0; n-=2) {
855
196
          ZEND_ASSERT(Z_TYPE_P(zv+1) == IS_PTR);
856
196
          zval *key = zv;
857
196
          zval *entry = (zval*) Z_PTR_P(zv+1);
858
196
          if (Z_OPT_COLLECTABLE_P(entry)) {
859
102
            GC_UNSET_FROM_WEAKMAP(entry);
860
102
            if (GC_REF_CHECK_COLOR(Z_COUNTED_P(key), GC_GREY)) {
861
              /* Key was scanned in gc_mark_roots, we must
862
               * ensure that it's eventually scanned in
863
               * gc_scan_roots as well. */
864
25
              if (!GC_REF_ADDRESS(Z_COUNTED_P(key))) {
865
5
                gc_extra_root(Z_COUNTED_P(key));
866
5
              }
867
77
            } else if (/* GC_REF_CHECK_COLOR(Z_COUNTED_P(key), GC_BLACK) && */ !GC_FROM_WEAKMAP_KEY(entry)) {
868
              /* Both the entry weakmap and key are BLACK, so we
869
               * can mark the entry BLACK as well.
870
               * !GC_FROM_WEAKMAP_KEY(entry) means that the key
871
               * was already scanned black (or will not be
872
               * scanned), so it's our responsibility to mark the
873
               * entry */
874
67
              ZEND_ASSERT(GC_REF_CHECK_COLOR(Z_COUNTED_P(key), GC_BLACK));
875
67
              ref = Z_COUNTED_P(entry);
876
67
              GC_ADDREF(ref);
877
67
              if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
878
62
                GC_REF_SET_BLACK(ref);
879
62
                GC_STACK_PUSH(ref);
880
62
              }
881
67
            }
882
102
          }
883
196
          zv += 2;
884
196
        }
885
109
        goto next;
886
109
      }
887
888
38.2k
      ht = obj->handlers->get_gc(obj, &table, &len);
889
38.2k
      n = len;
890
38.2k
      zv = table;
891
38.2k
      if (UNEXPECTED(ht)) {
892
13.2k
        GC_ADDREF(ht);
893
13.2k
        if (!GC_REF_CHECK_COLOR(ht, GC_BLACK)) {
894
13.2k
          GC_REF_SET_BLACK(ht);
895
16.8k
          for (; n != 0; n--) {
896
3.62k
            if (Z_COLLECTABLE_P(zv)) {
897
608
              ref = Z_COUNTED_P(zv);
898
608
              GC_ADDREF(ref);
899
608
              if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
900
479
                GC_REF_SET_BLACK(ref);
901
479
                GC_STACK_PUSH(ref);
902
479
              }
903
608
            }
904
3.62k
            zv++;
905
3.62k
          }
906
13.2k
          goto handle_ht;
907
13.2k
        }
908
13.2k
      }
909
910
133k
handle_zvals:
911
409k
      for (; n != 0; n--) {
912
300k
        if (Z_COLLECTABLE_P(zv)) {
913
27.6k
          ref = Z_COUNTED_P(zv);
914
27.6k
          GC_ADDREF(ref);
915
27.6k
          if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
916
23.8k
            GC_REF_SET_BLACK(ref);
917
23.8k
            zv++;
918
124k
            while (--n) {
919
101k
              if (Z_COLLECTABLE_P(zv)) {
920
90.3k
                zend_refcounted *ref = Z_COUNTED_P(zv);
921
90.3k
                GC_ADDREF(ref);
922
90.3k
                if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
923
89.0k
                  GC_REF_SET_BLACK(ref);
924
89.0k
                  GC_STACK_PUSH(ref);
925
89.0k
                }
926
90.3k
              }
927
101k
              zv++;
928
101k
            }
929
23.8k
            goto tail_call;
930
23.8k
          }
931
27.6k
        }
932
276k
        zv++;
933
276k
      }
934
133k
    }
935
199k
  } else if (GC_TYPE(ref) == IS_ARRAY) {
936
196k
    ZEND_ASSERT((zend_array*)ref != &EG(symbol_table));
937
196k
    ht = (zend_array*)ref;
938
209k
handle_ht:
939
209k
    n = ht->nNumUsed;
940
209k
    zv = ht->arPacked;
941
209k
    if (HT_IS_PACKED(ht)) {
942
108k
      goto handle_zvals;
943
108k
    }
944
945
101k
    p = (Bucket*)zv;
946
301k
    for (; n != 0; n--) {
947
283k
      zv = &p->val;
948
283k
      if (Z_TYPE_P(zv) == IS_INDIRECT) {
949
4.94k
        zv = Z_INDIRECT_P(zv);
950
4.94k
      }
951
283k
      if (Z_COLLECTABLE_P(zv)) {
952
85.0k
        ref = Z_COUNTED_P(zv);
953
85.0k
        GC_ADDREF(ref);
954
85.0k
        if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
955
83.5k
          GC_REF_SET_BLACK(ref);
956
83.5k
          p++;
957
168k
          while (--n) {
958
84.5k
            zv = &p->val;
959
84.5k
            if (Z_TYPE_P(zv) == IS_INDIRECT) {
960
369
              zv = Z_INDIRECT_P(zv);
961
369
            }
962
84.5k
            if (Z_COLLECTABLE_P(zv)) {
963
4.43k
              zend_refcounted *ref = Z_COUNTED_P(zv);
964
4.43k
              GC_ADDREF(ref);
965
4.43k
              if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
966
3.01k
                GC_REF_SET_BLACK(ref);
967
3.01k
                GC_STACK_PUSH(ref);
968
3.01k
              }
969
4.43k
            }
970
84.5k
            p++;
971
84.5k
          }
972
83.5k
          goto tail_call;
973
83.5k
        }
974
85.0k
      }
975
200k
      p++;
976
200k
    }
977
101k
  } else if (GC_TYPE(ref) == IS_REFERENCE) {
978
3.28k
    if (Z_COLLECTABLE(((zend_reference*)ref)->val)) {
979
1.65k
      ref = Z_COUNTED(((zend_reference*)ref)->val);
980
1.65k
      GC_ADDREF(ref);
981
1.65k
      if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
982
1.57k
        GC_REF_SET_BLACK(ref);
983
1.57k
        goto tail_call;
984
1.57k
      }
985
1.65k
    }
986
3.28k
  }
987
988
128k
next:
989
128k
  ref = GC_STACK_POP();
990
128k
  if (ref) {
991
92.6k
    goto tail_call;
992
92.6k
  }
993
128k
}
994
995
static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack)
996
68.0k
{
997
68.0k
  HashTable *ht;
998
68.0k
  Bucket *p;
999
68.0k
  zval *zv;
1000
68.0k
  uint32_t n;
1001
68.0k
  GC_STACK_DCL(stack);
1002
1003
1.42M
tail_call:
1004
1.42M
  GC_BENCH_INC(zval_marked_grey);
1005
1006
1.42M
  if (GC_TYPE(ref) == IS_OBJECT) {
1007
707k
    zend_object *obj = (zend_object*)ref;
1008
1009
707k
    if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
1010
707k
      zval *table;
1011
707k
      int len;
1012
1013
707k
      if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) {
1014
1.03k
        zend_weakmap_get_object_key_entry_gc(obj, &table, &len);
1015
1.03k
        n = len;
1016
1.03k
        zv = table;
1017
1.86k
        for (; n != 0; n-=2) {
1018
822
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
1019
822
          zval *entry = (zval*) Z_PTR_P(zv);
1020
822
          zval *weakmap = zv+1;
1021
822
          ZEND_ASSERT(Z_REFCOUNTED_P(weakmap));
1022
822
          if (Z_COLLECTABLE_P(entry)) {
1023
707
            GC_SET_FROM_WEAKMAP_KEY(entry);
1024
707
            ref = Z_COUNTED_P(entry);
1025
            /* Only DELREF if the contribution from the weakmap has
1026
             * not been cancelled yet */
1027
707
            if (!GC_FROM_WEAKMAP(entry)) {
1028
656
              GC_DELREF(ref);
1029
656
            }
1030
707
            if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1031
126
              GC_REF_SET_COLOR(ref, GC_GREY);
1032
126
              GC_STACK_PUSH(ref);
1033
126
            }
1034
707
          }
1035
822
          zv+=2;
1036
822
        }
1037
1.03k
      }
1038
1039
707k
      if (UNEXPECTED(obj->handlers->get_gc == zend_weakmap_get_gc)) {
1040
158
        zend_weakmap_get_entry_gc(obj, &table, &len);
1041
158
        n = len;
1042
158
        zv = table;
1043
418
        for (; n != 0; n--) {
1044
260
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
1045
260
          zval *entry = (zval*) Z_PTR_P(zv);
1046
260
          if (Z_COLLECTABLE_P(entry)) {
1047
166
            GC_SET_FROM_WEAKMAP(entry);
1048
166
            ref = Z_COUNTED_P(entry);
1049
            /* Only DELREF if the contribution from the weakmap key
1050
             * has not been cancelled yet */
1051
166
            if (!GC_FROM_WEAKMAP_KEY(entry)) {
1052
64
              GC_DELREF(ref);
1053
64
            }
1054
166
            if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1055
51
              GC_REF_SET_COLOR(ref, GC_GREY);
1056
51
              GC_STACK_PUSH(ref);
1057
51
            }
1058
166
          }
1059
260
          zv++;
1060
260
        }
1061
158
        goto next;
1062
158
      }
1063
1064
707k
      ht = obj->handlers->get_gc(obj, &table, &len);
1065
707k
      n = len;
1066
707k
      zv = table;
1067
707k
      if (UNEXPECTED(ht)) {
1068
666k
        GC_DELREF(ht);
1069
666k
        if (!GC_REF_CHECK_COLOR(ht, GC_GREY)) {
1070
665k
          GC_REF_SET_COLOR(ht, GC_GREY);
1071
891k
          for (; n != 0; n--) {
1072
225k
            if (Z_COLLECTABLE_P(zv)) {
1073
222k
              ref = Z_COUNTED_P(zv);
1074
222k
              GC_DELREF(ref);
1075
222k
              if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1076
218k
                GC_REF_SET_COLOR(ref, GC_GREY);
1077
218k
                GC_STACK_PUSH(ref);
1078
218k
              }
1079
222k
            }
1080
225k
            zv++;
1081
225k
          }
1082
665k
          goto handle_ht;
1083
665k
        }
1084
666k
      }
1085
168k
handle_zvals:
1086
596k
      for (; n != 0; n--) {
1087
464k
        if (Z_COLLECTABLE_P(zv)) {
1088
77.4k
          ref = Z_COUNTED_P(zv);
1089
77.4k
          GC_DELREF(ref);
1090
77.4k
          if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1091
36.8k
            GC_REF_SET_COLOR(ref, GC_GREY);
1092
36.8k
            zv++;
1093
144k
            while (--n) {
1094
107k
              if (Z_COLLECTABLE_P(zv)) {
1095
89.0k
                zend_refcounted *ref = Z_COUNTED_P(zv);
1096
89.0k
                GC_DELREF(ref);
1097
89.0k
                if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1098
83.9k
                  GC_REF_SET_COLOR(ref, GC_GREY);
1099
83.9k
                  GC_STACK_PUSH(ref);
1100
83.9k
                }
1101
89.0k
              }
1102
107k
              zv++;
1103
107k
            }
1104
36.8k
            goto tail_call;
1105
36.8k
          }
1106
77.4k
        }
1107
427k
        zv++;
1108
427k
      }
1109
168k
    }
1110
717k
  } else if (GC_TYPE(ref) == IS_ARRAY) {
1111
694k
    ZEND_ASSERT(((zend_array*)ref) != &EG(symbol_table));
1112
694k
    ht = (zend_array*)ref;
1113
1.36M
handle_ht:
1114
1.36M
    n = ht->nNumUsed;
1115
1.36M
    if (HT_IS_PACKED(ht)) {
1116
127k
            zv = ht->arPacked;
1117
127k
            goto handle_zvals;
1118
127k
    }
1119
1120
1.23M
    p = ht->arData;
1121
2.55M
    for (; n != 0; n--) {
1122
1.79M
      zv = &p->val;
1123
1.79M
      if (Z_TYPE_P(zv) == IS_INDIRECT) {
1124
815k
        zv = Z_INDIRECT_P(zv);
1125
815k
      }
1126
1.79M
      if (Z_COLLECTABLE_P(zv)) {
1127
612k
        ref = Z_COUNTED_P(zv);
1128
612k
        GC_DELREF(ref);
1129
612k
        if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1130
465k
          GC_REF_SET_COLOR(ref, GC_GREY);
1131
465k
          p++;
1132
1.72M
          while (--n) {
1133
1.25M
            zv = &p->val;
1134
1.25M
            if (Z_TYPE_P(zv) == IS_INDIRECT) {
1135
139k
              zv = Z_INDIRECT_P(zv);
1136
139k
            }
1137
1.25M
            if (Z_COLLECTABLE_P(zv)) {
1138
971k
              zend_refcounted *ref = Z_COUNTED_P(zv);
1139
971k
              GC_DELREF(ref);
1140
971k
              if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1141
544k
                GC_REF_SET_COLOR(ref, GC_GREY);
1142
544k
                GC_STACK_PUSH(ref);
1143
544k
              }
1144
971k
            }
1145
1.25M
            p++;
1146
1.25M
          }
1147
465k
          goto tail_call;
1148
465k
        }
1149
612k
      }
1150
1.32M
      p++;
1151
1.32M
    }
1152
1.23M
  } else if (GC_TYPE(ref) == IS_REFERENCE) {
1153
22.7k
    if (Z_COLLECTABLE(((zend_reference*)ref)->val)) {
1154
18.0k
      ref = Z_COUNTED(((zend_reference*)ref)->val);
1155
18.0k
      GC_DELREF(ref);
1156
18.0k
      if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1157
6.84k
        GC_REF_SET_COLOR(ref, GC_GREY);
1158
6.84k
        goto tail_call;
1159
6.84k
      }
1160
18.0k
    }
1161
22.7k
  }
1162
1163
915k
next:
1164
915k
  ref = GC_STACK_POP();
1165
915k
  if (ref) {
1166
847k
    goto tail_call;
1167
847k
  }
1168
915k
}
1169
1170
/* Two-Finger compaction algorithm */
1171
static void gc_compact(void)
1172
880k
{
1173
880k
  if (GC_G(num_roots) + GC_FIRST_ROOT != GC_G(first_unused)) {
1174
459k
    if (GC_G(num_roots)) {
1175
11.2k
      gc_root_buffer *free = GC_IDX2PTR(GC_FIRST_ROOT);
1176
11.2k
      gc_root_buffer *scan = GC_IDX2PTR(GC_G(first_unused) - 1);
1177
11.2k
      gc_root_buffer *end  = GC_IDX2PTR(GC_G(num_roots));
1178
11.2k
      uint32_t idx;
1179
11.2k
      zend_refcounted *p;
1180
1181
20.9k
      while (free < scan) {
1182
113k
        while (!GC_IS_UNUSED(free->ref)) {
1183
99.5k
          free++;
1184
99.5k
        }
1185
43.6k
        while (GC_IS_UNUSED(scan->ref)) {
1186
29.5k
          scan--;
1187
29.5k
        }
1188
14.1k
        if (scan > free) {
1189
7.47k
          p = scan->ref;
1190
7.47k
          free->ref = p;
1191
7.47k
          p = GC_GET_PTR(p);
1192
7.47k
          idx = gc_compress(GC_PTR2IDX(free));
1193
7.47k
          GC_REF_SET_INFO(p, idx | GC_REF_COLOR(p));
1194
7.47k
          free++;
1195
7.47k
          scan--;
1196
7.47k
          if (scan <= end) {
1197
4.38k
            break;
1198
4.38k
          }
1199
7.47k
        }
1200
14.1k
      }
1201
11.2k
    }
1202
459k
    GC_G(unused) = GC_INVALID;
1203
459k
    GC_G(first_unused) = GC_G(num_roots) + GC_FIRST_ROOT;
1204
459k
  }
1205
880k
}
1206
1207
static void gc_mark_roots(gc_stack *stack)
1208
23.6k
{
1209
23.6k
  gc_root_buffer *current, *last;
1210
1211
23.6k
  gc_compact();
1212
1213
23.6k
  current = GC_IDX2PTR(GC_FIRST_ROOT);
1214
23.6k
  last = GC_IDX2PTR(GC_G(first_unused));
1215
257k
  while (current != last) {
1216
233k
    if (GC_IS_ROOT(current->ref)) {
1217
233k
      if (GC_REF_CHECK_COLOR(current->ref, GC_PURPLE)) {
1218
68.0k
        GC_REF_SET_COLOR(current->ref, GC_GREY);
1219
68.0k
        gc_mark_grey(current->ref, stack);
1220
68.0k
      }
1221
233k
    }
1222
233k
    current++;
1223
233k
  }
1224
23.6k
}
1225
1226
static void gc_scan(zend_refcounted *ref, gc_stack *stack)
1227
68.0k
{
1228
68.0k
  HashTable *ht;
1229
68.0k
  Bucket *p;
1230
68.0k
  zval *zv;
1231
68.0k
  uint32_t n;
1232
68.0k
  GC_STACK_DCL(stack);
1233
1234
1.89M
tail_call:
1235
1.89M
  if (!GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1236
333
    goto next;
1237
333
  }
1238
1239
1.89M
  if (GC_REFCOUNT(ref) > 0) {
1240
36.2k
    if (!GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
1241
36.2k
      GC_REF_SET_BLACK(ref);
1242
36.2k
      if (UNEXPECTED(!_stack->next)) {
1243
19.3k
        gc_stack_next(_stack);
1244
19.3k
      }
1245
      /* Split stack and reuse the tail */
1246
36.2k
      _stack->next->prev = NULL;
1247
36.2k
      gc_scan_black(ref, _stack->next);
1248
36.2k
      _stack->next->prev = _stack;
1249
36.2k
    }
1250
36.2k
    goto next;
1251
36.2k
  }
1252
1253
1.85M
  if (GC_TYPE(ref) == IS_OBJECT) {
1254
671k
    zend_object *obj = (zend_object*)ref;
1255
671k
    if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
1256
671k
      zval *table;
1257
671k
      int len;
1258
1259
671k
      if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) {
1260
313
        zend_weakmap_get_object_entry_gc(obj, &table, &len);
1261
313
        n = len;
1262
313
        zv = table;
1263
494
        for (; n != 0; n--) {
1264
181
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
1265
181
          zval *entry = (zval*) Z_PTR_P(zv);
1266
181
          if (Z_OPT_COLLECTABLE_P(entry)) {
1267
113
            ref = Z_COUNTED_P(entry);
1268
113
            if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1269
50
              GC_REF_SET_COLOR(ref, GC_WHITE);
1270
50
              GC_STACK_PUSH(ref);
1271
50
            }
1272
113
          }
1273
181
          zv++;
1274
181
        }
1275
313
      }
1276
1277
671k
      ht = obj->handlers->get_gc(obj, &table, &len);
1278
671k
      n = len;
1279
671k
      zv = table;
1280
671k
      if (UNEXPECTED(ht)) {
1281
654k
        if (GC_REF_CHECK_COLOR(ht, GC_GREY)) {
1282
654k
          GC_REF_SET_COLOR(ht, GC_WHITE);
1283
654k
          GC_STACK_PUSH((zend_refcounted *) ht);
1284
876k
          for (; n != 0; n--) {
1285
222k
            if (Z_COLLECTABLE_P(zv)) {
1286
221k
              ref = Z_COUNTED_P(zv);
1287
221k
              if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1288
218k
                GC_REF_SET_COLOR(ref, GC_WHITE);
1289
218k
                GC_STACK_PUSH(ref);
1290
218k
              }
1291
221k
            }
1292
222k
            zv++;
1293
222k
          }
1294
654k
          goto handle_ht;
1295
654k
        }
1296
654k
      }
1297
1298
64.1k
handle_zvals:
1299
333k
      for (; n != 0; n--) {
1300
285k
        if (Z_COLLECTABLE_P(zv)) {
1301
79.0k
          ref = Z_COUNTED_P(zv);
1302
79.0k
          if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1303
15.8k
            GC_REF_SET_COLOR(ref, GC_WHITE);
1304
15.8k
            zv++;
1305
33.4k
            while (--n) {
1306
17.5k
              if (Z_COLLECTABLE_P(zv)) {
1307
8.76k
                zend_refcounted *ref = Z_COUNTED_P(zv);
1308
8.76k
                if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1309
5.03k
                  GC_REF_SET_COLOR(ref, GC_WHITE);
1310
5.03k
                  GC_STACK_PUSH(ref);
1311
5.03k
                }
1312
8.76k
              }
1313
17.5k
              zv++;
1314
17.5k
            }
1315
15.8k
            goto tail_call;
1316
15.8k
          }
1317
79.0k
        }
1318
269k
        zv++;
1319
269k
      }
1320
64.1k
    }
1321
1.18M
  } else if (GC_TYPE(ref) == IS_ARRAY) {
1322
1.16M
    ht = (HashTable *)ref;
1323
1.16M
    ZEND_ASSERT(ht != &EG(symbol_table));
1324
1325
1.82M
handle_ht:
1326
1.82M
    n = ht->nNumUsed;
1327
1.82M
    if (HT_IS_PACKED(ht)) {
1328
46.7k
            zv = ht->arPacked;
1329
46.7k
            goto handle_zvals;
1330
46.7k
    }
1331
1332
1.77M
    p = ht->arData;
1333
4.97M
    for (; n != 0; n--) {
1334
3.57M
      zv = &p->val;
1335
3.57M
      if (Z_TYPE_P(zv) == IS_INDIRECT) {
1336
1.76M
        zv = Z_INDIRECT_P(zv);
1337
1.76M
      }
1338
3.57M
      if (Z_COLLECTABLE_P(zv)) {
1339
1.50M
        ref = Z_COUNTED_P(zv);
1340
1.50M
        if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1341
383k
          GC_REF_SET_COLOR(ref, GC_WHITE);
1342
383k
          p++;
1343
1.56M
          while (--n) {
1344
1.17M
            zv = &p->val;
1345
1.17M
            if (Z_TYPE_P(zv) == IS_INDIRECT) {
1346
139k
              zv = Z_INDIRECT_P(zv);
1347
139k
            }
1348
1.17M
            if (Z_COLLECTABLE_P(zv)) {
1349
969k
              zend_refcounted *ref = Z_COUNTED_P(zv);
1350
969k
              if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1351
543k
                GC_REF_SET_COLOR(ref, GC_WHITE);
1352
543k
                GC_STACK_PUSH(ref);
1353
543k
              }
1354
969k
            }
1355
1.17M
            p++;
1356
1.17M
          }
1357
383k
          goto tail_call;
1358
383k
        }
1359
1.50M
      }
1360
3.19M
      p++;
1361
3.19M
    }
1362
1.77M
  } else if (GC_TYPE(ref) == IS_REFERENCE) {
1363
19.9k
    if (Z_COLLECTABLE(((zend_reference*)ref)->val)) {
1364
16.7k
      ref = Z_COUNTED(((zend_reference*)ref)->val);
1365
16.7k
      if (GC_REF_CHECK_COLOR(ref, GC_GREY)) {
1366
5.92k
        GC_REF_SET_COLOR(ref, GC_WHITE);
1367
5.92k
        goto tail_call;
1368
5.92k
      }
1369
16.7k
    }
1370
19.9k
  }
1371
1372
1.48M
next:
1373
1.48M
  ref = GC_STACK_POP();
1374
1.48M
  if (ref) {
1375
1.42M
    goto tail_call;
1376
1.42M
  }
1377
1.48M
}
1378
1379
static void gc_scan_roots(gc_stack *stack)
1380
23.6k
{
1381
23.6k
  uint32_t idx, end;
1382
23.6k
  gc_root_buffer *current;
1383
1384
  /* Root buffer might be reallocated during gc_scan,
1385
   * make sure to reload pointers. */
1386
23.6k
  idx = GC_FIRST_ROOT;
1387
23.6k
  end = GC_G(first_unused);
1388
257k
  while (idx != end) {
1389
233k
    current = GC_IDX2PTR(idx);
1390
233k
    if (GC_IS_ROOT(current->ref)) {
1391
233k
      if (GC_REF_CHECK_COLOR(current->ref, GC_GREY)) {
1392
68.0k
        GC_REF_SET_COLOR(current->ref, GC_WHITE);
1393
68.0k
        gc_scan(current->ref, stack);
1394
68.0k
      }
1395
233k
    }
1396
233k
    idx++;
1397
233k
  }
1398
1399
  /* Scan extra roots added during gc_scan */
1400
23.6k
  while (idx != GC_G(first_unused)) {
1401
50
    current = GC_IDX2PTR(idx);
1402
50
    if (GC_IS_ROOT(current->ref)) {
1403
50
      if (GC_REF_CHECK_COLOR(current->ref, GC_GREY)) {
1404
27
        GC_REF_SET_COLOR(current->ref, GC_WHITE);
1405
27
        gc_scan(current->ref, stack);
1406
27
      }
1407
50
    }
1408
50
    idx++;
1409
50
  }
1410
23.6k
}
1411
1412
static void gc_add_garbage(zend_refcounted *ref)
1413
982k
{
1414
982k
  uint32_t idx;
1415
982k
  gc_root_buffer *buf;
1416
1417
982k
  if (GC_HAS_UNUSED()) {
1418
0
    idx = GC_FETCH_UNUSED();
1419
982k
  } else if (GC_HAS_NEXT_UNUSED()) {
1420
982k
    idx = GC_FETCH_NEXT_UNUSED();
1421
982k
  } else {
1422
1
    gc_grow_root_buffer();
1423
1
    if (UNEXPECTED(!GC_HAS_NEXT_UNUSED())) {
1424
0
      return;
1425
0
    }
1426
1
    idx = GC_FETCH_NEXT_UNUSED();
1427
1
  }
1428
1429
982k
  buf = GC_IDX2PTR(idx);
1430
982k
  buf->ref = GC_MAKE_GARBAGE(ref);
1431
1432
982k
  idx = gc_compress(idx);
1433
982k
  GC_REF_SET_INFO(ref, idx | GC_BLACK);
1434
982k
  GC_G(num_roots)++;
1435
982k
}
1436
1437
static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *stack)
1438
23.5k
{
1439
23.5k
  int count = 0;
1440
23.5k
  HashTable *ht;
1441
23.5k
  Bucket *p;
1442
23.5k
  zval *zv;
1443
23.5k
  uint32_t n;
1444
23.5k
  GC_STACK_DCL(stack);
1445
1446
1.18M
tail_call:
1447
  /* don't count references for compatibility ??? */
1448
1.18M
  if (GC_TYPE(ref) != IS_REFERENCE) {
1449
1.16M
    count++;
1450
1.16M
  }
1451
1452
1.18M
  if (GC_TYPE(ref) == IS_OBJECT) {
1453
669k
    zend_object *obj = (zend_object*)ref;
1454
1455
669k
    if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
1456
669k
      int len;
1457
669k
      zval *table;
1458
1459
      /* optimization: color is GC_BLACK (0) */
1460
669k
      if (!GC_INFO(ref)) {
1461
493k
        gc_add_garbage(ref);
1462
493k
      }
1463
669k
      if (!(OBJ_FLAGS(obj) & IS_OBJ_DESTRUCTOR_CALLED)
1464
669k
       && (obj->handlers->dtor_obj != zend_objects_destroy_object
1465
525k
        || obj->ce->destructor != NULL)) {
1466
2.94k
        *flags |= GC_HAS_DESTRUCTORS;
1467
2.94k
      }
1468
1469
669k
      if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) {
1470
229
        zend_weakmap_get_object_entry_gc(obj, &table, &len);
1471
229
        n = len;
1472
229
        zv = table;
1473
328
        for (; n != 0; n--) {
1474
99
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
1475
99
          zval *entry = (zval*) Z_PTR_P(zv);
1476
99
          if (Z_COLLECTABLE_P(entry) && GC_FROM_WEAKMAP_KEY(entry)) {
1477
73
            GC_UNSET_FROM_WEAKMAP_KEY(entry);
1478
73
            GC_UNSET_FROM_WEAKMAP(entry);
1479
73
            ref = Z_COUNTED_P(entry);
1480
73
            GC_ADDREF(ref);
1481
73
            if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1482
26
              GC_REF_SET_BLACK(ref);
1483
26
              GC_STACK_PUSH(ref);
1484
26
            }
1485
73
          }
1486
99
          zv++;
1487
99
        }
1488
229
      }
1489
1490
669k
      if (UNEXPECTED(obj->handlers->get_gc == zend_weakmap_get_gc)) {
1491
49
        zend_weakmap_get_entry_gc(obj, &table, &len);
1492
49
        n = len;
1493
49
        zv = table;
1494
113
        for (; n != 0; n--) {
1495
64
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
1496
64
          zval *entry = (zval*) Z_PTR_P(zv);
1497
64
          if (Z_COLLECTABLE_P(entry) && GC_FROM_WEAKMAP(entry)) {
1498
36
            GC_UNSET_FROM_WEAKMAP_KEY(entry);
1499
36
            GC_UNSET_FROM_WEAKMAP(entry);
1500
36
            ref = Z_COUNTED_P(entry);
1501
36
            GC_ADDREF(ref);
1502
36
            if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1503
26
              GC_REF_SET_BLACK(ref);
1504
26
              GC_STACK_PUSH(ref);
1505
26
            }
1506
36
          }
1507
64
          zv++;
1508
64
        }
1509
49
        goto next;
1510
49
      }
1511
1512
669k
      ht = obj->handlers->get_gc(obj, &table, &len);
1513
669k
      n = len;
1514
669k
      zv = table;
1515
669k
      if (UNEXPECTED(ht)) {
1516
652k
        GC_ADDREF(ht);
1517
652k
        if (GC_REF_CHECK_COLOR(ht, GC_WHITE)) {
1518
652k
          GC_REF_SET_BLACK(ht);
1519
875k
          for (; n != 0; n--) {
1520
222k
            if (Z_COLLECTABLE_P(zv)) {
1521
221k
              ref = Z_COUNTED_P(zv);
1522
221k
              GC_ADDREF(ref);
1523
221k
              if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1524
218k
                GC_REF_SET_BLACK(ref);
1525
218k
                GC_STACK_PUSH(ref);
1526
218k
              }
1527
221k
            }
1528
222k
            zv++;
1529
222k
          }
1530
652k
          goto handle_ht;
1531
652k
        }
1532
652k
      }
1533
1534
35.5k
handle_zvals:
1535
180k
      for (; n != 0; n--) {
1536
159k
        if (Z_COLLECTABLE_P(zv)) {
1537
45.2k
          ref = Z_COUNTED_P(zv);
1538
45.2k
          GC_ADDREF(ref);
1539
45.2k
          if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1540
14.5k
            GC_REF_SET_BLACK(ref);
1541
14.5k
            zv++;
1542
25.3k
            while (--n) {
1543
10.8k
              if (Z_COLLECTABLE_P(zv)) {
1544
3.19k
                zend_refcounted *ref = Z_COUNTED_P(zv);
1545
3.19k
                GC_ADDREF(ref);
1546
3.19k
                if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1547
966
                  GC_REF_SET_BLACK(ref);
1548
966
                  GC_STACK_PUSH(ref);
1549
966
                }
1550
3.19k
              }
1551
10.8k
              zv++;
1552
10.8k
            }
1553
14.5k
            goto tail_call;
1554
14.5k
          }
1555
45.2k
        }
1556
144k
        zv++;
1557
144k
      }
1558
35.5k
    }
1559
669k
  } else if (GC_TYPE(ref) == IS_ARRAY) {
1560
    /* optimization: color is GC_BLACK (0) */
1561
498k
    if (!GC_INFO(ref)) {
1562
488k
      gc_add_garbage(ref);
1563
488k
    }
1564
498k
    ht = (zend_array*)ref;
1565
1566
1.15M
handle_ht:
1567
1.15M
    n = ht->nNumUsed;
1568
1.15M
    if (HT_IS_PACKED(ht)) {
1569
19.0k
      zv = ht->arPacked;
1570
19.0k
      goto handle_zvals;
1571
19.0k
    }
1572
1573
1.13M
    p = ht->arData;
1574
2.25M
    for (; n != 0; n--) {
1575
1.50M
      zv = &p->val;
1576
1.50M
      if (Z_TYPE_P(zv) == IS_INDIRECT) {
1577
810k
        zv = Z_INDIRECT_P(zv);
1578
810k
      }
1579
1.50M
      if (Z_COLLECTABLE_P(zv)) {
1580
527k
        ref = Z_COUNTED_P(zv);
1581
527k
        GC_ADDREF(ref);
1582
527k
        if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1583
382k
          GC_REF_SET_BLACK(ref);
1584
382k
          p++;
1585
1.55M
          while (--n) {
1586
1.17M
            zv = &p->val;
1587
1.17M
            if (Z_TYPE_P(zv) == IS_INDIRECT) {
1588
139k
              zv = Z_INDIRECT_P(zv);
1589
139k
            }
1590
1.17M
            if (Z_COLLECTABLE_P(zv)) {
1591
967k
              zend_refcounted *ref = Z_COUNTED_P(zv);
1592
967k
              GC_ADDREF(ref);
1593
967k
              if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1594
542k
                GC_REF_SET_BLACK(ref);
1595
542k
                GC_STACK_PUSH(ref);
1596
542k
              }
1597
967k
            }
1598
1.17M
            p++;
1599
1.17M
          }
1600
382k
          goto tail_call;
1601
382k
        }
1602
527k
      }
1603
1.12M
      p++;
1604
1.12M
    }
1605
1.13M
  } else if (GC_TYPE(ref) == IS_REFERENCE) {
1606
19.4k
    if (Z_COLLECTABLE(((zend_reference*)ref)->val)) {
1607
16.3k
      ref = Z_COUNTED(((zend_reference*)ref)->val);
1608
16.3k
      GC_ADDREF(ref);
1609
16.3k
      if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1610
5.43k
        GC_REF_SET_BLACK(ref);
1611
5.43k
        goto tail_call;
1612
5.43k
      }
1613
16.3k
    }
1614
19.4k
  }
1615
1616
785k
next:
1617
785k
  ref = GC_STACK_POP();
1618
785k
  if (ref) {
1619
761k
    goto tail_call;
1620
761k
  }
1621
1622
23.5k
  return count;
1623
785k
}
1624
1625
static int gc_collect_roots(uint32_t *flags, gc_stack *stack)
1626
23.6k
{
1627
23.6k
  uint32_t idx, end;
1628
23.6k
  zend_refcounted *ref;
1629
23.6k
  int count = 0;
1630
23.6k
  gc_root_buffer *current = GC_IDX2PTR(GC_FIRST_ROOT);
1631
23.6k
  gc_root_buffer *last = GC_IDX2PTR(GC_G(first_unused));
1632
1633
  /* remove non-garbage from the list */
1634
257k
  while (current != last) {
1635
234k
    if (GC_IS_ROOT(current->ref)) {
1636
234k
      if (GC_REF_CHECK_COLOR(current->ref, GC_BLACK)) {
1637
48.3k
        GC_REF_SET_INFO(current->ref, 0); /* reset GC_ADDRESS() and keep GC_BLACK */
1638
48.3k
        gc_remove_from_roots(current);
1639
48.3k
      }
1640
234k
    }
1641
234k
    current++;
1642
234k
  }
1643
1644
23.6k
  gc_compact();
1645
1646
  /* Root buffer might be reallocated during gc_collect_white,
1647
   * make sure to reload pointers. */
1648
23.6k
  idx = GC_FIRST_ROOT;
1649
23.6k
  end = GC_G(first_unused);
1650
209k
  while (idx != end) {
1651
185k
    current = GC_IDX2PTR(idx);
1652
185k
    ref = current->ref;
1653
185k
    ZEND_ASSERT(GC_IS_ROOT(ref));
1654
185k
    current->ref = GC_MAKE_GARBAGE(ref);
1655
185k
    if (GC_REF_CHECK_COLOR(ref, GC_WHITE)) {
1656
23.5k
      GC_REF_SET_BLACK(ref);
1657
23.5k
      count += gc_collect_white(ref, flags, stack);
1658
23.5k
    }
1659
185k
    idx++;
1660
185k
  }
1661
1662
23.6k
  return count;
1663
23.6k
}
1664
1665
static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffer *root, gc_stack *stack)
1666
2.94k
{
1667
2.94k
  HashTable *ht;
1668
2.94k
  Bucket *p;
1669
2.94k
  zval *zv;
1670
2.94k
  uint32_t n;
1671
2.94k
  int count = 0;
1672
2.94k
  GC_STACK_DCL(stack);
1673
1674
8.96k
tail_call:
1675
8.96k
  if (root) {
1676
2.94k
    root = NULL;
1677
2.94k
    count++;
1678
6.01k
  } else if (GC_REF_ADDRESS(ref) != 0
1679
6.01k
   && GC_REF_CHECK_COLOR(ref, GC_BLACK)) {
1680
1.84k
    GC_TRACE_REF(ref, "removing from buffer");
1681
1.84k
    GC_REMOVE_FROM_BUFFER(ref);
1682
1.84k
    count++;
1683
4.17k
  } else if (GC_TYPE(ref) == IS_REFERENCE) {
1684
464
    if (Z_COLLECTABLE(((zend_reference*)ref)->val)) {
1685
245
      ref = Z_COUNTED(((zend_reference*)ref)->val);
1686
245
      goto tail_call;
1687
245
    }
1688
219
    goto next;
1689
3.71k
  } else {
1690
3.71k
    goto next;
1691
3.71k
  }
1692
1693
4.79k
  if (GC_TYPE(ref) == IS_OBJECT) {
1694
4.48k
    zend_object *obj = (zend_object*)ref;
1695
1696
4.48k
    if (EXPECTED(!(OBJ_FLAGS(ref) & IS_OBJ_FREE_CALLED))) {
1697
4.48k
      int len;
1698
4.48k
      zval *table;
1699
1700
4.48k
      if (UNEXPECTED(GC_FLAGS(obj) & IS_OBJ_WEAKLY_REFERENCED)) {
1701
72
        zend_weakmap_get_object_entry_gc(obj, &table, &len);
1702
72
        n = len;
1703
72
        zv = table;
1704
77
        for (; n != 0; n--) {
1705
5
          ZEND_ASSERT(Z_TYPE_P(zv) == IS_PTR);
1706
5
          zval *entry = (zval*) Z_PTR_P(zv);
1707
5
          if (Z_OPT_COLLECTABLE_P(entry)) {
1708
5
            ref = Z_COUNTED_P(entry);
1709
5
            GC_STACK_PUSH(ref);
1710
5
          }
1711
5
          zv++;
1712
5
        }
1713
72
      }
1714
1715
4.48k
      ht = obj->handlers->get_gc(obj, &table, &len);
1716
4.48k
      n = len;
1717
4.48k
      zv = table;
1718
4.48k
      if (UNEXPECTED(ht)) {
1719
2.29k
        for (; n != 0; n--) {
1720
998
          if (Z_COLLECTABLE_P(zv)) {
1721
801
            ref = Z_COUNTED_P(zv);
1722
801
            GC_STACK_PUSH(ref);
1723
801
          }
1724
998
          zv++;
1725
998
        }
1726
1.29k
        if (GC_REF_ADDRESS(ht) != 0 && GC_REF_CHECK_COLOR(ht, GC_BLACK)) {
1727
5
          GC_TRACE_REF(ht, "removing from buffer");
1728
5
          GC_REMOVE_FROM_BUFFER(ht);
1729
5
        }
1730
1.29k
        goto handle_ht;
1731
1.29k
      }
1732
1733
3.46k
handle_zvals:
1734
3.57k
      for (; n != 0; n--) {
1735
3.23k
        if (Z_COLLECTABLE_P(zv)) {
1736
3.12k
          ref = Z_COUNTED_P(zv);
1737
3.12k
          zv++;
1738
3.72k
          while (--n) {
1739
599
            if (Z_COLLECTABLE_P(zv)) {
1740
553
              zend_refcounted *ref = Z_COUNTED_P(zv);
1741
553
              GC_STACK_PUSH(ref);
1742
553
            }
1743
599
            zv++;
1744
599
          }
1745
3.12k
          goto tail_call;
1746
3.12k
        }
1747
110
        zv++;
1748
110
      }
1749
3.46k
    }
1750
4.48k
  } else if (GC_TYPE(ref) == IS_ARRAY) {
1751
305
    ht = (zend_array*)ref;
1752
1753
1.59k
handle_ht:
1754
1.59k
    n = ht->nNumUsed;
1755
1.59k
    if (HT_IS_PACKED(ht)) {
1756
273
      zv = ht->arPacked;
1757
273
      goto handle_zvals;
1758
273
    }
1759
1760
1.32k
    p = ht->arData;
1761
1.72k
    for (; n != 0; n--) {
1762
1.48k
      zv = &p->val;
1763
1.48k
      if (Z_TYPE_P(zv) == IS_INDIRECT) {
1764
254
        zv = Z_INDIRECT_P(zv);
1765
254
      }
1766
1.48k
      if (Z_COLLECTABLE_P(zv)) {
1767
1.09k
        ref = Z_COUNTED_P(zv);
1768
1.09k
        p++;
1769
1.29k
        while (--n) {
1770
206
          zv = &p->val;
1771
206
          if (Z_TYPE_P(zv) == IS_INDIRECT) {
1772
20
            zv = Z_INDIRECT_P(zv);
1773
20
          }
1774
206
          if (Z_COLLECTABLE_P(zv)) {
1775
196
            zend_refcounted *ref = Z_COUNTED_P(zv);
1776
196
            GC_STACK_PUSH(ref);
1777
196
          }
1778
206
          p++;
1779
206
        }
1780
1.09k
        goto tail_call;
1781
1.09k
      }
1782
396
      p++;
1783
396
    }
1784
1.32k
  }
1785
1786
4.50k
next:
1787
4.50k
  ref = GC_STACK_POP();
1788
4.50k
  if (ref) {
1789
1.55k
    goto tail_call;
1790
1.55k
  }
1791
1792
2.94k
  return count;
1793
4.50k
}
1794
1795
static void zend_get_gc_buffer_release(void);
1796
static void zend_gc_check_root_tmpvars(void);
1797
static void zend_gc_remove_root_tmpvars(void);
1798
1799
static zend_internal_function gc_destructor_fiber;
1800
1801
static ZEND_COLD ZEND_NORETURN void gc_create_destructor_fiber_error(void)
1802
0
{
1803
0
  zend_error_noreturn(E_ERROR, "Unable to create destructor fiber");
1804
0
}
1805
1806
static ZEND_COLD ZEND_NORETURN void gc_start_destructor_fiber_error(void)
1807
0
{
1808
0
  zend_error_noreturn(E_ERROR, "Unable to start destructor fiber");
1809
0
}
1810
1811
static zend_always_inline zend_result gc_call_destructors(uint32_t idx, uint32_t end, zend_fiber *fiber)
1812
619
{
1813
619
  gc_root_buffer *current;
1814
619
  zend_refcounted *p;
1815
1816
  /* The root buffer might be reallocated during destructors calls,
1817
   * make sure to reload pointers as necessary. */
1818
5.19k
  while (idx != end) {
1819
4.63k
    current = GC_IDX2PTR(idx);
1820
4.63k
    if (GC_IS_DTOR_GARBAGE(current->ref)) {
1821
2.37k
      p = GC_GET_PTR(current->ref);
1822
      /* Mark this is as a normal root for the next GC run */
1823
2.37k
      current->ref = p;
1824
      /* Double check that the destructor hasn't been called yet. It
1825
       * could have already been invoked indirectly by some other
1826
       * destructor. */
1827
2.37k
      if (!(OBJ_FLAGS(p) & IS_OBJ_DESTRUCTOR_CALLED)) {
1828
2.37k
        if (fiber != NULL) {
1829
185
          GC_G(dtor_idx) = idx;
1830
185
        }
1831
2.37k
        zend_object *obj = (zend_object*)p;
1832
2.37k
        GC_TRACE_REF(obj, "calling destructor");
1833
2.37k
        GC_ADD_FLAGS(obj, IS_OBJ_DESTRUCTOR_CALLED);
1834
2.37k
        GC_ADDREF(obj);
1835
2.37k
        obj->handlers->dtor_obj(obj);
1836
2.37k
        GC_TRACE_REF(obj, "returned from destructor");
1837
2.37k
        GC_DELREF(obj);
1838
2.37k
        if (UNEXPECTED(fiber != NULL && GC_G(dtor_fiber) != fiber)) {
1839
          /* We resumed after suspension */
1840
53
          gc_check_possible_root((zend_refcounted*)&obj->gc);
1841
53
          return FAILURE;
1842
53
        }
1843
2.37k
      }
1844
2.37k
    }
1845
4.58k
    idx++;
1846
4.58k
  }
1847
1848
566
  return SUCCESS;
1849
619
}
1850
1851
static zend_fiber *gc_create_destructor_fiber(void)
1852
125
{
1853
125
  zval zobj;
1854
125
  zend_fiber *fiber;
1855
1856
125
  GC_TRACE("starting destructor fiber");
1857
1858
125
  if (UNEXPECTED(object_init_ex(&zobj, zend_ce_fiber) == FAILURE)) {
1859
0
    gc_create_destructor_fiber_error();
1860
0
  }
1861
1862
125
  fiber = (zend_fiber *)Z_OBJ(zobj);
1863
125
  fiber->fci.size = sizeof(fiber->fci);
1864
125
  fiber->fci_cache.function_handler = (zend_function*) &gc_destructor_fiber;
1865
1866
125
  GC_G(dtor_fiber) = fiber;
1867
1868
125
  if (UNEXPECTED(zend_fiber_start(fiber, NULL) == FAILURE)) {
1869
0
    gc_start_destructor_fiber_error();
1870
0
  }
1871
1872
125
  return fiber;
1873
125
}
1874
1875
static zend_never_inline void gc_call_destructors_in_fiber(uint32_t end)
1876
76
{
1877
76
  ZEND_ASSERT(!GC_G(dtor_fiber_running));
1878
1879
76
  zend_fiber *fiber = GC_G(dtor_fiber);
1880
1881
76
  GC_G(dtor_idx) = GC_FIRST_ROOT;
1882
76
  GC_G(dtor_end) = GC_G(first_unused);
1883
1884
76
  if (UNEXPECTED(!fiber)) {
1885
72
    fiber = gc_create_destructor_fiber();
1886
72
  } else {
1887
4
    zend_fiber_resume(fiber, NULL, NULL);
1888
4
  }
1889
1890
129
  for (;;) {
1891
    /* At this point, fiber has executed until suspension */
1892
129
    GC_TRACE("resumed from destructor fiber");
1893
1894
129
    if (UNEXPECTED(GC_G(dtor_fiber_running))) {
1895
      /* Fiber was suspended by a destructor. Start a new one for the
1896
       * remaining destructors. */
1897
53
      GC_TRACE("destructor fiber suspended by destructor");
1898
53
      GC_G(dtor_fiber) = NULL;
1899
53
      GC_G(dtor_idx)++;
1900
      /* We do not own the fiber anymore. It may be collected if the
1901
       * application does not reference it. */
1902
53
      zend_object_release(&fiber->std);
1903
53
      fiber = gc_create_destructor_fiber();
1904
53
      continue;
1905
76
    } else {
1906
      /* Fiber suspended itself after calling all destructors */
1907
76
      GC_TRACE("destructor fiber suspended itself");
1908
76
      break;
1909
76
    }
1910
129
  }
1911
76
}
1912
1913
ZEND_API int zend_gc_collect_cycles(void)
1914
851k
{
1915
851k
  int total_count = 0;
1916
851k
  bool should_rerun_gc = 0;
1917
851k
  bool did_rerun_gc = 0;
1918
1919
851k
  zend_hrtime_t start_time = zend_hrtime();
1920
851k
  if (GC_G(num_roots) && !GC_G(gc_active)) {
1921
23.2k
    zend_gc_remove_root_tmpvars();
1922
23.2k
  }
1923
1924
851k
rerun_gc:
1925
851k
  if (GC_G(num_roots)) {
1926
23.6k
    int count;
1927
23.6k
    gc_root_buffer *current, *last;
1928
23.6k
    zend_refcounted *p;
1929
23.6k
    uint32_t gc_flags = 0;
1930
23.6k
    uint32_t idx, end;
1931
23.6k
    gc_stack stack;
1932
1933
23.6k
    stack.prev = NULL;
1934
23.6k
    stack.next = NULL;
1935
1936
23.6k
    if (GC_G(gc_active)) {
1937
33
      GC_G(collector_time) += zend_hrtime() - start_time;
1938
33
      return 0;
1939
33
    }
1940
1941
23.6k
    GC_TRACE("Collecting cycles");
1942
23.6k
    GC_G(gc_runs)++;
1943
23.6k
    GC_G(gc_active) = 1;
1944
1945
23.6k
    GC_TRACE("Marking roots");
1946
23.6k
    gc_mark_roots(&stack);
1947
23.6k
    GC_TRACE("Scanning roots");
1948
23.6k
    gc_scan_roots(&stack);
1949
1950
23.6k
    GC_TRACE("Collecting roots");
1951
23.6k
    count = gc_collect_roots(&gc_flags, &stack);
1952
1953
23.6k
    if (!GC_G(num_roots)) {
1954
      /* nothing to free */
1955
18.7k
      GC_TRACE("Nothing to free");
1956
18.7k
      gc_stack_free(&stack);
1957
18.7k
      GC_G(gc_active) = 0;
1958
18.7k
      goto finish;
1959
18.7k
    }
1960
1961
4.86k
    end = GC_G(first_unused);
1962
1963
4.86k
    if (gc_flags & GC_HAS_DESTRUCTORS) {
1964
566
      GC_TRACE("Calling destructors");
1965
1966
      /* During a destructor call, new externally visible references to nested data may
1967
       * be introduced. These references can be introduced in a way that does not
1968
       * modify any refcounts, so we have no real way to detect this situation
1969
       * short of rerunning full GC tracing. What we do instead is to only run
1970
       * destructors at this point and automatically re-run GC afterwards. */
1971
566
      should_rerun_gc = 1;
1972
1973
      /* Mark all roots for which a dtor will be invoked as DTOR_GARBAGE. Additionally
1974
       * color them purple. This serves a double purpose: First, they should be
1975
       * considered new potential roots for the next GC run. Second, it will prevent
1976
       * their removal from the root buffer by nested data removal. */
1977
566
      idx = GC_FIRST_ROOT;
1978
566
      current = GC_IDX2PTR(GC_FIRST_ROOT);
1979
5.94k
      while (idx != end) {
1980
5.38k
        if (GC_IS_GARBAGE(current->ref)) {
1981
5.38k
          p = GC_GET_PTR(current->ref);
1982
5.38k
          if (GC_TYPE(p) == IS_OBJECT && !(OBJ_FLAGS(p) & IS_OBJ_DESTRUCTOR_CALLED)) {
1983
4.69k
            zend_object *obj = (zend_object *) p;
1984
4.69k
            if (obj->handlers->dtor_obj != zend_objects_destroy_object
1985
4.69k
              || obj->ce->destructor) {
1986
2.94k
              current->ref = GC_MAKE_DTOR_GARBAGE(obj);
1987
2.94k
              GC_REF_SET_COLOR(obj, GC_PURPLE);
1988
2.94k
            } else {
1989
1.74k
              GC_ADD_FLAGS(obj, IS_OBJ_DESTRUCTOR_CALLED);
1990
1.74k
            }
1991
4.69k
          }
1992
5.38k
        }
1993
5.38k
        current++;
1994
5.38k
        idx++;
1995
5.38k
      }
1996
1997
      /* Remove nested data for objects on which a destructor will be called.
1998
       * This will not remove the objects themselves, as they have been colored
1999
       * purple. */
2000
566
      idx = GC_FIRST_ROOT;
2001
566
      current = GC_IDX2PTR(GC_FIRST_ROOT);
2002
5.94k
      while (idx != end) {
2003
5.38k
        if (GC_IS_DTOR_GARBAGE(current->ref)) {
2004
2.94k
          p = GC_GET_PTR(current->ref);
2005
2.94k
          count -= gc_remove_nested_data_from_buffer(p, current, &stack);
2006
2.94k
        }
2007
5.38k
        current++;
2008
5.38k
        idx++;
2009
5.38k
      }
2010
2011
      /* Actually call destructors. */
2012
566
      zend_hrtime_t dtor_start_time = zend_hrtime();
2013
566
      if (EXPECTED(!EG(active_fiber))) {
2014
490
        gc_call_destructors(GC_FIRST_ROOT, end, NULL);
2015
490
      } else {
2016
76
        gc_call_destructors_in_fiber(end);
2017
76
      }
2018
566
      GC_G(dtor_time) += zend_hrtime() - dtor_start_time;
2019
2020
566
      if (GC_G(gc_protected)) {
2021
        /* something went wrong */
2022
15
        zend_get_gc_buffer_release();
2023
15
        GC_G(collector_time) += zend_hrtime() - start_time;
2024
15
        return 0;
2025
15
      }
2026
566
    }
2027
2028
4.84k
    gc_stack_free(&stack);
2029
2030
    /* Destroy zvals. The root buffer may be reallocated. */
2031
4.84k
    GC_TRACE("Destroying zvals");
2032
4.84k
    zend_hrtime_t free_start_time = zend_hrtime();
2033
4.84k
    idx = GC_FIRST_ROOT;
2034
1.16M
    while (idx != end) {
2035
1.16M
      current = GC_IDX2PTR(idx);
2036
1.16M
      if (GC_IS_GARBAGE(current->ref)) {
2037
54.8k
        p = GC_GET_PTR(current->ref);
2038
54.8k
        GC_TRACE_REF(p, "destroying");
2039
54.8k
        if (GC_TYPE(p) == IS_OBJECT) {
2040
45.2k
          zend_object *obj = (zend_object*)p;
2041
2042
45.2k
          EG(objects_store).object_buckets[obj->handle] = SET_OBJ_INVALID(obj);
2043
45.2k
          GC_TYPE_INFO(obj) = GC_NULL |
2044
45.2k
            (GC_TYPE_INFO(obj) & ~GC_TYPE_MASK);
2045
          /* Modify current before calling free_obj (bug #78811: free_obj() can cause the root buffer (with current) to be reallocated.) */
2046
45.2k
          current->ref = GC_MAKE_GARBAGE(((char*)obj) - obj->handlers->offset);
2047
45.2k
          if (!(OBJ_FLAGS(obj) & IS_OBJ_FREE_CALLED)) {
2048
45.2k
            GC_ADD_FLAGS(obj, IS_OBJ_FREE_CALLED);
2049
45.2k
            GC_ADDREF(obj);
2050
45.2k
            obj->handlers->free_obj(obj);
2051
45.2k
            GC_DELREF(obj);
2052
45.2k
          }
2053
2054
45.2k
          ZEND_OBJECTS_STORE_ADD_TO_FREE_LIST(obj->handle);
2055
45.2k
        } else if (GC_TYPE(p) == IS_ARRAY) {
2056
9.65k
          zend_array *arr = (zend_array*)p;
2057
2058
9.65k
          GC_TYPE_INFO(arr) = GC_NULL |
2059
9.65k
            (GC_TYPE_INFO(arr) & ~GC_TYPE_MASK);
2060
2061
          /* GC may destroy arrays with rc>1. This is valid and safe. */
2062
9.65k
          HT_ALLOW_COW_VIOLATION(arr);
2063
2064
9.65k
          zend_hash_destroy(arr);
2065
9.65k
        }
2066
54.8k
      }
2067
1.16M
      idx++;
2068
1.16M
    }
2069
2070
    /* Free objects */
2071
4.84k
    current = GC_IDX2PTR(GC_FIRST_ROOT);
2072
4.84k
    last = GC_IDX2PTR(end);
2073
1.16M
    while (current != last) {
2074
1.16M
      if (GC_IS_GARBAGE(current->ref)) {
2075
54.8k
        p = GC_GET_PTR(current->ref);
2076
54.8k
        GC_LINK_UNUSED(current);
2077
54.8k
        GC_G(num_roots)--;
2078
54.8k
        efree(p);
2079
54.8k
      }
2080
1.16M
      current++;
2081
1.16M
    }
2082
2083
4.84k
    GC_G(free_time) += zend_hrtime() - free_start_time;
2084
2085
4.84k
    GC_TRACE("Collection finished");
2086
4.84k
    GC_G(collected) += count;
2087
4.84k
    total_count += count;
2088
4.84k
    GC_G(gc_active) = 0;
2089
4.84k
  }
2090
2091
833k
  gc_compact();
2092
2093
  /* Objects with destructors were removed from this GC run. Rerun GC right away to clean them
2094
   * up. We do this only once: If we encounter more destructors on the second run, we'll not
2095
   * run GC another time. */
2096
833k
  if (should_rerun_gc && !did_rerun_gc) {
2097
441
    did_rerun_gc = 1;
2098
441
    goto rerun_gc;
2099
441
  }
2100
2101
851k
finish:
2102
851k
  zend_get_gc_buffer_release();
2103
2104
  /* Prevent GC from running during zend_gc_check_root_tmpvars, before
2105
   * gc_threshold is adjusted, as this may result in unbounded recursion */
2106
851k
  GC_G(gc_active) = 1;
2107
851k
  zend_gc_check_root_tmpvars();
2108
851k
  GC_G(gc_active) = 0;
2109
2110
851k
  GC_G(collector_time) += zend_hrtime() - start_time;
2111
851k
  return total_count;
2112
833k
}
2113
2114
ZEND_API void zend_gc_get_status(zend_gc_status *status)
2115
213
{
2116
213
  status->active = GC_G(gc_active);
2117
213
  status->gc_protected = GC_G(gc_protected);
2118
213
  status->full = GC_G(gc_full);
2119
213
  status->runs = GC_G(gc_runs);
2120
213
  status->collected = GC_G(collected);
2121
213
  status->threshold = GC_G(gc_threshold);
2122
213
  status->buf_size = GC_G(buf_size);
2123
213
  status->num_roots = GC_G(num_roots);
2124
213
  status->application_time = zend_hrtime() - GC_G(activated_at);
2125
213
  status->collector_time = GC_G(collector_time);
2126
213
  status->dtor_time = GC_G(dtor_time);
2127
213
  status->free_time = GC_G(free_time);
2128
213
}
2129
2130
73.4k
ZEND_API zend_get_gc_buffer *zend_get_gc_buffer_create(void) {
2131
  /* There can only be one get_gc() call active at a time,
2132
   * so there only needs to be one buffer. */
2133
73.4k
  zend_get_gc_buffer *gc_buffer = &EG(get_gc_buffer);
2134
73.4k
  gc_buffer->cur = gc_buffer->start;
2135
73.4k
  return gc_buffer;
2136
73.4k
}
2137
2138
2.07k
ZEND_API void zend_get_gc_buffer_grow(zend_get_gc_buffer *gc_buffer) {
2139
2.07k
  size_t old_capacity = gc_buffer->end - gc_buffer->start;
2140
2.07k
  size_t new_capacity = old_capacity == 0 ? 64 : old_capacity * 2;
2141
2.07k
  gc_buffer->start = erealloc(gc_buffer->start, new_capacity * sizeof(zval));
2142
2.07k
  gc_buffer->end = gc_buffer->start + new_capacity;
2143
2.07k
  gc_buffer->cur = gc_buffer->start + old_capacity;
2144
2.07k
}
2145
2146
851k
static void zend_get_gc_buffer_release(void) {
2147
851k
  zend_get_gc_buffer *gc_buffer = &EG(get_gc_buffer);
2148
851k
  efree(gc_buffer->start);
2149
851k
  gc_buffer->start = gc_buffer->end = gc_buffer->cur = NULL;
2150
851k
}
2151
2152
/* TMPVAR operands are destroyed using zval_ptr_dtor_nogc(), because they usually cannot contain
2153
 * cycles. However, there are some rare exceptions where this is possible, in which case we rely
2154
 * on the producing code to root the value. If a GC run occurs between the rooting and consumption
2155
 * of the value, we would end up leaking it. To avoid this, root all live TMPVAR values here. */
2156
851k
static void zend_gc_check_root_tmpvars(void) {
2157
851k
  zend_execute_data *ex = EG(current_execute_data);
2158
1.11M
  for (; ex; ex = ex->prev_execute_data) {
2159
265k
    zend_function *func = ex->func;
2160
265k
    if (!func || !ZEND_USER_CODE(func->type)) {
2161
262k
      continue;
2162
262k
    }
2163
2164
2.96k
    uint32_t op_num = ex->opline - ex->func->op_array.opcodes;
2165
12.0k
    for (uint32_t i = 0; i < func->op_array.last_live_range; i++) {
2166
9.39k
      const zend_live_range *range = &func->op_array.live_range[i];
2167
9.39k
      if (range->start > op_num) {
2168
261
        break;
2169
261
      }
2170
9.12k
      if (range->end <= op_num) {
2171
8.75k
        continue;
2172
8.75k
      }
2173
2174
374
      uint32_t kind = range->var & ZEND_LIVE_MASK;
2175
374
      if (kind == ZEND_LIVE_TMPVAR || kind == ZEND_LIVE_LOOP) {
2176
274
        uint32_t var_num = range->var & ~ZEND_LIVE_MASK;
2177
274
        zval *var = ZEND_CALL_VAR(ex, var_num);
2178
274
        if (Z_COLLECTABLE_P(var)) {
2179
263
          gc_check_possible_root(Z_COUNTED_P(var));
2180
263
        }
2181
274
      }
2182
374
    }
2183
2.96k
  }
2184
851k
}
2185
2186
23.2k
static void zend_gc_remove_root_tmpvars(void) {
2187
23.2k
  zend_execute_data *ex = EG(current_execute_data);
2188
29.8k
  for (; ex; ex = ex->prev_execute_data) {
2189
6.63k
    zend_function *func = ex->func;
2190
6.63k
    if (!func || !ZEND_USER_CODE(func->type)) {
2191
4.11k
      continue;
2192
4.11k
    }
2193
2194
2.52k
    uint32_t op_num = ex->opline - ex->func->op_array.opcodes;
2195
11.1k
    for (uint32_t i = 0; i < func->op_array.last_live_range; i++) {
2196
8.82k
      const zend_live_range *range = &func->op_array.live_range[i];
2197
8.82k
      if (range->start > op_num) {
2198
196
        break;
2199
196
      }
2200
8.63k
      if (range->end <= op_num) {
2201
8.36k
        continue;
2202
8.36k
      }
2203
2204
266
      uint32_t kind = range->var & ZEND_LIVE_MASK;
2205
266
      if (kind == ZEND_LIVE_TMPVAR || kind == ZEND_LIVE_LOOP) {
2206
266
        uint32_t var_num = range->var & ~ZEND_LIVE_MASK;
2207
266
        zval *var = ZEND_CALL_VAR(ex, var_num);
2208
266
        if (Z_COLLECTABLE_P(var)) {
2209
263
          GC_REMOVE_FROM_BUFFER(Z_COUNTED_P(var));
2210
263
        }
2211
266
      }
2212
266
    }
2213
2.52k
  }
2214
23.2k
}
2215
2216
#if GC_BENCH
2217
void gc_bench_print(void)
2218
{
2219
  fprintf(stderr, "GC Statistics\n");
2220
  fprintf(stderr, "-------------\n");
2221
  fprintf(stderr, "Runs:               %d\n", GC_G(gc_runs));
2222
  fprintf(stderr, "Collected:          %d\n", GC_G(collected));
2223
  fprintf(stderr, "Root buffer length: %d\n", GC_G(root_buf_length));
2224
  fprintf(stderr, "Root buffer peak:   %d\n\n", GC_G(root_buf_peak));
2225
  fprintf(stderr, "      Possible            Remove from  Marked\n");
2226
  fprintf(stderr, "        Root    Buffered     buffer     grey\n");
2227
  fprintf(stderr, "      --------  --------  -----------  ------\n");
2228
  fprintf(stderr, "ZVAL  %8d  %8d  %9d  %8d\n", GC_G(zval_possible_root), GC_G(zval_buffered), GC_G(zval_remove_from_buffer), GC_G(zval_marked_grey));
2229
}
2230
#endif
2231
2232
#ifdef ZTS
2233
size_t zend_gc_globals_size(void)
2234
{
2235
  return sizeof(zend_gc_globals);
2236
}
2237
#endif
2238
2239
static ZEND_FUNCTION(gc_destructor_fiber)
2240
125
{
2241
125
  uint32_t idx, end;
2242
2243
125
  zend_fiber *fiber = GC_G(dtor_fiber);
2244
125
  ZEND_ASSERT(fiber != NULL);
2245
125
  ZEND_ASSERT(fiber == EG(active_fiber));
2246
2247
129
  for (;;) {
2248
129
    GC_G(dtor_fiber_running) = true;
2249
2250
129
    idx = GC_G(dtor_idx);
2251
129
    end = GC_G(dtor_end);
2252
129
    if (UNEXPECTED(gc_call_destructors(idx, end, fiber) == FAILURE)) {
2253
      /* We resumed after being suspended by a destructor */
2254
53
      return;
2255
53
    }
2256
2257
    /* We have called all destructors. Suspend fiber until the next GC run
2258
     */
2259
76
    GC_G(dtor_fiber_running) = false;
2260
76
    zend_fiber_suspend(fiber, NULL, NULL);
2261
2262
76
    if (UNEXPECTED(fiber->flags & ZEND_FIBER_FLAG_DESTROYED)) {
2263
      /* Fiber is being destroyed by shutdown sequence */
2264
72
      if (GC_G(dtor_fiber) == fiber) {
2265
72
        GC_G(dtor_fiber) = NULL;
2266
72
      }
2267
72
      GC_DELREF(&fiber->std);
2268
72
      gc_check_possible_root((zend_refcounted*)&fiber->std.gc);
2269
72
      return;
2270
72
    }
2271
76
  }
2272
125
}
2273
2274
static zend_internal_function gc_destructor_fiber = {
2275
  .type = ZEND_INTERNAL_FUNCTION,
2276
  .fn_flags = ZEND_ACC_PUBLIC,
2277
  .handler = ZEND_FN(gc_destructor_fiber),
2278
};
2279
2280
void gc_init(void)
2281
16
{
2282
16
  gc_destructor_fiber.function_name = zend_string_init_interned(
2283
16
      "gc_destructor_fiber",
2284
16
      strlen("gc_destructor_fiber"),
2285
16
      true);
2286
16
}