/src/php-src/Zend/zend_alloc.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | +----------------------------------------------------------------------+ |
3 | | | Zend Engine | |
4 | | +----------------------------------------------------------------------+ |
5 | | | Copyright (c) Zend Technologies Ltd. (http://www.zend.com) | |
6 | | +----------------------------------------------------------------------+ |
7 | | | This source file is subject to version 2.00 of the Zend license, | |
8 | | | that is bundled with this package in the file LICENSE, and is | |
9 | | | available through the world-wide-web at the following url: | |
10 | | | http://www.zend.com/license/2_00.txt. | |
11 | | | If you did not receive a copy of the Zend license and are unable to | |
12 | | | obtain it through the world-wide-web, please send a note to | |
13 | | | license@zend.com so we can mail you a copy immediately. | |
14 | | +----------------------------------------------------------------------+ |
15 | | | Authors: Andi Gutmans <andi@php.net> | |
16 | | | Zeev Suraski <zeev@php.net> | |
17 | | | Dmitry Stogov <dmitry@php.net> | |
18 | | +----------------------------------------------------------------------+ |
19 | | */ |
20 | | |
21 | | /* |
22 | | * zend_alloc is designed to be a modern CPU cache friendly memory manager |
23 | | * for PHP. Most ideas are taken from jemalloc and tcmalloc implementations. |
24 | | * |
25 | | * All allocations are split into 3 categories: |
26 | | * |
27 | | * Huge - the size is greater than CHUNK size (~2M by default), allocation is |
28 | | * performed using mmap(). The result is aligned on 2M boundary. |
29 | | * |
30 | | * Large - a number of 4096K pages inside a CHUNK. Large blocks |
31 | | * are always aligned on page boundary. |
32 | | * |
33 | | * Small - less than 3/4 of page size. Small sizes are rounded up to nearest |
34 | | * greater predefined small size (there are 30 predefined sizes: |
35 | | * 8, 16, 24, 32, ... 3072). Small blocks are allocated from |
36 | | * RUNs. Each RUN is allocated as a single or few following pages. |
37 | | * Allocation inside RUNs implemented using linked list of free |
38 | | * elements. The result is aligned to 8 bytes. |
39 | | * |
40 | | * zend_alloc allocates memory from OS by CHUNKs, these CHUNKs and huge memory |
41 | | * blocks are always aligned to CHUNK boundary. So it's very easy to determine |
42 | | * the CHUNK owning the certain pointer. Regular CHUNKs reserve a single |
43 | | * page at start for special purpose. It contains bitset of free pages, |
44 | | * few bitset for available runs of predefined small sizes, map of pages that |
45 | | * keeps information about usage of each page in this CHUNK, etc. |
46 | | * |
47 | | * zend_alloc provides familiar emalloc/efree/erealloc API, but in addition it |
48 | | * provides specialized and optimized routines to allocate blocks of predefined |
49 | | * sizes (e.g. emalloc_2(), emallc_4(), ..., emalloc_large(), etc) |
50 | | * The library uses C preprocessor tricks that substitute calls to emalloc() |
51 | | * with more specialized routines when the requested size is known. |
52 | | */ |
53 | | |
54 | | #include "zend.h" |
55 | | #include "zend_alloc.h" |
56 | | #include "zend_globals.h" |
57 | | #include "zend_hrtime.h" |
58 | | #include "zend_operators.h" |
59 | | #include "zend_multiply.h" |
60 | | #include "zend_bitset.h" |
61 | | #include "zend_mmap.h" |
62 | | #include "zend_portability.h" |
63 | | #include <signal.h> |
64 | | |
65 | | #ifdef HAVE_UNISTD_H |
66 | | # include <unistd.h> |
67 | | #endif |
68 | | |
69 | | #ifdef ZEND_WIN32 |
70 | | # include <wincrypt.h> |
71 | | # include <process.h> |
72 | | # include "win32/winutil.h" |
73 | | # define getpid _getpid |
74 | | typedef int pid_t; |
75 | | #endif |
76 | | |
77 | | #include <stdio.h> |
78 | | #include <stdlib.h> |
79 | | #include <string.h> |
80 | | |
81 | | #include <sys/types.h> |
82 | | #include <sys/stat.h> |
83 | | #include <limits.h> |
84 | | #include <fcntl.h> |
85 | | #include <errno.h> |
86 | | #ifdef __SANITIZE_ADDRESS__ |
87 | | # include <sanitizer/asan_interface.h> |
88 | | #endif |
89 | | |
90 | | #ifndef _WIN32 |
91 | | # include <sys/mman.h> |
92 | | # ifndef MAP_ANON |
93 | | # ifdef MAP_ANONYMOUS |
94 | | # define MAP_ANON MAP_ANONYMOUS |
95 | | # endif |
96 | | # endif |
97 | | # ifndef MAP_FAILED |
98 | | # define MAP_FAILED ((void*)-1) |
99 | | # endif |
100 | | # ifndef MAP_POPULATE |
101 | | # define MAP_POPULATE 0 |
102 | | # endif |
103 | | # if defined(_SC_PAGESIZE) || (_SC_PAGE_SIZE) |
104 | 16 | # define REAL_PAGE_SIZE _real_page_size |
105 | | static size_t _real_page_size = ZEND_MM_PAGE_SIZE; |
106 | | # endif |
107 | | # ifdef MAP_ALIGNED_SUPER |
108 | | # define MAP_HUGETLB MAP_ALIGNED_SUPER |
109 | | # endif |
110 | | #endif |
111 | | |
112 | | #ifndef REAL_PAGE_SIZE |
113 | | # define REAL_PAGE_SIZE ZEND_MM_PAGE_SIZE |
114 | | #endif |
115 | | |
116 | | /* NetBSD has an mremap() function with a signature that is incompatible with Linux (WTF?), |
117 | | * so pretend it doesn't exist. */ |
118 | | #ifndef __linux__ |
119 | | # undef HAVE_MREMAP |
120 | | #endif |
121 | | |
122 | | #ifndef __APPLE__ |
123 | 0 | # define ZEND_MM_FD -1 |
124 | | #else |
125 | | # include <mach/vm_statistics.h> |
126 | | /* Mac allows to track anonymous page via vmmap per TAG id. |
127 | | * user land applications are allowed to take from 240 to 255. |
128 | | */ |
129 | | # define ZEND_MM_FD VM_MAKE_TAG(250U) |
130 | | #endif |
131 | | |
132 | | #ifndef ZEND_MM_STAT |
133 | | # define ZEND_MM_STAT 1 /* track current and peak memory usage */ |
134 | | #endif |
135 | | #ifndef ZEND_MM_LIMIT |
136 | | # define ZEND_MM_LIMIT 1 /* support for user-defined memory limit */ |
137 | | #endif |
138 | | #ifndef ZEND_MM_CUSTOM |
139 | | # define ZEND_MM_CUSTOM 1 /* support for custom memory allocator */ |
140 | | /* USE_ZEND_ALLOC=0 may switch to system malloc() */ |
141 | | #endif |
142 | | #ifndef ZEND_MM_STORAGE |
143 | | # define ZEND_MM_STORAGE 1 /* support for custom memory storage */ |
144 | | #endif |
145 | | #ifndef ZEND_MM_ERROR |
146 | | # define ZEND_MM_ERROR 1 /* report system errors */ |
147 | | #endif |
148 | | #ifndef ZEND_MM_HEAP_PROTECTION |
149 | | # define ZEND_MM_HEAP_PROTECTION 1 /* protect heap against corruptions */ |
150 | | #endif |
151 | | |
152 | | #if ZEND_MM_HEAP_PROTECTION |
153 | | /* Define ZEND_MM_MIN_USEABLE_BIN_SIZE to the size of two pointers */ |
154 | | # if UINTPTR_MAX == UINT64_MAX |
155 | 0 | # define ZEND_MM_MIN_USEABLE_BIN_SIZE 16 |
156 | | # elif UINTPTR_MAX == UINT32_MAX |
157 | | # define ZEND_MM_MIN_USEABLE_BIN_SIZE 8 |
158 | | # else |
159 | | # error |
160 | | # endif |
161 | | # if ZEND_MM_MIN_USEABLE_BIN_SIZE < ZEND_MM_MIN_SMALL_SIZE |
162 | | # error |
163 | | # endif |
164 | | #else /* ZEND_MM_HEAP_PROTECTION */ |
165 | | # define ZEND_MM_MIN_USEABLE_BIN_SIZE ZEND_MM_MIN_SMALL_SIZE |
166 | | #endif /* ZEND_MM_HEAP_PROTECTION */ |
167 | | |
168 | | #ifndef ZEND_MM_CHECK |
169 | 0 | # define ZEND_MM_CHECK(condition, message) do { \ |
170 | 0 | if (UNEXPECTED(!(condition))) { \ |
171 | 0 | zend_mm_panic(message); \ |
172 | 0 | } \ |
173 | 0 | } while (0) |
174 | | #endif |
175 | | |
176 | | typedef uint32_t zend_mm_page_info; /* 4-byte integer */ |
177 | | typedef zend_ulong zend_mm_bitset; /* 4-byte or 8-byte integer */ |
178 | | |
179 | | #define ZEND_MM_ALIGNED_OFFSET(size, alignment) \ |
180 | 0 | (((size_t)(size)) & ((alignment) - 1)) |
181 | | #define ZEND_MM_ALIGNED_BASE(size, alignment) \ |
182 | 0 | (((size_t)(size)) & ~((alignment) - 1)) |
183 | | #define ZEND_MM_SIZE_TO_NUM(size, alignment) \ |
184 | 0 | (((size_t)(size) + ((alignment) - 1)) / (alignment)) |
185 | | |
186 | 0 | #define ZEND_MM_BITSET_LEN (sizeof(zend_mm_bitset) * 8) /* 32 or 64 */ |
187 | | #define ZEND_MM_PAGE_MAP_LEN (ZEND_MM_PAGES / ZEND_MM_BITSET_LEN) /* 16 or 8 */ |
188 | | |
189 | | typedef zend_mm_bitset zend_mm_page_map[ZEND_MM_PAGE_MAP_LEN]; /* 64B */ |
190 | | |
191 | | #define ZEND_MM_IS_FRUN 0x00000000 |
192 | 0 | #define ZEND_MM_IS_LRUN 0x40000000 |
193 | 0 | #define ZEND_MM_IS_SRUN 0x80000000 |
194 | | |
195 | 0 | #define ZEND_MM_LRUN_PAGES_MASK 0x000003ff |
196 | 0 | #define ZEND_MM_LRUN_PAGES_OFFSET 0 |
197 | | |
198 | 0 | #define ZEND_MM_SRUN_BIN_NUM_MASK 0x0000001f |
199 | 0 | #define ZEND_MM_SRUN_BIN_NUM_OFFSET 0 |
200 | | |
201 | 0 | #define ZEND_MM_SRUN_FREE_COUNTER_MASK 0x01ff0000 |
202 | 0 | #define ZEND_MM_SRUN_FREE_COUNTER_OFFSET 16 |
203 | | |
204 | 0 | #define ZEND_MM_NRUN_OFFSET_MASK 0x01ff0000 |
205 | 0 | #define ZEND_MM_NRUN_OFFSET_OFFSET 16 |
206 | | |
207 | 0 | #define ZEND_MM_LRUN_PAGES(info) (((info) & ZEND_MM_LRUN_PAGES_MASK) >> ZEND_MM_LRUN_PAGES_OFFSET) |
208 | 0 | #define ZEND_MM_SRUN_BIN_NUM(info) (((info) & ZEND_MM_SRUN_BIN_NUM_MASK) >> ZEND_MM_SRUN_BIN_NUM_OFFSET) |
209 | 0 | #define ZEND_MM_SRUN_FREE_COUNTER(info) (((info) & ZEND_MM_SRUN_FREE_COUNTER_MASK) >> ZEND_MM_SRUN_FREE_COUNTER_OFFSET) |
210 | 0 | #define ZEND_MM_NRUN_OFFSET(info) (((info) & ZEND_MM_NRUN_OFFSET_MASK) >> ZEND_MM_NRUN_OFFSET_OFFSET) |
211 | | |
212 | | #define ZEND_MM_FRUN() ZEND_MM_IS_FRUN |
213 | 0 | #define ZEND_MM_LRUN(count) (ZEND_MM_IS_LRUN | ((count) << ZEND_MM_LRUN_PAGES_OFFSET)) |
214 | 0 | #define ZEND_MM_SRUN(bin_num) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET)) |
215 | 0 | #define ZEND_MM_SRUN_EX(bin_num, count) (ZEND_MM_IS_SRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((count) << ZEND_MM_SRUN_FREE_COUNTER_OFFSET)) |
216 | 0 | #define ZEND_MM_NRUN(bin_num, offset) (ZEND_MM_IS_SRUN | ZEND_MM_IS_LRUN | ((bin_num) << ZEND_MM_SRUN_BIN_NUM_OFFSET) | ((offset) << ZEND_MM_NRUN_OFFSET_OFFSET)) |
217 | | |
218 | 0 | #define ZEND_MM_BINS 30 |
219 | | |
220 | | #if UINTPTR_MAX == UINT64_MAX |
221 | 0 | # define BSWAPPTR(u) ZEND_BYTES_SWAP64(u) |
222 | | #else |
223 | | # define BSWAPPTR(u) ZEND_BYTES_SWAP32(u) |
224 | | #endif |
225 | | |
226 | | typedef struct _zend_mm_page zend_mm_page; |
227 | | typedef struct _zend_mm_bin zend_mm_bin; |
228 | | typedef struct _zend_mm_free_slot zend_mm_free_slot; |
229 | | typedef struct _zend_mm_chunk zend_mm_chunk; |
230 | | typedef struct _zend_mm_huge_list zend_mm_huge_list; |
231 | | |
232 | | static bool zend_mm_use_huge_pages = false; |
233 | | |
234 | | /* |
235 | | * Memory is retrieved from OS by chunks of fixed size 2MB. |
236 | | * Inside chunk it's managed by pages of fixed size 4096B. |
237 | | * So each chunk consists from 512 pages. |
238 | | * The first page of each chunk is reserved for chunk header. |
239 | | * It contains service information about all pages. |
240 | | * |
241 | | * free_pages - current number of free pages in this chunk |
242 | | * |
243 | | * free_tail - number of continuous free pages at the end of chunk |
244 | | * |
245 | | * free_map - bitset (a bit for each page). The bit is set if the corresponding |
246 | | * page is allocated. Allocator for "large sizes" may easily find a |
247 | | * free page (or a continuous number of pages) searching for zero |
248 | | * bits. |
249 | | * |
250 | | * map - contains service information for each page. (32-bits for each |
251 | | * page). |
252 | | * usage: |
253 | | * (2 bits) |
254 | | * FRUN - free page, |
255 | | * LRUN - first page of "large" allocation |
256 | | * SRUN - first page of a bin used for "small" allocation |
257 | | * |
258 | | * lrun_pages: |
259 | | * (10 bits) number of allocated pages |
260 | | * |
261 | | * srun_bin_num: |
262 | | * (5 bits) bin number (e.g. 0 for sizes 0-2, 1 for 3-4, |
263 | | * 2 for 5-8, 3 for 9-16 etc) see zend_alloc_sizes.h |
264 | | */ |
265 | | |
266 | | struct _zend_mm_heap { |
267 | | #if ZEND_MM_CUSTOM |
268 | | int use_custom_heap; |
269 | | #endif |
270 | | #if ZEND_MM_STORAGE |
271 | | zend_mm_storage *storage; |
272 | | #endif |
273 | | #if ZEND_MM_STAT |
274 | | size_t size; /* current memory usage */ |
275 | | size_t peak; /* peak memory usage */ |
276 | | #endif |
277 | | uintptr_t shadow_key; /* free slot shadow ptr xor key */ |
278 | | zend_mm_free_slot *free_slot[ZEND_MM_BINS]; /* free lists for small sizes */ |
279 | | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
280 | | size_t real_size; /* current size of allocated pages */ |
281 | | #endif |
282 | | #if ZEND_MM_STAT |
283 | | size_t real_peak; /* peak size of allocated pages */ |
284 | | #endif |
285 | | #if ZEND_MM_LIMIT |
286 | | size_t limit; /* memory limit */ |
287 | | int overflow; /* memory overflow flag */ |
288 | | #endif |
289 | | |
290 | | zend_mm_huge_list *huge_list; /* list of huge allocated blocks */ |
291 | | |
292 | | zend_mm_chunk *main_chunk; |
293 | | zend_mm_chunk *cached_chunks; /* list of unused chunks */ |
294 | | int chunks_count; /* number of allocated chunks */ |
295 | | int peak_chunks_count; /* peak number of allocated chunks for current request */ |
296 | | int cached_chunks_count; /* number of cached chunks */ |
297 | | double avg_chunks_count; /* average number of chunks allocated per request */ |
298 | | int last_chunks_delete_boundary; /* number of chunks after last deletion */ |
299 | | int last_chunks_delete_count; /* number of deletion over the last boundary */ |
300 | | #if ZEND_MM_CUSTOM |
301 | | struct { |
302 | | void *(*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
303 | | void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
304 | | void *(*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
305 | | size_t (*_gc)(void); |
306 | | void (*_shutdown)(bool full, bool silent); |
307 | | } custom_heap; |
308 | | union { |
309 | | HashTable *tracked_allocs; |
310 | | struct { |
311 | | bool poison_alloc; |
312 | | uint8_t poison_alloc_value; |
313 | | bool poison_free; |
314 | | uint8_t poison_free_value; |
315 | | uint8_t padding; |
316 | | bool check_freelists_on_shutdown; |
317 | | } debug; |
318 | | }; |
319 | | #endif |
320 | | pid_t pid; |
321 | | zend_random_bytes_insecure_state rand_state; |
322 | | }; |
323 | | |
324 | | struct _zend_mm_chunk { |
325 | | zend_mm_heap *heap; |
326 | | zend_mm_chunk *next; |
327 | | zend_mm_chunk *prev; |
328 | | uint32_t free_pages; /* number of free pages */ |
329 | | uint32_t free_tail; /* number of free pages at the end of chunk */ |
330 | | uint32_t num; |
331 | | char reserve[64 - (sizeof(void*) * 3 + sizeof(uint32_t) * 3)]; |
332 | | zend_mm_heap heap_slot; /* used only in main chunk */ |
333 | | zend_mm_page_map free_map; /* 512 bits or 64 bytes */ |
334 | | zend_mm_page_info map[ZEND_MM_PAGES]; /* 2 KB = 512 * 4 */ |
335 | | }; |
336 | | |
337 | | struct _zend_mm_page { |
338 | | char bytes[ZEND_MM_PAGE_SIZE]; |
339 | | }; |
340 | | |
341 | | /* |
342 | | * bin - is one or few continuous pages (up to 8) used for allocation of |
343 | | * a particular "small size". |
344 | | */ |
345 | | struct _zend_mm_bin { |
346 | | char bytes[ZEND_MM_PAGE_SIZE * 8]; |
347 | | }; |
348 | | |
349 | | struct _zend_mm_free_slot { |
350 | | zend_mm_free_slot *next_free_slot; |
351 | | }; |
352 | | |
353 | | struct _zend_mm_huge_list { |
354 | | void *ptr; |
355 | | size_t size; |
356 | | zend_mm_huge_list *next; |
357 | | #if ZEND_DEBUG |
358 | | zend_mm_debug_info dbg; |
359 | | #endif |
360 | | }; |
361 | | |
362 | | #define ZEND_MM_PAGE_ADDR(chunk, page_num) \ |
363 | 0 | ((void*)(((zend_mm_page*)(chunk)) + (page_num))) |
364 | | |
365 | | #define _BIN_DATA_SIZE(num, size, elements, pages, x, y) size, |
366 | | static const uint32_t bin_data_size[] = { |
367 | | ZEND_MM_BINS_INFO(_BIN_DATA_SIZE, x, y) |
368 | | }; |
369 | | |
370 | | #define _BIN_DATA_ELEMENTS(num, size, elements, pages, x, y) elements, |
371 | | static const uint32_t bin_elements[] = { |
372 | | ZEND_MM_BINS_INFO(_BIN_DATA_ELEMENTS, x, y) |
373 | | }; |
374 | | |
375 | | #define _BIN_DATA_PAGES(num, size, elements, pages, x, y) pages, |
376 | | static const uint32_t bin_pages[] = { |
377 | | ZEND_MM_BINS_INFO(_BIN_DATA_PAGES, x, y) |
378 | | }; |
379 | | |
380 | | #if ZEND_DEBUG |
381 | | ZEND_COLD void zend_debug_alloc_output(char *format, ...) |
382 | 0 | { |
383 | 0 | char output_buf[256]; |
384 | 0 | va_list args; |
385 | |
|
386 | 0 | va_start(args, format); |
387 | 0 | vsprintf(output_buf, format, args); |
388 | 0 | va_end(args); |
389 | |
|
390 | | #ifdef ZEND_WIN32 |
391 | | OutputDebugString(output_buf); |
392 | | #else |
393 | 0 | fprintf(stderr, "%s", output_buf); |
394 | 0 | #endif |
395 | 0 | } |
396 | | #endif |
397 | | |
398 | | static ZEND_COLD ZEND_NORETURN void zend_mm_panic(const char *message) |
399 | 0 | { |
400 | 0 | fprintf(stderr, "%s\n", message); |
401 | | /* See http://support.microsoft.com/kb/190351 */ |
402 | | #ifdef ZEND_WIN32 |
403 | | fflush(stderr); |
404 | | #endif |
405 | 0 | #if ZEND_DEBUG && defined(HAVE_KILL) && defined(HAVE_GETPID) |
406 | 0 | kill(getpid(), SIGSEGV); |
407 | 0 | #endif |
408 | 0 | abort(); |
409 | 0 | } |
410 | | |
411 | | static ZEND_COLD ZEND_NORETURN void zend_mm_safe_error(zend_mm_heap *heap, |
412 | | const char *format, |
413 | | size_t limit, |
414 | | #if ZEND_DEBUG |
415 | | const char *filename, |
416 | | uint32_t lineno, |
417 | | #endif |
418 | | size_t size) |
419 | 548 | { |
420 | | |
421 | 548 | heap->overflow = 1; |
422 | 548 | zend_try { |
423 | 548 | zend_error_noreturn(E_ERROR, |
424 | 548 | format, |
425 | 548 | limit, |
426 | 548 | #if ZEND_DEBUG |
427 | 548 | filename, |
428 | 548 | lineno, |
429 | 548 | #endif |
430 | 548 | size); |
431 | 548 | } zend_catch { |
432 | 0 | } zend_end_try(); |
433 | 0 | heap->overflow = 0; |
434 | 0 | zend_bailout(); |
435 | 0 | exit(1); |
436 | 548 | } |
437 | | |
438 | | #ifdef _WIN32 |
439 | | static void stderr_last_error(char *msg) |
440 | | { |
441 | | DWORD err = GetLastError(); |
442 | | char *buf = php_win32_error_to_msg(err); |
443 | | |
444 | | if (!buf[0]) { |
445 | | fprintf(stderr, "\n%s: [0x%08lx]\n", msg, err); |
446 | | } |
447 | | else { |
448 | | fprintf(stderr, "\n%s: [0x%08lx] %s\n", msg, err, buf); |
449 | | } |
450 | | |
451 | | php_win32_error_msg_free(buf); |
452 | | } |
453 | | #endif |
454 | | |
455 | | /*****************/ |
456 | | /* OS Allocation */ |
457 | | /*****************/ |
458 | | |
459 | | static void zend_mm_munmap(void *addr, size_t size) |
460 | 0 | { |
461 | | #ifdef _WIN32 |
462 | | if (VirtualFree(addr, 0, MEM_RELEASE) == 0) { |
463 | | /** ERROR_INVALID_ADDRESS is expected when addr is not range start address */ |
464 | | if (GetLastError() != ERROR_INVALID_ADDRESS) { |
465 | | #if ZEND_MM_ERROR |
466 | | stderr_last_error("VirtualFree() failed"); |
467 | | #endif |
468 | | return; |
469 | | } |
470 | | SetLastError(0); |
471 | | |
472 | | MEMORY_BASIC_INFORMATION mbi; |
473 | | if (VirtualQuery(addr, &mbi, sizeof(mbi)) == 0) { |
474 | | #if ZEND_MM_ERROR |
475 | | stderr_last_error("VirtualQuery() failed"); |
476 | | #endif |
477 | | return; |
478 | | } |
479 | | addr = mbi.AllocationBase; |
480 | | |
481 | | if (VirtualFree(addr, 0, MEM_RELEASE) == 0) { |
482 | | #if ZEND_MM_ERROR |
483 | | stderr_last_error("VirtualFree() failed"); |
484 | | #endif |
485 | | } |
486 | | } |
487 | | #else |
488 | 0 | if (munmap(addr, size) != 0) { |
489 | 0 | #if ZEND_MM_ERROR |
490 | 0 | fprintf(stderr, "\nmunmap() failed: [%d] %s\n", errno, strerror(errno)); |
491 | 0 | #endif |
492 | 0 | } |
493 | 0 | #endif |
494 | 0 | } |
495 | | |
496 | | #ifndef HAVE_MREMAP |
497 | | static void *zend_mm_mmap_fixed(void *addr, size_t size) |
498 | | { |
499 | | #ifdef _WIN32 |
500 | | void *ptr = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); |
501 | | |
502 | | if (ptr == NULL) { |
503 | | /** ERROR_INVALID_ADDRESS is expected when fixed addr range is not free */ |
504 | | if (GetLastError() != ERROR_INVALID_ADDRESS) { |
505 | | #if ZEND_MM_ERROR |
506 | | stderr_last_error("VirtualAlloc() fixed failed"); |
507 | | #endif |
508 | | } |
509 | | SetLastError(0); |
510 | | return NULL; |
511 | | } |
512 | | ZEND_ASSERT(ptr == addr); |
513 | | return ptr; |
514 | | #else |
515 | | int flags = MAP_PRIVATE | MAP_ANON; |
516 | | #if defined(MAP_EXCL) |
517 | | flags |= MAP_FIXED | MAP_EXCL; |
518 | | #elif defined(MAP_TRYFIXED) |
519 | | flags |= MAP_TRYFIXED; |
520 | | #endif |
521 | | /* MAP_FIXED leads to discarding of the old mapping, so it can't be used. */ |
522 | | void *ptr = mmap(addr, size, PROT_READ | PROT_WRITE, flags /*| MAP_POPULATE | MAP_HUGETLB*/, ZEND_MM_FD, 0); |
523 | | |
524 | | if (ptr == MAP_FAILED) { |
525 | | #if ZEND_MM_ERROR && !defined(MAP_EXCL) && !defined(MAP_TRYFIXED) |
526 | | fprintf(stderr, "\nmmap() fixed failed: [%d] %s\n", errno, strerror(errno)); |
527 | | #endif |
528 | | return NULL; |
529 | | } else if (ptr != addr) { |
530 | | zend_mm_munmap(ptr, size); |
531 | | return NULL; |
532 | | } |
533 | | return ptr; |
534 | | #endif |
535 | | } |
536 | | #endif |
537 | | |
538 | | static void *zend_mm_mmap(size_t size) |
539 | 0 | { |
540 | | #ifdef _WIN32 |
541 | | void *ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); |
542 | | |
543 | | if (ptr == NULL) { |
544 | | #if ZEND_MM_ERROR |
545 | | stderr_last_error("VirtualAlloc() failed"); |
546 | | #endif |
547 | | return NULL; |
548 | | } |
549 | | return ptr; |
550 | | #else |
551 | 0 | void *ptr; |
552 | |
|
553 | 0 | #if defined(MAP_HUGETLB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) |
554 | 0 | if (zend_mm_use_huge_pages && size == ZEND_MM_CHUNK_SIZE) { |
555 | 0 | int fd = -1; |
556 | 0 | int mflags = MAP_PRIVATE | MAP_ANON; |
557 | 0 | #if defined(MAP_HUGETLB) |
558 | 0 | mflags |= MAP_HUGETLB; |
559 | | #else |
560 | | fd = VM_FLAGS_SUPERPAGE_SIZE_2MB; |
561 | | #endif |
562 | 0 | ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, mflags, fd, 0); |
563 | 0 | if (ptr != MAP_FAILED) { |
564 | 0 | zend_mmap_set_name(ptr, size, "zend_alloc"); |
565 | 0 | return ptr; |
566 | 0 | } |
567 | 0 | } |
568 | 0 | #endif |
569 | | |
570 | 0 | ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, ZEND_MM_FD, 0); |
571 | |
|
572 | 0 | if (ptr == MAP_FAILED) { |
573 | 0 | #if ZEND_MM_ERROR |
574 | 0 | fprintf(stderr, "\nmmap() failed: [%d] %s\n", errno, strerror(errno)); |
575 | 0 | #endif |
576 | 0 | return NULL; |
577 | 0 | } |
578 | 0 | zend_mmap_set_name(ptr, size, "zend_alloc"); |
579 | 0 | return ptr; |
580 | 0 | #endif |
581 | 0 | } |
582 | | |
583 | | /***********/ |
584 | | /* Bitmask */ |
585 | | /***********/ |
586 | | |
587 | | /* number of trailing set (1) bits */ |
588 | | ZEND_ATTRIBUTE_CONST static zend_always_inline int zend_mm_bitset_nts(zend_mm_bitset bitset) |
589 | 0 | { |
590 | 0 | #if (defined(__GNUC__) || __has_builtin(__builtin_ctzl)) && SIZEOF_ZEND_LONG == SIZEOF_LONG && defined(PHP_HAVE_BUILTIN_CTZL) |
591 | 0 | return __builtin_ctzl(~bitset); |
592 | | #elif (defined(__GNUC__) || __has_builtin(__builtin_ctzll)) && defined(PHP_HAVE_BUILTIN_CTZLL) |
593 | | return __builtin_ctzll(~bitset); |
594 | | #elif defined(_WIN32) |
595 | | unsigned long index; |
596 | | |
597 | | #if defined(_WIN64) |
598 | | if (!BitScanForward64(&index, ~bitset)) { |
599 | | #else |
600 | | if (!BitScanForward(&index, ~bitset)) { |
601 | | #endif |
602 | | /* undefined behavior */ |
603 | | return 32; |
604 | | } |
605 | | |
606 | | return (int)index; |
607 | | #else |
608 | | int n; |
609 | | |
610 | | if (bitset == (zend_mm_bitset)-1) return ZEND_MM_BITSET_LEN; |
611 | | |
612 | | n = 0; |
613 | | #if SIZEOF_ZEND_LONG == 8 |
614 | | if (sizeof(zend_mm_bitset) == 8) { |
615 | | if ((bitset & 0xffffffff) == 0xffffffff) {n += 32; bitset = bitset >> Z_UL(32);} |
616 | | } |
617 | | #endif |
618 | | if ((bitset & 0x0000ffff) == 0x0000ffff) {n += 16; bitset = bitset >> 16;} |
619 | | if ((bitset & 0x000000ff) == 0x000000ff) {n += 8; bitset = bitset >> 8;} |
620 | | if ((bitset & 0x0000000f) == 0x0000000f) {n += 4; bitset = bitset >> 4;} |
621 | | if ((bitset & 0x00000003) == 0x00000003) {n += 2; bitset = bitset >> 2;} |
622 | | return n + (bitset & 1); |
623 | | #endif |
624 | 0 | } |
625 | | |
626 | | static zend_always_inline int zend_mm_bitset_is_set(zend_mm_bitset *bitset, int bit) |
627 | 0 | { |
628 | 0 | return ZEND_BIT_TEST(bitset, bit); |
629 | 0 | } |
630 | | |
631 | | static zend_always_inline void zend_mm_bitset_set_bit(zend_mm_bitset *bitset, int bit) |
632 | 0 | { |
633 | 0 | bitset[bit / ZEND_MM_BITSET_LEN] |= (Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1))); |
634 | 0 | } |
635 | | |
636 | | static zend_always_inline void zend_mm_bitset_reset_bit(zend_mm_bitset *bitset, int bit) |
637 | 0 | { |
638 | 0 | bitset[bit / ZEND_MM_BITSET_LEN] &= ~(Z_UL(1) << (bit & (ZEND_MM_BITSET_LEN-1))); |
639 | 0 | } |
640 | | |
641 | | static zend_always_inline void zend_mm_bitset_set_range(zend_mm_bitset *bitset, int start, int len) |
642 | 0 | { |
643 | 0 | if (len == 1) { |
644 | 0 | zend_mm_bitset_set_bit(bitset, start); |
645 | 0 | } else { |
646 | 0 | int pos = start / ZEND_MM_BITSET_LEN; |
647 | 0 | int end = (start + len - 1) / ZEND_MM_BITSET_LEN; |
648 | 0 | int bit = start & (ZEND_MM_BITSET_LEN - 1); |
649 | 0 | zend_mm_bitset tmp; |
650 | |
|
651 | 0 | if (pos != end) { |
652 | | /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */ |
653 | 0 | tmp = (zend_mm_bitset)-1 << bit; |
654 | 0 | bitset[pos++] |= tmp; |
655 | 0 | while (pos != end) { |
656 | | /* set all bits */ |
657 | 0 | bitset[pos++] = (zend_mm_bitset)-1; |
658 | 0 | } |
659 | 0 | end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1); |
660 | | /* set bits from "0" to "end" */ |
661 | 0 | tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end); |
662 | 0 | bitset[pos] |= tmp; |
663 | 0 | } else { |
664 | 0 | end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1); |
665 | | /* set bits from "bit" to "end" */ |
666 | 0 | tmp = (zend_mm_bitset)-1 << bit; |
667 | 0 | tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end); |
668 | 0 | bitset[pos] |= tmp; |
669 | 0 | } |
670 | 0 | } |
671 | 0 | } |
672 | | |
673 | | static zend_always_inline void zend_mm_bitset_reset_range(zend_mm_bitset *bitset, int start, int len) |
674 | 0 | { |
675 | 0 | if (len == 1) { |
676 | 0 | zend_mm_bitset_reset_bit(bitset, start); |
677 | 0 | } else { |
678 | 0 | int pos = start / ZEND_MM_BITSET_LEN; |
679 | 0 | int end = (start + len - 1) / ZEND_MM_BITSET_LEN; |
680 | 0 | int bit = start & (ZEND_MM_BITSET_LEN - 1); |
681 | 0 | zend_mm_bitset tmp; |
682 | |
|
683 | 0 | if (pos != end) { |
684 | | /* reset bits from "bit" to ZEND_MM_BITSET_LEN-1 */ |
685 | 0 | tmp = ~((Z_UL(1) << bit) - 1); |
686 | 0 | bitset[pos++] &= ~tmp; |
687 | 0 | while (pos != end) { |
688 | | /* set all bits */ |
689 | 0 | bitset[pos++] = 0; |
690 | 0 | } |
691 | 0 | end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1); |
692 | | /* reset bits from "0" to "end" */ |
693 | 0 | tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end); |
694 | 0 | bitset[pos] &= ~tmp; |
695 | 0 | } else { |
696 | 0 | end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1); |
697 | | /* reset bits from "bit" to "end" */ |
698 | 0 | tmp = (zend_mm_bitset)-1 << bit; |
699 | 0 | tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end); |
700 | 0 | bitset[pos] &= ~tmp; |
701 | 0 | } |
702 | 0 | } |
703 | 0 | } |
704 | | |
705 | | static zend_always_inline int zend_mm_bitset_is_free_range(zend_mm_bitset *bitset, int start, int len) |
706 | 0 | { |
707 | 0 | if (len == 1) { |
708 | 0 | return !zend_mm_bitset_is_set(bitset, start); |
709 | 0 | } else { |
710 | 0 | int pos = start / ZEND_MM_BITSET_LEN; |
711 | 0 | int end = (start + len - 1) / ZEND_MM_BITSET_LEN; |
712 | 0 | int bit = start & (ZEND_MM_BITSET_LEN - 1); |
713 | 0 | zend_mm_bitset tmp; |
714 | |
|
715 | 0 | if (pos != end) { |
716 | | /* set bits from "bit" to ZEND_MM_BITSET_LEN-1 */ |
717 | 0 | tmp = (zend_mm_bitset)-1 << bit; |
718 | 0 | if ((bitset[pos++] & tmp) != 0) { |
719 | 0 | return 0; |
720 | 0 | } |
721 | 0 | while (pos != end) { |
722 | | /* set all bits */ |
723 | 0 | if (bitset[pos++] != 0) { |
724 | 0 | return 0; |
725 | 0 | } |
726 | 0 | } |
727 | 0 | end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1); |
728 | | /* set bits from "0" to "end" */ |
729 | 0 | tmp = (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end); |
730 | 0 | return (bitset[pos] & tmp) == 0; |
731 | 0 | } else { |
732 | 0 | end = (start + len - 1) & (ZEND_MM_BITSET_LEN - 1); |
733 | | /* set bits from "bit" to "end" */ |
734 | 0 | tmp = (zend_mm_bitset)-1 << bit; |
735 | 0 | tmp &= (zend_mm_bitset)-1 >> ((ZEND_MM_BITSET_LEN - 1) - end); |
736 | 0 | return (bitset[pos] & tmp) == 0; |
737 | 0 | } |
738 | 0 | } |
739 | 0 | } |
740 | | |
741 | | /**********/ |
742 | | /* Chunks */ |
743 | | /**********/ |
744 | | |
745 | | static zend_always_inline void zend_mm_hugepage(void* ptr, size_t size) |
746 | 0 | { |
747 | 0 | #if defined(MADV_HUGEPAGE) |
748 | 0 | (void)madvise(ptr, size, MADV_HUGEPAGE); |
749 | | #elif defined(HAVE_MEMCNTL) |
750 | | struct memcntl_mha m = {.mha_cmd = MHA_MAPSIZE_VA, .mha_pagesize = ZEND_MM_CHUNK_SIZE, .mha_flags = 0}; |
751 | | (void)memcntl(ptr, size, MC_HAT_ADVISE, (char *)&m, 0, 0); |
752 | | #elif !defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) && !defined(MAP_ALIGNED_SUPER) |
753 | | zend_error_noreturn(E_ERROR, "huge_pages: thp unsupported on this platform"); |
754 | | #endif |
755 | 0 | } |
756 | | |
757 | | static void *zend_mm_chunk_alloc_int(size_t size, size_t alignment) |
758 | 0 | { |
759 | 0 | void *ptr = zend_mm_mmap(size); |
760 | |
|
761 | 0 | if (ptr == NULL) { |
762 | 0 | return NULL; |
763 | 0 | } else if (ZEND_MM_ALIGNED_OFFSET(ptr, alignment) == 0) { |
764 | 0 | if (zend_mm_use_huge_pages) { |
765 | 0 | zend_mm_hugepage(ptr, size); |
766 | 0 | } |
767 | | #ifdef __SANITIZE_ADDRESS__ |
768 | | ASAN_UNPOISON_MEMORY_REGION(ptr, size); |
769 | | #endif |
770 | 0 | return ptr; |
771 | 0 | } else { |
772 | 0 | size_t offset; |
773 | | |
774 | | /* chunk has to be aligned */ |
775 | 0 | zend_mm_munmap(ptr, size); |
776 | 0 | ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE); |
777 | | #ifdef _WIN32 |
778 | | offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment); |
779 | | if (offset != 0) { |
780 | | offset = alignment - offset; |
781 | | } |
782 | | zend_mm_munmap(ptr, size + alignment - REAL_PAGE_SIZE); |
783 | | ptr = zend_mm_mmap_fixed((void*)((char*)ptr + offset), size); |
784 | | if (ptr == NULL) { // fix GH-9650, fixed addr range is not free |
785 | | ptr = zend_mm_mmap(size + alignment - REAL_PAGE_SIZE); |
786 | | if (ptr == NULL) { |
787 | | return NULL; |
788 | | } |
789 | | offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment); |
790 | | if (offset != 0) { |
791 | | ptr = (void*)((char*)ptr + alignment - offset); |
792 | | } |
793 | | } |
794 | | return ptr; |
795 | | #else |
796 | 0 | offset = ZEND_MM_ALIGNED_OFFSET(ptr, alignment); |
797 | 0 | if (offset != 0) { |
798 | 0 | offset = alignment - offset; |
799 | 0 | zend_mm_munmap(ptr, offset); |
800 | 0 | ptr = (char*)ptr + offset; |
801 | 0 | alignment -= offset; |
802 | 0 | } |
803 | 0 | if (alignment > REAL_PAGE_SIZE) { |
804 | 0 | zend_mm_munmap((char*)ptr + size, alignment - REAL_PAGE_SIZE); |
805 | 0 | } |
806 | 0 | if (zend_mm_use_huge_pages) { |
807 | 0 | zend_mm_hugepage(ptr, size); |
808 | 0 | } |
809 | | # ifdef __SANITIZE_ADDRESS__ |
810 | | ASAN_UNPOISON_MEMORY_REGION(ptr, size); |
811 | | # endif |
812 | 0 | #endif |
813 | 0 | return ptr; |
814 | 0 | } |
815 | 0 | } |
816 | | |
817 | | static void *zend_mm_chunk_alloc(zend_mm_heap *heap, size_t size, size_t alignment) |
818 | 0 | { |
819 | 0 | #if ZEND_MM_STORAGE |
820 | 0 | if (UNEXPECTED(heap->storage)) { |
821 | 0 | void *ptr = heap->storage->handlers.chunk_alloc(heap->storage, size, alignment); |
822 | 0 | ZEND_ASSERT(((uintptr_t)((char*)ptr + (alignment-1)) & (alignment-1)) == (uintptr_t)ptr); |
823 | 0 | return ptr; |
824 | 0 | } |
825 | 0 | #endif |
826 | 0 | return zend_mm_chunk_alloc_int(size, alignment); |
827 | 0 | } |
828 | | |
829 | | static void zend_mm_chunk_free(zend_mm_heap *heap, void *addr, size_t size) |
830 | 0 | { |
831 | 0 | #if ZEND_MM_STORAGE |
832 | 0 | if (UNEXPECTED(heap->storage)) { |
833 | 0 | heap->storage->handlers.chunk_free(heap->storage, addr, size); |
834 | 0 | return; |
835 | 0 | } |
836 | 0 | #endif |
837 | 0 | zend_mm_munmap(addr, size); |
838 | 0 | } |
839 | | |
840 | | static int zend_mm_chunk_truncate(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size) |
841 | 0 | { |
842 | 0 | #if ZEND_MM_STORAGE |
843 | 0 | if (UNEXPECTED(heap->storage)) { |
844 | 0 | if (heap->storage->handlers.chunk_truncate) { |
845 | 0 | return heap->storage->handlers.chunk_truncate(heap->storage, addr, old_size, new_size); |
846 | 0 | } else { |
847 | 0 | return 0; |
848 | 0 | } |
849 | 0 | } |
850 | 0 | #endif |
851 | 0 | #ifndef _WIN32 |
852 | 0 | zend_mm_munmap((char*)addr + new_size, old_size - new_size); |
853 | 0 | return 1; |
854 | | #else |
855 | | return 0; |
856 | | #endif |
857 | 0 | } |
858 | | |
859 | | static int zend_mm_chunk_extend(zend_mm_heap *heap, void *addr, size_t old_size, size_t new_size) |
860 | 0 | { |
861 | 0 | #if ZEND_MM_STORAGE |
862 | 0 | if (UNEXPECTED(heap->storage)) { |
863 | 0 | if (heap->storage->handlers.chunk_extend) { |
864 | 0 | return heap->storage->handlers.chunk_extend(heap->storage, addr, old_size, new_size); |
865 | 0 | } else { |
866 | 0 | return 0; |
867 | 0 | } |
868 | 0 | } |
869 | 0 | #endif |
870 | 0 | #ifdef HAVE_MREMAP |
871 | | /* We don't use MREMAP_MAYMOVE due to alignment requirements. */ |
872 | 0 | void *ptr = mremap(addr, old_size, new_size, 0); |
873 | 0 | if (ptr == MAP_FAILED) { |
874 | 0 | return 0; |
875 | 0 | } |
876 | | /* Sanity check: The mapping shouldn't have moved. */ |
877 | 0 | ZEND_ASSERT(ptr == addr); |
878 | 0 | return 1; |
879 | | #elif !defined(_WIN32) |
880 | | return (zend_mm_mmap_fixed((char*)addr + old_size, new_size - old_size) != NULL); |
881 | | #else |
882 | | return 0; |
883 | | #endif |
884 | 0 | } |
885 | | |
886 | | static zend_always_inline void zend_mm_chunk_init(zend_mm_heap *heap, zend_mm_chunk *chunk) |
887 | 0 | { |
888 | 0 | chunk->heap = heap; |
889 | 0 | chunk->next = heap->main_chunk; |
890 | 0 | chunk->prev = heap->main_chunk->prev; |
891 | 0 | chunk->prev->next = chunk; |
892 | 0 | chunk->next->prev = chunk; |
893 | | /* mark first pages as allocated */ |
894 | 0 | chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE; |
895 | 0 | chunk->free_tail = ZEND_MM_FIRST_PAGE; |
896 | | /* the younger chunks have bigger number */ |
897 | 0 | chunk->num = chunk->prev->num + 1; |
898 | | /* mark first pages as allocated */ |
899 | 0 | chunk->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1; |
900 | 0 | chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE); |
901 | 0 | } |
902 | | |
903 | | /***********************/ |
904 | | /* Huge Runs (forward) */ |
905 | | /***********************/ |
906 | | |
907 | | static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
908 | | static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
909 | | static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
910 | | |
911 | | #if ZEND_DEBUG |
912 | | static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
913 | | #else |
914 | | static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
915 | | #endif |
916 | | |
917 | | /**************/ |
918 | | /* Large Runs */ |
919 | | /**************/ |
920 | | |
921 | | #if ZEND_DEBUG |
922 | | static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
923 | | #else |
924 | | static void *zend_mm_alloc_pages(zend_mm_heap *heap, uint32_t pages_count ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
925 | | #endif |
926 | 0 | { |
927 | 0 | zend_mm_chunk *chunk = heap->main_chunk; |
928 | 0 | uint32_t page_num, len; |
929 | 0 | int steps = 0; |
930 | |
|
931 | 0 | while (1) { |
932 | 0 | if (UNEXPECTED(chunk->free_pages < pages_count)) { |
933 | 0 | goto not_found; |
934 | | #if 0 |
935 | | } else if (UNEXPECTED(chunk->free_pages + chunk->free_tail == ZEND_MM_PAGES)) { |
936 | | if (UNEXPECTED(ZEND_MM_PAGES - chunk->free_tail < pages_count)) { |
937 | | goto not_found; |
938 | | } else { |
939 | | page_num = chunk->free_tail; |
940 | | goto found; |
941 | | } |
942 | | } else if (0) { |
943 | | /* First-Fit Search */ |
944 | | int free_tail = chunk->free_tail; |
945 | | zend_mm_bitset *bitset = chunk->free_map; |
946 | | zend_mm_bitset tmp = *(bitset++); |
947 | | int i = 0; |
948 | | |
949 | | while (1) { |
950 | | /* skip allocated blocks */ |
951 | | while (tmp == (zend_mm_bitset)-1) { |
952 | | i += ZEND_MM_BITSET_LEN; |
953 | | if (i == ZEND_MM_PAGES) { |
954 | | goto not_found; |
955 | | } |
956 | | tmp = *(bitset++); |
957 | | } |
958 | | /* find first 0 bit */ |
959 | | page_num = i + zend_mm_bitset_nts(tmp); |
960 | | /* reset bits from 0 to "bit" */ |
961 | | tmp &= tmp + 1; |
962 | | /* skip free blocks */ |
963 | | while (tmp == 0) { |
964 | | i += ZEND_MM_BITSET_LEN; |
965 | | len = i - page_num; |
966 | | if (len >= pages_count) { |
967 | | goto found; |
968 | | } else if (i >= free_tail) { |
969 | | goto not_found; |
970 | | } |
971 | | tmp = *(bitset++); |
972 | | } |
973 | | /* find first 1 bit */ |
974 | | len = (i + zend_ulong_ntz(tmp)) - page_num; |
975 | | if (len >= pages_count) { |
976 | | goto found; |
977 | | } |
978 | | /* set bits from 0 to "bit" */ |
979 | | tmp |= tmp - 1; |
980 | | } |
981 | | #endif |
982 | 0 | } else { |
983 | | /* Best-Fit Search */ |
984 | 0 | int best = -1; |
985 | 0 | uint32_t best_len = ZEND_MM_PAGES; |
986 | 0 | uint32_t free_tail = chunk->free_tail; |
987 | 0 | zend_mm_bitset *bitset = chunk->free_map; |
988 | 0 | zend_mm_bitset tmp = *(bitset++); |
989 | 0 | uint32_t i = 0; |
990 | |
|
991 | 0 | while (1) { |
992 | | /* skip allocated blocks */ |
993 | 0 | while (tmp == (zend_mm_bitset)-1) { |
994 | 0 | i += ZEND_MM_BITSET_LEN; |
995 | 0 | if (i == ZEND_MM_PAGES) { |
996 | 0 | if (best > 0) { |
997 | 0 | page_num = best; |
998 | 0 | goto found; |
999 | 0 | } else { |
1000 | 0 | goto not_found; |
1001 | 0 | } |
1002 | 0 | } |
1003 | 0 | tmp = *(bitset++); |
1004 | 0 | } |
1005 | | /* find first 0 bit */ |
1006 | 0 | page_num = i + zend_mm_bitset_nts(tmp); |
1007 | | /* reset bits from 0 to "bit" */ |
1008 | 0 | tmp &= tmp + 1; |
1009 | | /* skip free blocks */ |
1010 | 0 | while (tmp == 0) { |
1011 | 0 | i += ZEND_MM_BITSET_LEN; |
1012 | 0 | if (i >= free_tail || i == ZEND_MM_PAGES) { |
1013 | 0 | len = ZEND_MM_PAGES - page_num; |
1014 | 0 | if (len >= pages_count && len < best_len) { |
1015 | 0 | chunk->free_tail = page_num + pages_count; |
1016 | 0 | goto found; |
1017 | 0 | } else { |
1018 | | /* set accurate value */ |
1019 | 0 | chunk->free_tail = page_num; |
1020 | 0 | if (best > 0) { |
1021 | 0 | page_num = best; |
1022 | 0 | goto found; |
1023 | 0 | } else { |
1024 | 0 | goto not_found; |
1025 | 0 | } |
1026 | 0 | } |
1027 | 0 | } |
1028 | 0 | tmp = *(bitset++); |
1029 | 0 | } |
1030 | | /* find first 1 bit */ |
1031 | 0 | len = i + zend_ulong_ntz(tmp) - page_num; |
1032 | 0 | if (len >= pages_count) { |
1033 | 0 | if (len == pages_count) { |
1034 | 0 | goto found; |
1035 | 0 | } else if (len < best_len) { |
1036 | 0 | best_len = len; |
1037 | 0 | best = page_num; |
1038 | 0 | } |
1039 | 0 | } |
1040 | | /* set bits from 0 to "bit" */ |
1041 | 0 | tmp |= tmp - 1; |
1042 | 0 | } |
1043 | 0 | } |
1044 | | |
1045 | 0 | not_found: |
1046 | 0 | if (chunk->next == heap->main_chunk) { |
1047 | 0 | get_chunk: |
1048 | 0 | if (heap->cached_chunks) { |
1049 | 0 | heap->cached_chunks_count--; |
1050 | 0 | chunk = heap->cached_chunks; |
1051 | 0 | heap->cached_chunks = chunk->next; |
1052 | 0 | } else { |
1053 | 0 | #if ZEND_MM_LIMIT |
1054 | 0 | if (UNEXPECTED(ZEND_MM_CHUNK_SIZE > heap->limit - heap->real_size)) { |
1055 | 0 | if (zend_mm_gc(heap)) { |
1056 | 0 | goto get_chunk; |
1057 | 0 | } else if (heap->overflow == 0) { |
1058 | 0 | #if ZEND_DEBUG |
1059 | 0 | zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size); |
1060 | | #else |
1061 | | zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, ZEND_MM_PAGE_SIZE * pages_count); |
1062 | | #endif |
1063 | 0 | return NULL; |
1064 | 0 | } |
1065 | 0 | } |
1066 | 0 | #endif |
1067 | 0 | chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE); |
1068 | 0 | if (UNEXPECTED(chunk == NULL)) { |
1069 | | /* insufficient memory */ |
1070 | 0 | if (zend_mm_gc(heap) && |
1071 | 0 | (chunk = (zend_mm_chunk*)zend_mm_chunk_alloc(heap, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE)) != NULL) { |
1072 | | /* pass */ |
1073 | 0 | } else { |
1074 | | #if !ZEND_MM_LIMIT |
1075 | | zend_mm_safe_error(heap, "Out of memory"); |
1076 | | #elif ZEND_DEBUG |
1077 | | zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size); |
1078 | | #else |
1079 | | zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, ZEND_MM_PAGE_SIZE * pages_count); |
1080 | | #endif |
1081 | 0 | return NULL; |
1082 | 0 | } |
1083 | 0 | } |
1084 | 0 | #if ZEND_MM_STAT |
1085 | 0 | do { |
1086 | 0 | size_t size = heap->real_size + ZEND_MM_CHUNK_SIZE; |
1087 | 0 | size_t peak = MAX(heap->real_peak, size); |
1088 | 0 | heap->real_size = size; |
1089 | 0 | heap->real_peak = peak; |
1090 | 0 | } while (0); |
1091 | | #elif ZEND_MM_LIMIT |
1092 | | heap->real_size += ZEND_MM_CHUNK_SIZE; |
1093 | | |
1094 | | #endif |
1095 | 0 | } |
1096 | 0 | heap->chunks_count++; |
1097 | 0 | if (heap->chunks_count > heap->peak_chunks_count) { |
1098 | 0 | heap->peak_chunks_count = heap->chunks_count; |
1099 | 0 | } |
1100 | 0 | zend_mm_chunk_init(heap, chunk); |
1101 | 0 | page_num = ZEND_MM_FIRST_PAGE; |
1102 | 0 | len = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE; |
1103 | 0 | goto found; |
1104 | 0 | } else { |
1105 | 0 | chunk = chunk->next; |
1106 | 0 | steps++; |
1107 | 0 | } |
1108 | 0 | } |
1109 | | |
1110 | 0 | found: |
1111 | 0 | if (steps > 2 && pages_count < 8) { |
1112 | 0 | ZEND_MM_CHECK(chunk->next->prev == chunk, "zend_mm_heap corrupted"); |
1113 | 0 | ZEND_MM_CHECK(chunk->prev->next == chunk, "zend_mm_heap corrupted"); |
1114 | | |
1115 | | /* move chunk into the head of the linked-list */ |
1116 | 0 | chunk->prev->next = chunk->next; |
1117 | 0 | chunk->next->prev = chunk->prev; |
1118 | 0 | chunk->next = heap->main_chunk->next; |
1119 | 0 | chunk->prev = heap->main_chunk; |
1120 | 0 | chunk->prev->next = chunk; |
1121 | 0 | chunk->next->prev = chunk; |
1122 | 0 | } |
1123 | | /* mark run as allocated */ |
1124 | 0 | chunk->free_pages -= pages_count; |
1125 | 0 | zend_mm_bitset_set_range(chunk->free_map, page_num, pages_count); |
1126 | 0 | chunk->map[page_num] = ZEND_MM_LRUN(pages_count); |
1127 | 0 | if (page_num == chunk->free_tail) { |
1128 | 0 | chunk->free_tail = page_num + pages_count; |
1129 | 0 | } |
1130 | 0 | return ZEND_MM_PAGE_ADDR(chunk, page_num); |
1131 | 0 | } |
1132 | | |
1133 | | static zend_always_inline void *zend_mm_alloc_large_ex(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1134 | 0 | { |
1135 | 0 | int pages_count = (int)ZEND_MM_SIZE_TO_NUM(size, ZEND_MM_PAGE_SIZE); |
1136 | 0 | #if ZEND_DEBUG |
1137 | 0 | void *ptr = zend_mm_alloc_pages(heap, pages_count, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1138 | | #else |
1139 | | void *ptr = zend_mm_alloc_pages(heap, pages_count ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1140 | | #endif |
1141 | 0 | #if ZEND_MM_STAT |
1142 | 0 | do { |
1143 | 0 | size_t size = heap->size + pages_count * ZEND_MM_PAGE_SIZE; |
1144 | 0 | size_t peak = MAX(heap->peak, size); |
1145 | 0 | heap->size = size; |
1146 | 0 | heap->peak = peak; |
1147 | 0 | } while (0); |
1148 | 0 | #endif |
1149 | 0 | return ptr; |
1150 | 0 | } |
1151 | | |
1152 | | static zend_never_inline void *zend_mm_alloc_large(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1153 | 0 | { |
1154 | 0 | return zend_mm_alloc_large_ex(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1155 | 0 | } |
1156 | | |
1157 | | static zend_always_inline void zend_mm_delete_chunk(zend_mm_heap *heap, zend_mm_chunk *chunk) |
1158 | 0 | { |
1159 | 0 | ZEND_MM_CHECK(chunk->next->prev == chunk, "zend_mm_heap corrupted"); |
1160 | 0 | ZEND_MM_CHECK(chunk->prev->next == chunk, "zend_mm_heap corrupted"); |
1161 | | |
1162 | 0 | chunk->next->prev = chunk->prev; |
1163 | 0 | chunk->prev->next = chunk->next; |
1164 | 0 | heap->chunks_count--; |
1165 | 0 | if (heap->chunks_count + heap->cached_chunks_count < heap->avg_chunks_count + 0.1 |
1166 | 0 | || (heap->chunks_count == heap->last_chunks_delete_boundary |
1167 | 0 | && heap->last_chunks_delete_count >= 4)) { |
1168 | | /* delay deletion */ |
1169 | 0 | heap->cached_chunks_count++; |
1170 | 0 | chunk->next = heap->cached_chunks; |
1171 | 0 | heap->cached_chunks = chunk; |
1172 | 0 | } else { |
1173 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
1174 | 0 | heap->real_size -= ZEND_MM_CHUNK_SIZE; |
1175 | 0 | #endif |
1176 | 0 | if (!heap->cached_chunks) { |
1177 | 0 | if (heap->chunks_count != heap->last_chunks_delete_boundary) { |
1178 | 0 | heap->last_chunks_delete_boundary = heap->chunks_count; |
1179 | 0 | heap->last_chunks_delete_count = 0; |
1180 | 0 | } else { |
1181 | 0 | heap->last_chunks_delete_count++; |
1182 | 0 | } |
1183 | 0 | } |
1184 | 0 | if (!heap->cached_chunks || chunk->num > heap->cached_chunks->num) { |
1185 | 0 | zend_mm_chunk_free(heap, chunk, ZEND_MM_CHUNK_SIZE); |
1186 | 0 | } else { |
1187 | | //TODO: select the best chunk to delete??? |
1188 | 0 | chunk->next = heap->cached_chunks->next; |
1189 | 0 | zend_mm_chunk_free(heap, heap->cached_chunks, ZEND_MM_CHUNK_SIZE); |
1190 | 0 | heap->cached_chunks = chunk; |
1191 | 0 | } |
1192 | 0 | } |
1193 | 0 | } |
1194 | | |
1195 | | static zend_always_inline void zend_mm_free_pages_ex(zend_mm_heap *heap, zend_mm_chunk *chunk, uint32_t page_num, uint32_t pages_count, int free_chunk) |
1196 | 0 | { |
1197 | 0 | chunk->free_pages += pages_count; |
1198 | 0 | zend_mm_bitset_reset_range(chunk->free_map, page_num, pages_count); |
1199 | 0 | chunk->map[page_num] = 0; |
1200 | 0 | if (chunk->free_tail == page_num + pages_count) { |
1201 | | /* this setting may be not accurate */ |
1202 | 0 | chunk->free_tail = page_num; |
1203 | 0 | } |
1204 | 0 | if (free_chunk && chunk != heap->main_chunk && chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE) { |
1205 | 0 | zend_mm_delete_chunk(heap, chunk); |
1206 | 0 | } |
1207 | 0 | } |
1208 | | |
1209 | | static zend_never_inline void zend_mm_free_pages(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count) |
1210 | 0 | { |
1211 | 0 | zend_mm_free_pages_ex(heap, chunk, page_num, pages_count, 1); |
1212 | 0 | } |
1213 | | |
1214 | | static zend_always_inline void zend_mm_free_large(zend_mm_heap *heap, zend_mm_chunk *chunk, int page_num, int pages_count) |
1215 | 0 | { |
1216 | 0 | #if ZEND_MM_STAT |
1217 | 0 | heap->size -= pages_count * ZEND_MM_PAGE_SIZE; |
1218 | 0 | #endif |
1219 | 0 | zend_mm_free_pages(heap, chunk, page_num, pages_count); |
1220 | 0 | } |
1221 | | |
1222 | | /**************/ |
1223 | | /* Small Runs */ |
1224 | | /**************/ |
1225 | | |
1226 | | /* higher set bit number (0->N/A, 1->1, 2->2, 4->3, 8->4, 127->7, 128->8 etc) */ |
1227 | | static zend_always_inline int zend_mm_small_size_to_bit(int size) |
1228 | 0 | { |
1229 | 0 | #if (defined(__GNUC__) || __has_builtin(__builtin_clz)) && defined(PHP_HAVE_BUILTIN_CLZ) |
1230 | 0 | return (__builtin_clz(size) ^ 0x1f) + 1; |
1231 | | #elif defined(_WIN32) |
1232 | | unsigned long index; |
1233 | | |
1234 | | if (!BitScanReverse(&index, (unsigned long)size)) { |
1235 | | /* undefined behavior */ |
1236 | | return 64; |
1237 | | } |
1238 | | |
1239 | | return (((31 - (int)index) ^ 0x1f) + 1); |
1240 | | #else |
1241 | | int n = 16; |
1242 | | if (size <= 0x00ff) {n -= 8; size = size << 8;} |
1243 | | if (size <= 0x0fff) {n -= 4; size = size << 4;} |
1244 | | if (size <= 0x3fff) {n -= 2; size = size << 2;} |
1245 | | if (size <= 0x7fff) {n -= 1;} |
1246 | | return n; |
1247 | | #endif |
1248 | 0 | } |
1249 | | |
1250 | | #ifndef MAX |
1251 | | # define MAX(a, b) (((a) > (b)) ? (a) : (b)) |
1252 | | #endif |
1253 | | |
1254 | | #ifndef MIN |
1255 | | # define MIN(a, b) (((a) < (b)) ? (a) : (b)) |
1256 | | #endif |
1257 | | |
1258 | | static zend_always_inline int zend_mm_small_size_to_bin(size_t size) |
1259 | 0 | { |
1260 | | #if 0 |
1261 | | int n; |
1262 | | /*0, 1, 2, 3, 4, 5, 6, 7, 8, 9 10, 11, 12*/ |
1263 | | static const int f1[] = { 3, 3, 3, 3, 3, 3, 3, 4, 5, 6, 7, 8, 9}; |
1264 | | static const int f2[] = { 0, 0, 0, 0, 0, 0, 0, 4, 8, 12, 16, 20, 24}; |
1265 | | |
1266 | | if (UNEXPECTED(size <= 2)) return 0; |
1267 | | n = zend_mm_small_size_to_bit(size - 1); |
1268 | | return ((size-1) >> f1[n]) + f2[n]; |
1269 | | #else |
1270 | 0 | unsigned int t1, t2; |
1271 | |
|
1272 | 0 | if (size <= 64) { |
1273 | | /* we need to support size == 0 ... */ |
1274 | 0 | return (size - !!size) >> 3; |
1275 | 0 | } else { |
1276 | 0 | t1 = size - 1; |
1277 | 0 | t2 = zend_mm_small_size_to_bit(t1) - 3; |
1278 | 0 | t1 = t1 >> t2; |
1279 | 0 | t2 = t2 - 3; |
1280 | 0 | t2 = t2 << 2; |
1281 | 0 | return (int)(t1 + t2); |
1282 | 0 | } |
1283 | 0 | #endif |
1284 | 0 | } |
1285 | | |
1286 | 0 | #define ZEND_MM_SMALL_SIZE_TO_BIN(size) zend_mm_small_size_to_bin(size) |
1287 | | |
1288 | | #if ZEND_MM_HEAP_PROTECTION |
1289 | | /* We keep track of free slots by organizing them in a linked list, with the |
1290 | | * first word of every free slot being a pointer to the next one. |
1291 | | * |
1292 | | * In order to frustrate corruptions, we check the consistency of these pointers |
1293 | | * before dereference by comparing them with a shadow. |
1294 | | * |
1295 | | * The shadow is a copy of the pointer, stored at the end of the slot. It is |
1296 | | * XOR'ed with a random key, and converted to big-endian so that smaller |
1297 | | * corruptions affect the most significant bytes, which has a high chance of |
1298 | | * resulting in an invalid address instead of pointing to an adjacent slot. |
1299 | | */ |
1300 | | |
1301 | | #define ZEND_MM_FREE_SLOT_PTR_SHADOW(free_slot, bin_num) \ |
1302 | 0 | *((zend_mm_free_slot**)((char*)(free_slot) + bin_data_size[(bin_num)] - sizeof(zend_mm_free_slot*))) |
1303 | | |
1304 | | static zend_always_inline zend_mm_free_slot* zend_mm_encode_free_slot(const zend_mm_heap *heap, const zend_mm_free_slot *slot) |
1305 | 0 | { |
1306 | | #ifdef WORDS_BIGENDIAN |
1307 | | return (zend_mm_free_slot*)(((uintptr_t)slot) ^ heap->shadow_key); |
1308 | | #else |
1309 | 0 | return (zend_mm_free_slot*)(BSWAPPTR((uintptr_t)slot) ^ heap->shadow_key); |
1310 | 0 | #endif |
1311 | 0 | } |
1312 | | |
1313 | | static zend_always_inline zend_mm_free_slot* zend_mm_decode_free_slot(zend_mm_heap *heap, zend_mm_free_slot *slot) |
1314 | 0 | { |
1315 | | #ifdef WORDS_BIGENDIAN |
1316 | | return (zend_mm_free_slot*)((uintptr_t)slot ^ heap->shadow_key); |
1317 | | #else |
1318 | 0 | return (zend_mm_free_slot*)(BSWAPPTR((uintptr_t)slot ^ heap->shadow_key)); |
1319 | 0 | #endif |
1320 | 0 | } |
1321 | | |
1322 | | static zend_always_inline void zend_mm_set_next_free_slot(zend_mm_heap *heap, uint32_t bin_num, zend_mm_free_slot *slot, zend_mm_free_slot *next) |
1323 | 0 | { |
1324 | 0 | ZEND_ASSERT(bin_data_size[bin_num] >= ZEND_MM_MIN_USEABLE_BIN_SIZE); |
1325 | | |
1326 | 0 | slot->next_free_slot = next; |
1327 | 0 | ZEND_MM_FREE_SLOT_PTR_SHADOW(slot, bin_num) = zend_mm_encode_free_slot(heap, next); |
1328 | 0 | } |
1329 | | |
1330 | | static zend_always_inline zend_mm_free_slot *zend_mm_get_next_free_slot(zend_mm_heap *heap, uint32_t bin_num, zend_mm_free_slot* slot) |
1331 | 0 | { |
1332 | 0 | zend_mm_free_slot *next = slot->next_free_slot; |
1333 | 0 | if (EXPECTED(next != NULL)) { |
1334 | 0 | zend_mm_free_slot *shadow = ZEND_MM_FREE_SLOT_PTR_SHADOW(slot, bin_num); |
1335 | 0 | if (UNEXPECTED(next != zend_mm_decode_free_slot(heap, shadow))) { |
1336 | 0 | zend_mm_panic("zend_mm_heap corrupted"); |
1337 | 0 | } |
1338 | 0 | } |
1339 | 0 | return (zend_mm_free_slot*)next; |
1340 | 0 | } |
1341 | | |
1342 | | #else /* ZEND_MM_HEAP_PROTECTION */ |
1343 | | # define zend_mm_set_next_free_slot(heap, bin_num, slot, next) do { \ |
1344 | | (slot)->next_free_slot = (next); \ |
1345 | | } while (0) |
1346 | | # define zend_mm_get_next_free_slot(heap, bin_num, slot) (slot)->next_free_slot |
1347 | | #endif /* ZEND_MM_HEAP_PROTECTION */ |
1348 | | |
1349 | | static zend_never_inline void *zend_mm_alloc_small_slow(zend_mm_heap *heap, uint32_t bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1350 | 0 | { |
1351 | 0 | zend_mm_chunk *chunk; |
1352 | 0 | int page_num; |
1353 | 0 | zend_mm_bin *bin; |
1354 | 0 | zend_mm_free_slot *p, *end; |
1355 | |
|
1356 | 0 | #if ZEND_DEBUG |
1357 | 0 | bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num], bin_data_size[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1358 | | #else |
1359 | | bin = (zend_mm_bin*)zend_mm_alloc_pages(heap, bin_pages[bin_num] ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1360 | | #endif |
1361 | 0 | if (UNEXPECTED(bin == NULL)) { |
1362 | | /* insufficient memory */ |
1363 | 0 | return NULL; |
1364 | 0 | } |
1365 | | |
1366 | 0 | chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(bin, ZEND_MM_CHUNK_SIZE); |
1367 | 0 | page_num = ZEND_MM_ALIGNED_OFFSET(bin, ZEND_MM_CHUNK_SIZE) / ZEND_MM_PAGE_SIZE; |
1368 | 0 | chunk->map[page_num] = ZEND_MM_SRUN(bin_num); |
1369 | 0 | if (bin_pages[bin_num] > 1) { |
1370 | 0 | uint32_t i = 1; |
1371 | |
|
1372 | 0 | do { |
1373 | 0 | chunk->map[page_num+i] = ZEND_MM_NRUN(bin_num, i); |
1374 | 0 | i++; |
1375 | 0 | } while (i < bin_pages[bin_num]); |
1376 | 0 | } |
1377 | | |
1378 | | /* create a linked list of elements from 1 to last */ |
1379 | 0 | end = (zend_mm_free_slot*)((char*)bin + (bin_data_size[bin_num] * (bin_elements[bin_num] - 1))); |
1380 | 0 | heap->free_slot[bin_num] = p = (zend_mm_free_slot*)((char*)bin + bin_data_size[bin_num]); |
1381 | 0 | do { |
1382 | 0 | zend_mm_set_next_free_slot(heap, bin_num, p, (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num])); |
1383 | 0 | #if ZEND_DEBUG |
1384 | 0 | do { |
1385 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
1386 | 0 | dbg->size = 0; |
1387 | 0 | } while (0); |
1388 | 0 | #endif |
1389 | 0 | p = (zend_mm_free_slot*)((char*)p + bin_data_size[bin_num]); |
1390 | 0 | } while (p != end); |
1391 | | |
1392 | | /* terminate list using NULL */ |
1393 | 0 | p->next_free_slot = NULL; |
1394 | 0 | #if ZEND_DEBUG |
1395 | 0 | do { |
1396 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
1397 | 0 | dbg->size = 0; |
1398 | 0 | } while (0); |
1399 | 0 | #endif |
1400 | | |
1401 | | /* return first element */ |
1402 | 0 | return bin; |
1403 | 0 | } |
1404 | | |
1405 | | static zend_always_inline void *zend_mm_alloc_small(zend_mm_heap *heap, int bin_num ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1406 | 0 | { |
1407 | 0 | ZEND_ASSERT(bin_data_size[bin_num] >= ZEND_MM_MIN_USEABLE_BIN_SIZE); |
1408 | | |
1409 | 0 | #if ZEND_MM_STAT |
1410 | 0 | do { |
1411 | 0 | size_t size = heap->size + bin_data_size[bin_num]; |
1412 | 0 | size_t peak = MAX(heap->peak, size); |
1413 | 0 | heap->size = size; |
1414 | 0 | heap->peak = peak; |
1415 | 0 | } while (0); |
1416 | 0 | #endif |
1417 | |
|
1418 | 0 | if (EXPECTED(heap->free_slot[bin_num] != NULL)) { |
1419 | 0 | zend_mm_free_slot *p = heap->free_slot[bin_num]; |
1420 | 0 | heap->free_slot[bin_num] = zend_mm_get_next_free_slot(heap, bin_num, p); |
1421 | 0 | return p; |
1422 | 0 | } else { |
1423 | 0 | return zend_mm_alloc_small_slow(heap, bin_num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1424 | 0 | } |
1425 | 0 | } |
1426 | | |
1427 | | static zend_always_inline void zend_mm_free_small(zend_mm_heap *heap, void *ptr, int bin_num) |
1428 | 0 | { |
1429 | 0 | ZEND_ASSERT(bin_data_size[bin_num] >= ZEND_MM_MIN_USEABLE_BIN_SIZE); |
1430 | | |
1431 | 0 | zend_mm_free_slot *p; |
1432 | |
|
1433 | 0 | #if ZEND_MM_STAT |
1434 | 0 | heap->size -= bin_data_size[bin_num]; |
1435 | 0 | #endif |
1436 | |
|
1437 | 0 | #if ZEND_DEBUG |
1438 | 0 | do { |
1439 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
1440 | 0 | dbg->size = 0; |
1441 | 0 | } while (0); |
1442 | 0 | #endif |
1443 | |
|
1444 | 0 | p = (zend_mm_free_slot*)ptr; |
1445 | 0 | zend_mm_set_next_free_slot(heap, bin_num, p, heap->free_slot[bin_num]); |
1446 | 0 | heap->free_slot[bin_num] = p; |
1447 | 0 | } |
1448 | | |
1449 | | /********/ |
1450 | | /* Heap */ |
1451 | | /********/ |
1452 | | |
1453 | | #if ZEND_DEBUG |
1454 | | static zend_always_inline zend_mm_debug_info *zend_mm_get_debug_info(zend_mm_heap *heap, void *ptr) |
1455 | 0 | { |
1456 | 0 | size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); |
1457 | 0 | zend_mm_chunk *chunk; |
1458 | 0 | int page_num; |
1459 | 0 | zend_mm_page_info info; |
1460 | |
|
1461 | 0 | ZEND_MM_CHECK(page_offset != 0, "zend_mm_heap corrupted"); |
1462 | 0 | chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); |
1463 | 0 | page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); |
1464 | 0 | info = chunk->map[page_num]; |
1465 | 0 | ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); |
1466 | 0 | if (EXPECTED(info & ZEND_MM_IS_SRUN)) { |
1467 | 0 | int bin_num = ZEND_MM_SRUN_BIN_NUM(info); |
1468 | 0 | return (zend_mm_debug_info*)((char*)ptr + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
1469 | 0 | } else /* if (info & ZEND_MM_IS_LRUN) */ { |
1470 | 0 | int pages_count = ZEND_MM_LRUN_PAGES(info); |
1471 | |
|
1472 | 0 | return (zend_mm_debug_info*)((char*)ptr + ZEND_MM_PAGE_SIZE * pages_count - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
1473 | 0 | } |
1474 | 0 | } |
1475 | | #endif |
1476 | | |
1477 | | static zend_always_inline void *zend_mm_alloc_heap(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1478 | 0 | { |
1479 | 0 | void *ptr; |
1480 | 0 | #if ZEND_MM_HEAP_PROTECTION |
1481 | 0 | if (size < ZEND_MM_MIN_USEABLE_BIN_SIZE) { |
1482 | 0 | size = ZEND_MM_MIN_USEABLE_BIN_SIZE; |
1483 | 0 | } |
1484 | 0 | #endif /* ZEND_MM_HEAP_PROTECTION */ |
1485 | 0 | #if ZEND_DEBUG |
1486 | 0 | size_t real_size = size; |
1487 | 0 | zend_mm_debug_info *dbg; |
1488 | | |
1489 | | /* special handling for zero-size allocation */ |
1490 | 0 | size = MAX(size, 1); |
1491 | 0 | size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)); |
1492 | 0 | if (UNEXPECTED(size < real_size)) { |
1493 | 0 | zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", ZEND_MM_ALIGNED_SIZE(real_size), ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
1494 | 0 | return NULL; |
1495 | 0 | } |
1496 | 0 | #endif |
1497 | 0 | if (EXPECTED(size <= ZEND_MM_MAX_SMALL_SIZE)) { |
1498 | 0 | ptr = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1499 | 0 | #if ZEND_DEBUG |
1500 | 0 | dbg = zend_mm_get_debug_info(heap, ptr); |
1501 | 0 | dbg->size = real_size; |
1502 | 0 | dbg->filename = __zend_filename; |
1503 | 0 | dbg->orig_filename = __zend_orig_filename; |
1504 | 0 | dbg->lineno = __zend_lineno; |
1505 | 0 | dbg->orig_lineno = __zend_orig_lineno; |
1506 | 0 | #endif |
1507 | 0 | return ptr; |
1508 | 0 | } else if (EXPECTED(size <= ZEND_MM_MAX_LARGE_SIZE)) { |
1509 | 0 | ptr = zend_mm_alloc_large(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1510 | 0 | #if ZEND_DEBUG |
1511 | 0 | dbg = zend_mm_get_debug_info(heap, ptr); |
1512 | 0 | dbg->size = real_size; |
1513 | 0 | dbg->filename = __zend_filename; |
1514 | 0 | dbg->orig_filename = __zend_orig_filename; |
1515 | 0 | dbg->lineno = __zend_lineno; |
1516 | 0 | dbg->orig_lineno = __zend_orig_lineno; |
1517 | 0 | #endif |
1518 | 0 | return ptr; |
1519 | 0 | } else { |
1520 | 0 | #if ZEND_DEBUG |
1521 | 0 | size = real_size; |
1522 | 0 | #endif |
1523 | 0 | return zend_mm_alloc_huge(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1524 | 0 | } |
1525 | 0 | } |
1526 | | |
1527 | | static zend_always_inline void zend_mm_free_heap(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1528 | 0 | { |
1529 | 0 | size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); |
1530 | |
|
1531 | 0 | if (UNEXPECTED(page_offset == 0)) { |
1532 | 0 | if (ptr != NULL) { |
1533 | 0 | zend_mm_free_huge(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1534 | 0 | } |
1535 | 0 | } else { |
1536 | 0 | zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); |
1537 | 0 | int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); |
1538 | 0 | zend_mm_page_info info = chunk->map[page_num]; |
1539 | |
|
1540 | 0 | ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); |
1541 | 0 | if (EXPECTED(info & ZEND_MM_IS_SRUN)) { |
1542 | 0 | zend_mm_free_small(heap, ptr, ZEND_MM_SRUN_BIN_NUM(info)); |
1543 | 0 | } else /* if (info & ZEND_MM_IS_LRUN) */ { |
1544 | 0 | int pages_count = ZEND_MM_LRUN_PAGES(info); |
1545 | |
|
1546 | 0 | ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted"); |
1547 | 0 | zend_mm_free_large(heap, chunk, page_num, pages_count); |
1548 | 0 | } |
1549 | 0 | } |
1550 | 0 | } |
1551 | | |
1552 | | static size_t zend_mm_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1553 | 0 | { |
1554 | 0 | size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); |
1555 | |
|
1556 | 0 | if (UNEXPECTED(page_offset == 0)) { |
1557 | 0 | return zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1558 | 0 | } else { |
1559 | 0 | zend_mm_chunk *chunk; |
1560 | | #if 0 && ZEND_DEBUG |
1561 | | zend_mm_debug_info *dbg = zend_mm_get_debug_info(heap, ptr); |
1562 | | return dbg->size; |
1563 | | #else |
1564 | 0 | int page_num; |
1565 | 0 | zend_mm_page_info info; |
1566 | |
|
1567 | 0 | chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); |
1568 | 0 | page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); |
1569 | 0 | info = chunk->map[page_num]; |
1570 | 0 | ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); |
1571 | 0 | if (EXPECTED(info & ZEND_MM_IS_SRUN)) { |
1572 | 0 | return bin_data_size[ZEND_MM_SRUN_BIN_NUM(info)]; |
1573 | 0 | } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ { |
1574 | 0 | return ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE; |
1575 | 0 | } |
1576 | 0 | #endif |
1577 | 0 | } |
1578 | 0 | } |
1579 | | |
1580 | | static zend_never_inline void *zend_mm_realloc_slow(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1581 | 0 | { |
1582 | 0 | void *ret; |
1583 | |
|
1584 | 0 | #if ZEND_MM_STAT |
1585 | 0 | do { |
1586 | 0 | size_t orig_peak = heap->peak; |
1587 | 0 | #endif |
1588 | 0 | ret = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1589 | 0 | memcpy(ret, ptr, copy_size); |
1590 | 0 | zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1591 | 0 | #if ZEND_MM_STAT |
1592 | 0 | heap->peak = MAX(orig_peak, heap->size); |
1593 | 0 | } while (0); |
1594 | 0 | #endif |
1595 | 0 | return ret; |
1596 | 0 | } |
1597 | | |
1598 | | static zend_never_inline void *zend_mm_realloc_huge(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1599 | 0 | { |
1600 | 0 | size_t old_size; |
1601 | 0 | size_t new_size; |
1602 | 0 | #if ZEND_DEBUG |
1603 | 0 | size_t real_size; |
1604 | 0 | #endif |
1605 | |
|
1606 | 0 | old_size = zend_mm_get_huge_block_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1607 | 0 | #if ZEND_DEBUG |
1608 | 0 | real_size = size; |
1609 | 0 | size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)); |
1610 | 0 | #endif |
1611 | 0 | if (size > ZEND_MM_MAX_LARGE_SIZE) { |
1612 | 0 | #if ZEND_DEBUG |
1613 | 0 | size = real_size; |
1614 | 0 | #endif |
1615 | | #ifdef ZEND_WIN32 |
1616 | | /* On Windows we don't have ability to extend huge blocks in-place. |
1617 | | * We allocate them with 2MB size granularity, to avoid many |
1618 | | * reallocations when they are extended by small pieces |
1619 | | */ |
1620 | | new_size = ZEND_MM_ALIGNED_SIZE_EX(size, MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE)); |
1621 | | #else |
1622 | 0 | new_size = ZEND_MM_ALIGNED_SIZE_EX(size, REAL_PAGE_SIZE); |
1623 | 0 | #endif |
1624 | 0 | if (new_size == old_size) { |
1625 | 0 | #if ZEND_DEBUG |
1626 | 0 | zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1627 | | #else |
1628 | | zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1629 | | #endif |
1630 | 0 | return ptr; |
1631 | 0 | } else if (new_size < old_size) { |
1632 | | /* unmup tail */ |
1633 | 0 | if (zend_mm_chunk_truncate(heap, ptr, old_size, new_size)) { |
1634 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
1635 | 0 | heap->real_size -= old_size - new_size; |
1636 | 0 | #endif |
1637 | 0 | #if ZEND_MM_STAT |
1638 | 0 | heap->size -= old_size - new_size; |
1639 | 0 | #endif |
1640 | 0 | #if ZEND_DEBUG |
1641 | 0 | zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1642 | | #else |
1643 | | zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1644 | | #endif |
1645 | 0 | return ptr; |
1646 | 0 | } |
1647 | 0 | } else /* if (new_size > old_size) */ { |
1648 | 0 | #if ZEND_MM_LIMIT |
1649 | 0 | if (UNEXPECTED(new_size - old_size > heap->limit - heap->real_size)) { |
1650 | 0 | if (zend_mm_gc(heap) && new_size - old_size <= heap->limit - heap->real_size) { |
1651 | | /* pass */ |
1652 | 0 | } else if (heap->overflow == 0) { |
1653 | 0 | #if ZEND_DEBUG |
1654 | 0 | zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size); |
1655 | | #else |
1656 | | zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size); |
1657 | | #endif |
1658 | 0 | return NULL; |
1659 | 0 | } |
1660 | 0 | } |
1661 | 0 | #endif |
1662 | | /* try to map tail right after this block */ |
1663 | 0 | if (zend_mm_chunk_extend(heap, ptr, old_size, new_size)) { |
1664 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
1665 | 0 | heap->real_size += new_size - old_size; |
1666 | 0 | #endif |
1667 | 0 | #if ZEND_MM_STAT |
1668 | 0 | heap->real_peak = MAX(heap->real_peak, heap->real_size); |
1669 | 0 | heap->size += new_size - old_size; |
1670 | 0 | heap->peak = MAX(heap->peak, heap->size); |
1671 | 0 | #endif |
1672 | 0 | #if ZEND_DEBUG |
1673 | 0 | zend_mm_change_huge_block_size(heap, ptr, new_size, real_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1674 | | #else |
1675 | | zend_mm_change_huge_block_size(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1676 | | #endif |
1677 | 0 | return ptr; |
1678 | 0 | } |
1679 | 0 | } |
1680 | 0 | } |
1681 | | |
1682 | 0 | return zend_mm_realloc_slow(heap, ptr, size, MIN(old_size, copy_size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1683 | 0 | } |
1684 | | |
1685 | | static zend_always_inline void *zend_mm_realloc_heap(zend_mm_heap *heap, void *ptr, size_t size, bool use_copy_size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1686 | 0 | { |
1687 | 0 | size_t page_offset; |
1688 | 0 | size_t old_size; |
1689 | 0 | size_t new_size; |
1690 | 0 | void *ret; |
1691 | 0 | #if ZEND_DEBUG |
1692 | 0 | zend_mm_debug_info *dbg; |
1693 | 0 | #endif |
1694 | |
|
1695 | 0 | page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); |
1696 | 0 | if (UNEXPECTED(page_offset == 0)) { |
1697 | 0 | if (EXPECTED(ptr == NULL)) { |
1698 | 0 | return _zend_mm_alloc(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1699 | 0 | } else { |
1700 | 0 | return zend_mm_realloc_huge(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1701 | 0 | } |
1702 | 0 | } else { |
1703 | 0 | zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); |
1704 | 0 | int page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); |
1705 | 0 | zend_mm_page_info info = chunk->map[page_num]; |
1706 | 0 | #if ZEND_MM_HEAP_PROTECTION |
1707 | 0 | if (size < ZEND_MM_MIN_USEABLE_BIN_SIZE) { |
1708 | 0 | size = ZEND_MM_MIN_USEABLE_BIN_SIZE; |
1709 | 0 | } |
1710 | 0 | #endif /* ZEND_MM_HEAP_PROTECTION */ |
1711 | 0 | #if ZEND_DEBUG |
1712 | 0 | size_t real_size = size; |
1713 | |
|
1714 | 0 | size = ZEND_MM_ALIGNED_SIZE(size) + ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info)); |
1715 | 0 | #endif |
1716 | |
|
1717 | 0 | ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); |
1718 | 0 | if (info & ZEND_MM_IS_SRUN) { |
1719 | 0 | int old_bin_num = ZEND_MM_SRUN_BIN_NUM(info); |
1720 | |
|
1721 | 0 | do { |
1722 | 0 | old_size = bin_data_size[old_bin_num]; |
1723 | | |
1724 | | /* Check if requested size fits into current bin */ |
1725 | 0 | if (size <= old_size) { |
1726 | | /* Check if truncation is necessary */ |
1727 | 0 | if (old_bin_num > 0 && size < bin_data_size[old_bin_num - 1]) { |
1728 | | /* truncation */ |
1729 | 0 | ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1730 | 0 | copy_size = use_copy_size ? MIN(size, copy_size) : size; |
1731 | 0 | memcpy(ret, ptr, copy_size); |
1732 | 0 | zend_mm_free_small(heap, ptr, old_bin_num); |
1733 | 0 | } else { |
1734 | | /* reallocation in-place */ |
1735 | 0 | ret = ptr; |
1736 | 0 | } |
1737 | 0 | } else if (size <= ZEND_MM_MAX_SMALL_SIZE) { |
1738 | | /* small extension */ |
1739 | |
|
1740 | 0 | #if ZEND_MM_STAT |
1741 | 0 | do { |
1742 | 0 | size_t orig_peak = heap->peak; |
1743 | 0 | #endif |
1744 | 0 | ret = zend_mm_alloc_small(heap, ZEND_MM_SMALL_SIZE_TO_BIN(size) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1745 | 0 | copy_size = use_copy_size ? MIN(old_size, copy_size) : old_size; |
1746 | 0 | memcpy(ret, ptr, copy_size); |
1747 | 0 | zend_mm_free_small(heap, ptr, old_bin_num); |
1748 | 0 | #if ZEND_MM_STAT |
1749 | 0 | heap->peak = MAX(orig_peak, heap->size); |
1750 | 0 | } while (0); |
1751 | 0 | #endif |
1752 | 0 | } else { |
1753 | | /* slow reallocation */ |
1754 | 0 | break; |
1755 | 0 | } |
1756 | | |
1757 | 0 | #if ZEND_DEBUG |
1758 | 0 | dbg = zend_mm_get_debug_info(heap, ret); |
1759 | 0 | dbg->size = real_size; |
1760 | 0 | dbg->filename = __zend_filename; |
1761 | 0 | dbg->orig_filename = __zend_orig_filename; |
1762 | 0 | dbg->lineno = __zend_lineno; |
1763 | 0 | dbg->orig_lineno = __zend_orig_lineno; |
1764 | 0 | #endif |
1765 | 0 | return ret; |
1766 | 0 | } while (0); |
1767 | |
|
1768 | 0 | } else /* if (info & ZEND_MM_IS_LARGE_RUN) */ { |
1769 | 0 | ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted"); |
1770 | 0 | old_size = ZEND_MM_LRUN_PAGES(info) * ZEND_MM_PAGE_SIZE; |
1771 | 0 | if (size > ZEND_MM_MAX_SMALL_SIZE && size <= ZEND_MM_MAX_LARGE_SIZE) { |
1772 | 0 | new_size = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE); |
1773 | 0 | if (new_size == old_size) { |
1774 | 0 | #if ZEND_DEBUG |
1775 | 0 | dbg = zend_mm_get_debug_info(heap, ptr); |
1776 | 0 | dbg->size = real_size; |
1777 | 0 | dbg->filename = __zend_filename; |
1778 | 0 | dbg->orig_filename = __zend_orig_filename; |
1779 | 0 | dbg->lineno = __zend_lineno; |
1780 | 0 | dbg->orig_lineno = __zend_orig_lineno; |
1781 | 0 | #endif |
1782 | 0 | return ptr; |
1783 | 0 | } else if (new_size < old_size) { |
1784 | | /* free tail pages */ |
1785 | 0 | int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE); |
1786 | 0 | int rest_pages_count = (int)((old_size - new_size) / ZEND_MM_PAGE_SIZE); |
1787 | |
|
1788 | 0 | #if ZEND_MM_STAT |
1789 | 0 | heap->size -= rest_pages_count * ZEND_MM_PAGE_SIZE; |
1790 | 0 | #endif |
1791 | 0 | chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count); |
1792 | 0 | chunk->free_pages += rest_pages_count; |
1793 | 0 | zend_mm_bitset_reset_range(chunk->free_map, page_num + new_pages_count, rest_pages_count); |
1794 | 0 | #if ZEND_DEBUG |
1795 | 0 | dbg = zend_mm_get_debug_info(heap, ptr); |
1796 | 0 | dbg->size = real_size; |
1797 | 0 | dbg->filename = __zend_filename; |
1798 | 0 | dbg->orig_filename = __zend_orig_filename; |
1799 | 0 | dbg->lineno = __zend_lineno; |
1800 | 0 | dbg->orig_lineno = __zend_orig_lineno; |
1801 | 0 | #endif |
1802 | 0 | return ptr; |
1803 | 0 | } else /* if (new_size > old_size) */ { |
1804 | 0 | int new_pages_count = (int)(new_size / ZEND_MM_PAGE_SIZE); |
1805 | 0 | int old_pages_count = (int)(old_size / ZEND_MM_PAGE_SIZE); |
1806 | | |
1807 | | /* try to allocate tail pages after this block */ |
1808 | 0 | if (page_num + new_pages_count <= ZEND_MM_PAGES && |
1809 | 0 | zend_mm_bitset_is_free_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count)) { |
1810 | 0 | #if ZEND_MM_STAT |
1811 | 0 | do { |
1812 | 0 | size_t size = heap->size + (new_size - old_size); |
1813 | 0 | size_t peak = MAX(heap->peak, size); |
1814 | 0 | heap->size = size; |
1815 | 0 | heap->peak = peak; |
1816 | 0 | } while (0); |
1817 | 0 | #endif |
1818 | 0 | chunk->free_pages -= new_pages_count - old_pages_count; |
1819 | 0 | zend_mm_bitset_set_range(chunk->free_map, page_num + old_pages_count, new_pages_count - old_pages_count); |
1820 | 0 | chunk->map[page_num] = ZEND_MM_LRUN(new_pages_count); |
1821 | 0 | #if ZEND_DEBUG |
1822 | 0 | dbg = zend_mm_get_debug_info(heap, ptr); |
1823 | 0 | dbg->size = real_size; |
1824 | 0 | dbg->filename = __zend_filename; |
1825 | 0 | dbg->orig_filename = __zend_orig_filename; |
1826 | 0 | dbg->lineno = __zend_lineno; |
1827 | 0 | dbg->orig_lineno = __zend_orig_lineno; |
1828 | 0 | #endif |
1829 | 0 | return ptr; |
1830 | 0 | } |
1831 | 0 | } |
1832 | 0 | } |
1833 | 0 | } |
1834 | 0 | #if ZEND_DEBUG |
1835 | 0 | size = real_size; |
1836 | 0 | #endif |
1837 | 0 | } |
1838 | | |
1839 | 0 | copy_size = MIN(old_size, copy_size); |
1840 | 0 | return zend_mm_realloc_slow(heap, ptr, size, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1841 | 0 | } |
1842 | | |
1843 | | /*********************/ |
1844 | | /* Huge Runs (again) */ |
1845 | | /*********************/ |
1846 | | |
1847 | | #if ZEND_DEBUG |
1848 | | static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1849 | | #else |
1850 | | static void zend_mm_add_huge_block(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1851 | | #endif |
1852 | 0 | { |
1853 | 0 | zend_mm_huge_list *list = (zend_mm_huge_list*)zend_mm_alloc_heap(heap, sizeof(zend_mm_huge_list) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1854 | 0 | list->ptr = ptr; |
1855 | 0 | list->size = size; |
1856 | 0 | list->next = heap->huge_list; |
1857 | 0 | #if ZEND_DEBUG |
1858 | 0 | list->dbg.size = dbg_size; |
1859 | 0 | list->dbg.filename = __zend_filename; |
1860 | 0 | list->dbg.orig_filename = __zend_orig_filename; |
1861 | 0 | list->dbg.lineno = __zend_lineno; |
1862 | 0 | list->dbg.orig_lineno = __zend_orig_lineno; |
1863 | 0 | #endif |
1864 | 0 | heap->huge_list = list; |
1865 | 0 | } |
1866 | | |
1867 | | static size_t zend_mm_del_huge_block(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1868 | 0 | { |
1869 | 0 | zend_mm_huge_list *prev = NULL; |
1870 | 0 | zend_mm_huge_list *list = heap->huge_list; |
1871 | 0 | while (list != NULL) { |
1872 | 0 | if (list->ptr == ptr) { |
1873 | 0 | size_t size; |
1874 | |
|
1875 | 0 | if (prev) { |
1876 | 0 | prev->next = list->next; |
1877 | 0 | } else { |
1878 | 0 | heap->huge_list = list->next; |
1879 | 0 | } |
1880 | 0 | size = list->size; |
1881 | 0 | zend_mm_free_heap(heap, list ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1882 | 0 | return size; |
1883 | 0 | } |
1884 | 0 | prev = list; |
1885 | 0 | list = list->next; |
1886 | 0 | } |
1887 | 0 | ZEND_MM_CHECK(0, "zend_mm_heap corrupted"); |
1888 | 0 | return 0; |
1889 | 0 | } |
1890 | | |
1891 | | static size_t zend_mm_get_huge_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1892 | 0 | { |
1893 | 0 | zend_mm_huge_list *list = heap->huge_list; |
1894 | 0 | while (list != NULL) { |
1895 | 0 | if (list->ptr == ptr) { |
1896 | 0 | return list->size; |
1897 | 0 | } |
1898 | 0 | list = list->next; |
1899 | 0 | } |
1900 | 0 | ZEND_MM_CHECK(0, "zend_mm_heap corrupted"); |
1901 | 0 | return 0; |
1902 | 0 | } |
1903 | | |
1904 | | #if ZEND_DEBUG |
1905 | | static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size, size_t dbg_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1906 | | #else |
1907 | | static void zend_mm_change_huge_block_size(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1908 | | #endif |
1909 | 0 | { |
1910 | 0 | zend_mm_huge_list *list = heap->huge_list; |
1911 | 0 | while (list != NULL) { |
1912 | 0 | if (list->ptr == ptr) { |
1913 | 0 | list->size = size; |
1914 | 0 | #if ZEND_DEBUG |
1915 | 0 | list->dbg.size = dbg_size; |
1916 | 0 | list->dbg.filename = __zend_filename; |
1917 | 0 | list->dbg.orig_filename = __zend_orig_filename; |
1918 | 0 | list->dbg.lineno = __zend_lineno; |
1919 | 0 | list->dbg.orig_lineno = __zend_orig_lineno; |
1920 | 0 | #endif |
1921 | 0 | return; |
1922 | 0 | } |
1923 | 0 | list = list->next; |
1924 | 0 | } |
1925 | 0 | } |
1926 | | |
1927 | | static void *zend_mm_alloc_huge(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
1928 | 0 | { |
1929 | | #ifdef ZEND_WIN32 |
1930 | | /* On Windows we don't have ability to extend huge blocks in-place. |
1931 | | * We allocate them with 2MB size granularity, to avoid many |
1932 | | * reallocations when they are extended by small pieces |
1933 | | */ |
1934 | | size_t alignment = MAX(REAL_PAGE_SIZE, ZEND_MM_CHUNK_SIZE); |
1935 | | #else |
1936 | 0 | size_t alignment = REAL_PAGE_SIZE; |
1937 | 0 | #endif |
1938 | 0 | size_t new_size = ZEND_MM_ALIGNED_SIZE_EX(size, alignment); |
1939 | 0 | void *ptr; |
1940 | |
|
1941 | 0 | if (UNEXPECTED(new_size < size)) { |
1942 | 0 | zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (%zu + %zu)", size, alignment); |
1943 | 0 | } |
1944 | | |
1945 | 0 | #if ZEND_MM_LIMIT |
1946 | 0 | if (UNEXPECTED(new_size > heap->limit - heap->real_size)) { |
1947 | 0 | if (zend_mm_gc(heap) && new_size <= heap->limit - heap->real_size) { |
1948 | | /* pass */ |
1949 | 0 | } else if (heap->overflow == 0) { |
1950 | 0 | #if ZEND_DEBUG |
1951 | 0 | zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", heap->limit, __zend_filename, __zend_lineno, size); |
1952 | | #else |
1953 | | zend_mm_safe_error(heap, "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", heap->limit, size); |
1954 | | #endif |
1955 | 0 | return NULL; |
1956 | 0 | } |
1957 | 0 | } |
1958 | 0 | #endif |
1959 | 0 | ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE); |
1960 | 0 | if (UNEXPECTED(ptr == NULL)) { |
1961 | | /* insufficient memory */ |
1962 | 0 | if (zend_mm_gc(heap) && |
1963 | 0 | (ptr = zend_mm_chunk_alloc(heap, new_size, ZEND_MM_CHUNK_SIZE)) != NULL) { |
1964 | | /* pass */ |
1965 | 0 | } else { |
1966 | | #if !ZEND_MM_LIMIT |
1967 | | zend_mm_safe_error(heap, "Out of memory"); |
1968 | | #elif ZEND_DEBUG |
1969 | | zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) at %s:%d (tried to allocate %zu bytes)", heap->real_size, __zend_filename, __zend_lineno, size); |
1970 | | #else |
1971 | | zend_mm_safe_error(heap, "Out of memory (allocated %zu bytes) (tried to allocate %zu bytes)", heap->real_size, size); |
1972 | | #endif |
1973 | 0 | return NULL; |
1974 | 0 | } |
1975 | 0 | } |
1976 | 0 | #if ZEND_DEBUG |
1977 | 0 | zend_mm_add_huge_block(heap, ptr, new_size, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1978 | | #else |
1979 | | zend_mm_add_huge_block(heap, ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
1980 | | #endif |
1981 | 0 | #if ZEND_MM_STAT |
1982 | 0 | do { |
1983 | 0 | size_t size = heap->real_size + new_size; |
1984 | 0 | size_t peak = MAX(heap->real_peak, size); |
1985 | 0 | heap->real_size = size; |
1986 | 0 | heap->real_peak = peak; |
1987 | 0 | } while (0); |
1988 | 0 | do { |
1989 | 0 | size_t size = heap->size + new_size; |
1990 | 0 | size_t peak = MAX(heap->peak, size); |
1991 | 0 | heap->size = size; |
1992 | 0 | heap->peak = peak; |
1993 | 0 | } while (0); |
1994 | | #elif ZEND_MM_LIMIT |
1995 | | heap->real_size += new_size; |
1996 | | #endif |
1997 | 0 | return ptr; |
1998 | 0 | } |
1999 | | |
2000 | | static void zend_mm_free_huge(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2001 | 0 | { |
2002 | 0 | size_t size; |
2003 | |
|
2004 | 0 | ZEND_MM_CHECK(ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE) == 0, "zend_mm_heap corrupted"); |
2005 | 0 | size = zend_mm_del_huge_block(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2006 | 0 | zend_mm_chunk_free(heap, ptr, size); |
2007 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
2008 | 0 | heap->real_size -= size; |
2009 | 0 | #endif |
2010 | 0 | #if ZEND_MM_STAT |
2011 | 0 | heap->size -= size; |
2012 | 0 | #endif |
2013 | 0 | } |
2014 | | |
2015 | | /******************/ |
2016 | | /* Initialization */ |
2017 | | /******************/ |
2018 | | |
2019 | | static void zend_mm_refresh_key(zend_mm_heap *heap) |
2020 | 0 | { |
2021 | 0 | zend_random_bytes_insecure(&heap->rand_state, &heap->shadow_key, sizeof(heap->shadow_key)); |
2022 | 0 | } |
2023 | | |
2024 | | static void zend_mm_init_key(zend_mm_heap *heap) |
2025 | 0 | { |
2026 | 0 | memset(&heap->rand_state, 0, sizeof(heap->rand_state)); |
2027 | 0 | zend_mm_refresh_key(heap); |
2028 | 0 | } |
2029 | | |
2030 | | static zend_mm_heap *zend_mm_init(void) |
2031 | 0 | { |
2032 | 0 | zend_mm_chunk *chunk = (zend_mm_chunk*)zend_mm_chunk_alloc_int(ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE); |
2033 | 0 | zend_mm_heap *heap; |
2034 | |
|
2035 | 0 | if (UNEXPECTED(chunk == NULL)) { |
2036 | 0 | #if ZEND_MM_ERROR |
2037 | 0 | fprintf(stderr, "Can't initialize heap\n"); |
2038 | 0 | #endif |
2039 | 0 | return NULL; |
2040 | 0 | } |
2041 | 0 | heap = &chunk->heap_slot; |
2042 | 0 | chunk->heap = heap; |
2043 | 0 | chunk->next = chunk; |
2044 | 0 | chunk->prev = chunk; |
2045 | 0 | chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE; |
2046 | 0 | chunk->free_tail = ZEND_MM_FIRST_PAGE; |
2047 | 0 | chunk->num = 0; |
2048 | 0 | chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1; |
2049 | 0 | chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE); |
2050 | 0 | heap->main_chunk = chunk; |
2051 | 0 | heap->cached_chunks = NULL; |
2052 | 0 | heap->chunks_count = 1; |
2053 | 0 | heap->peak_chunks_count = 1; |
2054 | 0 | heap->cached_chunks_count = 0; |
2055 | 0 | heap->avg_chunks_count = 1.0; |
2056 | 0 | heap->last_chunks_delete_boundary = 0; |
2057 | 0 | heap->last_chunks_delete_count = 0; |
2058 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
2059 | 0 | heap->real_size = ZEND_MM_CHUNK_SIZE; |
2060 | 0 | #endif |
2061 | 0 | #if ZEND_MM_STAT |
2062 | 0 | heap->real_peak = ZEND_MM_CHUNK_SIZE; |
2063 | 0 | heap->size = 0; |
2064 | 0 | heap->peak = 0; |
2065 | 0 | #endif |
2066 | 0 | zend_mm_init_key(heap); |
2067 | 0 | #if ZEND_MM_LIMIT |
2068 | 0 | heap->limit = (size_t)Z_L(-1) >> 1; |
2069 | 0 | heap->overflow = 0; |
2070 | 0 | #endif |
2071 | 0 | #if ZEND_MM_CUSTOM |
2072 | 0 | heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE; |
2073 | 0 | #endif |
2074 | 0 | #if ZEND_MM_STORAGE |
2075 | 0 | heap->storage = NULL; |
2076 | 0 | #endif |
2077 | 0 | heap->huge_list = NULL; |
2078 | 0 | heap->pid = getpid(); |
2079 | 0 | return heap; |
2080 | 0 | } |
2081 | | |
2082 | | ZEND_API size_t zend_mm_gc(zend_mm_heap *heap) |
2083 | 0 | { |
2084 | 0 | zend_mm_free_slot *p, *q; |
2085 | 0 | zend_mm_chunk *chunk; |
2086 | 0 | size_t page_offset; |
2087 | 0 | int page_num; |
2088 | 0 | zend_mm_page_info info; |
2089 | 0 | uint32_t i, free_counter; |
2090 | 0 | bool has_free_pages; |
2091 | 0 | size_t collected = 0; |
2092 | |
|
2093 | 0 | #if ZEND_MM_CUSTOM |
2094 | 0 | if (heap->use_custom_heap) { |
2095 | 0 | size_t (*gc)(void) = heap->custom_heap._gc; |
2096 | 0 | if (gc) { |
2097 | 0 | return gc(); |
2098 | 0 | } |
2099 | 0 | return 0; |
2100 | 0 | } |
2101 | 0 | #endif |
2102 | | |
2103 | 0 | for (i = 0; i < ZEND_MM_BINS; i++) { |
2104 | 0 | has_free_pages = false; |
2105 | 0 | p = heap->free_slot[i]; |
2106 | 0 | while (p != NULL) { |
2107 | 0 | chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE); |
2108 | 0 | ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); |
2109 | 0 | page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE); |
2110 | 0 | ZEND_ASSERT(page_offset != 0); |
2111 | 0 | page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); |
2112 | 0 | info = chunk->map[page_num]; |
2113 | 0 | ZEND_ASSERT(info & ZEND_MM_IS_SRUN); |
2114 | 0 | if (info & ZEND_MM_IS_LRUN) { |
2115 | 0 | page_num -= ZEND_MM_NRUN_OFFSET(info); |
2116 | 0 | info = chunk->map[page_num]; |
2117 | 0 | ZEND_ASSERT(info & ZEND_MM_IS_SRUN); |
2118 | 0 | ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN)); |
2119 | 0 | } |
2120 | 0 | ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i); |
2121 | 0 | free_counter = ZEND_MM_SRUN_FREE_COUNTER(info) + 1; |
2122 | 0 | if (free_counter == bin_elements[i]) { |
2123 | 0 | has_free_pages = true; |
2124 | 0 | } |
2125 | 0 | chunk->map[page_num] = ZEND_MM_SRUN_EX(i, free_counter); |
2126 | 0 | p = zend_mm_get_next_free_slot(heap, i, p); |
2127 | 0 | } |
2128 | | |
2129 | 0 | if (!has_free_pages) { |
2130 | 0 | continue; |
2131 | 0 | } |
2132 | | |
2133 | 0 | q = (zend_mm_free_slot*)&heap->free_slot[i]; |
2134 | 0 | p = q->next_free_slot; |
2135 | 0 | while (p != NULL) { |
2136 | 0 | chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(p, ZEND_MM_CHUNK_SIZE); |
2137 | 0 | ZEND_MM_CHECK(chunk->heap == heap, "zend_mm_heap corrupted"); |
2138 | 0 | page_offset = ZEND_MM_ALIGNED_OFFSET(p, ZEND_MM_CHUNK_SIZE); |
2139 | 0 | ZEND_ASSERT(page_offset != 0); |
2140 | 0 | page_num = (int)(page_offset / ZEND_MM_PAGE_SIZE); |
2141 | 0 | info = chunk->map[page_num]; |
2142 | 0 | ZEND_ASSERT(info & ZEND_MM_IS_SRUN); |
2143 | 0 | if (info & ZEND_MM_IS_LRUN) { |
2144 | 0 | page_num -= ZEND_MM_NRUN_OFFSET(info); |
2145 | 0 | info = chunk->map[page_num]; |
2146 | 0 | ZEND_ASSERT(info & ZEND_MM_IS_SRUN); |
2147 | 0 | ZEND_ASSERT(!(info & ZEND_MM_IS_LRUN)); |
2148 | 0 | } |
2149 | 0 | ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(info) == i); |
2150 | 0 | if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[i]) { |
2151 | | /* remove from cache */ |
2152 | 0 | p = zend_mm_get_next_free_slot(heap, i, p); |
2153 | 0 | if (q == (zend_mm_free_slot*)&heap->free_slot[i]) { |
2154 | 0 | q->next_free_slot = p; |
2155 | 0 | } else { |
2156 | 0 | zend_mm_set_next_free_slot(heap, i, q, p); |
2157 | 0 | } |
2158 | 0 | } else { |
2159 | 0 | q = p; |
2160 | 0 | if (q == (zend_mm_free_slot*)&heap->free_slot[i]) { |
2161 | 0 | p = q->next_free_slot; |
2162 | 0 | } else { |
2163 | 0 | p = zend_mm_get_next_free_slot(heap, i, q); |
2164 | 0 | } |
2165 | 0 | } |
2166 | 0 | } |
2167 | 0 | } |
2168 | | |
2169 | 0 | chunk = heap->main_chunk; |
2170 | 0 | do { |
2171 | 0 | i = ZEND_MM_FIRST_PAGE; |
2172 | 0 | while (i < chunk->free_tail) { |
2173 | 0 | if (zend_mm_bitset_is_set(chunk->free_map, i)) { |
2174 | 0 | info = chunk->map[i]; |
2175 | 0 | if (info & ZEND_MM_IS_SRUN) { |
2176 | 0 | int bin_num = ZEND_MM_SRUN_BIN_NUM(info); |
2177 | 0 | int pages_count = bin_pages[bin_num]; |
2178 | |
|
2179 | 0 | if (ZEND_MM_SRUN_FREE_COUNTER(info) == bin_elements[bin_num]) { |
2180 | | /* all elements are free */ |
2181 | 0 | zend_mm_free_pages_ex(heap, chunk, i, pages_count, 0); |
2182 | 0 | collected += pages_count; |
2183 | 0 | } else { |
2184 | | /* reset counter */ |
2185 | 0 | chunk->map[i] = ZEND_MM_SRUN(bin_num); |
2186 | 0 | } |
2187 | 0 | i += bin_pages[bin_num]; |
2188 | 0 | } else /* if (info & ZEND_MM_IS_LRUN) */ { |
2189 | 0 | i += ZEND_MM_LRUN_PAGES(info); |
2190 | 0 | } |
2191 | 0 | } else { |
2192 | 0 | i++; |
2193 | 0 | } |
2194 | 0 | } |
2195 | 0 | if (chunk->free_pages == ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE && chunk != heap->main_chunk) { |
2196 | 0 | zend_mm_chunk *next_chunk = chunk->next; |
2197 | |
|
2198 | 0 | zend_mm_delete_chunk(heap, chunk); |
2199 | 0 | chunk = next_chunk; |
2200 | 0 | } else { |
2201 | 0 | chunk = chunk->next; |
2202 | 0 | } |
2203 | 0 | } while (chunk != heap->main_chunk); |
2204 | |
|
2205 | 0 | return collected * ZEND_MM_PAGE_SIZE; |
2206 | 0 | } |
2207 | | |
2208 | | #if ZEND_DEBUG |
2209 | | /******************/ |
2210 | | /* Leak detection */ |
2211 | | /******************/ |
2212 | | |
2213 | | static zend_long zend_mm_find_leaks_small(zend_mm_chunk *p, uint32_t i, uint32_t j, zend_leak_info *leak) |
2214 | 0 | { |
2215 | 0 | bool empty = true; |
2216 | 0 | zend_long count = 0; |
2217 | 0 | int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]); |
2218 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * (j + 1) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
2219 | |
|
2220 | 0 | while (j < bin_elements[bin_num]) { |
2221 | 0 | if (dbg->size != 0) { |
2222 | 0 | if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) { |
2223 | 0 | count++; |
2224 | 0 | dbg->size = 0; |
2225 | 0 | dbg->filename = NULL; |
2226 | 0 | dbg->lineno = 0; |
2227 | 0 | } else { |
2228 | 0 | empty = false; |
2229 | 0 | } |
2230 | 0 | } |
2231 | 0 | j++; |
2232 | 0 | dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]); |
2233 | 0 | } |
2234 | 0 | if (empty) { |
2235 | 0 | zend_mm_bitset_reset_range(p->free_map, i, bin_pages[bin_num]); |
2236 | 0 | } |
2237 | 0 | return count; |
2238 | 0 | } |
2239 | | |
2240 | | static zend_long zend_mm_find_leaks(zend_mm_heap *heap, zend_mm_chunk *p, uint32_t i, zend_leak_info *leak) |
2241 | 0 | { |
2242 | 0 | zend_long count = 0; |
2243 | |
|
2244 | 0 | do { |
2245 | 0 | while (i < p->free_tail) { |
2246 | 0 | if (zend_mm_bitset_is_set(p->free_map, i)) { |
2247 | 0 | if (p->map[i] & ZEND_MM_IS_SRUN) { |
2248 | 0 | int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]); |
2249 | 0 | count += zend_mm_find_leaks_small(p, i, 0, leak); |
2250 | 0 | i += bin_pages[bin_num]; |
2251 | 0 | } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ { |
2252 | 0 | int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]); |
2253 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
2254 | |
|
2255 | 0 | if (dbg->filename == leak->filename && dbg->lineno == leak->lineno) { |
2256 | 0 | count++; |
2257 | 0 | } |
2258 | 0 | zend_mm_bitset_reset_range(p->free_map, i, pages_count); |
2259 | 0 | i += pages_count; |
2260 | 0 | } |
2261 | 0 | } else { |
2262 | 0 | i++; |
2263 | 0 | } |
2264 | 0 | } |
2265 | 0 | p = p->next; |
2266 | 0 | i = ZEND_MM_FIRST_PAGE; |
2267 | 0 | } while (p != heap->main_chunk); |
2268 | 0 | return count; |
2269 | 0 | } |
2270 | | |
2271 | | static zend_long zend_mm_find_leaks_huge(zend_mm_heap *heap, zend_mm_huge_list *list) |
2272 | 0 | { |
2273 | 0 | zend_long count = 0; |
2274 | 0 | zend_mm_huge_list *prev = list; |
2275 | 0 | zend_mm_huge_list *p = list->next; |
2276 | |
|
2277 | 0 | while (p) { |
2278 | 0 | if (p->dbg.filename == list->dbg.filename && p->dbg.lineno == list->dbg.lineno) { |
2279 | 0 | prev->next = p->next; |
2280 | 0 | zend_mm_chunk_free(heap, p->ptr, p->size); |
2281 | 0 | zend_mm_free_heap(heap, p, NULL, 0, NULL, 0); |
2282 | 0 | count++; |
2283 | 0 | } else { |
2284 | 0 | prev = p; |
2285 | 0 | } |
2286 | 0 | p = prev->next; |
2287 | 0 | } |
2288 | |
|
2289 | 0 | return count; |
2290 | 0 | } |
2291 | | |
2292 | | static void zend_mm_check_leaks(zend_mm_heap *heap) |
2293 | 0 | { |
2294 | 0 | zend_mm_huge_list *list; |
2295 | 0 | zend_mm_chunk *p; |
2296 | 0 | zend_leak_info leak; |
2297 | 0 | zend_long repeated = 0; |
2298 | 0 | uint32_t total = 0; |
2299 | 0 | uint32_t i, j; |
2300 | | |
2301 | | /* find leaked huge blocks and free them */ |
2302 | 0 | list = heap->huge_list; |
2303 | 0 | while (list) { |
2304 | 0 | zend_mm_huge_list *q = list; |
2305 | |
|
2306 | 0 | leak.addr = list->ptr; |
2307 | 0 | leak.size = list->dbg.size; |
2308 | 0 | leak.filename = list->dbg.filename; |
2309 | 0 | leak.orig_filename = list->dbg.orig_filename; |
2310 | 0 | leak.lineno = list->dbg.lineno; |
2311 | 0 | leak.orig_lineno = list->dbg.orig_lineno; |
2312 | |
|
2313 | 0 | zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL); |
2314 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak); |
2315 | 0 | repeated = zend_mm_find_leaks_huge(heap, list); |
2316 | 0 | total += 1 + repeated; |
2317 | 0 | if (repeated) { |
2318 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated); |
2319 | 0 | } |
2320 | |
|
2321 | 0 | heap->huge_list = list = list->next; |
2322 | 0 | zend_mm_chunk_free(heap, q->ptr, q->size); |
2323 | 0 | zend_mm_free_heap(heap, q, NULL, 0, NULL, 0); |
2324 | 0 | } |
2325 | | |
2326 | | /* for each chunk */ |
2327 | 0 | p = heap->main_chunk; |
2328 | 0 | do { |
2329 | 0 | i = ZEND_MM_FIRST_PAGE; |
2330 | 0 | while (i < p->free_tail) { |
2331 | 0 | if (zend_mm_bitset_is_set(p->free_map, i)) { |
2332 | 0 | if (p->map[i] & ZEND_MM_IS_SRUN) { |
2333 | 0 | int bin_num = ZEND_MM_SRUN_BIN_NUM(p->map[i]); |
2334 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
2335 | |
|
2336 | 0 | j = 0; |
2337 | 0 | while (j < bin_elements[bin_num]) { |
2338 | 0 | if (dbg->size != 0) { |
2339 | 0 | leak.addr = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * i + bin_data_size[bin_num] * j); |
2340 | 0 | leak.size = dbg->size; |
2341 | 0 | leak.filename = dbg->filename; |
2342 | 0 | leak.orig_filename = dbg->orig_filename; |
2343 | 0 | leak.lineno = dbg->lineno; |
2344 | 0 | leak.orig_lineno = dbg->orig_lineno; |
2345 | |
|
2346 | 0 | zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL); |
2347 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak); |
2348 | |
|
2349 | 0 | dbg->size = 0; |
2350 | 0 | dbg->filename = NULL; |
2351 | 0 | dbg->lineno = 0; |
2352 | |
|
2353 | 0 | repeated = zend_mm_find_leaks_small(p, i, j + 1, &leak) + |
2354 | 0 | zend_mm_find_leaks(heap, p, i + bin_pages[bin_num], &leak); |
2355 | 0 | total += 1 + repeated; |
2356 | 0 | if (repeated) { |
2357 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated); |
2358 | 0 | } |
2359 | 0 | } |
2360 | 0 | dbg = (zend_mm_debug_info*)((char*)dbg + bin_data_size[bin_num]); |
2361 | 0 | j++; |
2362 | 0 | } |
2363 | 0 | i += bin_pages[bin_num]; |
2364 | 0 | } else /* if (p->map[i] & ZEND_MM_IS_LRUN) */ { |
2365 | 0 | int pages_count = ZEND_MM_LRUN_PAGES(p->map[i]); |
2366 | 0 | zend_mm_debug_info *dbg = (zend_mm_debug_info*)((char*)p + ZEND_MM_PAGE_SIZE * (i + pages_count) - ZEND_MM_ALIGNED_SIZE(sizeof(zend_mm_debug_info))); |
2367 | |
|
2368 | 0 | leak.addr = (void*)((char*)p + ZEND_MM_PAGE_SIZE * i); |
2369 | 0 | leak.size = dbg->size; |
2370 | 0 | leak.filename = dbg->filename; |
2371 | 0 | leak.orig_filename = dbg->orig_filename; |
2372 | 0 | leak.lineno = dbg->lineno; |
2373 | 0 | leak.orig_lineno = dbg->orig_lineno; |
2374 | |
|
2375 | 0 | zend_message_dispatcher(ZMSG_LOG_SCRIPT_NAME, NULL); |
2376 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAK_DETECTED, &leak); |
2377 | |
|
2378 | 0 | zend_mm_bitset_reset_range(p->free_map, i, pages_count); |
2379 | |
|
2380 | 0 | repeated = zend_mm_find_leaks(heap, p, i + pages_count, &leak); |
2381 | 0 | total += 1 + repeated; |
2382 | 0 | if (repeated) { |
2383 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAK_REPEATED, (void *)(uintptr_t)repeated); |
2384 | 0 | } |
2385 | 0 | i += pages_count; |
2386 | 0 | } |
2387 | 0 | } else { |
2388 | 0 | i++; |
2389 | 0 | } |
2390 | 0 | } |
2391 | 0 | p = p->next; |
2392 | 0 | } while (p != heap->main_chunk); |
2393 | 0 | if (total) { |
2394 | 0 | zend_message_dispatcher(ZMSG_MEMORY_LEAKS_GRAND_TOTAL, &total); |
2395 | 0 | } |
2396 | 0 | } |
2397 | | #endif |
2398 | | |
2399 | | #if ZEND_MM_CUSTOM |
2400 | | static void *tracked_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
2401 | | static void tracked_free_all(zend_mm_heap *heap); |
2402 | | static void *poison_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
2403 | | |
2404 | | static void zend_mm_check_freelists(zend_mm_heap *heap) |
2405 | 0 | { |
2406 | 0 | for (uint32_t bin_num = 0; bin_num < ZEND_MM_BINS; bin_num++) { |
2407 | 0 | zend_mm_free_slot *slot = heap->free_slot[bin_num]; |
2408 | 0 | while (slot) { |
2409 | 0 | slot = zend_mm_get_next_free_slot(heap, bin_num, slot); |
2410 | 0 | } |
2411 | 0 | } |
2412 | 0 | } |
2413 | | #endif |
2414 | | |
2415 | | ZEND_API void zend_mm_shutdown(zend_mm_heap *heap, bool full, bool silent) |
2416 | 300k | { |
2417 | 300k | zend_mm_chunk *p; |
2418 | 300k | zend_mm_huge_list *list; |
2419 | | |
2420 | 300k | #if ZEND_MM_CUSTOM |
2421 | 300k | if (heap->use_custom_heap) { |
2422 | 300k | if (heap->custom_heap._malloc == tracked_malloc) { |
2423 | 156k | if (silent) { |
2424 | 25.1k | tracked_free_all(heap); |
2425 | 25.1k | } |
2426 | 156k | zend_hash_clean(heap->tracked_allocs); |
2427 | 156k | if (full) { |
2428 | 0 | zend_hash_destroy(heap->tracked_allocs); |
2429 | 0 | free(heap->tracked_allocs); |
2430 | | /* Make sure the heap free below does not use tracked_free(). */ |
2431 | 0 | heap->custom_heap._free = __zend_free; |
2432 | 0 | } |
2433 | 156k | #if ZEND_MM_STAT |
2434 | 156k | heap->size = 0; |
2435 | 156k | #endif |
2436 | 156k | } |
2437 | | |
2438 | 300k | void (*shutdown)(bool, bool) = heap->custom_heap._shutdown; |
2439 | | |
2440 | 300k | if (full) { |
2441 | 0 | heap->custom_heap._free(heap ZEND_FILE_LINE_CC ZEND_FILE_LINE_EMPTY_CC); |
2442 | 0 | } |
2443 | | |
2444 | 300k | if (shutdown) { |
2445 | 0 | shutdown(full, silent); |
2446 | 0 | } |
2447 | | |
2448 | 300k | return; |
2449 | 300k | } |
2450 | 0 | #endif |
2451 | | |
2452 | 0 | #if ZEND_DEBUG |
2453 | 0 | if (!silent) { |
2454 | 0 | char *tmp = getenv("ZEND_ALLOC_PRINT_LEAKS"); |
2455 | 0 | if (!tmp || ZEND_ATOL(tmp)) { |
2456 | 0 | zend_mm_check_leaks(heap); |
2457 | 0 | } |
2458 | 0 | } |
2459 | 0 | #endif |
2460 | | |
2461 | | /* free huge blocks */ |
2462 | 0 | list = heap->huge_list; |
2463 | 0 | heap->huge_list = NULL; |
2464 | 0 | while (list) { |
2465 | 0 | zend_mm_huge_list *q = list; |
2466 | 0 | list = list->next; |
2467 | 0 | zend_mm_chunk_free(heap, q->ptr, q->size); |
2468 | 0 | } |
2469 | | |
2470 | | /* move all chunks except of the first one into the cache */ |
2471 | 0 | p = heap->main_chunk->next; |
2472 | 0 | while (p != heap->main_chunk) { |
2473 | 0 | zend_mm_chunk *q = p->next; |
2474 | 0 | p->next = heap->cached_chunks; |
2475 | 0 | heap->cached_chunks = p; |
2476 | 0 | p = q; |
2477 | 0 | heap->chunks_count--; |
2478 | 0 | heap->cached_chunks_count++; |
2479 | 0 | } |
2480 | |
|
2481 | 0 | if (full) { |
2482 | | /* free all cached chunks */ |
2483 | 0 | while (heap->cached_chunks) { |
2484 | 0 | p = heap->cached_chunks; |
2485 | 0 | heap->cached_chunks = p->next; |
2486 | 0 | zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE); |
2487 | 0 | } |
2488 | | /* free the first chunk */ |
2489 | 0 | zend_mm_chunk_free(heap, heap->main_chunk, ZEND_MM_CHUNK_SIZE); |
2490 | 0 | } else { |
2491 | | /* free some cached chunks to keep average count */ |
2492 | 0 | heap->avg_chunks_count = (heap->avg_chunks_count + (double)heap->peak_chunks_count) / 2.0; |
2493 | 0 | while ((double)heap->cached_chunks_count + 0.9 > heap->avg_chunks_count && |
2494 | 0 | heap->cached_chunks) { |
2495 | 0 | p = heap->cached_chunks; |
2496 | 0 | heap->cached_chunks = p->next; |
2497 | 0 | zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE); |
2498 | 0 | heap->cached_chunks_count--; |
2499 | 0 | } |
2500 | | /* clear cached chunks */ |
2501 | 0 | p = heap->cached_chunks; |
2502 | 0 | while (p != NULL) { |
2503 | 0 | zend_mm_chunk *q = p->next; |
2504 | 0 | memset(p, 0, sizeof(zend_mm_chunk)); |
2505 | 0 | p->next = q; |
2506 | 0 | p = q; |
2507 | 0 | } |
2508 | | |
2509 | | /* reinitialize the first chunk and heap */ |
2510 | 0 | p = heap->main_chunk; |
2511 | 0 | p->heap = &p->heap_slot; |
2512 | 0 | p->next = p; |
2513 | 0 | p->prev = p; |
2514 | 0 | p->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE; |
2515 | 0 | p->free_tail = ZEND_MM_FIRST_PAGE; |
2516 | 0 | p->num = 0; |
2517 | |
|
2518 | 0 | #if ZEND_MM_STAT |
2519 | 0 | heap->size = heap->peak = 0; |
2520 | 0 | #endif |
2521 | 0 | memset(heap->free_slot, 0, sizeof(heap->free_slot)); |
2522 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
2523 | 0 | heap->real_size = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE; |
2524 | 0 | #endif |
2525 | 0 | #if ZEND_MM_STAT |
2526 | 0 | heap->real_peak = (heap->cached_chunks_count + 1) * ZEND_MM_CHUNK_SIZE; |
2527 | 0 | #endif |
2528 | 0 | heap->chunks_count = 1; |
2529 | 0 | heap->peak_chunks_count = 1; |
2530 | 0 | heap->last_chunks_delete_boundary = 0; |
2531 | 0 | heap->last_chunks_delete_count = 0; |
2532 | |
|
2533 | 0 | memset(p->free_map, 0, sizeof(p->free_map) + sizeof(p->map)); |
2534 | 0 | p->free_map[0] = (1L << ZEND_MM_FIRST_PAGE) - 1; |
2535 | 0 | p->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE); |
2536 | |
|
2537 | 0 | pid_t pid = getpid(); |
2538 | 0 | if (heap->pid != pid) { |
2539 | 0 | zend_mm_init_key(heap); |
2540 | 0 | heap->pid = pid; |
2541 | 0 | } else { |
2542 | 0 | zend_mm_refresh_key(heap); |
2543 | 0 | } |
2544 | 0 | } |
2545 | 0 | } |
2546 | | |
2547 | | /**************/ |
2548 | | /* PUBLIC API */ |
2549 | | /**************/ |
2550 | | |
2551 | | ZEND_API void* ZEND_FASTCALL _zend_mm_alloc(zend_mm_heap *heap, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2552 | 0 | { |
2553 | 0 | return zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2554 | 0 | } |
2555 | | |
2556 | | ZEND_API void ZEND_FASTCALL _zend_mm_free(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2557 | 0 | { |
2558 | 0 | zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2559 | 0 | } |
2560 | | |
2561 | | void* ZEND_FASTCALL _zend_mm_realloc(zend_mm_heap *heap, void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2562 | 0 | { |
2563 | 0 | return zend_mm_realloc_heap(heap, ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2564 | 0 | } |
2565 | | |
2566 | | void* ZEND_FASTCALL _zend_mm_realloc2(zend_mm_heap *heap, void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2567 | 0 | { |
2568 | 0 | return zend_mm_realloc_heap(heap, ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2569 | 0 | } |
2570 | | |
2571 | | ZEND_API size_t ZEND_FASTCALL _zend_mm_block_size(zend_mm_heap *heap, void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2572 | 0 | { |
2573 | 0 | #if ZEND_MM_CUSTOM |
2574 | 0 | if (UNEXPECTED(heap->use_custom_heap)) { |
2575 | 0 | if (heap->custom_heap._malloc == tracked_malloc) { |
2576 | 0 | zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2; |
2577 | 0 | zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h); |
2578 | 0 | if (size_zv) { |
2579 | 0 | return Z_LVAL_P(size_zv); |
2580 | 0 | } |
2581 | 0 | } else if (heap->custom_heap._malloc != poison_malloc) { |
2582 | 0 | return 0; |
2583 | 0 | } |
2584 | 0 | } |
2585 | 0 | #endif |
2586 | 0 | return zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2587 | 0 | } |
2588 | | |
2589 | | /**********************/ |
2590 | | /* Allocation Manager */ |
2591 | | /**********************/ |
2592 | | |
2593 | | typedef struct _zend_alloc_globals { |
2594 | | zend_mm_heap *mm_heap; |
2595 | | } zend_alloc_globals; |
2596 | | |
2597 | | #ifdef ZTS |
2598 | | static int alloc_globals_id; |
2599 | | static size_t alloc_globals_offset; |
2600 | | # define AG(v) ZEND_TSRMG_FAST(alloc_globals_offset, zend_alloc_globals *, v) |
2601 | | #else |
2602 | 7.07G | # define AG(v) (alloc_globals.v) |
2603 | | static zend_alloc_globals alloc_globals; |
2604 | | #endif |
2605 | | |
2606 | | ZEND_API bool is_zend_mm(void) |
2607 | 5 | { |
2608 | 5 | #if ZEND_MM_CUSTOM |
2609 | 5 | return !AG(mm_heap)->use_custom_heap; |
2610 | | #else |
2611 | | return 1; |
2612 | | #endif |
2613 | 5 | } |
2614 | | |
2615 | | ZEND_API bool is_zend_ptr(const void *ptr) |
2616 | 0 | { |
2617 | 0 | #if ZEND_MM_CUSTOM |
2618 | 0 | if (AG(mm_heap)->use_custom_heap) { |
2619 | 0 | if (AG(mm_heap)->custom_heap._malloc == tracked_malloc) { |
2620 | 0 | zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2; |
2621 | 0 | zval *size_zv = zend_hash_index_find(AG(mm_heap)->tracked_allocs, h); |
2622 | 0 | if (size_zv) { |
2623 | 0 | return 1; |
2624 | 0 | } |
2625 | 0 | } |
2626 | 0 | return 0; |
2627 | 0 | } |
2628 | 0 | #endif |
2629 | | |
2630 | 0 | if (AG(mm_heap)->main_chunk) { |
2631 | 0 | zend_mm_chunk *chunk = AG(mm_heap)->main_chunk; |
2632 | |
|
2633 | 0 | do { |
2634 | 0 | if (ptr >= (void*)chunk |
2635 | 0 | && ptr < (void*)((char*)chunk + ZEND_MM_CHUNK_SIZE)) { |
2636 | 0 | return 1; |
2637 | 0 | } |
2638 | 0 | chunk = chunk->next; |
2639 | 0 | } while (chunk != AG(mm_heap)->main_chunk); |
2640 | 0 | } |
2641 | | |
2642 | 0 | zend_mm_huge_list *block = AG(mm_heap)->huge_list; |
2643 | 0 | while (block) { |
2644 | 0 | if (ptr >= block->ptr |
2645 | 0 | && ptr < (void*)((char*)block->ptr + block->size)) { |
2646 | 0 | return 1; |
2647 | 0 | } |
2648 | 0 | block = block->next; |
2649 | 0 | } |
2650 | | |
2651 | 0 | return 0; |
2652 | 0 | } |
2653 | | |
2654 | | #if !ZEND_DEBUG && defined(HAVE_BUILTIN_CONSTANT_P) |
2655 | | #undef _emalloc |
2656 | | |
2657 | | #if ZEND_MM_CUSTOM |
2658 | | # define ZEND_MM_CUSTOM_ALLOCATOR(size) do { \ |
2659 | | if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \ |
2660 | | return AG(mm_heap)->custom_heap._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \ |
2661 | | } \ |
2662 | | } while (0) |
2663 | | # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) do { \ |
2664 | | if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { \ |
2665 | | AG(mm_heap)->custom_heap._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \ |
2666 | | return; \ |
2667 | | } \ |
2668 | | } while (0) |
2669 | | #else |
2670 | | # define ZEND_MM_CUSTOM_ALLOCATOR(size) |
2671 | | # define ZEND_MM_CUSTOM_DEALLOCATOR(ptr) |
2672 | | #endif |
2673 | | |
2674 | | # define _ZEND_BIN_ALLOCATOR(_num, _size, _elements, _pages, _min_size, y) \ |
2675 | | ZEND_API void* ZEND_FASTCALL _emalloc_ ## _size(void) { \ |
2676 | | ZEND_MM_CUSTOM_ALLOCATOR(_size); \ |
2677 | | if (_size < _min_size) { \ |
2678 | | return _emalloc_ ## _min_size(); \ |
2679 | | } \ |
2680 | | return zend_mm_alloc_small(AG(mm_heap), _num ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \ |
2681 | | } |
2682 | | |
2683 | | ZEND_MM_BINS_INFO(_ZEND_BIN_ALLOCATOR, ZEND_MM_MIN_USEABLE_BIN_SIZE, y) |
2684 | | |
2685 | | ZEND_API void* ZEND_FASTCALL _emalloc_large(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2686 | | { |
2687 | | ZEND_MM_CUSTOM_ALLOCATOR(size); |
2688 | | return zend_mm_alloc_large_ex(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2689 | | } |
2690 | | |
2691 | | ZEND_API void* ZEND_FASTCALL _emalloc_huge(size_t size) |
2692 | | { |
2693 | | ZEND_MM_CUSTOM_ALLOCATOR(size); |
2694 | | return zend_mm_alloc_huge(AG(mm_heap), size); |
2695 | | } |
2696 | | |
2697 | | #if ZEND_DEBUG |
2698 | | # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, _min_size, y) \ |
2699 | | ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \ |
2700 | | ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \ |
2701 | | if (_size < _min_size) { \ |
2702 | | _efree_ ## _min_size(ptr); \ |
2703 | | return; \ |
2704 | | } \ |
2705 | | { \ |
2706 | | size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); \ |
2707 | | zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \ |
2708 | | int page_num = page_offset / ZEND_MM_PAGE_SIZE; \ |
2709 | | ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \ |
2710 | | ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_SRUN); \ |
2711 | | ZEND_ASSERT(ZEND_MM_SRUN_BIN_NUM(chunk->map[page_num]) == _num); \ |
2712 | | zend_mm_free_small(AG(mm_heap), ptr, _num); \ |
2713 | | } \ |
2714 | | } |
2715 | | #else |
2716 | | # define _ZEND_BIN_FREE(_num, _size, _elements, _pages, _min_size, y) \ |
2717 | | ZEND_API void ZEND_FASTCALL _efree_ ## _size(void *ptr) { \ |
2718 | | ZEND_MM_CUSTOM_DEALLOCATOR(ptr); \ |
2719 | | if (_size < _min_size) { \ |
2720 | | _efree_ ## _min_size(ptr); \ |
2721 | | return; \ |
2722 | | } \ |
2723 | | { \ |
2724 | | zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); \ |
2725 | | ZEND_MM_CHECK(chunk->heap == AG(mm_heap), "zend_mm_heap corrupted"); \ |
2726 | | zend_mm_free_small(AG(mm_heap), ptr, _num); \ |
2727 | | } \ |
2728 | | } |
2729 | | #endif |
2730 | | |
2731 | | ZEND_MM_BINS_INFO(_ZEND_BIN_FREE, ZEND_MM_MIN_USEABLE_BIN_SIZE, y) |
2732 | | |
2733 | | ZEND_API void ZEND_FASTCALL _efree_large(void *ptr, size_t size) |
2734 | | { |
2735 | | ZEND_MM_CUSTOM_DEALLOCATOR(ptr); |
2736 | | { |
2737 | | size_t page_offset = ZEND_MM_ALIGNED_OFFSET(ptr, ZEND_MM_CHUNK_SIZE); |
2738 | | zend_mm_chunk *chunk = (zend_mm_chunk*)ZEND_MM_ALIGNED_BASE(ptr, ZEND_MM_CHUNK_SIZE); |
2739 | | int page_num = page_offset / ZEND_MM_PAGE_SIZE; |
2740 | | uint32_t pages_count = ZEND_MM_ALIGNED_SIZE_EX(size, ZEND_MM_PAGE_SIZE) / ZEND_MM_PAGE_SIZE; |
2741 | | |
2742 | | ZEND_MM_CHECK(chunk->heap == AG(mm_heap) && ZEND_MM_ALIGNED_OFFSET(page_offset, ZEND_MM_PAGE_SIZE) == 0, "zend_mm_heap corrupted"); |
2743 | | ZEND_ASSERT(chunk->map[page_num] & ZEND_MM_IS_LRUN); |
2744 | | ZEND_ASSERT(ZEND_MM_LRUN_PAGES(chunk->map[page_num]) == pages_count); |
2745 | | zend_mm_free_large(AG(mm_heap), chunk, page_num, pages_count); |
2746 | | } |
2747 | | } |
2748 | | |
2749 | | ZEND_API void ZEND_FASTCALL _efree_huge(void *ptr, size_t size) |
2750 | | { |
2751 | | |
2752 | | ZEND_MM_CUSTOM_DEALLOCATOR(ptr); |
2753 | | zend_mm_free_huge(AG(mm_heap), ptr); |
2754 | | } |
2755 | | #endif |
2756 | | |
2757 | | ZEND_API void* ZEND_FASTCALL _emalloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2758 | 2.06G | { |
2759 | 2.06G | #if ZEND_MM_CUSTOM |
2760 | 2.06G | if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { |
2761 | 2.06G | return AG(mm_heap)->custom_heap._malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); \ |
2762 | 2.06G | } |
2763 | 0 | #endif |
2764 | 0 | return zend_mm_alloc_heap(AG(mm_heap), size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2765 | 2.06G | } |
2766 | | |
2767 | | ZEND_API void ZEND_FASTCALL _efree(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2768 | 2.06G | { |
2769 | 2.06G | #if ZEND_MM_CUSTOM |
2770 | 2.06G | if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { |
2771 | 2.06G | AG(mm_heap)->custom_heap._free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2772 | 2.06G | return; |
2773 | 2.06G | } |
2774 | 0 | #endif |
2775 | 0 | zend_mm_free_heap(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2776 | 0 | } |
2777 | | |
2778 | | ZEND_API void* ZEND_FASTCALL _erealloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2779 | 20.2M | { |
2780 | 20.2M | #if ZEND_MM_CUSTOM |
2781 | 20.2M | if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { |
2782 | 20.2M | return AG(mm_heap)->custom_heap._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2783 | 20.2M | } |
2784 | 0 | #endif |
2785 | 0 | return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 0, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2786 | 20.2M | } |
2787 | | |
2788 | | ZEND_API void* ZEND_FASTCALL _erealloc2(void *ptr, size_t size, size_t copy_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2789 | 232k | { |
2790 | 232k | #if ZEND_MM_CUSTOM |
2791 | 232k | if (UNEXPECTED(AG(mm_heap)->use_custom_heap)) { |
2792 | 232k | return AG(mm_heap)->custom_heap._realloc(ptr, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2793 | 232k | } |
2794 | 0 | #endif |
2795 | 0 | return zend_mm_realloc_heap(AG(mm_heap), ptr, size, 1, copy_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2796 | 232k | } |
2797 | | |
2798 | | ZEND_API size_t ZEND_FASTCALL _zend_mem_block_size(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2799 | 0 | { |
2800 | 0 | return _zend_mm_block_size(AG(mm_heap), ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2801 | 0 | } |
2802 | | |
2803 | | ZEND_API void* ZEND_FASTCALL _safe_emalloc(size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2804 | 1.40M | { |
2805 | 1.40M | return _emalloc(zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2806 | 1.40M | } |
2807 | | |
2808 | | ZEND_API void* ZEND_FASTCALL _safe_malloc(size_t nmemb, size_t size, size_t offset) |
2809 | 0 | { |
2810 | 0 | return pemalloc(zend_safe_address_guarded(nmemb, size, offset), 1); |
2811 | 0 | } |
2812 | | |
2813 | | ZEND_API void* ZEND_FASTCALL _safe_erealloc(void *ptr, size_t nmemb, size_t size, size_t offset ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2814 | 1.53M | { |
2815 | 1.53M | return _erealloc(ptr, zend_safe_address_guarded(nmemb, size, offset) ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2816 | 1.53M | } |
2817 | | |
2818 | | ZEND_API void* ZEND_FASTCALL _safe_realloc(void *ptr, size_t nmemb, size_t size, size_t offset) |
2819 | 0 | { |
2820 | 0 | return perealloc(ptr, zend_safe_address_guarded(nmemb, size, offset), 1); |
2821 | 0 | } |
2822 | | |
2823 | | ZEND_API void* ZEND_FASTCALL _ecalloc(size_t nmemb, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2824 | 381M | { |
2825 | 381M | void *p; |
2826 | | |
2827 | 381M | size = zend_safe_address_guarded(nmemb, size, 0); |
2828 | 381M | p = _emalloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2829 | 381M | memset(p, 0, size); |
2830 | 381M | return p; |
2831 | 381M | } |
2832 | | |
2833 | | ZEND_API char* ZEND_FASTCALL _estrdup(const char *s ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2834 | 1.60G | { |
2835 | 1.60G | size_t length; |
2836 | 1.60G | char *p; |
2837 | | |
2838 | 1.60G | length = strlen(s); |
2839 | 1.60G | if (UNEXPECTED(length + 1 == 0)) { |
2840 | 0 | zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length); |
2841 | 0 | } |
2842 | 1.60G | p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2843 | 1.60G | memcpy(p, s, length+1); |
2844 | 1.60G | return p; |
2845 | 1.60G | } |
2846 | | |
2847 | | ZEND_API char* ZEND_FASTCALL _estrndup(const char *s, size_t length ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2848 | 5.96M | { |
2849 | 5.96M | char *p; |
2850 | | |
2851 | 5.96M | if (UNEXPECTED(length + 1 == 0)) { |
2852 | 0 | zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length); |
2853 | 0 | } |
2854 | 5.96M | p = (char *) _emalloc(length + 1 ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
2855 | 5.96M | memcpy(p, s, length); |
2856 | 5.96M | p[length] = 0; |
2857 | 5.96M | return p; |
2858 | 5.96M | } |
2859 | | |
2860 | | static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void); |
2861 | | |
2862 | | ZEND_API char* ZEND_FASTCALL zend_strndup(const char *s, size_t length) |
2863 | 8 | { |
2864 | 8 | char *p; |
2865 | | |
2866 | 8 | if (UNEXPECTED(length + 1 == 0)) { |
2867 | 0 | zend_error_noreturn(E_ERROR, "Possible integer overflow in memory allocation (1 * %zu + 1)", length); |
2868 | 0 | } |
2869 | 8 | p = (char *) malloc(length + 1); |
2870 | 8 | if (UNEXPECTED(p == NULL)) { |
2871 | 0 | zend_out_of_memory(); |
2872 | 0 | } |
2873 | 8 | if (EXPECTED(length)) { |
2874 | 8 | memcpy(p, s, length); |
2875 | 8 | } |
2876 | 8 | p[length] = 0; |
2877 | 8 | return p; |
2878 | 8 | } |
2879 | | |
2880 | | ZEND_API zend_result zend_set_memory_limit(size_t memory_limit) |
2881 | 308k | { |
2882 | 308k | #if ZEND_MM_LIMIT |
2883 | 308k | zend_mm_heap *heap = AG(mm_heap); |
2884 | | |
2885 | 308k | if (UNEXPECTED(memory_limit < heap->real_size)) { |
2886 | 0 | if (memory_limit >= heap->real_size - heap->cached_chunks_count * ZEND_MM_CHUNK_SIZE) { |
2887 | | /* free some cached chunks to fit into new memory limit */ |
2888 | 0 | do { |
2889 | 0 | zend_mm_chunk *p = heap->cached_chunks; |
2890 | 0 | heap->cached_chunks = p->next; |
2891 | 0 | zend_mm_chunk_free(heap, p, ZEND_MM_CHUNK_SIZE); |
2892 | 0 | heap->cached_chunks_count--; |
2893 | 0 | heap->real_size -= ZEND_MM_CHUNK_SIZE; |
2894 | 0 | } while (memory_limit < heap->real_size); |
2895 | 0 | return SUCCESS; |
2896 | 0 | } |
2897 | 0 | return FAILURE; |
2898 | 0 | } |
2899 | 308k | AG(mm_heap)->limit = memory_limit; |
2900 | 308k | #endif |
2901 | 308k | return SUCCESS; |
2902 | 308k | } |
2903 | | |
2904 | | ZEND_API bool zend_alloc_in_memory_limit_error_reporting(void) |
2905 | 3.35M | { |
2906 | 3.35M | #if ZEND_MM_LIMIT |
2907 | 3.35M | return AG(mm_heap)->overflow; |
2908 | | #else |
2909 | | return false; |
2910 | | #endif |
2911 | 3.35M | } |
2912 | | |
2913 | | ZEND_API size_t zend_memory_usage(bool real_usage) |
2914 | 25 | { |
2915 | 25 | #if ZEND_MM_STAT |
2916 | 25 | if (real_usage) { |
2917 | 0 | return AG(mm_heap)->real_size; |
2918 | 25 | } else { |
2919 | 25 | size_t usage = AG(mm_heap)->size; |
2920 | 25 | return usage; |
2921 | 25 | } |
2922 | 0 | #endif |
2923 | 0 | return 0; |
2924 | 25 | } |
2925 | | |
2926 | | ZEND_API size_t zend_memory_peak_usage(bool real_usage) |
2927 | 0 | { |
2928 | 0 | #if ZEND_MM_STAT |
2929 | 0 | if (real_usage) { |
2930 | 0 | return AG(mm_heap)->real_peak; |
2931 | 0 | } else { |
2932 | 0 | return AG(mm_heap)->peak; |
2933 | 0 | } |
2934 | 0 | #endif |
2935 | 0 | return 0; |
2936 | 0 | } |
2937 | | |
2938 | | ZEND_API void zend_memory_reset_peak_usage(void) |
2939 | 0 | { |
2940 | 0 | #if ZEND_MM_STAT |
2941 | 0 | AG(mm_heap)->real_peak = AG(mm_heap)->real_size; |
2942 | 0 | AG(mm_heap)->peak = AG(mm_heap)->size; |
2943 | 0 | #endif |
2944 | 0 | } |
2945 | | |
2946 | | ZEND_API void shutdown_memory_manager(bool silent, bool full_shutdown) |
2947 | 300k | { |
2948 | 300k | zend_mm_shutdown(AG(mm_heap), full_shutdown, silent); |
2949 | 300k | } |
2950 | | |
2951 | | static ZEND_COLD ZEND_NORETURN void zend_out_of_memory(void) |
2952 | 0 | { |
2953 | 0 | fprintf(stderr, "Out of memory\n"); |
2954 | 0 | exit(1); |
2955 | 0 | } |
2956 | | |
2957 | | #if ZEND_MM_CUSTOM |
2958 | 1.46G | static zend_always_inline void tracked_add(zend_mm_heap *heap, void *ptr, size_t size) { |
2959 | 1.46G | zval size_zv; |
2960 | 1.46G | zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2; |
2961 | 1.46G | ZEND_ASSERT((void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2) == ptr); |
2962 | 1.46G | ZVAL_LONG(&size_zv, size); |
2963 | 1.46G | zend_hash_index_add_new(heap->tracked_allocs, h, &size_zv); |
2964 | 1.46G | } |
2965 | | |
2966 | 1.46G | static zend_always_inline zval *tracked_get_size_zv(zend_mm_heap *heap, void *ptr) { |
2967 | 1.46G | zend_ulong h = ((uintptr_t) ptr) >> ZEND_MM_ALIGNMENT_LOG2; |
2968 | 1.46G | zval *size_zv = zend_hash_index_find(heap->tracked_allocs, h); |
2969 | 1.46G | ZEND_ASSERT(size_zv && "Trying to free pointer not allocated through ZendMM"); |
2970 | 1.46G | return size_zv; |
2971 | 1.46G | } |
2972 | | |
2973 | 1.46G | static zend_always_inline void tracked_check_limit(zend_mm_heap *heap, size_t add_size) { |
2974 | 1.46G | #if ZEND_MM_STAT |
2975 | 1.46G | if (add_size > heap->limit - heap->size && !heap->overflow) { |
2976 | 548 | #if ZEND_DEBUG |
2977 | 548 | zend_mm_safe_error(heap, |
2978 | 548 | "Allowed memory size of %zu bytes exhausted at %s:%d (tried to allocate %zu bytes)", |
2979 | 548 | heap->limit, "file", 0, add_size); |
2980 | | #else |
2981 | | zend_mm_safe_error(heap, |
2982 | | "Allowed memory size of %zu bytes exhausted (tried to allocate %zu bytes)", |
2983 | | heap->limit, add_size); |
2984 | | #endif |
2985 | 548 | } |
2986 | 1.46G | #endif |
2987 | 1.46G | } |
2988 | | |
2989 | | static void *tracked_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
2990 | 1.45G | { |
2991 | 1.45G | zend_mm_heap *heap = AG(mm_heap); |
2992 | 1.45G | tracked_check_limit(heap, size); |
2993 | | |
2994 | 1.45G | void *ptr = malloc(size); |
2995 | 1.45G | if (!ptr) { |
2996 | 0 | zend_out_of_memory(); |
2997 | 0 | } |
2998 | | |
2999 | 1.45G | tracked_add(heap, ptr, size); |
3000 | 1.45G | #if ZEND_MM_STAT |
3001 | 1.45G | heap->size += size; |
3002 | 1.45G | #endif |
3003 | 1.45G | return ptr; |
3004 | 1.45G | } |
3005 | | |
3006 | 1.45G | static void tracked_free(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) { |
3007 | 1.45G | if (!ptr) { |
3008 | 456k | return; |
3009 | 456k | } |
3010 | | |
3011 | 1.45G | zend_mm_heap *heap = AG(mm_heap); |
3012 | 1.45G | zval *size_zv = tracked_get_size_zv(heap, ptr); |
3013 | 1.45G | #if ZEND_MM_STAT |
3014 | 1.45G | heap->size -= Z_LVAL_P(size_zv); |
3015 | 1.45G | #endif |
3016 | 1.45G | zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) size_zv); |
3017 | 1.45G | free(ptr); |
3018 | 1.45G | } |
3019 | | |
3020 | 17.3M | static void *tracked_realloc(void *ptr, size_t new_size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) { |
3021 | 17.3M | zend_mm_heap *heap = AG(mm_heap); |
3022 | 17.3M | zval *old_size_zv = NULL; |
3023 | 17.3M | size_t old_size = 0; |
3024 | 17.3M | if (ptr) { |
3025 | 14.7M | old_size_zv = tracked_get_size_zv(heap, ptr); |
3026 | 14.7M | old_size = Z_LVAL_P(old_size_zv); |
3027 | 14.7M | } |
3028 | | |
3029 | 17.3M | if (new_size > old_size) { |
3030 | 13.4M | tracked_check_limit(heap, new_size - old_size); |
3031 | 13.4M | } |
3032 | | |
3033 | | /* Delete information about old allocation only after checking the memory limit. */ |
3034 | 17.3M | if (old_size_zv) { |
3035 | 14.7M | zend_hash_del_bucket(heap->tracked_allocs, (Bucket *) old_size_zv); |
3036 | 14.7M | } |
3037 | | |
3038 | 17.3M | ptr = __zend_realloc(ptr, new_size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3039 | 17.3M | tracked_add(heap, ptr, new_size); |
3040 | 17.3M | #if ZEND_MM_STAT |
3041 | 17.3M | heap->size += new_size - old_size; |
3042 | 17.3M | #endif |
3043 | 17.3M | return ptr; |
3044 | 17.3M | } |
3045 | | |
3046 | 25.1k | static void tracked_free_all(zend_mm_heap *heap) { |
3047 | 25.1k | HashTable *tracked_allocs = heap->tracked_allocs; |
3048 | 25.1k | zend_ulong h; |
3049 | 346M | ZEND_HASH_FOREACH_NUM_KEY(tracked_allocs, h) { |
3050 | 346M | void *ptr = (void *) (uintptr_t) (h << ZEND_MM_ALIGNMENT_LOG2); |
3051 | 346M | free(ptr); |
3052 | 346M | } ZEND_HASH_FOREACH_END(); |
3053 | 25.1k | } |
3054 | | |
3055 | | static void* poison_malloc(size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3056 | 0 | { |
3057 | 0 | zend_mm_heap *heap = AG(mm_heap); |
3058 | |
|
3059 | 0 | if (SIZE_MAX - heap->debug.padding * 2 < size) { |
3060 | 0 | zend_mm_panic("Integer overflow in memory allocation"); |
3061 | 0 | } |
3062 | 0 | size += heap->debug.padding * 2; |
3063 | |
|
3064 | 0 | void *ptr = zend_mm_alloc_heap(heap, size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3065 | |
|
3066 | 0 | if (EXPECTED(ptr)) { |
3067 | 0 | if (heap->debug.poison_alloc) { |
3068 | 0 | memset(ptr, heap->debug.poison_alloc_value, size); |
3069 | 0 | } |
3070 | |
|
3071 | 0 | ptr = (char*)ptr + heap->debug.padding; |
3072 | 0 | } |
3073 | |
|
3074 | 0 | return ptr; |
3075 | 0 | } |
3076 | | |
3077 | | static void poison_free(void *ptr ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3078 | 0 | { |
3079 | 0 | zend_mm_heap *heap = AG(mm_heap); |
3080 | |
|
3081 | 0 | if (EXPECTED(ptr)) { |
3082 | | /* zend_mm_shutdown() will try to free the heap when custom handlers |
3083 | | * are installed */ |
3084 | 0 | if (UNEXPECTED(ptr == heap)) { |
3085 | 0 | return; |
3086 | 0 | } |
3087 | | |
3088 | 0 | ptr = (char*)ptr - heap->debug.padding; |
3089 | |
|
3090 | 0 | size_t size = zend_mm_size(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3091 | |
|
3092 | 0 | if (heap->debug.poison_free) { |
3093 | 0 | memset(ptr, heap->debug.poison_free_value, size); |
3094 | 0 | } |
3095 | 0 | } |
3096 | | |
3097 | 0 | zend_mm_free_heap(heap, ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3098 | 0 | } |
3099 | | |
3100 | | static void* poison_realloc(void *ptr, size_t size ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3101 | 0 | { |
3102 | 0 | zend_mm_heap *heap = AG(mm_heap); |
3103 | |
|
3104 | 0 | void *new = poison_malloc(size ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3105 | |
|
3106 | 0 | if (ptr) { |
3107 | | /* Determine the size of the old allocation from the unpadded pointer. */ |
3108 | 0 | size_t oldsize = zend_mm_size(heap, (char*)ptr - heap->debug.padding ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3109 | | |
3110 | | /* Remove the padding size to determine the size that is available to the user. */ |
3111 | 0 | oldsize -= (2 * heap->debug.padding); |
3112 | |
|
3113 | 0 | #if ZEND_DEBUG |
3114 | 0 | oldsize -= sizeof(zend_mm_debug_info); |
3115 | 0 | #endif |
3116 | |
|
3117 | 0 | memcpy(new, ptr, MIN(oldsize, size)); |
3118 | 0 | poison_free(ptr ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3119 | 0 | } |
3120 | |
|
3121 | 0 | return new; |
3122 | 0 | } |
3123 | | |
3124 | | static size_t poison_gc(void) |
3125 | 0 | { |
3126 | 0 | zend_mm_heap *heap = AG(mm_heap); |
3127 | |
|
3128 | 0 | void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
3129 | 0 | void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
3130 | 0 | void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
3131 | 0 | size_t (*_gc)(void); |
3132 | 0 | void (*_shutdown)(bool, bool); |
3133 | |
|
3134 | 0 | zend_mm_get_custom_handlers_ex(heap, &_malloc, &_free, &_realloc, &_gc, &_shutdown); |
3135 | 0 | zend_mm_set_custom_handlers_ex(heap, NULL, NULL, NULL, NULL, NULL); |
3136 | |
|
3137 | 0 | size_t collected = zend_mm_gc(heap); |
3138 | |
|
3139 | 0 | zend_mm_set_custom_handlers_ex(heap, _malloc, _free, _realloc, _gc, _shutdown); |
3140 | |
|
3141 | 0 | return collected; |
3142 | 0 | } |
3143 | | |
3144 | | static void poison_shutdown(bool full, bool silent) |
3145 | 0 | { |
3146 | 0 | zend_mm_heap *heap = AG(mm_heap); |
3147 | |
|
3148 | 0 | void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
3149 | 0 | void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
3150 | 0 | void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC); |
3151 | 0 | size_t (*_gc)(void); |
3152 | 0 | void (*_shutdown)(bool, bool); |
3153 | |
|
3154 | 0 | zend_mm_get_custom_handlers_ex(heap, &_malloc, &_free, &_realloc, &_gc, &_shutdown); |
3155 | 0 | zend_mm_set_custom_handlers_ex(heap, NULL, NULL, NULL, NULL, NULL); |
3156 | |
|
3157 | 0 | if (heap->debug.check_freelists_on_shutdown) { |
3158 | 0 | zend_mm_check_freelists(heap); |
3159 | 0 | } |
3160 | |
|
3161 | 0 | zend_mm_shutdown(heap, full, silent); |
3162 | |
|
3163 | 0 | if (!full) { |
3164 | 0 | zend_mm_set_custom_handlers_ex(heap, _malloc, _free, _realloc, _gc, _shutdown); |
3165 | 0 | } |
3166 | 0 | } |
3167 | | |
3168 | | static void poison_enable(zend_mm_heap *heap, char *parameters) |
3169 | 0 | { |
3170 | 0 | char *tmp = parameters; |
3171 | 0 | char *end = tmp + strlen(tmp); |
3172 | | |
3173 | | /* Trim heading/trailing whitespaces */ |
3174 | 0 | while (*tmp == ' ' || *tmp == '\t' || *tmp == '\n') { |
3175 | 0 | tmp++; |
3176 | 0 | } |
3177 | 0 | while (end != tmp && (*(end-1) == ' ' || *(end-1) == '\t' || *(end-1) == '\n')) { |
3178 | 0 | end--; |
3179 | 0 | } |
3180 | |
|
3181 | 0 | if (tmp == end) { |
3182 | 0 | return; |
3183 | 0 | } |
3184 | | |
3185 | 0 | while (1) { |
3186 | 0 | char *key = tmp; |
3187 | |
|
3188 | 0 | tmp = memchr(tmp, '=', end - tmp); |
3189 | 0 | if (!tmp) { |
3190 | 0 | size_t key_len = end - key; |
3191 | 0 | fprintf(stderr, "Unexpected EOF after ZEND_MM_DEBUG parameter '%.*s', expected '='\n", |
3192 | 0 | (int)key_len, key); |
3193 | 0 | return; |
3194 | 0 | } |
3195 | | |
3196 | 0 | size_t key_len = tmp - key; |
3197 | 0 | char *value = tmp + 1; |
3198 | |
|
3199 | 0 | if (key_len == strlen("poison_alloc") |
3200 | 0 | && !memcmp(key, "poison_alloc", key_len)) { |
3201 | |
|
3202 | 0 | heap->debug.poison_alloc = true; |
3203 | 0 | heap->debug.poison_alloc_value = (uint8_t) ZEND_STRTOUL(value, &tmp, 0); |
3204 | |
|
3205 | 0 | } else if (key_len == strlen("poison_free") |
3206 | 0 | && !memcmp(key, "poison_free", key_len)) { |
3207 | |
|
3208 | 0 | heap->debug.poison_free = true; |
3209 | 0 | heap->debug.poison_free_value = (uint8_t) ZEND_STRTOUL(value, &tmp, 0); |
3210 | |
|
3211 | 0 | } else if (key_len == strlen("padding") |
3212 | 0 | && !memcmp(key, "padding", key_len)) { |
3213 | |
|
3214 | 0 | uint8_t padding = ZEND_STRTOUL(value, &tmp, 0); |
3215 | 0 | if (ZEND_MM_ALIGNED_SIZE(padding) != padding) { |
3216 | 0 | fprintf(stderr, "ZEND_MM_DEBUG padding must be a multiple of %u, %u given\n", |
3217 | 0 | (unsigned int)ZEND_MM_ALIGNMENT, |
3218 | 0 | (unsigned int)padding); |
3219 | 0 | return; |
3220 | 0 | } |
3221 | 0 | heap->debug.padding = padding; |
3222 | |
|
3223 | 0 | } else if (key_len == strlen("check_freelists_on_shutdown") |
3224 | 0 | && !memcmp(key, "check_freelists_on_shutdown", key_len)) { |
3225 | |
|
3226 | 0 | heap->debug.check_freelists_on_shutdown = (bool) ZEND_STRTOUL(value, &tmp, 0); |
3227 | |
|
3228 | 0 | } else { |
3229 | 0 | fprintf(stderr, "Unknown ZEND_MM_DEBUG parameter: '%.*s'\n", |
3230 | 0 | (int)key_len, key); |
3231 | 0 | return; |
3232 | 0 | } |
3233 | | |
3234 | 0 | if (tmp == end) { |
3235 | 0 | break; |
3236 | 0 | } |
3237 | 0 | if (*tmp != ',') { |
3238 | 0 | fprintf(stderr, "Unexpected '%c' after value of ZEND_MM_DEBUG parameter '%.*s', expected ','\n", |
3239 | 0 | *tmp, (int)key_len, key); |
3240 | 0 | return; |
3241 | 0 | } |
3242 | 0 | tmp++; |
3243 | 0 | } |
3244 | | |
3245 | 0 | zend_mm_set_custom_handlers_ex(heap, poison_malloc, poison_free, |
3246 | 0 | poison_realloc, poison_gc, poison_shutdown); |
3247 | 0 | } |
3248 | | #endif |
3249 | | |
3250 | | static void alloc_globals_ctor(zend_alloc_globals *alloc_globals) |
3251 | 16 | { |
3252 | 16 | char *tmp; |
3253 | | |
3254 | 16 | #if ZEND_MM_CUSTOM |
3255 | 16 | tmp = getenv("USE_ZEND_ALLOC"); |
3256 | 16 | if (tmp && !ZEND_ATOL(tmp)) { |
3257 | 16 | bool tracked = (tmp = getenv("USE_TRACKED_ALLOC")) && ZEND_ATOL(tmp); |
3258 | 16 | zend_mm_heap *mm_heap = alloc_globals->mm_heap = malloc(sizeof(zend_mm_heap)); |
3259 | 16 | memset(mm_heap, 0, sizeof(zend_mm_heap)); |
3260 | 16 | mm_heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD; |
3261 | 16 | mm_heap->limit = (size_t)Z_L(-1) >> 1; |
3262 | 16 | mm_heap->overflow = 0; |
3263 | | |
3264 | 16 | if (!tracked) { |
3265 | | /* Use system allocator. */ |
3266 | 8 | mm_heap->custom_heap._malloc = __zend_malloc; |
3267 | 8 | mm_heap->custom_heap._free = __zend_free; |
3268 | 8 | mm_heap->custom_heap._realloc = __zend_realloc; |
3269 | 8 | } else { |
3270 | | /* Use system allocator and track allocations for auto-free. */ |
3271 | 8 | mm_heap->custom_heap._malloc = tracked_malloc; |
3272 | 8 | mm_heap->custom_heap._free = tracked_free; |
3273 | 8 | mm_heap->custom_heap._realloc = tracked_realloc; |
3274 | 8 | mm_heap->tracked_allocs = malloc(sizeof(HashTable)); |
3275 | 8 | zend_hash_init(mm_heap->tracked_allocs, 1024, NULL, NULL, 1); |
3276 | 8 | } |
3277 | 16 | return; |
3278 | 16 | } |
3279 | 0 | #endif |
3280 | | |
3281 | 0 | tmp = getenv("USE_ZEND_ALLOC_HUGE_PAGES"); |
3282 | 0 | if (tmp && ZEND_ATOL(tmp)) { |
3283 | 0 | zend_mm_use_huge_pages = true; |
3284 | 0 | } |
3285 | 0 | alloc_globals->mm_heap = zend_mm_init(); |
3286 | |
|
3287 | 0 | #if ZEND_MM_CUSTOM |
3288 | 0 | ZEND_ASSERT(!alloc_globals->mm_heap->tracked_allocs); |
3289 | 0 | tmp = getenv("ZEND_MM_DEBUG"); |
3290 | 0 | if (tmp) { |
3291 | 0 | poison_enable(alloc_globals->mm_heap, tmp); |
3292 | 0 | } |
3293 | 0 | #endif |
3294 | 0 | } |
3295 | | |
3296 | | #ifdef ZTS |
3297 | | static void alloc_globals_dtor(zend_alloc_globals *alloc_globals) |
3298 | | { |
3299 | | zend_mm_shutdown(alloc_globals->mm_heap, 1, 1); |
3300 | | } |
3301 | | #endif |
3302 | | |
3303 | | ZEND_API void start_memory_manager(void) |
3304 | 16 | { |
3305 | 16 | #ifndef _WIN32 |
3306 | 16 | # if defined(_SC_PAGESIZE) |
3307 | 16 | REAL_PAGE_SIZE = sysconf(_SC_PAGESIZE); |
3308 | | # elif defined(_SC_PAGE_SIZE) |
3309 | | REAL_PAGE_SIZE = sysconf(_SC_PAGE_SIZE); |
3310 | | # endif |
3311 | 16 | #endif |
3312 | | #ifdef ZTS |
3313 | | ts_allocate_fast_id(&alloc_globals_id, &alloc_globals_offset, sizeof(zend_alloc_globals), (ts_allocate_ctor) alloc_globals_ctor, (ts_allocate_dtor) alloc_globals_dtor); |
3314 | | #else |
3315 | 16 | alloc_globals_ctor(&alloc_globals); |
3316 | 16 | #endif |
3317 | 16 | } |
3318 | | |
3319 | | ZEND_API zend_mm_heap *zend_mm_set_heap(zend_mm_heap *new_heap) |
3320 | 0 | { |
3321 | 0 | zend_mm_heap *old_heap; |
3322 | |
|
3323 | 0 | old_heap = AG(mm_heap); |
3324 | 0 | AG(mm_heap) = (zend_mm_heap*)new_heap; |
3325 | 0 | return (zend_mm_heap*)old_heap; |
3326 | 0 | } |
3327 | | |
3328 | | ZEND_API zend_mm_heap *zend_mm_get_heap(void) |
3329 | 0 | { |
3330 | 0 | return AG(mm_heap); |
3331 | 0 | } |
3332 | | |
3333 | | ZEND_API bool zend_mm_is_custom_heap(zend_mm_heap *new_heap) |
3334 | 0 | { |
3335 | 0 | #if ZEND_MM_CUSTOM |
3336 | 0 | return AG(mm_heap)->use_custom_heap; |
3337 | | #else |
3338 | | return 0; |
3339 | | #endif |
3340 | 0 | } |
3341 | | |
3342 | | ZEND_API void zend_mm_set_custom_handlers(zend_mm_heap *heap, |
3343 | | void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3344 | | void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3345 | | void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)) |
3346 | 0 | { |
3347 | 0 | #if ZEND_MM_CUSTOM |
3348 | 0 | zend_mm_set_custom_handlers_ex(heap, _malloc, _free, _realloc, NULL, NULL); |
3349 | 0 | #endif |
3350 | 0 | } |
3351 | | |
3352 | | ZEND_API void zend_mm_set_custom_handlers_ex(zend_mm_heap *heap, |
3353 | | void* (*_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3354 | | void (*_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3355 | | void* (*_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3356 | | size_t (*_gc)(void), |
3357 | | void (*_shutdown)(bool, bool)) |
3358 | 0 | { |
3359 | 0 | #if ZEND_MM_CUSTOM |
3360 | 0 | zend_mm_heap *_heap = (zend_mm_heap*)heap; |
3361 | |
|
3362 | 0 | if (!_malloc && !_free && !_realloc) { |
3363 | 0 | _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_NONE; |
3364 | 0 | } else { |
3365 | 0 | _heap->use_custom_heap = ZEND_MM_CUSTOM_HEAP_STD; |
3366 | 0 | _heap->custom_heap._malloc = _malloc; |
3367 | 0 | _heap->custom_heap._free = _free; |
3368 | 0 | _heap->custom_heap._realloc = _realloc; |
3369 | 0 | _heap->custom_heap._gc = _gc; |
3370 | 0 | _heap->custom_heap._shutdown = _shutdown; |
3371 | 0 | } |
3372 | 0 | #endif |
3373 | 0 | } |
3374 | | |
3375 | | ZEND_API void zend_mm_get_custom_handlers(zend_mm_heap *heap, |
3376 | | void* (**_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3377 | | void (**_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3378 | | void* (**_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC)) |
3379 | 0 | { |
3380 | 0 | #if ZEND_MM_CUSTOM |
3381 | 0 | zend_mm_get_custom_handlers_ex(heap, _malloc, _free, _realloc, NULL, NULL); |
3382 | 0 | #endif |
3383 | 0 | } |
3384 | | |
3385 | | ZEND_API void zend_mm_get_custom_handlers_ex(zend_mm_heap *heap, |
3386 | | void* (**_malloc)(size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3387 | | void (**_free)(void* ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3388 | | void* (**_realloc)(void*, size_t ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC), |
3389 | | size_t (**_gc)(void), |
3390 | | void (**_shutdown)(bool, bool)) |
3391 | 0 | { |
3392 | 0 | #if ZEND_MM_CUSTOM |
3393 | 0 | zend_mm_heap *_heap = (zend_mm_heap*)heap; |
3394 | |
|
3395 | 0 | if (heap->use_custom_heap) { |
3396 | 0 | *_malloc = _heap->custom_heap._malloc; |
3397 | 0 | *_free = _heap->custom_heap._free; |
3398 | 0 | *_realloc = _heap->custom_heap._realloc; |
3399 | 0 | if (_gc != NULL) { |
3400 | 0 | *_gc = _heap->custom_heap._gc; |
3401 | 0 | } |
3402 | 0 | if (_shutdown != NULL) { |
3403 | 0 | *_shutdown = _heap->custom_heap._shutdown; |
3404 | 0 | } |
3405 | 0 | } else { |
3406 | 0 | *_malloc = NULL; |
3407 | 0 | *_free = NULL; |
3408 | 0 | *_realloc = NULL; |
3409 | 0 | if (_gc != NULL) { |
3410 | 0 | *_gc = NULL; |
3411 | 0 | } |
3412 | 0 | if (_shutdown != NULL) { |
3413 | 0 | *_shutdown = NULL; |
3414 | 0 | } |
3415 | 0 | } |
3416 | | #else |
3417 | | *_malloc = NULL; |
3418 | | *_free = NULL; |
3419 | | *_realloc = NULL; |
3420 | | *_gc = NULL; |
3421 | | *_shutdown = NULL; |
3422 | | #endif |
3423 | 0 | } |
3424 | | |
3425 | | ZEND_API zend_mm_storage *zend_mm_get_storage(zend_mm_heap *heap) |
3426 | 0 | { |
3427 | 0 | #if ZEND_MM_STORAGE |
3428 | 0 | return heap->storage; |
3429 | | #else |
3430 | | return NULL; |
3431 | | #endif |
3432 | 0 | } |
3433 | | |
3434 | | ZEND_API zend_mm_heap *zend_mm_startup(void) |
3435 | 0 | { |
3436 | 0 | return zend_mm_init(); |
3437 | 0 | } |
3438 | | |
3439 | | ZEND_API zend_mm_heap *zend_mm_startup_ex(const zend_mm_handlers *handlers, void *data, size_t data_size) |
3440 | 0 | { |
3441 | 0 | #if ZEND_MM_STORAGE |
3442 | 0 | zend_mm_storage tmp_storage, *storage; |
3443 | 0 | zend_mm_chunk *chunk; |
3444 | 0 | zend_mm_heap *heap; |
3445 | |
|
3446 | 0 | memcpy((zend_mm_handlers*)&tmp_storage.handlers, handlers, sizeof(zend_mm_handlers)); |
3447 | 0 | tmp_storage.data = data; |
3448 | 0 | chunk = (zend_mm_chunk*)handlers->chunk_alloc(&tmp_storage, ZEND_MM_CHUNK_SIZE, ZEND_MM_CHUNK_SIZE); |
3449 | 0 | if (UNEXPECTED(chunk == NULL)) { |
3450 | 0 | #if ZEND_MM_ERROR |
3451 | 0 | fprintf(stderr, "Can't initialize heap\n"); |
3452 | 0 | #endif |
3453 | 0 | return NULL; |
3454 | 0 | } |
3455 | 0 | heap = &chunk->heap_slot; |
3456 | 0 | chunk->heap = heap; |
3457 | 0 | chunk->next = chunk; |
3458 | 0 | chunk->prev = chunk; |
3459 | 0 | chunk->free_pages = ZEND_MM_PAGES - ZEND_MM_FIRST_PAGE; |
3460 | 0 | chunk->free_tail = ZEND_MM_FIRST_PAGE; |
3461 | 0 | chunk->num = 0; |
3462 | 0 | chunk->free_map[0] = (Z_L(1) << ZEND_MM_FIRST_PAGE) - 1; |
3463 | 0 | chunk->map[0] = ZEND_MM_LRUN(ZEND_MM_FIRST_PAGE); |
3464 | 0 | heap->main_chunk = chunk; |
3465 | 0 | heap->cached_chunks = NULL; |
3466 | 0 | heap->chunks_count = 1; |
3467 | 0 | heap->peak_chunks_count = 1; |
3468 | 0 | heap->cached_chunks_count = 0; |
3469 | 0 | heap->avg_chunks_count = 1.0; |
3470 | 0 | heap->last_chunks_delete_boundary = 0; |
3471 | 0 | heap->last_chunks_delete_count = 0; |
3472 | 0 | #if ZEND_MM_STAT || ZEND_MM_LIMIT |
3473 | 0 | heap->real_size = ZEND_MM_CHUNK_SIZE; |
3474 | 0 | #endif |
3475 | 0 | #if ZEND_MM_STAT |
3476 | 0 | heap->real_peak = ZEND_MM_CHUNK_SIZE; |
3477 | 0 | heap->size = 0; |
3478 | 0 | heap->peak = 0; |
3479 | 0 | #endif |
3480 | 0 | zend_mm_init_key(heap); |
3481 | 0 | #if ZEND_MM_LIMIT |
3482 | 0 | heap->limit = (size_t)Z_L(-1) >> 1; |
3483 | 0 | heap->overflow = 0; |
3484 | 0 | #endif |
3485 | 0 | #if ZEND_MM_CUSTOM |
3486 | 0 | heap->use_custom_heap = 0; |
3487 | 0 | #endif |
3488 | 0 | heap->storage = &tmp_storage; |
3489 | 0 | heap->huge_list = NULL; |
3490 | 0 | memset(heap->free_slot, 0, sizeof(heap->free_slot)); |
3491 | 0 | storage = _zend_mm_alloc(heap, sizeof(zend_mm_storage) + data_size ZEND_FILE_LINE_CC ZEND_FILE_LINE_CC); |
3492 | 0 | if (!storage) { |
3493 | 0 | handlers->chunk_free(&tmp_storage, chunk, ZEND_MM_CHUNK_SIZE); |
3494 | 0 | #if ZEND_MM_ERROR |
3495 | 0 | fprintf(stderr, "Can't initialize heap\n"); |
3496 | 0 | #endif |
3497 | 0 | return NULL; |
3498 | 0 | } |
3499 | 0 | memcpy(storage, &tmp_storage, sizeof(zend_mm_storage)); |
3500 | 0 | if (data) { |
3501 | 0 | storage->data = (void*)(((char*)storage + sizeof(zend_mm_storage))); |
3502 | 0 | memcpy(storage->data, data, data_size); |
3503 | 0 | } |
3504 | 0 | heap->storage = storage; |
3505 | 0 | heap->pid = getpid(); |
3506 | 0 | return heap; |
3507 | | #else |
3508 | | return NULL; |
3509 | | #endif |
3510 | 0 | } |
3511 | | |
3512 | | ZEND_API void * __zend_malloc(size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3513 | 611M | { |
3514 | 611M | void *tmp = malloc(len); |
3515 | 611M | if (EXPECTED(tmp || !len)) { |
3516 | 611M | return tmp; |
3517 | 611M | } |
3518 | 0 | zend_out_of_memory(); |
3519 | 611M | } |
3520 | | |
3521 | | ZEND_API void * __zend_calloc(size_t nmemb, size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3522 | 0 | { |
3523 | 0 | void *tmp; |
3524 | |
|
3525 | 0 | len = zend_safe_address_guarded(nmemb, len, 0); |
3526 | 0 | tmp = __zend_malloc(len ZEND_FILE_LINE_RELAY_CC ZEND_FILE_LINE_ORIG_RELAY_CC); |
3527 | 0 | memset(tmp, 0, len); |
3528 | 0 | return tmp; |
3529 | 0 | } |
3530 | | |
3531 | | ZEND_API void * __zend_realloc(void *p, size_t len ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3532 | 20.5M | { |
3533 | 20.5M | p = realloc(p, len); |
3534 | 20.5M | if (EXPECTED(p || !len)) { |
3535 | 20.5M | return p; |
3536 | 20.5M | } |
3537 | 0 | zend_out_of_memory(); |
3538 | 20.5M | } |
3539 | | |
3540 | | ZEND_API void __zend_free(void *p ZEND_FILE_LINE_DC ZEND_FILE_LINE_ORIG_DC) |
3541 | 611M | { |
3542 | 611M | free(p); |
3543 | 611M | return; |
3544 | 611M | } |
3545 | | |
3546 | | ZEND_API char * __zend_strdup(const char *s) |
3547 | 0 | { |
3548 | 0 | char *tmp = strdup(s); |
3549 | 0 | if (EXPECTED(tmp)) { |
3550 | 0 | return tmp; |
3551 | 0 | } |
3552 | 0 | zend_out_of_memory(); |
3553 | 0 | } |
3554 | | |
3555 | | #ifdef ZTS |
3556 | | size_t zend_mm_globals_size(void) |
3557 | | { |
3558 | | return sizeof(zend_alloc_globals); |
3559 | | } |
3560 | | #endif |