Line data Source code
1 : // Copyright 2011 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_HEAP_SPACES_H_
6 : #define V8_HEAP_SPACES_H_
7 :
8 : #include <list>
9 : #include <map>
10 : #include <memory>
11 : #include <unordered_map>
12 : #include <unordered_set>
13 : #include <vector>
14 :
15 : #include "src/allocation.h"
16 : #include "src/base/atomic-utils.h"
17 : #include "src/base/iterator.h"
18 : #include "src/base/platform/mutex.h"
19 : #include "src/cancelable-task.h"
20 : #include "src/flags.h"
21 : #include "src/globals.h"
22 : #include "src/heap/heap.h"
23 : #include "src/heap/invalidated-slots.h"
24 : #include "src/heap/marking.h"
25 : #include "src/objects.h"
26 : #include "src/objects/map.h"
27 : #include "src/utils.h"
28 :
29 : namespace v8 {
30 : namespace internal {
31 :
32 : namespace heap {
33 : class HeapTester;
34 : class TestCodeRangeScope;
35 : } // namespace heap
36 :
37 : class AllocationInfo;
38 : class AllocationObserver;
39 : class CompactionSpace;
40 : class CompactionSpaceCollection;
41 : class FreeList;
42 : class Isolate;
43 : class LocalArrayBufferTracker;
44 : class MemoryAllocator;
45 : class MemoryChunk;
46 : class Page;
47 : class PagedSpace;
48 : class SemiSpace;
49 : class SkipList;
50 : class SlotsBuffer;
51 : class SlotSet;
52 : class TypedSlotSet;
53 : class Space;
54 :
55 : // -----------------------------------------------------------------------------
56 : // Heap structures:
57 : //
58 : // A JS heap consists of a young generation, an old generation, and a large
59 : // object space. The young generation is divided into two semispaces. A
60 : // scavenger implements Cheney's copying algorithm. The old generation is
61 : // separated into a map space and an old object space. The map space contains
62 : // all (and only) map objects, the rest of old objects go into the old space.
63 : // The old generation is collected by a mark-sweep-compact collector.
64 : //
65 : // The semispaces of the young generation are contiguous. The old and map
66 : // spaces consists of a list of pages. A page has a page header and an object
67 : // area.
68 : //
69 : // There is a separate large object space for objects larger than
70 : // kMaxRegularHeapObjectSize, so that they do not have to move during
71 : // collection. The large object space is paged. Pages in large object space
72 : // may be larger than the page size.
73 : //
74 : // A store-buffer based write barrier is used to keep track of intergenerational
75 : // references. See heap/store-buffer.h.
76 : //
77 : // During scavenges and mark-sweep collections we sometimes (after a store
78 : // buffer overflow) iterate intergenerational pointers without decoding heap
79 : // object maps so if the page belongs to old space or large object space
80 : // it is essential to guarantee that the page does not contain any
81 : // garbage pointers to new space: every pointer aligned word which satisfies
82 : // the Heap::InNewSpace() predicate must be a pointer to a live heap object in
83 : // new space. Thus objects in old space and large object spaces should have a
84 : // special layout (e.g. no bare integer fields). This requirement does not
85 : // apply to map space which is iterated in a special fashion. However we still
86 : // require pointer fields of dead maps to be cleaned.
87 : //
88 : // To enable lazy cleaning of old space pages we can mark chunks of the page
89 : // as being garbage. Garbage sections are marked with a special map. These
90 : // sections are skipped when scanning the page, even if we are otherwise
91 : // scanning without regard for object boundaries. Garbage sections are chained
92 : // together to form a free list after a GC. Garbage sections created outside
93 : // of GCs by object trunctation etc. may not be in the free list chain. Very
94 : // small free spaces are ignored, they need only be cleaned of bogus pointers
95 : // into new space.
96 : //
97 : // Each page may have up to one special garbage section. The start of this
98 : // section is denoted by the top field in the space. The end of the section
99 : // is denoted by the limit field in the space. This special garbage section
100 : // is not marked with a free space map in the data. The point of this section
101 : // is to enable linear allocation without having to constantly update the byte
102 : // array every time the top field is updated and a new object is created. The
103 : // special garbage section is not in the chain of garbage sections.
104 : //
105 : // Since the top and limit fields are in the space, not the page, only one page
106 : // has a special garbage section, and if the top and limit are equal then there
107 : // is no special garbage section.
108 :
109 : // Some assertion macros used in the debugging mode.
110 :
111 : #define DCHECK_PAGE_ALIGNED(address) \
112 : DCHECK((OffsetFrom(address) & Page::kPageAlignmentMask) == 0)
113 :
114 : #define DCHECK_OBJECT_ALIGNED(address) \
115 : DCHECK((OffsetFrom(address) & kObjectAlignmentMask) == 0)
116 :
117 : #define DCHECK_OBJECT_SIZE(size) \
118 : DCHECK((0 < size) && (size <= kMaxRegularHeapObjectSize))
119 :
120 : #define DCHECK_CODEOBJECT_SIZE(size, code_space) \
121 : DCHECK((0 < size) && (size <= code_space->AreaSize()))
122 :
123 : #define DCHECK_PAGE_OFFSET(offset) \
124 : DCHECK((Page::kObjectStartOffset <= offset) && (offset <= Page::kPageSize))
125 :
126 : enum FreeListCategoryType {
127 : kTiniest,
128 : kTiny,
129 : kSmall,
130 : kMedium,
131 : kLarge,
132 : kHuge,
133 :
134 : kFirstCategory = kTiniest,
135 : kLastCategory = kHuge,
136 : kNumberOfCategories = kLastCategory + 1,
137 : kInvalidCategory
138 : };
139 :
140 : enum FreeMode { kLinkCategory, kDoNotLinkCategory };
141 :
142 : enum RememberedSetType {
143 : OLD_TO_NEW,
144 : OLD_TO_OLD,
145 : NUMBER_OF_REMEMBERED_SET_TYPES = OLD_TO_OLD + 1
146 : };
147 :
148 : // A free list category maintains a linked list of free memory blocks.
149 : class FreeListCategory {
150 : public:
151 : static const int kSize = kIntSize + // FreeListCategoryType type_
152 : kIntSize + // padding for type_
153 : kSizetSize + // size_t available_
154 : kPointerSize + // FreeSpace* top_
155 : kPointerSize + // FreeListCategory* prev_
156 : kPointerSize; // FreeListCategory* next_
157 :
158 : FreeListCategory()
159 : : type_(kInvalidCategory),
160 : available_(0),
161 : top_(nullptr),
162 : prev_(nullptr),
163 3528480 : next_(nullptr) {}
164 :
165 : void Initialize(FreeListCategoryType type) {
166 2349678 : type_ = type;
167 2349678 : available_ = 0;
168 2349678 : top_ = nullptr;
169 2349678 : prev_ = nullptr;
170 2349678 : next_ = nullptr;
171 : }
172 :
173 : void Invalidate();
174 :
175 : void Reset();
176 :
177 0 : void ResetStats() { Reset(); }
178 :
179 : void RepairFreeList(Heap* heap);
180 :
181 : // Relinks the category into the currently owning free list. Requires that the
182 : // category is currently unlinked.
183 : void Relink();
184 :
185 : void Free(FreeSpace* node, size_t size_in_bytes, FreeMode mode);
186 :
187 : // Picks a node from the list and stores its size in |node_size|. Returns
188 : // nullptr if the category is empty.
189 : FreeSpace* PickNodeFromList(size_t* node_size);
190 :
191 : // Performs a single try to pick a node of at least |minimum_size| from the
192 : // category. Stores the actual size in |node_size|. Returns nullptr if no
193 : // node is found.
194 : FreeSpace* TryPickNodeFromList(size_t minimum_size, size_t* node_size);
195 :
196 : // Picks a node of at least |minimum_size| from the category. Stores the
197 : // actual size in |node_size|. Returns nullptr if no node is found.
198 : FreeSpace* SearchForNodeInList(size_t minimum_size, size_t* node_size);
199 :
200 : inline FreeList* owner();
201 : inline Page* page() const;
202 : inline bool is_linked();
203 6397372 : bool is_empty() { return top() == nullptr; }
204 : size_t available() const { return available_; }
205 :
206 : #ifdef DEBUG
207 : size_t SumFreeList();
208 : int FreeListLength();
209 : #endif
210 :
211 : private:
212 : // For debug builds we accurately compute free lists lengths up until
213 : // {kVeryLongFreeList} by manually walking the list.
214 : static const int kVeryLongFreeList = 500;
215 :
216 : FreeSpace* top() { return top_; }
217 21765565 : void set_top(FreeSpace* top) { top_ = top; }
218 : FreeListCategory* prev() { return prev_; }
219 2268470 : void set_prev(FreeListCategory* prev) { prev_ = prev; }
220 : FreeListCategory* next() { return next_; }
221 3410948 : void set_next(FreeListCategory* next) { next_ = next; }
222 :
223 : // |type_|: The type of this free list category.
224 : FreeListCategoryType type_;
225 :
226 : // |available_|: Total available bytes in all blocks of this free list
227 : // category.
228 : size_t available_;
229 :
230 : // |top_|: Points to the top FreeSpace* in the free list category.
231 : FreeSpace* top_;
232 :
233 : FreeListCategory* prev_;
234 : FreeListCategory* next_;
235 :
236 : friend class FreeList;
237 : friend class PagedSpace;
238 : };
239 :
240 : // MemoryChunk represents a memory region owned by a specific space.
241 : // It is divided into the header and the body. Chunk start is always
242 : // 1MB aligned. Start of the body is aligned so it can accommodate
243 : // any heap object.
244 5371002 : class MemoryChunk {
245 : public:
246 : // Use with std data structures.
247 : struct Hasher {
248 : size_t operator()(MemoryChunk* const chunk) const {
249 350946171 : return reinterpret_cast<size_t>(chunk) >> kPageSizeBits;
250 : }
251 : };
252 :
253 : enum Flag {
254 : NO_FLAGS = 0u,
255 : IS_EXECUTABLE = 1u << 0,
256 : POINTERS_TO_HERE_ARE_INTERESTING = 1u << 1,
257 : POINTERS_FROM_HERE_ARE_INTERESTING = 1u << 2,
258 : // A page in new space has one of the next to flags set.
259 : IN_FROM_SPACE = 1u << 3,
260 : IN_TO_SPACE = 1u << 4,
261 : NEW_SPACE_BELOW_AGE_MARK = 1u << 5,
262 : EVACUATION_CANDIDATE = 1u << 6,
263 : NEVER_EVACUATE = 1u << 7,
264 :
265 : // Large objects can have a progress bar in their page header. These object
266 : // are scanned in increments and will be kept black while being scanned.
267 : // Even if the mutator writes to them they will be kept black and a white
268 : // to grey transition is performed in the value.
269 : HAS_PROGRESS_BAR = 1u << 8,
270 :
271 : // |PAGE_NEW_OLD_PROMOTION|: A page tagged with this flag has been promoted
272 : // from new to old space during evacuation.
273 : PAGE_NEW_OLD_PROMOTION = 1u << 9,
274 :
275 : // |PAGE_NEW_NEW_PROMOTION|: A page tagged with this flag has been moved
276 : // within the new space during evacuation.
277 : PAGE_NEW_NEW_PROMOTION = 1u << 10,
278 :
279 : // This flag is intended to be used for testing. Works only when both
280 : // FLAG_stress_compaction and FLAG_manual_evacuation_candidates_selection
281 : // are set. It forces the page to become an evacuation candidate at next
282 : // candidates selection cycle.
283 : FORCE_EVACUATION_CANDIDATE_FOR_TESTING = 1u << 11,
284 :
285 : // This flag is intended to be used for testing.
286 : NEVER_ALLOCATE_ON_PAGE = 1u << 12,
287 :
288 : // The memory chunk is already logically freed, however the actual freeing
289 : // still has to be performed.
290 : PRE_FREED = 1u << 13,
291 :
292 : // |POOLED|: When actually freeing this chunk, only uncommit and do not
293 : // give up the reservation as we still reuse the chunk at some point.
294 : POOLED = 1u << 14,
295 :
296 : // |COMPACTION_WAS_ABORTED|: Indicates that the compaction in this page
297 : // has been aborted and needs special handling by the sweeper.
298 : COMPACTION_WAS_ABORTED = 1u << 15,
299 :
300 : // |COMPACTION_WAS_ABORTED_FOR_TESTING|: During stress testing evacuation
301 : // on pages is sometimes aborted. The flag is used to avoid repeatedly
302 : // triggering on the same page.
303 : COMPACTION_WAS_ABORTED_FOR_TESTING = 1u << 16,
304 :
305 : // |ANCHOR|: Flag is set if page is an anchor.
306 : ANCHOR = 1u << 17,
307 :
308 : // |SWEEP_TO_ITERATE|: The page requires sweeping using external markbits
309 : // to iterate the page.
310 : SWEEP_TO_ITERATE = 1u << 18
311 : };
312 :
313 : using Flags = uintptr_t;
314 :
315 : static const Flags kPointersToHereAreInterestingMask =
316 : POINTERS_TO_HERE_ARE_INTERESTING;
317 :
318 : static const Flags kPointersFromHereAreInterestingMask =
319 : POINTERS_FROM_HERE_ARE_INTERESTING;
320 :
321 : static const Flags kEvacuationCandidateMask = EVACUATION_CANDIDATE;
322 :
323 : static const Flags kIsInNewSpaceMask = IN_FROM_SPACE | IN_TO_SPACE;
324 :
325 : static const Flags kSkipEvacuationSlotsRecordingMask =
326 : kEvacuationCandidateMask | kIsInNewSpaceMask;
327 :
328 : // |kSweepingDone|: The page state when sweeping is complete or sweeping must
329 : // not be performed on that page. Sweeper threads that are done with their
330 : // work will set this value and not touch the page anymore.
331 : // |kSweepingPending|: This page is ready for parallel sweeping.
332 : // |kSweepingInProgress|: This page is currently swept by a sweeper thread.
333 : enum ConcurrentSweepingState {
334 : kSweepingDone,
335 : kSweepingPending,
336 : kSweepingInProgress,
337 : };
338 :
339 : static const intptr_t kAlignment =
340 : (static_cast<uintptr_t>(1) << kPageSizeBits);
341 :
342 : static const intptr_t kAlignmentMask = kAlignment - 1;
343 :
344 : static const intptr_t kSizeOffset = 0;
345 : static const intptr_t kFlagsOffset = kSizeOffset + kSizetSize;
346 : static const intptr_t kAreaStartOffset = kFlagsOffset + kIntptrSize;
347 : static const intptr_t kAreaEndOffset = kAreaStartOffset + kPointerSize;
348 : static const intptr_t kReservationOffset = kAreaEndOffset + kPointerSize;
349 : static const intptr_t kOwnerOffset = kReservationOffset + 2 * kPointerSize;
350 :
351 : static const size_t kMinHeaderSize =
352 : kSizeOffset // NOLINT
353 : + kSizetSize // size_t size
354 : + kUIntptrSize // uintptr_t flags_
355 : + kPointerSize // Address area_start_
356 : + kPointerSize // Address area_end_
357 : + 2 * kPointerSize // VirtualMemory reservation_
358 : + kPointerSize // Address owner_
359 : + kPointerSize // Heap* heap_
360 : + kIntptrSize // intptr_t progress_bar_
361 : + kIntptrSize // intptr_t live_byte_count_
362 : + kPointerSize * NUMBER_OF_REMEMBERED_SET_TYPES // SlotSet* array
363 : + kPointerSize * NUMBER_OF_REMEMBERED_SET_TYPES // TypedSlotSet* array
364 : + kPointerSize // InvalidatedSlots* invalidated_slots_
365 : + kPointerSize // SkipList* skip_list_
366 : + kPointerSize // AtomicValue high_water_mark_
367 : + kPointerSize // base::RecursiveMutex* mutex_
368 : + kPointerSize // base::AtomicWord concurrent_sweeping_
369 : + kPointerSize // base::Mutex* page_protection_change_mutex_
370 : + kPointerSize // unitptr_t write_unprotect_counter_
371 : + kSizetSize // size_t allocated_bytes_
372 : + kSizetSize // size_t wasted_memory_
373 : + kPointerSize // AtomicValue next_chunk_
374 : + kPointerSize // AtomicValue prev_chunk_
375 : + FreeListCategory::kSize * kNumberOfCategories
376 : // FreeListCategory categories_[kNumberOfCategories]
377 : + kPointerSize // LocalArrayBufferTracker* local_tracker_
378 : + kIntptrSize // intptr_t young_generation_live_byte_count_
379 : + kPointerSize; // Bitmap* young_generation_bitmap_
380 :
381 : // We add some more space to the computed header size to amount for missing
382 : // alignment requirements in our computation.
383 : // Try to get kHeaderSize properly aligned on 32-bit and 64-bit machines.
384 : static const size_t kHeaderSize = kMinHeaderSize;
385 :
386 : static const int kBodyOffset =
387 : CODE_POINTER_ALIGN(kHeaderSize + Bitmap::kSize);
388 :
389 : // The start offset of the object area in a page. Aligned to both maps and
390 : // code alignment to be suitable for both. Also aligned to 32 words because
391 : // the marking bitmap is arranged in 32 bit chunks.
392 : static const int kObjectStartAlignment = 32 * kPointerSize;
393 : static const int kObjectStartOffset =
394 : kBodyOffset - 1 +
395 : (kObjectStartAlignment - (kBodyOffset - 1) % kObjectStartAlignment);
396 :
397 : // Page size in bytes. This must be a multiple of the OS page size.
398 : static const int kPageSize = 1 << kPageSizeBits;
399 : static const intptr_t kPageAlignmentMask = (1 << kPageSizeBits) - 1;
400 :
401 : static const int kAllocatableMemory = kPageSize - kObjectStartOffset;
402 :
403 : // Only works if the pointer is in the first kPageSize of the MemoryChunk.
404 5758860300 : static MemoryChunk* FromAddress(Address a) {
405 13042037080 : return reinterpret_cast<MemoryChunk*>(OffsetFrom(a) & ~kAlignmentMask);
406 : }
407 :
408 : static inline MemoryChunk* FromAnyPointerAddress(Heap* heap, Address addr);
409 :
410 3356305 : static inline void UpdateHighWaterMark(Address mark) {
411 6712610 : if (mark == nullptr) return;
412 : // Need to subtract one from the mark because when a chunk is full the
413 : // top points to the next address after the chunk, which effectively belongs
414 : // to another chunk. See the comment to Page::FromTopOrLimit.
415 1761443 : MemoryChunk* chunk = MemoryChunk::FromAddress(mark - 1);
416 1761443 : intptr_t new_mark = static_cast<intptr_t>(mark - chunk->address());
417 : intptr_t old_mark = 0;
418 1761443 : do {
419 : old_mark = chunk->high_water_mark_.Value();
420 2263337 : } while ((new_mark > old_mark) &&
421 : !chunk->high_water_mark_.TrySetValue(old_mark, new_mark));
422 : }
423 :
424 : static bool IsValid(MemoryChunk* chunk) { return chunk != nullptr; }
425 :
426 : Address address() const {
427 : return reinterpret_cast<Address>(const_cast<MemoryChunk*>(this));
428 : }
429 :
430 : base::RecursiveMutex* mutex() { return mutex_; }
431 :
432 70708671 : bool Contains(Address addr) {
433 70766906 : return addr >= area_start() && addr < area_end();
434 : }
435 :
436 : // Checks whether |addr| can be a limit of addresses in this page. It's a
437 : // limit if it's in the page, or if it's just after the last byte of the page.
438 119478546 : bool ContainsLimit(Address addr) {
439 119478546 : return addr >= area_start() && addr <= area_end();
440 : }
441 :
442 : base::AtomicValue<ConcurrentSweepingState>& concurrent_sweeping_state() {
443 : return concurrent_sweeping_;
444 : }
445 :
446 3888900 : bool SweepingDone() {
447 3888985 : return concurrent_sweeping_state().Value() == kSweepingDone;
448 : }
449 :
450 : size_t size() const { return size_; }
451 : void set_size(size_t size) { size_ = size; }
452 :
453 : inline Heap* heap() const { return heap_; }
454 :
455 : Heap* synchronized_heap();
456 :
457 : inline SkipList* skip_list() { return skip_list_; }
458 :
459 142501 : inline void set_skip_list(SkipList* skip_list) { skip_list_ = skip_list; }
460 :
461 : template <RememberedSetType type, AccessMode access_mode = AccessMode::ATOMIC>
462 : SlotSet* slot_set() {
463 : if (access_mode == AccessMode::ATOMIC)
464 219162963 : return base::AsAtomicPointer::Acquire_Load(&slot_set_[type]);
465 : return slot_set_[type];
466 : }
467 :
468 : template <RememberedSetType type, AccessMode access_mode = AccessMode::ATOMIC>
469 : TypedSlotSet* typed_slot_set() {
470 : if (access_mode == AccessMode::ATOMIC)
471 3470051 : return base::AsAtomicPointer::Acquire_Load(&typed_slot_set_[type]);
472 : return typed_slot_set_[type];
473 : }
474 :
475 : template <RememberedSetType type>
476 : SlotSet* AllocateSlotSet();
477 : // Not safe to be called concurrently.
478 : template <RememberedSetType type>
479 : void ReleaseSlotSet();
480 : template <RememberedSetType type>
481 : TypedSlotSet* AllocateTypedSlotSet();
482 : // Not safe to be called concurrently.
483 : template <RememberedSetType type>
484 : void ReleaseTypedSlotSet();
485 :
486 : InvalidatedSlots* AllocateInvalidatedSlots();
487 : void ReleaseInvalidatedSlots();
488 : void RegisterObjectWithInvalidatedSlots(HeapObject* object, int size);
489 : InvalidatedSlots* invalidated_slots() { return invalidated_slots_; }
490 :
491 : void AllocateLocalTracker();
492 : void ReleaseLocalTracker();
493 : inline LocalArrayBufferTracker* local_tracker() { return local_tracker_; }
494 : bool contains_array_buffers();
495 :
496 : void AllocateYoungGenerationBitmap();
497 : void ReleaseYoungGenerationBitmap();
498 :
499 : Address area_start() { return area_start_; }
500 : Address area_end() { return area_end_; }
501 2615317 : size_t area_size() { return static_cast<size_t>(area_end() - area_start()); }
502 :
503 : bool CommitArea(size_t requested);
504 :
505 : // Approximate amount of physical memory committed for this chunk.
506 : size_t CommittedPhysicalMemory();
507 :
508 219840 : Address HighWaterMark() { return address() + high_water_mark_.Value(); }
509 :
510 2990 : int progress_bar() {
511 : DCHECK(IsFlagSet(HAS_PROGRESS_BAR));
512 5995 : return static_cast<int>(progress_bar_);
513 : }
514 :
515 2933 : void set_progress_bar(int progress_bar) {
516 : DCHECK(IsFlagSet(HAS_PROGRESS_BAR));
517 6206 : progress_bar_ = progress_bar;
518 2933 : }
519 :
520 : void ResetProgressBar() {
521 4056 : if (IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR)) {
522 : set_progress_bar(0);
523 : }
524 : }
525 :
526 5758595367 : inline uint32_t AddressToMarkbitIndex(Address addr) const {
527 6371326787 : return static_cast<uint32_t>(addr - this->address()) >> kPointerSizeLog2;
528 : }
529 :
530 : inline Address MarkbitIndexToAddress(uint32_t index) const {
531 : return this->address() + (index << kPointerSizeLog2);
532 : }
533 :
534 : template <AccessMode access_mode = AccessMode::NON_ATOMIC>
535 : void SetFlag(Flag flag) {
536 : if (access_mode == AccessMode::NON_ATOMIC) {
537 3756675 : flags_ |= flag;
538 : } else {
539 7039 : base::AsAtomicWord::SetBits<uintptr_t>(&flags_, flag, flag);
540 : }
541 : }
542 :
543 : template <AccessMode access_mode = AccessMode::NON_ATOMIC>
544 813150870 : bool IsFlagSet(Flag flag) {
545 4838858449 : return (GetFlags<access_mode>() & flag) != 0;
546 : }
547 :
548 1450530 : void ClearFlag(Flag flag) { flags_ &= ~flag; }
549 : // Set or clear multiple flags at a time. The flags in the mask are set to
550 : // the value in "flags", the rest retain the current value in |flags_|.
551 : void SetFlags(uintptr_t flags, uintptr_t mask) {
552 1094891 : flags_ = (flags_ & ~mask) | (flags & mask);
553 : }
554 :
555 : // Return all current flags.
556 : template <AccessMode access_mode = AccessMode::NON_ATOMIC>
557 : uintptr_t GetFlags() {
558 : if (access_mode == AccessMode::NON_ATOMIC) {
559 : return flags_;
560 : } else {
561 3941875579 : return base::AsAtomicWord::Relaxed_Load(&flags_);
562 : }
563 : }
564 :
565 : bool NeverEvacuate() { return IsFlagSet(NEVER_EVACUATE); }
566 :
567 : void MarkNeverEvacuate() { SetFlag(NEVER_EVACUATE); }
568 :
569 : bool CanAllocate() {
570 38624462 : return !IsEvacuationCandidate() && !IsFlagSet(NEVER_ALLOCATE_ON_PAGE);
571 : }
572 :
573 : template <AccessMode access_mode = AccessMode::NON_ATOMIC>
574 3931975041 : bool IsEvacuationCandidate() {
575 : DCHECK(!(IsFlagSet<access_mode>(NEVER_EVACUATE) &&
576 : IsFlagSet<access_mode>(EVACUATION_CANDIDATE)));
577 3931975041 : return IsFlagSet<access_mode>(EVACUATION_CANDIDATE);
578 : }
579 :
580 : template <AccessMode access_mode = AccessMode::NON_ATOMIC>
581 9990536 : bool ShouldSkipEvacuationSlotRecording() {
582 : uintptr_t flags = GetFlags<access_mode>();
583 : return ((flags & kSkipEvacuationSlotsRecordingMask) != 0) &&
584 9990536 : ((flags & COMPACTION_WAS_ABORTED) == 0);
585 : }
586 :
587 : Executability executable() {
588 1237062 : return IsFlagSet(IS_EXECUTABLE) ? EXECUTABLE : NOT_EXECUTABLE;
589 : }
590 :
591 3292770958 : bool InNewSpace() { return (flags_ & kIsInNewSpaceMask) != 0; }
592 :
593 : bool InToSpace() { return IsFlagSet(IN_TO_SPACE); }
594 :
595 : bool InFromSpace() { return IsFlagSet(IN_FROM_SPACE); }
596 :
597 : MemoryChunk* next_chunk() { return next_chunk_.Value(); }
598 :
599 : MemoryChunk* prev_chunk() { return prev_chunk_.Value(); }
600 :
601 : void set_next_chunk(MemoryChunk* next) { next_chunk_.SetValue(next); }
602 :
603 : void set_prev_chunk(MemoryChunk* prev) { prev_chunk_.SetValue(prev); }
604 :
605 : Space* owner() const {
606 : uintptr_t owner_value = base::AsAtomicWord::Acquire_Load(
607 250895142 : reinterpret_cast<const uintptr_t*>(&owner_));
608 270619398 : return ((owner_value & kPageHeaderTagMask) == kPageHeaderTag)
609 236251260 : ? reinterpret_cast<Space*>(owner_value - kPageHeaderTag)
610 506870658 : : nullptr;
611 : }
612 :
613 : void set_owner(Space* space) {
614 : DCHECK_EQ(0, reinterpret_cast<uintptr_t>(space) & kPageHeaderTagMask);
615 : base::AsAtomicWord::Release_Store(
616 : reinterpret_cast<uintptr_t*>(&owner_),
617 2520961 : reinterpret_cast<uintptr_t>(space) + kPageHeaderTag);
618 : DCHECK_EQ(kPageHeaderTag, base::AsAtomicWord::Relaxed_Load(
619 : reinterpret_cast<const uintptr_t*>(&owner_)) &
620 : kPageHeaderTagMask);
621 : }
622 :
623 407242524 : bool HasPageHeader() { return owner() != nullptr; }
624 :
625 : void InsertAfter(MemoryChunk* other);
626 : void Unlink();
627 :
628 : // Emits a memory barrier. For TSAN builds the other thread needs to perform
629 : // MemoryChunk::synchronized_heap() to simulate the barrier.
630 : void InitializationMemoryFence();
631 :
632 : void SetReadAndExecutable();
633 : void SetReadAndWritable();
634 :
635 : protected:
636 : static MemoryChunk* Initialize(Heap* heap, Address base, size_t size,
637 : Address area_start, Address area_end,
638 : Executability executable, Space* owner,
639 : VirtualMemory* reservation);
640 :
641 : // Should be called when memory chunk is about to be freed.
642 : void ReleaseAllocatedMemory();
643 :
644 : VirtualMemory* reserved_memory() { return &reservation_; }
645 :
646 : size_t size_;
647 : uintptr_t flags_;
648 :
649 : // Start and end of allocatable memory on this chunk.
650 : Address area_start_;
651 : Address area_end_;
652 :
653 : // If the chunk needs to remember its memory reservation, it is stored here.
654 : VirtualMemory reservation_;
655 :
656 : // The identity of the owning space. This is tagged as a failure pointer, but
657 : // no failure can be in an object, so this can be distinguished from any entry
658 : // in a fixed array.
659 : Address owner_;
660 :
661 : Heap* heap_;
662 :
663 : // Used by the incremental marker to keep track of the scanning progress in
664 : // large objects that have a progress bar and are scanned in increments.
665 : intptr_t progress_bar_;
666 :
667 : // Count of bytes marked black on page.
668 : intptr_t live_byte_count_;
669 :
670 : // A single slot set for small pages (of size kPageSize) or an array of slot
671 : // set for large pages. In the latter case the number of entries in the array
672 : // is ceil(size() / kPageSize).
673 : SlotSet* slot_set_[NUMBER_OF_REMEMBERED_SET_TYPES];
674 : TypedSlotSet* typed_slot_set_[NUMBER_OF_REMEMBERED_SET_TYPES];
675 : InvalidatedSlots* invalidated_slots_;
676 :
677 : SkipList* skip_list_;
678 :
679 : // Assuming the initial allocation on a page is sequential,
680 : // count highest number of bytes ever allocated on the page.
681 : base::AtomicValue<intptr_t> high_water_mark_;
682 :
683 : base::RecursiveMutex* mutex_;
684 :
685 : base::AtomicValue<ConcurrentSweepingState> concurrent_sweeping_;
686 :
687 : base::Mutex* page_protection_change_mutex_;
688 :
689 : // This field is only relevant for code pages. It depicts the number of
690 : // times a component requested this page to be read+writeable. The
691 : // counter is decremented when a component resets to read+executable.
692 : // If Value() == 0 => The memory is read and executable.
693 : // If Value() >= 1 => The Memory is read and writable.
694 : // The maximum value can right now only be 2.
695 : uintptr_t write_unprotect_counter_;
696 :
697 : // Byte allocated on the page, which includes all objects on the page
698 : // and the linear allocation area.
699 : size_t allocated_bytes_;
700 : // Freed memory that was not added to the free list.
701 : size_t wasted_memory_;
702 :
703 : // next_chunk_ holds a pointer of type MemoryChunk
704 : base::AtomicValue<MemoryChunk*> next_chunk_;
705 : // prev_chunk_ holds a pointer of type MemoryChunk
706 : base::AtomicValue<MemoryChunk*> prev_chunk_;
707 :
708 : FreeListCategory categories_[kNumberOfCategories];
709 :
710 : LocalArrayBufferTracker* local_tracker_;
711 :
712 : intptr_t young_generation_live_byte_count_;
713 : Bitmap* young_generation_bitmap_;
714 :
715 : private:
716 762713 : void InitializeReservedMemory() { reservation_.Reset(); }
717 :
718 : friend class ConcurrentMarkingState;
719 : friend class IncrementalMarkingState;
720 : friend class MajorAtomicMarkingState;
721 : friend class MajorNonAtomicMarkingState;
722 : friend class MemoryAllocator;
723 : friend class MemoryChunkValidator;
724 : friend class MinorMarkingState;
725 : friend class MinorNonAtomicMarkingState;
726 : friend class PagedSpace;
727 : };
728 :
729 : static_assert(kMaxRegularHeapObjectSize <= MemoryChunk::kAllocatableMemory,
730 : "kMaxRegularHeapObjectSize <= MemoryChunk::kAllocatableMemory");
731 :
732 :
733 : // -----------------------------------------------------------------------------
734 : // A page is a memory chunk of a size 512K. Large object pages may be larger.
735 : //
736 : // The only way to get a page pointer is by calling factory methods:
737 : // Page* p = Page::FromAddress(addr); or
738 : // Page* p = Page::FromTopOrLimit(top);
739 : class Page : public MemoryChunk {
740 : public:
741 : static const intptr_t kCopyAllFlags = ~0;
742 :
743 : // Page flags copied from from-space to to-space when flipping semispaces.
744 : static const intptr_t kCopyOnFlipFlagsMask =
745 : static_cast<intptr_t>(MemoryChunk::POINTERS_TO_HERE_ARE_INTERESTING) |
746 : static_cast<intptr_t>(MemoryChunk::POINTERS_FROM_HERE_ARE_INTERESTING);
747 :
748 : // Returns the page containing a given address. The address ranges
749 : // from [page_addr .. page_addr + kPageSize[. This only works if the object
750 : // is in fact in a page.
751 7802413800 : static Page* FromAddress(Address addr) {
752 11822922671 : return reinterpret_cast<Page*>(OffsetFrom(addr) & ~kPageAlignmentMask);
753 : }
754 :
755 : // Returns the page containing the address provided. The address can
756 : // potentially point righter after the page. To be also safe for tagged values
757 : // we subtract a hole word. The valid address ranges from
758 : // [page_addr + kObjectStartOffset .. page_addr + kPageSize + kPointerSize].
759 : static Page* FromAllocationAreaAddress(Address address) {
760 1046820 : return Page::FromAddress(address - kPointerSize);
761 : }
762 :
763 : // Checks if address1 and address2 are on the same new space page.
764 : static bool OnSamePage(Address address1, Address address2) {
765 : return Page::FromAddress(address1) == Page::FromAddress(address2);
766 : }
767 :
768 : // Checks whether an address is page aligned.
769 : static bool IsAlignedToPageSize(Address addr) {
770 3277857 : return (OffsetFrom(addr) & kPageAlignmentMask) == 0;
771 : }
772 :
773 : static bool IsAtObjectStart(Address addr) {
774 : return (reinterpret_cast<intptr_t>(addr) & kPageAlignmentMask) ==
775 : kObjectStartOffset;
776 : }
777 :
778 : static Page* ConvertNewToOld(Page* old_page);
779 :
780 : inline static Page* FromAnyPointerAddress(Heap* heap, Address addr);
781 :
782 : // Create a Page object that is only used as anchor for the doubly-linked
783 : // list of real pages.
784 588080 : explicit Page(Space* owner) { InitializeAsAnchor(owner); }
785 :
786 : inline void MarkNeverAllocateForTesting();
787 : inline void MarkEvacuationCandidate();
788 : inline void ClearEvacuationCandidate();
789 :
790 : Page* next_page() { return static_cast<Page*>(next_chunk()); }
791 : Page* prev_page() { return static_cast<Page*>(prev_chunk()); }
792 : void set_next_page(Page* page) { set_next_chunk(page); }
793 : void set_prev_page(Page* page) { set_prev_chunk(page); }
794 :
795 : template <typename Callback>
796 903585 : inline void ForAllFreeListCategories(Callback callback) {
797 7122345 : for (int i = kFirstCategory; i < kNumberOfCategories; i++) {
798 6218760 : callback(&categories_[i]);
799 : }
800 903585 : }
801 :
802 : // Returns the offset of a given address to this page.
803 5 : inline size_t Offset(Address a) { return static_cast<size_t>(a - address()); }
804 :
805 : // Returns the address for a given offset to the this page.
806 : Address OffsetToAddress(size_t offset) {
807 : DCHECK_PAGE_OFFSET(offset);
808 : return address() + offset;
809 : }
810 :
811 : // WaitUntilSweepingCompleted only works when concurrent sweeping is in
812 : // progress. In particular, when we know that right before this call a
813 : // sweeper thread was sweeping this page.
814 : void WaitUntilSweepingCompleted() {
815 0 : mutex_->Lock();
816 0 : mutex_->Unlock();
817 : DCHECK(SweepingDone());
818 : }
819 :
820 : void ResetFreeListStatistics();
821 :
822 : size_t AvailableInFreeList();
823 :
824 : size_t AvailableInFreeListFromAllocatedBytes() {
825 : DCHECK_GE(area_size(), wasted_memory() + allocated_bytes());
826 : return area_size() - wasted_memory() - allocated_bytes();
827 : }
828 :
829 : FreeListCategory* free_list_category(FreeListCategoryType type) {
830 19202015 : return &categories_[type];
831 : }
832 :
833 : inline void InitializeFreeListCategories();
834 :
835 : bool is_anchor() { return IsFlagSet(Page::ANCHOR); }
836 :
837 : size_t wasted_memory() { return wasted_memory_; }
838 11042693 : void add_wasted_memory(size_t waste) { wasted_memory_ += waste; }
839 : size_t allocated_bytes() { return allocated_bytes_; }
840 : void IncreaseAllocatedBytes(size_t bytes) {
841 : DCHECK_LE(bytes, area_size());
842 1539919 : allocated_bytes_ += bytes;
843 : }
844 : void DecreaseAllocatedBytes(size_t bytes) {
845 : DCHECK_LE(bytes, area_size());
846 : DCHECK_GE(allocated_bytes(), bytes);
847 30245848 : allocated_bytes_ -= bytes;
848 : }
849 :
850 : void ResetAllocatedBytes();
851 :
852 : size_t ShrinkToHighWaterMark();
853 :
854 : V8_EXPORT_PRIVATE void CreateBlackArea(Address start, Address end);
855 : void DestroyBlackArea(Address start, Address end);
856 :
857 : #ifdef DEBUG
858 : void Print();
859 : #endif // DEBUG
860 :
861 : private:
862 : enum InitializationMode { kFreeMemory, kDoNotFreeMemory };
863 :
864 : void InitializeAsAnchor(Space* owner);
865 :
866 : friend class MemoryAllocator;
867 : };
868 :
869 : class LargePage : public MemoryChunk {
870 : public:
871 30434 : HeapObject* GetObject() { return HeapObject::FromAddress(area_start()); }
872 :
873 : inline LargePage* next_page() {
874 : return static_cast<LargePage*>(next_chunk());
875 : }
876 :
877 : inline void set_next_page(LargePage* page) { set_next_chunk(page); }
878 :
879 : // Uncommit memory that is not in use anymore by the object. If the object
880 : // cannot be shrunk 0 is returned.
881 : Address GetAddressToShrink(Address object_address, size_t object_size);
882 :
883 : void ClearOutOfLiveRangeSlots(Address free_start);
884 :
885 : // A limit to guarantee that we do not overflow typed slot offset in
886 : // the old to old remembered set.
887 : // Note that this limit is higher than what assembler already imposes on
888 : // x64 and ia32 architectures.
889 : static const int kMaxCodePageSize = 512 * MB;
890 :
891 : private:
892 : static LargePage* Initialize(Heap* heap, MemoryChunk* chunk,
893 : Executability executable, Space* owner);
894 :
895 : friend class MemoryAllocator;
896 : };
897 :
898 :
899 : // ----------------------------------------------------------------------------
900 : // Space is the abstract superclass for all allocation spaces.
901 : class Space : public Malloced {
902 : public:
903 : Space(Heap* heap, AllocationSpace id, Executability executable)
904 : : allocation_observers_paused_(false),
905 : heap_(heap),
906 : id_(id),
907 : executable_(executable),
908 : committed_(0),
909 1396168 : max_committed_(0) {}
910 :
911 686646 : virtual ~Space() {}
912 :
913 : Heap* heap() const { return heap_; }
914 :
915 : // Does the space need executable memory?
916 : Executability executable() { return executable_; }
917 :
918 : // Identity used in error reporting.
919 : AllocationSpace identity() { return id_; }
920 :
921 : void AddAllocationObserver(AllocationObserver* observer);
922 :
923 : void RemoveAllocationObserver(AllocationObserver* observer);
924 :
925 : V8_EXPORT_PRIVATE virtual void PauseAllocationObservers();
926 :
927 : V8_EXPORT_PRIVATE virtual void ResumeAllocationObservers();
928 :
929 38367 : V8_EXPORT_PRIVATE virtual void StartNextInlineAllocationStep() {}
930 :
931 : void AllocationStep(int bytes_since_last, Address soon_object, int size);
932 :
933 : // Return the total amount committed memory for this space, i.e., allocatable
934 : // memory and page headers.
935 4210246 : virtual size_t CommittedMemory() { return committed_; }
936 :
937 0 : virtual size_t MaximumCommittedMemory() { return max_committed_; }
938 :
939 : // Returns allocated size.
940 : virtual size_t Size() = 0;
941 :
942 : // Returns size of objects. Can differ from the allocated size
943 : // (e.g. see LargeObjectSpace).
944 0 : virtual size_t SizeOfObjects() { return Size(); }
945 :
946 : // Approximate amount of physical memory committed for this space.
947 : virtual size_t CommittedPhysicalMemory() = 0;
948 :
949 : // Return the available bytes without growing.
950 : virtual size_t Available() = 0;
951 :
952 21870346 : virtual int RoundSizeDownToObjectAlignment(int size) {
953 21870346 : if (id_ == CODE_SPACE) {
954 0 : return RoundDown(size, kCodeAlignment);
955 : } else {
956 21870346 : return RoundDown(size, kPointerSize);
957 : }
958 : }
959 :
960 : virtual std::unique_ptr<ObjectIterator> GetObjectIterator() = 0;
961 :
962 : void AccountCommitted(size_t bytes) {
963 : DCHECK_GE(committed_ + bytes, committed_);
964 614726 : committed_ += bytes;
965 614726 : if (committed_ > max_committed_) {
966 546381 : max_committed_ = committed_;
967 : }
968 : }
969 :
970 : void AccountUncommitted(size_t bytes) {
971 : DCHECK_GE(committed_, committed_ - bytes);
972 435449 : committed_ -= bytes;
973 : }
974 :
975 : V8_EXPORT_PRIVATE void* GetRandomMmapAddr();
976 :
977 : #ifdef DEBUG
978 : virtual void Print() = 0;
979 : #endif
980 :
981 : protected:
982 : intptr_t GetNextInlineAllocationStepSize();
983 :
984 : std::vector<AllocationObserver*> allocation_observers_;
985 : bool allocation_observers_paused_;
986 :
987 : private:
988 : Heap* heap_;
989 : AllocationSpace id_;
990 : Executability executable_;
991 :
992 : // Keeps track of committed memory in a space.
993 : size_t committed_;
994 : size_t max_committed_;
995 :
996 : DISALLOW_COPY_AND_ASSIGN(Space);
997 : };
998 :
999 :
1000 : class MemoryChunkValidator {
1001 : // Computed offsets should match the compiler generated ones.
1002 : STATIC_ASSERT(MemoryChunk::kSizeOffset == offsetof(MemoryChunk, size_));
1003 :
1004 : // Validate our estimates on the header size.
1005 : STATIC_ASSERT(sizeof(MemoryChunk) <= MemoryChunk::kHeaderSize);
1006 : STATIC_ASSERT(sizeof(LargePage) <= MemoryChunk::kHeaderSize);
1007 : STATIC_ASSERT(sizeof(Page) <= MemoryChunk::kHeaderSize);
1008 : };
1009 :
1010 :
1011 : // ----------------------------------------------------------------------------
1012 : // All heap objects containing executable code (code objects) must be allocated
1013 : // from a 2 GB range of memory, so that they can call each other using 32-bit
1014 : // displacements. This happens automatically on 32-bit platforms, where 32-bit
1015 : // displacements cover the entire 4GB virtual address space. On 64-bit
1016 : // platforms, we support this using the CodeRange object, which reserves and
1017 : // manages a range of virtual memory.
1018 : class CodeRange {
1019 : public:
1020 : explicit CodeRange(Isolate* isolate);
1021 114002 : ~CodeRange() {
1022 57001 : if (virtual_memory_.IsReserved()) virtual_memory_.Release();
1023 57001 : }
1024 :
1025 : // Reserves a range of virtual memory, but does not commit any of it.
1026 : // Can only be called once, at heap initialization time.
1027 : // Returns false on failure.
1028 : bool SetUp(size_t requested_size);
1029 :
1030 928710 : bool valid() { return virtual_memory_.IsReserved(); }
1031 : Address start() {
1032 : DCHECK(valid());
1033 1708918 : return static_cast<Address>(virtual_memory_.address());
1034 : }
1035 : size_t size() {
1036 : DCHECK(valid());
1037 153809 : return virtual_memory_.size();
1038 : }
1039 : bool contains(Address address) {
1040 463324 : if (!valid()) return false;
1041 456802 : Address start = static_cast<Address>(virtual_memory_.address());
1042 920126 : return start <= address && address < start + virtual_memory_.size();
1043 : }
1044 :
1045 : // Allocates a chunk of memory from the large-object portion of
1046 : // the code range. On platforms with no separate code range, should
1047 : // not be called.
1048 : MUST_USE_RESULT Address AllocateRawMemory(const size_t requested_size,
1049 : const size_t commit_size,
1050 : size_t* allocated);
1051 : bool CommitRawMemory(Address start, size_t length);
1052 : bool UncommitRawMemory(Address start, size_t length);
1053 : void FreeRawMemory(Address buf, size_t length);
1054 :
1055 : private:
1056 : class FreeBlock {
1057 : public:
1058 309339 : FreeBlock() : start(0), size(0) {}
1059 : FreeBlock(Address start_arg, size_t size_arg)
1060 357680 : : start(start_arg), size(size_arg) {
1061 : DCHECK(IsAddressAligned(start, MemoryChunk::kAlignment));
1062 : DCHECK(size >= static_cast<size_t>(Page::kPageSize));
1063 : }
1064 : FreeBlock(void* start_arg, size_t size_arg)
1065 : : start(static_cast<Address>(start_arg)), size(size_arg) {
1066 : DCHECK(IsAddressAligned(start, MemoryChunk::kAlignment));
1067 : DCHECK(size >= static_cast<size_t>(Page::kPageSize));
1068 : }
1069 :
1070 : Address start;
1071 : size_t size;
1072 : };
1073 :
1074 : // Finds a block on the allocation list that contains at least the
1075 : // requested amount of memory. If none is found, sorts and merges
1076 : // the existing free memory blocks, and searches again.
1077 : // If none can be found, returns false.
1078 : bool GetNextAllocationBlock(size_t requested);
1079 : // Compares the start addresses of two free blocks.
1080 : static bool CompareFreeBlockAddress(const FreeBlock& left,
1081 : const FreeBlock& right);
1082 : bool ReserveBlock(const size_t requested_size, FreeBlock* block);
1083 : void ReleaseBlock(const FreeBlock* block);
1084 :
1085 : Isolate* isolate_;
1086 :
1087 : // The reserved range of virtual memory that all code objects are put in.
1088 : VirtualMemory virtual_memory_;
1089 :
1090 : // The global mutex guards free_list_ and allocation_list_ as GC threads may
1091 : // access both lists concurrently to the main thread.
1092 : base::Mutex code_range_mutex_;
1093 :
1094 : // Freed blocks of memory are added to the free list. When the allocation
1095 : // list is exhausted, the free list is sorted and merged to make the new
1096 : // allocation list.
1097 : std::vector<FreeBlock> free_list_;
1098 :
1099 : // Memory is allocated from the free blocks on the allocation list.
1100 : // The block at current_allocation_block_index_ is the current block.
1101 : std::vector<FreeBlock> allocation_list_;
1102 : size_t current_allocation_block_index_;
1103 :
1104 : DISALLOW_COPY_AND_ASSIGN(CodeRange);
1105 : };
1106 :
1107 :
1108 : class SkipList {
1109 : public:
1110 : SkipList() { Clear(); }
1111 :
1112 : void Clear() {
1113 18417270 : for (int idx = 0; idx < kSize; idx++) {
1114 18417270 : starts_[idx] = reinterpret_cast<Address>(-1);
1115 : }
1116 : }
1117 :
1118 1472175 : Address StartFor(Address addr) { return starts_[RegionNumber(addr)]; }
1119 :
1120 : void AddObject(Address addr, int size) {
1121 : int start_region = RegionNumber(addr);
1122 54560311 : int end_region = RegionNumber(addr + size - kPointerSize);
1123 67823338 : for (int idx = start_region; idx <= end_region; idx++) {
1124 67823338 : if (starts_[idx] > addr) {
1125 9637696 : starts_[idx] = addr;
1126 : } else {
1127 : // In the first region, there may already be an object closer to the
1128 : // start of the region. Do not change the start in that case. If this
1129 : // is not the first region, you probably added overlapping objects.
1130 : DCHECK_EQ(start_region, idx);
1131 : }
1132 : }
1133 : }
1134 :
1135 : static inline int RegionNumber(Address addr) {
1136 221103831 : return (OffsetFrom(addr) & Page::kPageAlignmentMask) >> kRegionSizeLog2;
1137 : }
1138 :
1139 54560311 : static void Update(Address addr, int size) {
1140 : Page* page = Page::FromAddress(addr);
1141 54560311 : SkipList* list = page->skip_list();
1142 54560311 : if (list == nullptr) {
1143 142501 : list = new SkipList();
1144 : page->set_skip_list(list);
1145 : }
1146 :
1147 : list->AddObject(addr, size);
1148 54560311 : }
1149 :
1150 : private:
1151 : static const int kRegionSizeLog2 = 13;
1152 : static const int kRegionSize = 1 << kRegionSizeLog2;
1153 : static const int kSize = Page::kPageSize / kRegionSize;
1154 :
1155 : STATIC_ASSERT(Page::kPageSize % kRegionSize == 0);
1156 :
1157 : Address starts_[kSize];
1158 : };
1159 :
1160 :
1161 : // ----------------------------------------------------------------------------
1162 : // A space acquires chunks of memory from the operating system. The memory
1163 : // allocator allocates and deallocates pages for the paged heap spaces and large
1164 : // pages for large object space.
1165 55789 : class V8_EXPORT_PRIVATE MemoryAllocator {
1166 : public:
1167 : // Unmapper takes care of concurrently unmapping and uncommitting memory
1168 : // chunks.
1169 167367 : class Unmapper {
1170 : public:
1171 : class UnmapFreeMemoryTask;
1172 :
1173 57422 : Unmapper(Heap* heap, MemoryAllocator* allocator)
1174 : : heap_(heap),
1175 : allocator_(allocator),
1176 : pending_unmapping_tasks_semaphore_(0),
1177 287114 : concurrent_unmapping_tasks_active_(0) {
1178 57423 : chunks_[kRegular].reserve(kReservedQueueingSlots);
1179 57423 : chunks_[kPooled].reserve(kReservedQueueingSlots);
1180 57423 : }
1181 :
1182 197485 : void AddMemoryChunkSafe(MemoryChunk* chunk) {
1183 390269 : if ((chunk->size() == Page::kPageSize) &&
1184 : (chunk->executable() != EXECUTABLE)) {
1185 192544 : AddMemoryChunkSafe<kRegular>(chunk);
1186 : } else {
1187 4941 : AddMemoryChunkSafe<kNonRegular>(chunk);
1188 : }
1189 197485 : }
1190 :
1191 186834 : MemoryChunk* TryGetPooledMemoryChunkSafe() {
1192 : // Procedure:
1193 : // (1) Try to get a chunk that was declared as pooled and already has
1194 : // been uncommitted.
1195 : // (2) Try to steal any memory chunk of kPageSize that would've been
1196 : // unmapped.
1197 186834 : MemoryChunk* chunk = GetMemoryChunkSafe<kPooled>();
1198 186834 : if (chunk == nullptr) {
1199 170874 : chunk = GetMemoryChunkSafe<kRegular>();
1200 170874 : if (chunk != nullptr) {
1201 : // For stolen chunks we need to manually free any allocated memory.
1202 1408 : chunk->ReleaseAllocatedMemory();
1203 : }
1204 : }
1205 186834 : return chunk;
1206 : }
1207 :
1208 : void FreeQueuedChunks();
1209 : void WaitUntilCompleted();
1210 : void TearDown();
1211 :
1212 : bool has_delayed_chunks() { return delayed_regular_chunks_.size() > 0; }
1213 :
1214 764 : int NumberOfDelayedChunks() {
1215 764 : base::LockGuard<base::Mutex> guard(&mutex_);
1216 1528 : return static_cast<int>(delayed_regular_chunks_.size());
1217 : }
1218 :
1219 : int NumberOfChunks();
1220 :
1221 : private:
1222 : static const int kReservedQueueingSlots = 64;
1223 : static const int kMaxUnmapperTasks = 24;
1224 :
1225 : enum ChunkQueueType {
1226 : kRegular, // Pages of kPageSize that do not live in a CodeRange and
1227 : // can thus be used for stealing.
1228 : kNonRegular, // Large chunks and executable chunks.
1229 : kPooled, // Pooled chunks, already uncommited and ready for reuse.
1230 : kNumberOfChunkQueues,
1231 : };
1232 :
1233 : enum class FreeMode {
1234 : kUncommitPooled,
1235 : kReleasePooled,
1236 : };
1237 :
1238 : template <ChunkQueueType type>
1239 447657 : void AddMemoryChunkSafe(MemoryChunk* chunk) {
1240 447657 : base::LockGuard<base::Mutex> guard(&mutex_);
1241 524364 : if (type != kRegular || allocator_->CanFreeMemoryChunk(chunk)) {
1242 378021 : chunks_[type].push_back(chunk);
1243 : } else {
1244 : DCHECK_EQ(type, kRegular);
1245 69638 : delayed_regular_chunks_.push_back(chunk);
1246 : }
1247 447659 : }
1248 :
1249 : template <ChunkQueueType type>
1250 1303010 : MemoryChunk* GetMemoryChunkSafe() {
1251 1303010 : base::LockGuard<base::Mutex> guard(&mutex_);
1252 1303378 : if (chunks_[type].empty()) return nullptr;
1253 377863 : MemoryChunk* chunk = chunks_[type].back();
1254 : chunks_[type].pop_back();
1255 377863 : return chunk;
1256 : }
1257 :
1258 : void ReconsiderDelayedChunks();
1259 : template <FreeMode mode>
1260 : void PerformFreeMemoryOnQueuedChunks();
1261 :
1262 : Heap* const heap_;
1263 : MemoryAllocator* const allocator_;
1264 : base::Mutex mutex_;
1265 : std::vector<MemoryChunk*> chunks_[kNumberOfChunkQueues];
1266 : // Delayed chunks cannot be processed in the current unmapping cycle because
1267 : // of dependencies such as an active sweeper.
1268 : // See MemoryAllocator::CanFreeMemoryChunk.
1269 : std::list<MemoryChunk*> delayed_regular_chunks_;
1270 : CancelableTaskManager::Id task_ids_[kMaxUnmapperTasks];
1271 : base::Semaphore pending_unmapping_tasks_semaphore_;
1272 : intptr_t concurrent_unmapping_tasks_active_;
1273 :
1274 : friend class MemoryAllocator;
1275 : };
1276 :
1277 : enum AllocationMode {
1278 : kRegular,
1279 : kPooled,
1280 : };
1281 :
1282 : enum FreeMode {
1283 : kFull,
1284 : kAlreadyPooled,
1285 : kPreFreeAndQueue,
1286 : kPooledAndQueue,
1287 : };
1288 :
1289 : static size_t CodePageGuardStartOffset();
1290 :
1291 : static size_t CodePageGuardSize();
1292 :
1293 : static size_t CodePageAreaStartOffset();
1294 :
1295 : static size_t CodePageAreaEndOffset();
1296 :
1297 211529 : static size_t CodePageAreaSize() {
1298 293146 : return CodePageAreaEndOffset() - CodePageAreaStartOffset();
1299 : }
1300 :
1301 1338801 : static size_t PageAreaSize(AllocationSpace space) {
1302 : DCHECK_NE(LO_SPACE, space);
1303 : return (space == CODE_SPACE) ? CodePageAreaSize()
1304 3155672 : : Page::kAllocatableMemory;
1305 : }
1306 :
1307 : static intptr_t GetCommitPageSize();
1308 :
1309 : explicit MemoryAllocator(Isolate* isolate);
1310 :
1311 : // Initializes its internal bookkeeping structures.
1312 : // Max capacity of the total space and executable memory limit.
1313 : bool SetUp(size_t max_capacity, size_t code_range_size);
1314 :
1315 : void TearDown();
1316 :
1317 : // Allocates a Page from the allocator. AllocationMode is used to indicate
1318 : // whether pooled allocation, which only works for MemoryChunk::kPageSize,
1319 : // should be tried first.
1320 : template <MemoryAllocator::AllocationMode alloc_mode = kRegular,
1321 : typename SpaceType>
1322 : Page* AllocatePage(size_t size, SpaceType* owner, Executability executable);
1323 :
1324 : LargePage* AllocateLargePage(size_t size, LargeObjectSpace* owner,
1325 : Executability executable);
1326 :
1327 : template <MemoryAllocator::FreeMode mode = kFull>
1328 : void Free(MemoryChunk* chunk);
1329 :
1330 : bool CanFreeMemoryChunk(MemoryChunk* chunk);
1331 :
1332 : // Returns allocated spaces in bytes.
1333 : size_t Size() { return size_.Value(); }
1334 :
1335 : // Returns allocated executable spaces in bytes.
1336 : size_t SizeExecutable() { return size_executable_.Value(); }
1337 :
1338 : // Returns the maximum available bytes of heaps.
1339 : size_t Available() {
1340 : const size_t size = Size();
1341 132931 : return capacity_ < size ? 0 : capacity_ - size;
1342 : }
1343 :
1344 : // Returns maximum available bytes that the old space can have.
1345 29652 : size_t MaxAvailable() {
1346 29652 : return (Available() / Page::kPageSize) * Page::kAllocatableMemory;
1347 : }
1348 :
1349 : // Returns an indication of whether a pointer is in a space that has
1350 : // been allocated by this MemoryAllocator.
1351 : V8_INLINE bool IsOutsideAllocatedSpace(const void* address) {
1352 5462294 : return address < lowest_ever_allocated_.Value() ||
1353 : address >= highest_ever_allocated_.Value();
1354 : }
1355 :
1356 : // Returns a MemoryChunk in which the memory region from commit_area_size to
1357 : // reserve_area_size of the chunk area is reserved but not committed, it
1358 : // could be committed later by calling MemoryChunk::CommitArea.
1359 : MemoryChunk* AllocateChunk(size_t reserve_area_size, size_t commit_area_size,
1360 : Executability executable, Space* space);
1361 :
1362 : Address ReserveAlignedMemory(size_t requested, size_t alignment, void* hint,
1363 : VirtualMemory* controller);
1364 : Address AllocateAlignedMemory(size_t reserve_size, size_t commit_size,
1365 : size_t alignment, Executability executable,
1366 : void* hint, VirtualMemory* controller);
1367 :
1368 : bool CommitMemory(Address addr, size_t size, Executability executable);
1369 :
1370 : void FreeMemory(VirtualMemory* reservation, Executability executable);
1371 : void FreeMemory(Address addr, size_t size, Executability executable);
1372 :
1373 : // Partially release |bytes_to_free| bytes starting at |start_free|. Note that
1374 : // internally memory is freed from |start_free| to the end of the reservation.
1375 : // Additional memory beyond the page is not accounted though, so
1376 : // |bytes_to_free| is computed by the caller.
1377 : void PartialFreeMemory(MemoryChunk* chunk, Address start_free,
1378 : size_t bytes_to_free, Address new_area_end);
1379 :
1380 : // Commit a contiguous block of memory from the initial chunk. Assumes that
1381 : // the address is not nullptr, the size is greater than zero, and that the
1382 : // block is contained in the initial chunk. Returns true if it succeeded
1383 : // and false otherwise.
1384 : bool CommitBlock(Address start, size_t size, Executability executable);
1385 :
1386 : // Uncommit a contiguous block of memory [start..(start+size)[.
1387 : // start is not nullptr, the size is greater than zero, and the
1388 : // block is contained in the initial chunk. Returns true if it succeeded
1389 : // and false otherwise.
1390 : bool UncommitBlock(Address start, size_t size);
1391 :
1392 : // Zaps a contiguous block of memory [start..(start+size)[ thus
1393 : // filling it up with a recognizable non-nullptr bit pattern.
1394 : void ZapBlock(Address start, size_t size);
1395 :
1396 : MUST_USE_RESULT bool CommitExecutableMemory(VirtualMemory* vm, Address start,
1397 : size_t commit_size,
1398 : size_t reserved_size);
1399 :
1400 : CodeRange* code_range() { return code_range_; }
1401 : Unmapper* unmapper() { return &unmapper_; }
1402 :
1403 : #ifdef DEBUG
1404 : // Reports statistic info of the space.
1405 : void ReportStatistics();
1406 : #endif
1407 :
1408 : private:
1409 : // PreFree logically frees the object, i.e., it takes care of the size
1410 : // bookkeeping and calls the allocation callback.
1411 : void PreFreeMemory(MemoryChunk* chunk);
1412 :
1413 : // FreeMemory can be called concurrently when PreFree was executed before.
1414 : void PerformFreeMemory(MemoryChunk* chunk);
1415 :
1416 : // See AllocatePage for public interface. Note that currently we only support
1417 : // pools for NOT_EXECUTABLE pages of size MemoryChunk::kPageSize.
1418 : template <typename SpaceType>
1419 : MemoryChunk* AllocatePagePooled(SpaceType* owner);
1420 :
1421 : // Initializes pages in a chunk. Returns the first page address.
1422 : // This function and GetChunkId() are provided for the mark-compact
1423 : // collector to rebuild page headers in the from space, which is
1424 : // used as a marking stack and its page headers are destroyed.
1425 : Page* InitializePagesInChunk(int chunk_id, int pages_in_chunk,
1426 : PagedSpace* owner);
1427 :
1428 801386 : void UpdateAllocatedSpaceLimits(void* low, void* high) {
1429 : // The use of atomic primitives does not guarantee correctness (wrt.
1430 : // desired semantics) by default. The loop here ensures that we update the
1431 : // values only if they did not change in between.
1432 : void* ptr = nullptr;
1433 801386 : do {
1434 : ptr = lowest_ever_allocated_.Value();
1435 974476 : } while ((low < ptr) && !lowest_ever_allocated_.TrySetValue(ptr, low));
1436 801386 : do {
1437 : ptr = highest_ever_allocated_.Value();
1438 923378 : } while ((high > ptr) && !highest_ever_allocated_.TrySetValue(ptr, high));
1439 801386 : }
1440 :
1441 : Isolate* isolate_;
1442 : CodeRange* code_range_;
1443 :
1444 : // Maximum space size in bytes.
1445 : size_t capacity_;
1446 :
1447 : // Allocated space size in bytes.
1448 : base::AtomicNumber<size_t> size_;
1449 : // Allocated executable space size in bytes.
1450 : base::AtomicNumber<size_t> size_executable_;
1451 :
1452 : // We keep the lowest and highest addresses allocated as a quick way
1453 : // of determining that pointers are outside the heap. The estimate is
1454 : // conservative, i.e. not all addresses in 'allocated' space are allocated
1455 : // to our heap. The range is [lowest, highest[, inclusive on the low end
1456 : // and exclusive on the high end.
1457 : base::AtomicValue<void*> lowest_ever_allocated_;
1458 : base::AtomicValue<void*> highest_ever_allocated_;
1459 :
1460 : VirtualMemory last_chunk_;
1461 : Unmapper unmapper_;
1462 :
1463 : friend class heap::TestCodeRangeScope;
1464 :
1465 : DISALLOW_IMPLICIT_CONSTRUCTORS(MemoryAllocator);
1466 : };
1467 :
1468 : extern template Page*
1469 : MemoryAllocator::AllocatePage<MemoryAllocator::kRegular, PagedSpace>(
1470 : size_t size, PagedSpace* owner, Executability executable);
1471 : extern template Page*
1472 : MemoryAllocator::AllocatePage<MemoryAllocator::kRegular, SemiSpace>(
1473 : size_t size, SemiSpace* owner, Executability executable);
1474 : extern template Page*
1475 : MemoryAllocator::AllocatePage<MemoryAllocator::kPooled, SemiSpace>(
1476 : size_t size, SemiSpace* owner, Executability executable);
1477 :
1478 : // -----------------------------------------------------------------------------
1479 : // Interface for heap object iterator to be implemented by all object space
1480 : // object iterators.
1481 : //
1482 : // NOTE: The space specific object iterators also implements the own next()
1483 : // method which is used to avoid using virtual functions
1484 : // iterating a specific space.
1485 :
1486 113180 : class V8_EXPORT_PRIVATE ObjectIterator : public Malloced {
1487 : public:
1488 55465 : virtual ~ObjectIterator() {}
1489 : virtual HeapObject* Next() = 0;
1490 : };
1491 :
1492 : template <class PAGE_TYPE>
1493 : class PageIteratorImpl
1494 : : public base::iterator<std::forward_iterator_tag, PAGE_TYPE> {
1495 : public:
1496 3021012 : explicit PageIteratorImpl(PAGE_TYPE* p) : p_(p) {}
1497 : PageIteratorImpl(const PageIteratorImpl<PAGE_TYPE>& other) : p_(other.p_) {}
1498 0 : PAGE_TYPE* operator*() { return p_; }
1499 : bool operator==(const PageIteratorImpl<PAGE_TYPE>& rhs) {
1500 111617 : return rhs.p_ == p_;
1501 : }
1502 : bool operator!=(const PageIteratorImpl<PAGE_TYPE>& rhs) {
1503 8739020 : return rhs.p_ != p_;
1504 : }
1505 : inline PageIteratorImpl<PAGE_TYPE>& operator++();
1506 : inline PageIteratorImpl<PAGE_TYPE> operator++(int);
1507 :
1508 : private:
1509 : PAGE_TYPE* p_;
1510 : };
1511 :
1512 : typedef PageIteratorImpl<Page> PageIterator;
1513 : typedef PageIteratorImpl<LargePage> LargePageIterator;
1514 :
1515 : class PageRange {
1516 : public:
1517 : typedef PageIterator iterator;
1518 33282 : PageRange(Page* begin, Page* end) : begin_(begin), end_(end) {}
1519 0 : explicit PageRange(Page* page) : PageRange(page, page->next_page()) {}
1520 : inline PageRange(Address start, Address limit);
1521 :
1522 : iterator begin() { return iterator(begin_); }
1523 : iterator end() { return iterator(end_); }
1524 :
1525 : private:
1526 : Page* begin_;
1527 : Page* end_;
1528 : };
1529 :
1530 : // -----------------------------------------------------------------------------
1531 : // Heap object iterator in new/old/map spaces.
1532 : //
1533 : // A HeapObjectIterator iterates objects from the bottom of the given space
1534 : // to its top or from the bottom of the given page to its top.
1535 : //
1536 : // If objects are allocated in the page during iteration the iterator may
1537 : // or may not iterate over those objects. The caller must create a new
1538 : // iterator in order to be sure to visit these new objects.
1539 66558 : class V8_EXPORT_PRIVATE HeapObjectIterator : public ObjectIterator {
1540 : public:
1541 : // Creates a new object iterator in a given space.
1542 : explicit HeapObjectIterator(PagedSpace* space);
1543 : explicit HeapObjectIterator(Page* page);
1544 :
1545 : // Advance to the next object, skipping free spaces and other fillers and
1546 : // skipping the special garbage section of which there is one per space.
1547 : // Returns nullptr when the iteration has ended.
1548 : inline HeapObject* Next() override;
1549 :
1550 : private:
1551 : // Fast (inlined) path of next().
1552 : inline HeapObject* FromCurrentPage();
1553 :
1554 : // Slow path of next(), goes into the next page. Returns false if the
1555 : // iteration has ended.
1556 : bool AdvanceToNextPage();
1557 :
1558 : Address cur_addr_; // Current iteration point.
1559 : Address cur_end_; // End iteration point.
1560 : PagedSpace* space_;
1561 : PageRange page_range_;
1562 : PageRange::iterator current_page_;
1563 : };
1564 :
1565 :
1566 : // -----------------------------------------------------------------------------
1567 : // A space has a circular list of pages. The next page can be accessed via
1568 : // Page::next_page() call.
1569 :
1570 : // An abstraction of allocation and relocation pointers in a page-structured
1571 : // space.
1572 : class AllocationInfo {
1573 : public:
1574 719130 : AllocationInfo() : top_(nullptr), limit_(nullptr) {}
1575 342675 : AllocationInfo(Address top, Address limit) : top_(top), limit_(limit) {}
1576 :
1577 : void Reset(Address top, Address limit) {
1578 : set_top(top);
1579 : set_limit(limit);
1580 : }
1581 :
1582 : INLINE(void set_top(Address top)) {
1583 : SLOW_DCHECK(top == nullptr ||
1584 : (reinterpret_cast<intptr_t>(top) & kHeapObjectTagMask) == 0);
1585 579696293 : top_ = top;
1586 : }
1587 :
1588 : INLINE(Address top()) const {
1589 : SLOW_DCHECK(top_ == nullptr ||
1590 : (reinterpret_cast<intptr_t>(top_) & kHeapObjectTagMask) == 0);
1591 : return top_;
1592 : }
1593 :
1594 : Address* top_address() { return &top_; }
1595 :
1596 : INLINE(void set_limit(Address limit)) {
1597 4882019 : limit_ = limit;
1598 : }
1599 :
1600 : INLINE(Address limit()) const {
1601 : return limit_;
1602 : }
1603 :
1604 : Address* limit_address() { return &limit_; }
1605 :
1606 : #ifdef DEBUG
1607 : bool VerifyPagedAllocation() {
1608 : return (Page::FromAllocationAreaAddress(top_) ==
1609 : Page::FromAllocationAreaAddress(limit_)) &&
1610 : (top_ <= limit_);
1611 : }
1612 : #endif
1613 :
1614 : private:
1615 : // Current allocation top.
1616 : Address top_;
1617 : // Current allocation limit.
1618 : Address limit_;
1619 : };
1620 :
1621 :
1622 : // An abstraction of the accounting statistics of a page-structured space.
1623 : //
1624 : // The stats are only set by functions that ensure they stay balanced. These
1625 : // functions increase or decrease one of the non-capacity stats in conjunction
1626 : // with capacity, or else they always balance increases and decreases to the
1627 : // non-capacity stats.
1628 : class AllocationStats BASE_EMBEDDED {
1629 : public:
1630 : AllocationStats() { Clear(); }
1631 :
1632 : // Zero out all the allocation statistics (i.e., no capacity).
1633 : void Clear() {
1634 : capacity_ = 0;
1635 1429308 : max_capacity_ = 0;
1636 : ClearSize();
1637 : }
1638 :
1639 : void ClearSize() {
1640 1599708 : size_ = 0;
1641 : #ifdef DEBUG
1642 : allocated_on_page_.clear();
1643 : #endif
1644 : }
1645 :
1646 : // Accessors for the allocation statistics.
1647 : size_t Capacity() { return capacity_.Value(); }
1648 : size_t MaxCapacity() { return max_capacity_; }
1649 : size_t Size() { return size_; }
1650 : #ifdef DEBUG
1651 : size_t AllocatedOnPage(Page* page) { return allocated_on_page_[page]; }
1652 : #endif
1653 :
1654 : void IncreaseAllocatedBytes(size_t bytes, Page* page) {
1655 : DCHECK_GE(size_ + bytes, size_);
1656 2472818 : size_ += bytes;
1657 : #ifdef DEBUG
1658 : allocated_on_page_[page] += bytes;
1659 : #endif
1660 : }
1661 :
1662 : void DecreaseAllocatedBytes(size_t bytes, Page* page) {
1663 : DCHECK_GE(size_, bytes);
1664 2063428 : size_ -= bytes;
1665 : #ifdef DEBUG
1666 : DCHECK_GE(allocated_on_page_[page], bytes);
1667 : allocated_on_page_[page] -= bytes;
1668 : #endif
1669 : }
1670 :
1671 : void DecreaseCapacity(size_t bytes) {
1672 : size_t capacity = capacity_.Value();
1673 : DCHECK_GE(capacity, bytes);
1674 : DCHECK_GE(capacity - bytes, size_);
1675 : USE(capacity);
1676 : capacity_.Decrement(bytes);
1677 : }
1678 :
1679 509516 : void IncreaseCapacity(size_t bytes) {
1680 : size_t capacity = capacity_.Value();
1681 : DCHECK_GE(capacity + bytes, capacity);
1682 : capacity_.Increment(bytes);
1683 509516 : if (capacity > max_capacity_) {
1684 249487 : max_capacity_ = capacity;
1685 : }
1686 509516 : }
1687 :
1688 : private:
1689 : // |capacity_|: The number of object-area bytes (i.e., not including page
1690 : // bookkeeping structures) currently in the space.
1691 : // During evacuation capacity of the main spaces is accessed from multiple
1692 : // threads to check the old generation hard limit.
1693 : base::AtomicNumber<size_t> capacity_;
1694 :
1695 : // |max_capacity_|: The maximum capacity ever observed.
1696 : size_t max_capacity_;
1697 :
1698 : // |size_|: The number of allocated bytes.
1699 : size_t size_;
1700 :
1701 : #ifdef DEBUG
1702 : std::unordered_map<Page*, size_t, Page::Hasher> allocated_on_page_;
1703 : #endif
1704 : };
1705 :
1706 : // A free list maintaining free blocks of memory. The free list is organized in
1707 : // a way to encourage objects allocated around the same time to be near each
1708 : // other. The normal way to allocate is intended to be by bumping a 'top'
1709 : // pointer until it hits a 'limit' pointer. When the limit is hit we need to
1710 : // find a new space to allocate from. This is done with the free list, which is
1711 : // divided up into rough categories to cut down on waste. Having finer
1712 : // categories would scatter allocation more.
1713 :
1714 : // The free list is organized in categories as follows:
1715 : // kMinBlockSize-10 words (tiniest): The tiniest blocks are only used for
1716 : // allocation, when categories >= small do not have entries anymore.
1717 : // 11-31 words (tiny): The tiny blocks are only used for allocation, when
1718 : // categories >= small do not have entries anymore.
1719 : // 32-255 words (small): Used for allocating free space between 1-31 words in
1720 : // size.
1721 : // 256-2047 words (medium): Used for allocating free space between 32-255 words
1722 : // in size.
1723 : // 1048-16383 words (large): Used for allocating free space between 256-2047
1724 : // words in size.
1725 : // At least 16384 words (huge): This list is for objects of 2048 words or
1726 : // larger. Empty pages are also added to this list.
1727 : class V8_EXPORT_PRIVATE FreeList {
1728 : public:
1729 : // This method returns how much memory can be allocated after freeing
1730 : // maximum_freed memory.
1731 : static inline size_t GuaranteedAllocatable(size_t maximum_freed) {
1732 423222 : if (maximum_freed <= kTiniestListMax) {
1733 : // Since we are not iterating over all list entries, we cannot guarantee
1734 : // that we can find the maximum freed block in that free list.
1735 : return 0;
1736 362868 : } else if (maximum_freed <= kTinyListMax) {
1737 : return kTinyAllocationMax;
1738 350564 : } else if (maximum_freed <= kSmallListMax) {
1739 : return kSmallAllocationMax;
1740 337233 : } else if (maximum_freed <= kMediumListMax) {
1741 : return kMediumAllocationMax;
1742 238259 : } else if (maximum_freed <= kLargeListMax) {
1743 : return kLargeAllocationMax;
1744 : }
1745 : return maximum_freed;
1746 : }
1747 :
1748 : static FreeListCategoryType SelectFreeListCategoryType(size_t size_in_bytes) {
1749 19591484 : if (size_in_bytes <= kTiniestListMax) {
1750 : return kTiniest;
1751 13048356 : } else if (size_in_bytes <= kTinyListMax) {
1752 : return kTiny;
1753 5107248 : } else if (size_in_bytes <= kSmallListMax) {
1754 : return kSmall;
1755 1596466 : } else if (size_in_bytes <= kMediumListMax) {
1756 : return kMedium;
1757 1120017 : } else if (size_in_bytes <= kLargeListMax) {
1758 : return kLarge;
1759 : }
1760 : return kHuge;
1761 : }
1762 :
1763 : explicit FreeList(PagedSpace* owner);
1764 :
1765 : // Adds a node on the free list. The block of size {size_in_bytes} starting
1766 : // at {start} is placed on the free list. The return value is the number of
1767 : // bytes that were not added to the free list, because they freed memory block
1768 : // was too small. Bookkeeping information will be written to the block, i.e.,
1769 : // its contents will be destroyed. The start address should be word aligned,
1770 : // and the size should be a non-zero multiple of the word size.
1771 : size_t Free(Address start, size_t size_in_bytes, FreeMode mode);
1772 :
1773 : // Finds a node of size at least size_in_bytes and sets up a linear allocation
1774 : // area using this node. Returns false if there is no such node and the caller
1775 : // has to retry allocation after collecting garbage.
1776 : MUST_USE_RESULT bool Allocate(size_t size_in_bytes);
1777 :
1778 : // Clear the free list.
1779 : void Reset();
1780 :
1781 813649 : void ResetStats() {
1782 : wasted_bytes_.SetValue(0);
1783 : ForAllFreeListCategories(
1784 170400 : [](FreeListCategory* category) { category->ResetStats(); });
1785 813649 : }
1786 :
1787 : // Return the number of bytes available on the free list.
1788 : size_t Available() {
1789 : size_t available = 0;
1790 1698121 : ForAllFreeListCategories([&available](FreeListCategory* category) {
1791 1698121 : available += category->available();
1792 : });
1793 : return available;
1794 : }
1795 :
1796 : bool IsEmpty() {
1797 : bool empty = true;
1798 : ForAllFreeListCategories([&empty](FreeListCategory* category) {
1799 : if (!category->is_empty()) empty = false;
1800 : });
1801 : return empty;
1802 : }
1803 :
1804 : // Used after booting the VM.
1805 : void RepairLists(Heap* heap);
1806 :
1807 : size_t EvictFreeListItems(Page* page);
1808 : bool ContainsPageFreeListItems(Page* page);
1809 :
1810 : PagedSpace* owner() { return owner_; }
1811 : size_t wasted_bytes() { return wasted_bytes_.Value(); }
1812 :
1813 : template <typename Callback>
1814 : void ForAllFreeListCategories(FreeListCategoryType type, Callback callback) {
1815 15406866 : FreeListCategory* current = categories_[type];
1816 18167312 : while (current != nullptr) {
1817 : FreeListCategory* next = current->next();
1818 : callback(current);
1819 : current = next;
1820 : }
1821 : }
1822 :
1823 : template <typename Callback>
1824 335304 : void ForAllFreeListCategories(Callback callback) {
1825 15571770 : for (int i = kFirstCategory; i < kNumberOfCategories; i++) {
1826 15406866 : ForAllFreeListCategories(static_cast<FreeListCategoryType>(i), callback);
1827 : }
1828 335304 : }
1829 :
1830 : bool AddCategory(FreeListCategory* category);
1831 : void RemoveCategory(FreeListCategory* category);
1832 : void PrintCategories(FreeListCategoryType type);
1833 :
1834 : // Returns a page containing an entry for a given type, or nullptr otherwise.
1835 : inline Page* GetPageForCategoryType(FreeListCategoryType type);
1836 :
1837 : #ifdef DEBUG
1838 : size_t SumFreeLists();
1839 : bool IsVeryLong();
1840 : #endif
1841 :
1842 : private:
1843 : class FreeListCategoryIterator {
1844 : public:
1845 : FreeListCategoryIterator(FreeList* free_list, FreeListCategoryType type)
1846 4069969 : : current_(free_list->categories_[type]) {}
1847 :
1848 : bool HasNext() { return current_ != nullptr; }
1849 :
1850 : FreeListCategory* Next() {
1851 : DCHECK(HasNext());
1852 : FreeListCategory* tmp = current_;
1853 1666779 : current_ = current_->next();
1854 : return tmp;
1855 : }
1856 :
1857 : private:
1858 : FreeListCategory* current_;
1859 : };
1860 :
1861 : // The size range of blocks, in bytes.
1862 : static const size_t kMinBlockSize = 3 * kPointerSize;
1863 : static const size_t kMaxBlockSize = Page::kAllocatableMemory;
1864 :
1865 : static const size_t kTiniestListMax = 0xa * kPointerSize;
1866 : static const size_t kTinyListMax = 0x1f * kPointerSize;
1867 : static const size_t kSmallListMax = 0xff * kPointerSize;
1868 : static const size_t kMediumListMax = 0x7ff * kPointerSize;
1869 : static const size_t kLargeListMax = 0x3fff * kPointerSize;
1870 : static const size_t kTinyAllocationMax = kTiniestListMax;
1871 : static const size_t kSmallAllocationMax = kTinyListMax;
1872 : static const size_t kMediumAllocationMax = kSmallListMax;
1873 : static const size_t kLargeAllocationMax = kMediumListMax;
1874 :
1875 : FreeSpace* FindNodeFor(size_t size_in_bytes, size_t* node_size);
1876 :
1877 : // Walks all available categories for a given |type| and tries to retrieve
1878 : // a node. Returns nullptr if the category is empty.
1879 : FreeSpace* FindNodeIn(FreeListCategoryType type, size_t* node_size);
1880 :
1881 : // Tries to retrieve a node from the first category in a given |type|.
1882 : // Returns nullptr if the category is empty.
1883 : FreeSpace* TryFindNodeIn(FreeListCategoryType type, size_t* node_size,
1884 : size_t minimum_size);
1885 :
1886 : // Searches a given |type| for a node of at least |minimum_size|.
1887 : FreeSpace* SearchForNodeInList(FreeListCategoryType type, size_t* node_size,
1888 : size_t minimum_size);
1889 :
1890 : // The tiny categories are not used for fast allocation.
1891 : FreeListCategoryType SelectFastAllocationFreeListCategoryType(
1892 : size_t size_in_bytes) {
1893 2044549 : if (size_in_bytes <= kSmallAllocationMax) {
1894 : return kSmall;
1895 542798 : } else if (size_in_bytes <= kMediumAllocationMax) {
1896 : return kMedium;
1897 345437 : } else if (size_in_bytes <= kLargeAllocationMax) {
1898 : return kLarge;
1899 : }
1900 : return kHuge;
1901 : }
1902 :
1903 : FreeListCategory* top(FreeListCategoryType type) const {
1904 58957 : return categories_[type];
1905 : }
1906 :
1907 : PagedSpace* owner_;
1908 : base::AtomicNumber<size_t> wasted_bytes_;
1909 : FreeListCategory* categories_[kNumberOfCategories];
1910 :
1911 : friend class FreeListCategory;
1912 :
1913 : DISALLOW_IMPLICIT_CONSTRUCTORS(FreeList);
1914 : };
1915 :
1916 : // LocalAllocationBuffer represents a linear allocation area that is created
1917 : // from a given {AllocationResult} and can be used to allocate memory without
1918 : // synchronization.
1919 : //
1920 : // The buffer is properly closed upon destruction and reassignment.
1921 : // Example:
1922 : // {
1923 : // AllocationResult result = ...;
1924 : // LocalAllocationBuffer a(heap, result, size);
1925 : // LocalAllocationBuffer b = a;
1926 : // CHECK(!a.IsValid());
1927 : // CHECK(b.IsValid());
1928 : // // {a} is invalid now and cannot be used for further allocations.
1929 : // }
1930 : // // Since {b} went out of scope, the LAB is closed, resulting in creating a
1931 : // // filler object for the remaining area.
1932 : class LocalAllocationBuffer {
1933 : public:
1934 : // Indicates that a buffer cannot be used for allocations anymore. Can result
1935 : // from either reassigning a buffer, or trying to construct it from an
1936 : // invalid {AllocationResult}.
1937 : static inline LocalAllocationBuffer InvalidBuffer();
1938 :
1939 : // Creates a new LAB from a given {AllocationResult}. Results in
1940 : // InvalidBuffer if the result indicates a retry.
1941 : static inline LocalAllocationBuffer FromResult(Heap* heap,
1942 : AllocationResult result,
1943 : intptr_t size);
1944 :
1945 528731 : ~LocalAllocationBuffer() { Close(); }
1946 :
1947 : // Convert to C++11 move-semantics once allowed by the style guide.
1948 : LocalAllocationBuffer(const LocalAllocationBuffer& other);
1949 : LocalAllocationBuffer& operator=(const LocalAllocationBuffer& other);
1950 :
1951 : MUST_USE_RESULT inline AllocationResult AllocateRawAligned(
1952 : int size_in_bytes, AllocationAlignment alignment);
1953 :
1954 79674931 : inline bool IsValid() { return allocation_info_.top() != nullptr; }
1955 :
1956 : // Try to merge LABs, which is only possible when they are adjacent in memory.
1957 : // Returns true if the merge was successful, false otherwise.
1958 : inline bool TryMerge(LocalAllocationBuffer* other);
1959 :
1960 : inline bool TryFreeLast(HeapObject* object, int object_size);
1961 :
1962 : // Close a LAB, effectively invalidating it. Returns the unused area.
1963 : AllocationInfo Close();
1964 :
1965 : private:
1966 : LocalAllocationBuffer(Heap* heap, AllocationInfo allocation_info);
1967 :
1968 : Heap* heap_;
1969 : AllocationInfo allocation_info_;
1970 : };
1971 :
1972 : class V8_EXPORT_PRIVATE PagedSpace : NON_EXPORTED_BASE(public Space) {
1973 : public:
1974 : typedef PageIterator iterator;
1975 :
1976 : static const size_t kCompactionMemoryWanted = 500 * KB;
1977 :
1978 : // Creates a space with an id.
1979 : PagedSpace(Heap* heap, AllocationSpace id, Executability executable);
1980 :
1981 946336 : ~PagedSpace() override { TearDown(); }
1982 :
1983 : // Set up the space using the given address range of virtual memory (from
1984 : // the memory allocator's initial chunk) if possible. If the block of
1985 : // addresses is not big enough to contain a single page-aligned page, a
1986 : // fresh chunk will be allocated.
1987 : bool SetUp();
1988 :
1989 : // Returns true if the space has been successfully set up and not
1990 : // subsequently torn down.
1991 : bool HasBeenSetUp();
1992 :
1993 : // Checks whether an object/address is in this space.
1994 : inline bool Contains(Address a);
1995 : inline bool Contains(Object* o);
1996 : bool ContainsSlow(Address addr);
1997 :
1998 : // During boot the free_space_map is created, and afterwards we may need
1999 : // to write it into the free list nodes that were already created.
2000 : void RepairFreeListsAfterDeserialization();
2001 :
2002 : // Prepares for a mark-compact GC.
2003 : void PrepareForMarkCompact();
2004 :
2005 : // Current capacity without growing (Size() + Available()).
2006 : size_t Capacity() { return accounting_stats_.Capacity(); }
2007 :
2008 : // Approximate amount of physical memory committed for this space.
2009 : size_t CommittedPhysicalMemory() override;
2010 :
2011 : void ResetFreeListStatistics();
2012 :
2013 : // Sets the capacity, the available space and the wasted space to zero.
2014 : // The stats are rebuilt during sweeping by adding each page to the
2015 : // capacity and the size when it is encountered. As free spaces are
2016 : // discovered during the sweeping they are subtracted from the size and added
2017 : // to the available and wasted totals.
2018 170400 : void ClearStats() {
2019 : accounting_stats_.ClearSize();
2020 170400 : free_list_.ResetStats();
2021 170400 : ResetFreeListStatistics();
2022 170400 : }
2023 :
2024 : // Available bytes without growing. These are the bytes on the free list.
2025 : // The bytes in the linear allocation area are not included in this total
2026 : // because updating the stats would slow down allocation. New pages are
2027 : // immediately added to the free list so they show up here.
2028 1210420 : size_t Available() override { return free_list_.Available(); }
2029 :
2030 : // Allocated bytes in this space. Garbage bytes that were not found due to
2031 : // concurrent sweeping are counted as being allocated! The bytes in the
2032 : // current linear allocation area (between top and limit) are also counted
2033 : // here.
2034 8422876 : size_t Size() override { return accounting_stats_.Size(); }
2035 :
2036 : // As size, but the bytes in lazily swept pages are estimated and the bytes
2037 : // in the current linear allocation area are not included.
2038 : size_t SizeOfObjects() override;
2039 :
2040 : // Wasted bytes in this space. These are just the bytes that were thrown away
2041 : // due to being too small to use for allocation.
2042 691440 : virtual size_t Waste() { return free_list_.wasted_bytes(); }
2043 :
2044 : // Returns the allocation pointer in this space.
2045 509665501 : Address top() { return allocation_info_.top(); }
2046 328232554 : Address limit() { return allocation_info_.limit(); }
2047 :
2048 : // The allocation top address.
2049 : Address* allocation_top_address() { return allocation_info_.top_address(); }
2050 :
2051 : // The allocation limit address.
2052 : Address* allocation_limit_address() {
2053 : return allocation_info_.limit_address();
2054 : }
2055 :
2056 : enum UpdateSkipList { UPDATE_SKIP_LIST, IGNORE_SKIP_LIST };
2057 :
2058 : // Allocate the requested number of bytes in the space if possible, return a
2059 : // failure object if not. Only use IGNORE_SKIP_LIST if the skip list is going
2060 : // to be manually updated later.
2061 : MUST_USE_RESULT inline AllocationResult AllocateRawUnaligned(
2062 : int size_in_bytes, UpdateSkipList update_skip_list = UPDATE_SKIP_LIST);
2063 :
2064 : // Allocate the requested number of bytes in the space double aligned if
2065 : // possible, return a failure object if not.
2066 : MUST_USE_RESULT inline AllocationResult AllocateRawAligned(
2067 : int size_in_bytes, AllocationAlignment alignment);
2068 :
2069 : // Allocate the requested number of bytes in the space and consider allocation
2070 : // alignment if needed.
2071 : MUST_USE_RESULT inline AllocationResult AllocateRaw(
2072 : int size_in_bytes, AllocationAlignment alignment);
2073 :
2074 : // Give a block of memory to the space's free list. It might be added to
2075 : // the free list or accounted as waste.
2076 : // If add_to_freelist is false then just accounting stats are updated and
2077 : // no attempt to add area to free list is made.
2078 : size_t Free(Address start, size_t size_in_bytes) {
2079 1931370 : size_t wasted = free_list_.Free(start, size_in_bytes, kLinkCategory);
2080 : Page* page = Page::FromAddress(start);
2081 : accounting_stats_.DecreaseAllocatedBytes(size_in_bytes, page);
2082 : DCHECK_GE(size_in_bytes, wasted);
2083 : return size_in_bytes - wasted;
2084 : }
2085 :
2086 : size_t UnaccountedFree(Address start, size_t size_in_bytes) {
2087 28862120 : size_t wasted = free_list_.Free(start, size_in_bytes, kDoNotLinkCategory);
2088 : DCHECK_GE(size_in_bytes, wasted);
2089 28827663 : return size_in_bytes - wasted;
2090 : }
2091 :
2092 : inline bool TryFreeLast(HeapObject* object, int object_size);
2093 :
2094 165179 : void ResetFreeList() { free_list_.Reset(); }
2095 :
2096 : void PauseAllocationObservers() override;
2097 : void ResumeAllocationObservers() override;
2098 :
2099 : // Empty space allocation info, returning unused area to free list.
2100 : void EmptyAllocationInfo();
2101 :
2102 : void MarkAllocationInfoBlack();
2103 : void UnmarkAllocationInfo();
2104 :
2105 : void DecreaseAllocatedBytes(size_t bytes, Page* page) {
2106 : accounting_stats_.DecreaseAllocatedBytes(bytes, page);
2107 : }
2108 : void IncreaseAllocatedBytes(size_t bytes, Page* page) {
2109 : accounting_stats_.IncreaseAllocatedBytes(bytes, page);
2110 : }
2111 : void DecreaseCapacity(size_t bytes) {
2112 : accounting_stats_.DecreaseCapacity(bytes);
2113 : }
2114 : void IncreaseCapacity(size_t bytes) {
2115 509516 : accounting_stats_.IncreaseCapacity(bytes);
2116 : }
2117 :
2118 : void RefineAllocatedBytesAfterSweeping(Page* page);
2119 :
2120 : // The dummy page that anchors the linked list of pages.
2121 : Page* anchor() { return &anchor_; }
2122 :
2123 : Page* InitializePage(MemoryChunk* chunk, Executability executable);
2124 : void ReleasePage(Page* page);
2125 : // Adds the page to this space and returns the number of bytes added to the
2126 : // free list of the space.
2127 : size_t AddPage(Page* page);
2128 : void RemovePage(Page* page);
2129 : // Remove a page if it has at least |size_in_bytes| bytes available that can
2130 : // be used for allocation.
2131 : Page* RemovePageSafe(int size_in_bytes);
2132 :
2133 : void SetReadAndExecutable();
2134 : void SetReadAndWritable();
2135 :
2136 : #ifdef VERIFY_HEAP
2137 : // Verify integrity of this space.
2138 : virtual void Verify(ObjectVisitor* visitor);
2139 :
2140 : void VerifyLiveBytes();
2141 :
2142 : // Overridden by subclasses to verify space-specific object
2143 : // properties (e.g., only maps or free-list nodes are in map space).
2144 : virtual void VerifyObject(HeapObject* obj) {}
2145 : #endif
2146 :
2147 : #ifdef DEBUG
2148 : void VerifyCountersAfterSweeping();
2149 : void VerifyCountersBeforeConcurrentSweeping();
2150 : // Print meta info and objects in this space.
2151 : void Print() override;
2152 :
2153 : // Reports statistics for the space
2154 : void ReportStatistics();
2155 :
2156 : // Report code object related statistics
2157 : static void ReportCodeStatistics(Isolate* isolate);
2158 : static void ResetCodeStatistics(Isolate* isolate);
2159 : #endif
2160 :
2161 : Page* FirstPage() { return anchor_.next_page(); }
2162 : Page* LastPage() { return anchor_.prev_page(); }
2163 :
2164 : bool CanExpand(size_t size);
2165 :
2166 : // Returns the number of total pages in this space.
2167 : int CountTotalPages();
2168 :
2169 : // Return size of allocatable area on a page in this space.
2170 2371155 : inline int AreaSize() { return static_cast<int>(area_size_); }
2171 :
2172 178455141 : virtual bool is_local() { return false; }
2173 :
2174 : // Merges {other} into the current space. Note that this modifies {other},
2175 : // e.g., removes its bump pointer area and resets statistics.
2176 : void MergeCompactionSpace(CompactionSpace* other);
2177 :
2178 : // Refills the free list from the corresponding free list filled by the
2179 : // sweeper.
2180 : virtual void RefillFreeList();
2181 :
2182 : FreeList* free_list() { return &free_list_; }
2183 :
2184 : base::Mutex* mutex() { return &space_mutex_; }
2185 :
2186 : inline void UnlinkFreeListCategories(Page* page);
2187 : inline size_t RelinkFreeListCategories(Page* page);
2188 :
2189 4166304 : iterator begin() { return iterator(anchor_.next_page()); }
2190 3858424 : iterator end() { return iterator(&anchor_); }
2191 :
2192 : // Shrink immortal immovable pages of the space to be exactly the size needed
2193 : // using the high water mark.
2194 : void ShrinkImmortalImmovablePages();
2195 :
2196 : size_t ShrinkPageToHighWaterMark(Page* page);
2197 :
2198 : std::unique_ptr<ObjectIterator> GetObjectIterator() override;
2199 :
2200 : // Sets the page that is currently locked by the task using the space. This
2201 : // page will be preferred for sweeping to avoid a potential deadlock where
2202 : // multiple tasks hold locks on pages while trying to sweep each others pages.
2203 107368 : void AnnounceLockedPage(Page* page) { locked_page_ = page; }
2204 :
2205 : Address ComputeLimit(Address start, Address end, size_t size_in_bytes);
2206 : void SetAllocationInfo(Address top, Address limit);
2207 :
2208 : private:
2209 : // Set space allocation info.
2210 : void SetTopAndLimit(Address top, Address limit) {
2211 : DCHECK(top == limit ||
2212 : Page::FromAddress(top) == Page::FromAddress(limit - 1));
2213 2976412 : MemoryChunk::UpdateHighWaterMark(allocation_info_.top());
2214 : allocation_info_.Reset(top, limit);
2215 : }
2216 : void DecreaseLimit(Address new_limit);
2217 : void StartNextInlineAllocationStep() override;
2218 153776527 : bool SupportsInlineAllocation() { return identity() == OLD_SPACE; }
2219 :
2220 : protected:
2221 : // PagedSpaces that should be included in snapshots have different, i.e.,
2222 : // smaller, initial pages.
2223 0 : virtual bool snapshotable() { return true; }
2224 :
2225 : bool HasPages() { return anchor_.next_page() != &anchor_; }
2226 :
2227 : // Cleans up the space, frees all pages in this space except those belonging
2228 : // to the initial chunk, uncommits addresses in the initial chunk.
2229 : void TearDown();
2230 :
2231 : // Expands the space by allocating a fixed number of pages. Returns false if
2232 : // it cannot allocate requested number of pages from OS, or if the hard heap
2233 : // size limit has been hit.
2234 : bool Expand();
2235 :
2236 : // Sets up a linear allocation area that fits the given number of bytes.
2237 : // Returns false if there is not enough space and the caller has to retry
2238 : // after collecting garbage.
2239 : inline bool EnsureLinearAllocationArea(int size_in_bytes);
2240 : // Allocates an object from the linear allocation area. Assumes that the
2241 : // linear allocation area is large enought to fit the object.
2242 : inline HeapObject* AllocateLinearly(int size_in_bytes);
2243 : // Tries to allocate an aligned object from the linear allocation area.
2244 : // Returns nullptr if the linear allocation area does not fit the object.
2245 : // Otherwise, returns the object pointer and writes the allocation size
2246 : // (object size + alignment filler size) to the size_in_bytes.
2247 : inline HeapObject* TryAllocateLinearlyAligned(int* size_in_bytes,
2248 : AllocationAlignment alignment);
2249 : // If sweeping is still in progress try to sweep unswept pages. If that is
2250 : // not successful, wait for the sweeper threads and retry free-list
2251 : // allocation. Returns false if there is not enough space and the caller
2252 : // has to retry after collecting garbage.
2253 : MUST_USE_RESULT virtual bool SweepAndRetryAllocation(int size_in_bytes);
2254 :
2255 : // Slow path of AllocateRaw. This function is space-dependent. Returns false
2256 : // if there is not enough space and the caller has to retry after
2257 : // collecting garbage.
2258 : MUST_USE_RESULT virtual bool SlowAllocateRaw(int size_in_bytes);
2259 :
2260 : // Implementation of SlowAllocateRaw. Returns false if there is not enough
2261 : // space and the caller has to retry after collecting garbage.
2262 : MUST_USE_RESULT bool RawSlowAllocateRaw(int size_in_bytes);
2263 :
2264 : size_t area_size_;
2265 :
2266 : // Accounting information for this space.
2267 : AllocationStats accounting_stats_;
2268 :
2269 : // The dummy page that anchors the double linked list of pages.
2270 : Page anchor_;
2271 :
2272 : // The space's free list.
2273 : FreeList free_list_;
2274 :
2275 : // Normal allocation information.
2276 : AllocationInfo allocation_info_;
2277 :
2278 : // Mutex guarding any concurrent access to the space.
2279 : base::Mutex space_mutex_;
2280 :
2281 : Page* locked_page_;
2282 : Address top_on_previous_step_;
2283 :
2284 : friend class IncrementalMarking;
2285 : friend class MarkCompactCollector;
2286 :
2287 : // Used in cctest.
2288 : friend class heap::HeapTester;
2289 : };
2290 :
2291 : enum SemiSpaceId { kFromSpace = 0, kToSpace = 1 };
2292 :
2293 : // -----------------------------------------------------------------------------
2294 : // SemiSpace in young generation
2295 : //
2296 : // A SemiSpace is a contiguous chunk of memory holding page-like memory chunks.
2297 : // The mark-compact collector uses the memory of the first page in the from
2298 : // space as a marking stack when tracing live objects.
2299 213484 : class SemiSpace : public Space {
2300 : public:
2301 : typedef PageIterator iterator;
2302 :
2303 : static void Swap(SemiSpace* from, SemiSpace* to);
2304 :
2305 110010 : SemiSpace(Heap* heap, SemiSpaceId semispace)
2306 : : Space(heap, NEW_SPACE, NOT_EXECUTABLE),
2307 : current_capacity_(0),
2308 : maximum_capacity_(0),
2309 : minimum_capacity_(0),
2310 : age_mark_(nullptr),
2311 : committed_(false),
2312 : id_(semispace),
2313 : anchor_(this),
2314 : current_page_(nullptr),
2315 220020 : pages_used_(0) {}
2316 :
2317 : inline bool Contains(HeapObject* o);
2318 : inline bool Contains(Object* o);
2319 : inline bool ContainsSlow(Address a);
2320 :
2321 : void SetUp(size_t initial_capacity, size_t maximum_capacity);
2322 : void TearDown();
2323 : bool HasBeenSetUp() { return maximum_capacity_ != 0; }
2324 :
2325 : bool Commit();
2326 : bool Uncommit();
2327 : bool is_committed() { return committed_; }
2328 :
2329 : // Grow the semispace to the new capacity. The new capacity requested must
2330 : // be larger than the current capacity and less than the maximum capacity.
2331 : bool GrowTo(size_t new_capacity);
2332 :
2333 : // Shrinks the semispace to the new capacity. The new capacity requested
2334 : // must be more than the amount of used memory in the semispace and less
2335 : // than the current capacity.
2336 : bool ShrinkTo(size_t new_capacity);
2337 :
2338 : bool EnsureCurrentCapacity();
2339 :
2340 : // Returns the start address of the first page of the space.
2341 269739 : Address space_start() {
2342 : DCHECK_NE(anchor_.next_page(), anchor());
2343 269739 : return anchor_.next_page()->area_start();
2344 : }
2345 :
2346 : Page* first_page() { return anchor_.next_page(); }
2347 : Page* current_page() { return current_page_; }
2348 : int pages_used() { return pages_used_; }
2349 :
2350 : // Returns one past the end address of the space.
2351 117188 : Address space_end() { return anchor_.prev_page()->area_end(); }
2352 :
2353 : // Returns the start address of the current page of the space.
2354 1177455 : Address page_low() { return current_page_->area_start(); }
2355 :
2356 : // Returns one past the end address of the current page of the space.
2357 1228145 : Address page_high() { return current_page_->area_end(); }
2358 :
2359 200270 : bool AdvancePage() {
2360 100135 : Page* next_page = current_page_->next_page();
2361 : // We cannot expand if we reached the maximum number of pages already. Note
2362 : // that we need to account for the next page already for this check as we
2363 : // could potentially fill the whole page after advancing.
2364 100135 : const bool reached_max_pages = (pages_used_ + 1) == max_pages();
2365 100135 : if (next_page == anchor() || reached_max_pages) {
2366 : return false;
2367 : }
2368 73663 : current_page_ = next_page;
2369 73663 : pages_used_++;
2370 73663 : return true;
2371 : }
2372 :
2373 : // Resets the space to using the first page.
2374 : void Reset();
2375 :
2376 : void RemovePage(Page* page);
2377 : void PrependPage(Page* page);
2378 : Page* InitializePage(MemoryChunk* chunk, Executability executable);
2379 :
2380 : // Age mark accessors.
2381 : Address age_mark() { return age_mark_; }
2382 : void set_age_mark(Address mark);
2383 :
2384 : // Returns the current capacity of the semispace.
2385 : size_t current_capacity() { return current_capacity_; }
2386 :
2387 : // Returns the maximum capacity of the semispace.
2388 : size_t maximum_capacity() { return maximum_capacity_; }
2389 :
2390 : // Returns the initial capacity of the semispace.
2391 : size_t minimum_capacity() { return minimum_capacity_; }
2392 :
2393 : SemiSpaceId id() { return id_; }
2394 :
2395 : // Approximate amount of physical memory committed for this space.
2396 : size_t CommittedPhysicalMemory() override;
2397 :
2398 : // If we don't have these here then SemiSpace will be abstract. However
2399 : // they should never be called:
2400 :
2401 0 : size_t Size() override {
2402 0 : UNREACHABLE();
2403 : }
2404 :
2405 0 : size_t SizeOfObjects() override { return Size(); }
2406 :
2407 0 : size_t Available() override {
2408 0 : UNREACHABLE();
2409 : }
2410 :
2411 993038 : iterator begin() { return iterator(anchor_.next_page()); }
2412 502505 : iterator end() { return iterator(anchor()); }
2413 :
2414 : std::unique_ptr<ObjectIterator> GetObjectIterator() override;
2415 :
2416 : #ifdef DEBUG
2417 : void Print() override;
2418 : // Validate a range of of addresses in a SemiSpace.
2419 : // The "from" address must be on a page prior to the "to" address,
2420 : // in the linked page order, or it must be earlier on the same page.
2421 : static void AssertValidRange(Address from, Address to);
2422 : #else
2423 : // Do nothing.
2424 : inline static void AssertValidRange(Address from, Address to) {}
2425 : #endif
2426 :
2427 : #ifdef VERIFY_HEAP
2428 : virtual void Verify();
2429 : #endif
2430 :
2431 : private:
2432 : void RewindPages(Page* start, int num_pages);
2433 :
2434 : inline Page* anchor() { return &anchor_; }
2435 : inline int max_pages() {
2436 100135 : return static_cast<int>(current_capacity_ / Page::kPageSize);
2437 : }
2438 :
2439 : // Copies the flags into the masked positions on all pages in the space.
2440 : void FixPagesFlags(intptr_t flags, intptr_t flag_mask);
2441 :
2442 : // The currently committed space capacity.
2443 : size_t current_capacity_;
2444 :
2445 : // The maximum capacity that can be used by this space. A space cannot grow
2446 : // beyond that size.
2447 : size_t maximum_capacity_;
2448 :
2449 : // The minimum capacity for the space. A space cannot shrink below this size.
2450 : size_t minimum_capacity_;
2451 :
2452 : // Used to govern object promotion during mark-compact collection.
2453 : Address age_mark_;
2454 :
2455 : bool committed_;
2456 : SemiSpaceId id_;
2457 :
2458 : Page anchor_;
2459 : Page* current_page_;
2460 : int pages_used_;
2461 :
2462 : friend class NewSpace;
2463 : friend class SemiSpaceIterator;
2464 : };
2465 :
2466 :
2467 : // A SemiSpaceIterator is an ObjectIterator that iterates over the active
2468 : // semispace of the heap's new space. It iterates over the objects in the
2469 : // semispace from a given start address (defaulting to the bottom of the
2470 : // semispace) to the top of the semispace. New objects allocated after the
2471 : // iterator is created are not iterated.
2472 22186 : class SemiSpaceIterator : public ObjectIterator {
2473 : public:
2474 : // Create an iterator over the allocated objects in the given to-space.
2475 : explicit SemiSpaceIterator(NewSpace* space);
2476 :
2477 : inline HeapObject* Next() override;
2478 :
2479 : private:
2480 : void Initialize(Address start, Address end);
2481 :
2482 : // The current iteration point.
2483 : Address current_;
2484 : // The end of iteration.
2485 : Address limit_;
2486 : };
2487 :
2488 : // -----------------------------------------------------------------------------
2489 : // The young generation space.
2490 : //
2491 : // The new space consists of a contiguous pair of semispaces. It simply
2492 : // forwards most functions to the appropriate semispace.
2493 :
2494 213472 : class NewSpace : public Space {
2495 : public:
2496 : typedef PageIterator iterator;
2497 :
2498 55005 : explicit NewSpace(Heap* heap)
2499 : : Space(heap, NEW_SPACE, NOT_EXECUTABLE),
2500 : top_on_previous_step_(0),
2501 : to_space_(heap, kToSpace),
2502 : from_space_(heap, kFromSpace),
2503 : reservation_(),
2504 : allocated_histogram_(nullptr),
2505 165015 : promoted_histogram_(nullptr) {}
2506 :
2507 : inline bool Contains(HeapObject* o);
2508 : inline bool ContainsSlow(Address a);
2509 : inline bool Contains(Object* o);
2510 :
2511 : bool SetUp(size_t initial_semispace_capacity, size_t max_semispace_capacity);
2512 :
2513 : // Tears down the space. Heap memory was not allocated by the space, so it
2514 : // is not deallocated here.
2515 : void TearDown();
2516 :
2517 : // True if the space has been set up but not torn down.
2518 : bool HasBeenSetUp() {
2519 6 : return to_space_.HasBeenSetUp() && from_space_.HasBeenSetUp();
2520 : }
2521 :
2522 : // Flip the pair of spaces.
2523 : void Flip();
2524 :
2525 : // Grow the capacity of the semispaces. Assumes that they are not at
2526 : // their maximum capacity.
2527 : void Grow();
2528 :
2529 : // Shrink the capacity of the semispaces.
2530 : void Shrink();
2531 :
2532 : // Return the allocated bytes in the active semispace.
2533 962335 : size_t Size() override {
2534 : DCHECK_GE(top(), to_space_.page_low());
2535 2887005 : return to_space_.pages_used() * Page::kAllocatableMemory +
2536 2887005 : static_cast<size_t>(top() - to_space_.page_low());
2537 : }
2538 :
2539 644956 : size_t SizeOfObjects() override { return Size(); }
2540 :
2541 : // Return the allocatable capacity of a semispace.
2542 : size_t Capacity() {
2543 : SLOW_DCHECK(to_space_.current_capacity() == from_space_.current_capacity());
2544 390006 : return (to_space_.current_capacity() / Page::kPageSize) *
2545 390006 : Page::kAllocatableMemory;
2546 : }
2547 :
2548 : // Return the current size of a semispace, allocatable and non-allocatable
2549 : // memory.
2550 : size_t TotalCapacity() {
2551 : DCHECK(to_space_.current_capacity() == from_space_.current_capacity());
2552 313384 : return to_space_.current_capacity();
2553 : }
2554 :
2555 : // Committed memory for NewSpace is the committed memory of both semi-spaces
2556 : // combined.
2557 750489 : size_t CommittedMemory() override {
2558 750489 : return from_space_.CommittedMemory() + to_space_.CommittedMemory();
2559 : }
2560 :
2561 0 : size_t MaximumCommittedMemory() override {
2562 : return from_space_.MaximumCommittedMemory() +
2563 0 : to_space_.MaximumCommittedMemory();
2564 : }
2565 :
2566 : // Approximate amount of physical memory committed for this space.
2567 : size_t CommittedPhysicalMemory() override;
2568 :
2569 : // Return the available bytes without growing.
2570 86478 : size_t Available() override {
2571 : DCHECK_GE(Capacity(), Size());
2572 86496 : return Capacity() - Size();
2573 : }
2574 :
2575 178866 : size_t AllocatedSinceLastGC() {
2576 178866 : const Address age_mark = to_space_.age_mark();
2577 : DCHECK_NOT_NULL(age_mark);
2578 : DCHECK_NOT_NULL(top());
2579 : Page* const age_mark_page = Page::FromAllocationAreaAddress(age_mark);
2580 : Page* const last_page = Page::FromAllocationAreaAddress(top());
2581 : Page* current_page = age_mark_page;
2582 : size_t allocated = 0;
2583 178866 : if (current_page != last_page) {
2584 : DCHECK_EQ(current_page, age_mark_page);
2585 : DCHECK_GE(age_mark_page->area_end(), age_mark);
2586 60837 : allocated += age_mark_page->area_end() - age_mark;
2587 : current_page = current_page->next_page();
2588 : } else {
2589 : DCHECK_GE(top(), age_mark);
2590 118029 : return top() - age_mark;
2591 : }
2592 180657 : while (current_page != last_page) {
2593 : DCHECK_NE(current_page, age_mark_page);
2594 58983 : allocated += Page::kAllocatableMemory;
2595 : current_page = current_page->next_page();
2596 : }
2597 : DCHECK_GE(top(), current_page->area_start());
2598 60837 : allocated += top() - current_page->area_start();
2599 : DCHECK_LE(allocated, Size());
2600 60837 : return allocated;
2601 : }
2602 :
2603 : void MovePageFromSpaceToSpace(Page* page) {
2604 : DCHECK(page->InFromSpace());
2605 995 : from_space_.RemovePage(page);
2606 995 : to_space_.PrependPage(page);
2607 : }
2608 :
2609 : bool Rebalance();
2610 :
2611 : // Return the maximum capacity of a semispace.
2612 : size_t MaximumCapacity() {
2613 : DCHECK(to_space_.maximum_capacity() == from_space_.maximum_capacity());
2614 262147 : return to_space_.maximum_capacity();
2615 : }
2616 :
2617 : bool IsAtMaximumCapacity() { return TotalCapacity() == MaximumCapacity(); }
2618 :
2619 : // Returns the initial capacity of a semispace.
2620 : size_t InitialTotalCapacity() {
2621 : DCHECK(to_space_.minimum_capacity() == from_space_.minimum_capacity());
2622 22738 : return to_space_.minimum_capacity();
2623 : }
2624 :
2625 : // Return the address of the allocation pointer in the active semispace.
2626 : Address top() {
2627 : DCHECK(to_space_.current_page()->ContainsLimit(allocation_info_.top()));
2628 2314431 : return allocation_info_.top();
2629 : }
2630 :
2631 : // Return the address of the allocation pointer limit in the active semispace.
2632 : Address limit() {
2633 : DCHECK(to_space_.current_page()->ContainsLimit(allocation_info_.limit()));
2634 215120 : return allocation_info_.limit();
2635 : }
2636 :
2637 80913 : void ResetOriginalTop() {
2638 : DCHECK_GE(top(), original_top());
2639 : DCHECK_LE(top(), original_limit());
2640 : original_top_.SetValue(top());
2641 80913 : }
2642 :
2643 : Address original_top() { return original_top_.Value(); }
2644 : Address original_limit() { return original_limit_.Value(); }
2645 :
2646 : // Return the address of the first object in the active semispace.
2647 124693 : Address bottom() { return to_space_.space_start(); }
2648 :
2649 : // Get the age mark of the inactive semispace.
2650 130848825 : Address age_mark() { return from_space_.age_mark(); }
2651 : // Set the age mark in the active semispace.
2652 86452 : void set_age_mark(Address mark) { to_space_.set_age_mark(mark); }
2653 :
2654 : // The allocation top and limit address.
2655 : Address* allocation_top_address() { return allocation_info_.top_address(); }
2656 :
2657 : // The allocation limit address.
2658 : Address* allocation_limit_address() {
2659 : return allocation_info_.limit_address();
2660 : }
2661 :
2662 : MUST_USE_RESULT INLINE(AllocationResult AllocateRawAligned(
2663 : int size_in_bytes, AllocationAlignment alignment));
2664 :
2665 : MUST_USE_RESULT INLINE(
2666 : AllocationResult AllocateRawUnaligned(int size_in_bytes));
2667 :
2668 : MUST_USE_RESULT INLINE(AllocationResult AllocateRaw(
2669 : int size_in_bytes, AllocationAlignment alignment));
2670 :
2671 : MUST_USE_RESULT inline AllocationResult AllocateRawSynchronized(
2672 : int size_in_bytes, AllocationAlignment alignment);
2673 :
2674 : // Reset the allocation pointer to the beginning of the active semispace.
2675 : void ResetAllocationInfo();
2676 :
2677 : // When inline allocation stepping is active, either because of incremental
2678 : // marking, idle scavenge, or allocation statistics gathering, we 'interrupt'
2679 : // inline allocation every once in a while. This is done by setting
2680 : // allocation_info_.limit to be lower than the actual limit and and increasing
2681 : // it in steps to guarantee that the observers are notified periodically.
2682 : void UpdateInlineAllocationLimit(int size_in_bytes);
2683 :
2684 : void DisableInlineAllocationSteps() {
2685 1269 : top_on_previous_step_ = 0;
2686 1269 : UpdateInlineAllocationLimit(0);
2687 : }
2688 :
2689 : // Get the extent of the inactive semispace (for use as a marking stack,
2690 : // or to zap it). Notice: space-addresses are not necessarily on the
2691 : // same page, so FromSpaceStart() might be above FromSpaceEnd().
2692 : Address FromSpacePageLow() { return from_space_.page_low(); }
2693 : Address FromSpacePageHigh() { return from_space_.page_high(); }
2694 58570 : Address FromSpaceStart() { return from_space_.space_start(); }
2695 58570 : Address FromSpaceEnd() { return from_space_.space_end(); }
2696 :
2697 : // Get the extent of the active semispace's pages' memory.
2698 24 : Address ToSpaceStart() { return to_space_.space_start(); }
2699 24 : Address ToSpaceEnd() { return to_space_.space_end(); }
2700 :
2701 : inline bool ToSpaceContainsSlow(Address a);
2702 : inline bool FromSpaceContainsSlow(Address a);
2703 : inline bool ToSpaceContains(Object* o);
2704 : inline bool FromSpaceContains(Object* o);
2705 :
2706 : // Try to switch the active semispace to a new, empty, page.
2707 : // Returns false if this isn't possible or reasonable (i.e., there
2708 : // are no pages, or the current page is already empty), or true
2709 : // if successful.
2710 : bool AddFreshPage();
2711 : bool AddFreshPageSynchronized();
2712 :
2713 : #ifdef VERIFY_HEAP
2714 : // Verify the active semispace.
2715 : virtual void Verify();
2716 : #endif
2717 :
2718 : #ifdef DEBUG
2719 : // Print the active semispace.
2720 : void Print() override { to_space_.Print(); }
2721 : #endif
2722 :
2723 : // Iterates the active semispace to collect statistics.
2724 : void CollectStatistics();
2725 : // Reports previously collected statistics of the active semispace.
2726 : void ReportStatistics();
2727 : // Clears previously collected statistics.
2728 : void ClearHistograms();
2729 :
2730 : // Record the allocation or promotion of a heap object. Note that we don't
2731 : // record every single allocation, but only those that happen in the
2732 : // to space during a scavenge GC.
2733 : void RecordAllocation(HeapObject* obj);
2734 : void RecordPromotion(HeapObject* obj);
2735 :
2736 : // Return whether the operation succeeded.
2737 : bool CommitFromSpaceIfNeeded() {
2738 86452 : if (from_space_.is_committed()) return true;
2739 28738 : return from_space_.Commit();
2740 : }
2741 :
2742 : bool UncommitFromSpace() {
2743 22708 : if (!from_space_.is_committed()) return true;
2744 16137 : return from_space_.Uncommit();
2745 : }
2746 :
2747 0 : bool IsFromSpaceCommitted() { return from_space_.is_committed(); }
2748 :
2749 : SemiSpace* active_space() { return &to_space_; }
2750 :
2751 : void PauseAllocationObservers() override;
2752 : void ResumeAllocationObservers() override;
2753 :
2754 34976 : iterator begin() { return to_space_.begin(); }
2755 : iterator end() { return to_space_.end(); }
2756 :
2757 : std::unique_ptr<ObjectIterator> GetObjectIterator() override;
2758 :
2759 : SemiSpace& from_space() { return from_space_; }
2760 : SemiSpace& to_space() { return to_space_; }
2761 :
2762 : private:
2763 : // Update allocation info to match the current to-space page.
2764 : void UpdateAllocationInfo();
2765 :
2766 : base::Mutex mutex_;
2767 :
2768 : // Allocation pointer and limit for normal allocation and allocation during
2769 : // mark-compact collection.
2770 : AllocationInfo allocation_info_;
2771 : Address top_on_previous_step_;
2772 : // The top and the limit at the time of setting the allocation info.
2773 : // These values can be accessed by background tasks.
2774 : base::AtomicValue<Address> original_top_;
2775 : base::AtomicValue<Address> original_limit_;
2776 :
2777 : // The semispaces.
2778 : SemiSpace to_space_;
2779 : SemiSpace from_space_;
2780 : VirtualMemory reservation_;
2781 :
2782 : HistogramInfo* allocated_histogram_;
2783 : HistogramInfo* promoted_histogram_;
2784 :
2785 : bool EnsureAllocation(int size_in_bytes, AllocationAlignment alignment);
2786 :
2787 : // If we are doing inline allocation in steps, this method performs the 'step'
2788 : // operation. top is the memory address of the bump pointer at the last
2789 : // inline allocation (i.e. it determines the numbers of bytes actually
2790 : // allocated since the last step.) new_top is the address of the bump pointer
2791 : // where the next byte is going to be allocated from. top and new_top may be
2792 : // different when we cross a page boundary or reset the space.
2793 : void InlineAllocationStep(Address top, Address new_top, Address soon_object,
2794 : size_t size);
2795 : void StartNextInlineAllocationStep() override;
2796 :
2797 : friend class SemiSpaceIterator;
2798 : };
2799 :
2800 : class PauseAllocationObserversScope {
2801 : public:
2802 : explicit PauseAllocationObserversScope(Heap* heap);
2803 : ~PauseAllocationObserversScope();
2804 :
2805 : private:
2806 : Heap* heap_;
2807 : DISALLOW_COPY_AND_ASSIGN(PauseAllocationObserversScope);
2808 : };
2809 :
2810 : // -----------------------------------------------------------------------------
2811 : // Compaction space that is used temporarily during compaction.
2812 :
2813 156532 : class V8_EXPORT_PRIVATE CompactionSpace : public PagedSpace {
2814 : public:
2815 : CompactionSpace(Heap* heap, AllocationSpace id, Executability executable)
2816 156531 : : PagedSpace(heap, id, executable) {}
2817 :
2818 58905040 : bool is_local() override { return true; }
2819 :
2820 : protected:
2821 : // The space is temporary and not included in any snapshots.
2822 0 : bool snapshotable() override { return false; }
2823 :
2824 : MUST_USE_RESULT bool SweepAndRetryAllocation(int size_in_bytes) override;
2825 :
2826 : MUST_USE_RESULT bool SlowAllocateRaw(int size_in_bytes) override;
2827 : };
2828 :
2829 :
2830 : // A collection of |CompactionSpace|s used by a single compaction task.
2831 : class CompactionSpaceCollection : public Malloced {
2832 : public:
2833 156530 : explicit CompactionSpaceCollection(Heap* heap)
2834 : : old_space_(heap, OLD_SPACE, Executability::NOT_EXECUTABLE),
2835 156530 : code_space_(heap, CODE_SPACE, Executability::EXECUTABLE) {}
2836 :
2837 59455936 : CompactionSpace* Get(AllocationSpace space) {
2838 59455936 : switch (space) {
2839 : case OLD_SPACE:
2840 59352469 : return &old_space_;
2841 : case CODE_SPACE:
2842 103467 : return &code_space_;
2843 : default:
2844 0 : UNREACHABLE();
2845 : }
2846 : UNREACHABLE();
2847 : }
2848 :
2849 : private:
2850 : CompactionSpace old_space_;
2851 : CompactionSpace code_space_;
2852 : };
2853 :
2854 :
2855 : // -----------------------------------------------------------------------------
2856 : // Old object space (includes the old space of objects and code space)
2857 :
2858 213478 : class OldSpace : public PagedSpace {
2859 : public:
2860 : // Creates an old space object. The constructor does not allocate pages
2861 : // from OS.
2862 : OldSpace(Heap* heap, AllocationSpace id, Executability executable)
2863 110010 : : PagedSpace(heap, id, executable) {}
2864 : };
2865 :
2866 :
2867 : // For contiguous spaces, top should be in the space (or at the end) and limit
2868 : // should be the end of the space.
2869 : #define DCHECK_SEMISPACE_ALLOCATION_INFO(info, space) \
2870 : SLOW_DCHECK((space).page_low() <= (info).top() && \
2871 : (info).top() <= (space).page_high() && \
2872 : (info).limit() <= (space).page_high())
2873 :
2874 :
2875 : // -----------------------------------------------------------------------------
2876 : // Old space for all map objects
2877 :
2878 106730 : class MapSpace : public PagedSpace {
2879 : public:
2880 : // Creates a map space object.
2881 : MapSpace(Heap* heap, AllocationSpace id)
2882 54999 : : PagedSpace(heap, id, NOT_EXECUTABLE) {}
2883 :
2884 0 : int RoundSizeDownToObjectAlignment(int size) override {
2885 : if (base::bits::IsPowerOfTwo(Map::kSize)) {
2886 : return RoundDown(size, Map::kSize);
2887 : } else {
2888 0 : return (size / Map::kSize) * Map::kSize;
2889 : }
2890 : }
2891 :
2892 : #ifdef VERIFY_HEAP
2893 : void VerifyObject(HeapObject* obj) override;
2894 : #endif
2895 : };
2896 :
2897 :
2898 : // -----------------------------------------------------------------------------
2899 : // Large objects ( > kMaxRegularHeapObjectSize ) are allocated and
2900 : // managed by the large object space. A large object is allocated from OS
2901 : // heap with extra padding bytes (Page::kPageSize + Page::kObjectStartOffset).
2902 : // A large object always starts at Page::kObjectStartOffset to a page.
2903 : // Large objects do not move during garbage collections.
2904 :
2905 : class LargeObjectSpace : public Space {
2906 : public:
2907 : typedef LargePageIterator iterator;
2908 :
2909 : LargeObjectSpace(Heap* heap, AllocationSpace id);
2910 : virtual ~LargeObjectSpace();
2911 :
2912 : // Initializes internal data structures.
2913 : bool SetUp();
2914 :
2915 : // Releases internal resources, frees objects in this space.
2916 : void TearDown();
2917 :
2918 : static size_t ObjectSizeFor(size_t chunk_size) {
2919 103261 : if (chunk_size <= (Page::kPageSize + Page::kObjectStartOffset)) return 0;
2920 103252 : return chunk_size - Page::kPageSize - Page::kObjectStartOffset;
2921 : }
2922 :
2923 : // Shared implementation of AllocateRaw, AllocateRawCode and
2924 : // AllocateRawFixedArray.
2925 : MUST_USE_RESULT AllocationResult
2926 : AllocateRaw(int object_size, Executability executable);
2927 :
2928 : // Available bytes for objects in this space.
2929 : inline size_t Available() override;
2930 :
2931 777675 : size_t Size() override { return size_; }
2932 :
2933 3221971 : size_t SizeOfObjects() override { return objects_size_; }
2934 :
2935 : // Approximate amount of physical memory committed for this space.
2936 : size_t CommittedPhysicalMemory() override;
2937 :
2938 : int PageCount() { return page_count_; }
2939 :
2940 : // Finds an object for a given address, returns a Smi if it is not found.
2941 : // The function iterates through all objects in this space, may be slow.
2942 : Object* FindObject(Address a);
2943 :
2944 : // Takes the chunk_map_mutex_ and calls FindPage after that.
2945 : LargePage* FindPageThreadSafe(Address a);
2946 :
2947 : // Finds a large object page containing the given address, returns nullptr
2948 : // if such a page doesn't exist.
2949 : LargePage* FindPage(Address a);
2950 :
2951 : // Clears the marking state of live objects.
2952 : void ClearMarkingStateOfLiveObjects();
2953 :
2954 : // Frees unmarked objects.
2955 : void FreeUnmarkedObjects();
2956 :
2957 : void InsertChunkMapEntries(LargePage* page);
2958 : void RemoveChunkMapEntries(LargePage* page);
2959 : void RemoveChunkMapEntries(LargePage* page, Address free_start);
2960 :
2961 : // Checks whether a heap object is in this space; O(1).
2962 : bool Contains(HeapObject* obj);
2963 : // Checks whether an address is in the object area in this space. Iterates
2964 : // all objects in the space. May be slow.
2965 0 : bool ContainsSlow(Address addr) { return FindObject(addr)->IsHeapObject(); }
2966 :
2967 : // Checks whether the space is empty.
2968 6 : bool IsEmpty() { return first_page_ == nullptr; }
2969 :
2970 : LargePage* first_page() { return first_page_; }
2971 :
2972 : // Collect code statistics.
2973 : void CollectCodeStatistics();
2974 :
2975 : iterator begin() { return iterator(first_page_); }
2976 : iterator end() { return iterator(nullptr); }
2977 :
2978 : std::unique_ptr<ObjectIterator> GetObjectIterator() override;
2979 :
2980 : #ifdef VERIFY_HEAP
2981 : virtual void Verify();
2982 : #endif
2983 :
2984 : #ifdef DEBUG
2985 : void Print() override;
2986 : void ReportStatistics();
2987 : #endif
2988 :
2989 : private:
2990 : // The head of the linked list of large object chunks.
2991 : LargePage* first_page_;
2992 : size_t size_; // allocated bytes
2993 : int page_count_; // number of chunks
2994 : size_t objects_size_; // size of objects
2995 : // The chunk_map_mutex_ has to be used when the chunk map is accessed
2996 : // concurrently.
2997 : base::Mutex chunk_map_mutex_;
2998 : // Page-aligned addresses to their corresponding LargePage.
2999 : std::unordered_map<Address, LargePage*> chunk_map_;
3000 :
3001 : friend class LargeObjectIterator;
3002 : };
3003 :
3004 :
3005 22186 : class LargeObjectIterator : public ObjectIterator {
3006 : public:
3007 : explicit LargeObjectIterator(LargeObjectSpace* space);
3008 :
3009 : HeapObject* Next() override;
3010 :
3011 : private:
3012 : LargePage* current_;
3013 : };
3014 :
3015 : // Iterates over the chunks (pages and large object pages) that can contain
3016 : // pointers to new space or to evacuation candidates.
3017 : class MemoryChunkIterator BASE_EMBEDDED {
3018 : public:
3019 : inline explicit MemoryChunkIterator(Heap* heap);
3020 :
3021 : // Return nullptr when the iterator is done.
3022 : inline MemoryChunk* next();
3023 :
3024 : private:
3025 : enum State {
3026 : kOldSpaceState,
3027 : kMapState,
3028 : kCodeState,
3029 : kLargeObjectState,
3030 : kFinishedState
3031 : };
3032 : Heap* heap_;
3033 : State state_;
3034 : PageIterator old_iterator_;
3035 : PageIterator code_iterator_;
3036 : PageIterator map_iterator_;
3037 : LargePageIterator lo_iterator_;
3038 : };
3039 :
3040 : } // namespace internal
3041 : } // namespace v8
3042 :
3043 : #endif // V8_HEAP_SPACES_H_
|
