Line data Source code
1 : // Copyright 2012 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_HEAP_MARK_COMPACT_INL_H_
6 : #define V8_HEAP_MARK_COMPACT_INL_H_
7 :
8 : #include "src/assembler-inl.h"
9 : #include "src/base/bits.h"
10 : #include "src/heap/mark-compact.h"
11 : #include "src/heap/objects-visiting-inl.h"
12 : #include "src/heap/remembered-set.h"
13 : #include "src/objects/js-collection-inl.h"
14 : #include "src/objects/js-weak-refs-inl.h"
15 : #include "src/objects/slots-inl.h"
16 : #include "src/transitions.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 :
21 : template <typename ConcreteState, AccessMode access_mode>
22 : bool MarkingStateBase<ConcreteState, access_mode>::GreyToBlack(HeapObject obj) {
23 769609469 : MemoryChunk* p = MemoryChunk::FromHeapObject(obj);
24 769831999 : MarkBit markbit = MarkBitFrom(p, obj->address());
25 777633672 : if (!Marking::GreyToBlack<access_mode>(markbit)) return false;
26 747938608 : static_cast<ConcreteState*>(this)->IncrementLiveBytes(p, obj->Size());
27 : return true;
28 : }
29 :
30 : template <typename ConcreteState, AccessMode access_mode>
31 : bool MarkingStateBase<ConcreteState, access_mode>::WhiteToGrey(HeapObject obj) {
32 : return Marking::WhiteToGrey<access_mode>(MarkBitFrom(obj));
33 : }
34 :
35 : template <typename ConcreteState, AccessMode access_mode>
36 : bool MarkingStateBase<ConcreteState, access_mode>::WhiteToBlack(
37 : HeapObject obj) {
38 3693496 : return WhiteToGrey(obj) && GreyToBlack(obj);
39 : }
40 :
41 : template <FixedArrayVisitationMode fixed_array_mode,
42 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
43 : MarkingVisitor<fixed_array_mode, retaining_path_mode,
44 83597451 : MarkingState>::MarkingVisitor(MarkCompactCollector* collector,
45 : MarkingState* marking_state)
46 83597460 : : heap_(collector->heap()),
47 : collector_(collector),
48 : marking_state_(marking_state),
49 250792363 : mark_compact_epoch_(collector->epoch()) {}
50 :
51 : template <FixedArrayVisitationMode fixed_array_mode,
52 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
53 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
54 : MarkingState>::VisitBytecodeArray(Map map,
55 : BytecodeArray array) {
56 1789328 : int size = BytecodeArray::BodyDescriptor::SizeOf(map, array);
57 1789328 : BytecodeArray::BodyDescriptor::IterateBody(map, array, size, this);
58 :
59 1789328 : if (!heap_->is_current_gc_forced()) {
60 1674246 : array->MakeOlder();
61 : }
62 : return size;
63 : }
64 :
65 : template <FixedArrayVisitationMode fixed_array_mode,
66 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
67 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
68 : MarkingState>::VisitDescriptorArray(Map map,
69 : DescriptorArray array) {
70 4585141 : int size = DescriptorArray::BodyDescriptor::SizeOf(map, array);
71 : VisitPointers(array, array->GetFirstPointerSlot(),
72 4585140 : array->GetDescriptorSlot(0));
73 4585140 : VisitDescriptors(array, array->number_of_descriptors());
74 : return size;
75 : }
76 :
77 : template <FixedArrayVisitationMode fixed_array_mode,
78 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
79 : int MarkingVisitor<fixed_array_mode, retaining_path_mode, MarkingState>::
80 : VisitSharedFunctionInfo(Map map, SharedFunctionInfo shared_info) {
81 19495519 : int size = SharedFunctionInfo::BodyDescriptor::SizeOf(map, shared_info);
82 19495519 : SharedFunctionInfo::BodyDescriptor::IterateBody(map, shared_info, size, this);
83 :
84 : // If the SharedFunctionInfo has old bytecode, mark it as flushable,
85 : // otherwise visit the function data field strongly.
86 19495519 : if (shared_info->ShouldFlushBytecode()) {
87 116 : collector_->AddBytecodeFlushingCandidate(shared_info);
88 : } else {
89 : VisitPointer(shared_info,
90 : HeapObject::RawField(shared_info,
91 19495403 : SharedFunctionInfo::kFunctionDataOffset));
92 : }
93 : return size;
94 : }
95 :
96 : template <FixedArrayVisitationMode fixed_array_mode,
97 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
98 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
99 : MarkingState>::VisitFixedArray(Map map, FixedArray object) {
100 : return (fixed_array_mode == FixedArrayVisitationMode::kRegular)
101 : ? Parent::VisitFixedArray(map, object)
102 : : VisitFixedArrayIncremental(map, object);
103 : }
104 :
105 : template <FixedArrayVisitationMode fixed_array_mode,
106 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
107 : template <typename T>
108 : V8_INLINE int
109 : MarkingVisitor<fixed_array_mode, retaining_path_mode,
110 : MarkingState>::VisitEmbedderTracingSubclass(Map map, T object) {
111 223634 : if (heap_->local_embedder_heap_tracer()->InUse()) {
112 30 : marking_worklist()->embedder()->Push(MarkCompactCollectorBase::kMainThread,
113 : object);
114 : }
115 223634 : int size = T::BodyDescriptor::SizeOf(map, object);
116 223634 : T::BodyDescriptor::IterateBody(map, object, size, this);
117 : return size;
118 : }
119 :
120 : template <FixedArrayVisitationMode fixed_array_mode,
121 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
122 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
123 : MarkingState>::VisitJSApiObject(Map map, JSObject object) {
124 : return VisitEmbedderTracingSubclass(map, object);
125 : }
126 :
127 : template <FixedArrayVisitationMode fixed_array_mode,
128 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
129 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
130 : MarkingState>::VisitJSArrayBuffer(Map map,
131 : JSArrayBuffer object) {
132 : return VisitEmbedderTracingSubclass(map, object);
133 : }
134 :
135 : template <FixedArrayVisitationMode fixed_array_mode,
136 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
137 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
138 : MarkingState>::VisitJSDataView(Map map, JSDataView object) {
139 : return VisitEmbedderTracingSubclass(map, object);
140 : }
141 :
142 : template <FixedArrayVisitationMode fixed_array_mode,
143 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
144 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
145 : MarkingState>::VisitJSTypedArray(Map map,
146 : JSTypedArray object) {
147 : return VisitEmbedderTracingSubclass(map, object);
148 : }
149 :
150 : template <FixedArrayVisitationMode fixed_array_mode,
151 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
152 : int MarkingVisitor<fixed_array_mode, retaining_path_mode, MarkingState>::
153 5021 : VisitEphemeronHashTable(Map map, EphemeronHashTable table) {
154 16219 : collector_->AddEphemeronHashTable(table);
155 :
156 70172 : for (int i = 0; i < table->Capacity(); i++) {
157 : ObjectSlot key_slot =
158 70172 : table->RawFieldOfElementAt(EphemeronHashTable::EntryToIndex(i));
159 70172 : HeapObject key = HeapObject::cast(table->KeyAt(i));
160 : collector_->RecordSlot(table, key_slot, key);
161 :
162 : ObjectSlot value_slot =
163 70172 : table->RawFieldOfElementAt(EphemeronHashTable::EntryToValueIndex(i));
164 :
165 75084 : if (marking_state()->IsBlackOrGrey(key)) {
166 : VisitPointer(table, value_slot);
167 :
168 : } else {
169 506 : Object value_obj = *value_slot;
170 :
171 1012 : if (value_obj->IsHeapObject()) {
172 : HeapObject value = HeapObject::cast(value_obj);
173 : collector_->RecordSlot(table, value_slot, value);
174 :
175 : // Revisit ephemerons with both key and value unreachable at end
176 : // of concurrent marking cycle.
177 237 : if (marking_state()->IsWhite(value)) {
178 85 : collector_->AddEphemeron(key, value);
179 : }
180 : }
181 : }
182 : }
183 :
184 16219 : return table->SizeFromMap(map);
185 : }
186 :
187 : template <FixedArrayVisitationMode fixed_array_mode,
188 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
189 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
190 : MarkingState>::VisitMap(Map meta_map, Map map) {
191 2127795 : int size = Map::BodyDescriptor::SizeOf(meta_map, map);
192 11632700 : if (map->CanTransition()) {
193 : // Maps that can transition share their descriptor arrays and require
194 : // special visiting logic to avoid memory leaks.
195 : // Since descriptor arrays are potentially shared, ensure that only the
196 : // descriptors that belong to this map are marked. The first time a
197 : // non-empty descriptor array is marked, its header is also visited. The
198 : // slot holding the descriptor array will be implicitly recorded when the
199 : // pointer fields of this map are visited.
200 11551351 : DescriptorArray descriptors = map->instance_descriptors();
201 : MarkDescriptorArrayBlack(map, descriptors);
202 11551356 : int number_of_own_descriptors = map->NumberOfOwnDescriptors();
203 11551355 : if (number_of_own_descriptors) {
204 : DCHECK_LE(number_of_own_descriptors,
205 : descriptors->number_of_descriptors());
206 : VisitDescriptors(descriptors, number_of_own_descriptors);
207 : }
208 : // Mark the pointer fields of the Map. Since the transitions array has
209 : // been marked already, it is fine that one of these fields contains a
210 : // pointer to it.
211 : }
212 11632704 : Map::BodyDescriptor::IterateBody(meta_map, map, size, this);
213 : return size;
214 : }
215 :
216 : template <FixedArrayVisitationMode fixed_array_mode,
217 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
218 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
219 : MarkingState>::VisitTransitionArray(Map map,
220 : TransitionArray array) {
221 258785 : int size = TransitionArray::BodyDescriptor::SizeOf(map, array);
222 258785 : TransitionArray::BodyDescriptor::IterateBody(map, array, size, this);
223 258785 : collector_->AddTransitionArray(array);
224 : return size;
225 : }
226 :
227 : template <FixedArrayVisitationMode fixed_array_mode,
228 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
229 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
230 : MarkingState>::VisitJSWeakRef(Map map, JSWeakRef weak_ref) {
231 314 : if (weak_ref->target()->IsHeapObject()) {
232 157 : HeapObject target = HeapObject::cast(weak_ref->target());
233 294 : if (marking_state()->IsBlackOrGrey(target)) {
234 : // Record the slot inside the JSWeakRef, since the IterateBody below
235 : // won't visit it.
236 : ObjectSlot slot =
237 62 : HeapObject::RawField(weak_ref, JSWeakCell::kTargetOffset);
238 : collector_->RecordSlot(weak_ref, slot, target);
239 : } else {
240 : // JSWeakRef points to a potentially dead object. We have to process
241 : // them when we know the liveness of the whole transitive closure.
242 95 : collector_->AddWeakRef(weak_ref);
243 : }
244 : }
245 157 : int size = JSWeakRef::BodyDescriptor::SizeOf(map, weak_ref);
246 157 : JSWeakRef::BodyDescriptor::IterateBody(map, weak_ref, size, this);
247 : return size;
248 : }
249 :
250 : template <FixedArrayVisitationMode fixed_array_mode,
251 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
252 : int MarkingVisitor<fixed_array_mode, retaining_path_mode,
253 : MarkingState>::VisitJSWeakCell(Map map,
254 : JSWeakCell weak_cell) {
255 336 : if (weak_cell->target()->IsHeapObject()) {
256 168 : HeapObject target = HeapObject::cast(weak_cell->target());
257 336 : if (marking_state()->IsBlackOrGrey(target)) {
258 : // Record the slot inside the JSWeakCell, since the IterateBody below
259 : // won't visit it.
260 : ObjectSlot slot =
261 52 : HeapObject::RawField(weak_cell, JSWeakCell::kTargetOffset);
262 : collector_->RecordSlot(weak_cell, slot, target);
263 : } else {
264 : // JSWeakCell points to a potentially dead object. We have to process
265 : // them when we know the liveness of the whole transitive closure.
266 116 : collector_->AddWeakCell(weak_cell);
267 : }
268 : }
269 168 : int size = JSWeakCell::BodyDescriptor::SizeOf(map, weak_cell);
270 168 : JSWeakCell::BodyDescriptor::IterateBody(map, weak_cell, size, this);
271 : return size;
272 : }
273 :
274 : // class template arguments
275 : template <FixedArrayVisitationMode fixed_array_mode,
276 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
277 : // method template arguments
278 : template <typename TSlot>
279 : void MarkingVisitor<fixed_array_mode, retaining_path_mode,
280 : MarkingState>::VisitPointerImpl(HeapObject host,
281 25966351 : TSlot slot) {
282 : static_assert(std::is_same<TSlot, ObjectSlot>::value ||
283 : std::is_same<TSlot, MaybeObjectSlot>::value,
284 : "Only ObjectSlot and MaybeObjectSlot are expected here");
285 3286843661 : typename TSlot::TObject object = *slot;
286 3286843579 : HeapObject target_object;
287 3286843579 : if (object.GetHeapObjectIfStrong(&target_object)) {
288 5988564245 : collector_->RecordSlot(host, HeapObjectSlot(slot), target_object);
289 : MarkObject(host, target_object);
290 141828675 : } else if (TSlot::kCanBeWeak && object.GetHeapObjectIfWeak(&target_object)) {
291 59891743 : if (marking_state()->IsBlackOrGrey(target_object)) {
292 : // Weak references with live values are directly processed here to reduce
293 : // the processing time of weak cells during the main GC pause.
294 46810825 : collector_->RecordSlot(host, HeapObjectSlot(slot), target_object);
295 : } else {
296 : // If we do not know about liveness of values of weak cells, we have to
297 : // process them when we know the liveness of the whole transitive
298 : // closure.
299 10519980 : collector_->AddWeakReference(host, HeapObjectSlot(slot));
300 : }
301 : }
302 : }
303 :
304 : // class template arguments
305 : template <FixedArrayVisitationMode fixed_array_mode,
306 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
307 : // method template arguments
308 : template <typename TSlot>
309 : void MarkingVisitor<fixed_array_mode, retaining_path_mode,
310 : MarkingState>::VisitPointersImpl(HeapObject host,
311 : TSlot start, TSlot end) {
312 3454448747 : for (TSlot p = start; p < end; ++p) {
313 : VisitPointer(host, p);
314 : }
315 : }
316 :
317 : template <FixedArrayVisitationMode fixed_array_mode,
318 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
319 2451291 : void MarkingVisitor<fixed_array_mode, retaining_path_mode,
320 : MarkingState>::VisitEmbeddedPointer(Code host,
321 4834599 : RelocInfo* rinfo) {
322 : DCHECK(rinfo->rmode() == RelocInfo::EMBEDDED_OBJECT);
323 : HeapObject object = HeapObject::cast(rinfo->target_object());
324 2451291 : collector_->RecordRelocSlot(host, rinfo, object);
325 4834599 : if (!marking_state()->IsBlackOrGrey(object)) {
326 228919 : if (host->IsWeakObject(object)) {
327 68694 : collector_->AddWeakObjectInCode(object, host);
328 : } else {
329 : MarkObject(host, object);
330 : }
331 : }
332 2451291 : }
333 :
334 : template <FixedArrayVisitationMode fixed_array_mode,
335 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
336 47180 : void MarkingVisitor<fixed_array_mode, retaining_path_mode,
337 : MarkingState>::VisitCodeTarget(Code host,
338 47180 : RelocInfo* rinfo) {
339 : DCHECK(RelocInfo::IsCodeTargetMode(rinfo->rmode()));
340 47180 : Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
341 47180 : collector_->RecordRelocSlot(host, rinfo, target);
342 : MarkObject(host, target);
343 47180 : }
344 :
345 : template <FixedArrayVisitationMode fixed_array_mode,
346 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
347 : void MarkingVisitor<fixed_array_mode, retaining_path_mode, MarkingState>::
348 12678604 : MarkDescriptorArrayBlack(HeapObject host, DescriptorArray descriptors) {
349 : // Note that WhiteToBlack is not sufficient here because it fails if the
350 : // descriptor array is grey. So we need to do two steps: WhiteToGrey and
351 : // GreyToBlack. Alternatively, we could check WhiteToGrey || WhiteToBlack.
352 12678604 : if (marking_state()->WhiteToGrey(descriptors)) {
353 3974663 : if (retaining_path_mode == TraceRetainingPathMode::kEnabled &&
354 3974663 : V8_UNLIKELY(FLAG_track_retaining_path)) {
355 0 : heap_->AddRetainer(host, descriptors);
356 : }
357 : }
358 21367715 : if (marking_state()->GreyToBlack(descriptors)) {
359 : VisitPointers(descriptors, descriptors->GetFirstPointerSlot(),
360 7106981 : descriptors->GetDescriptorSlot(0));
361 : }
362 : DCHECK(marking_state()->IsBlack(descriptors));
363 : }
364 :
365 : template <FixedArrayVisitationMode fixed_array_mode,
366 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
367 : void MarkingVisitor<fixed_array_mode, retaining_path_mode,
368 : MarkingState>::MarkObject(HeapObject host,
369 2536371899 : HeapObject object) {
370 5392675645 : if (marking_state()->WhiteToGrey(object)) {
371 138186143 : marking_worklist()->Push(object);
372 112166501 : if (retaining_path_mode == TraceRetainingPathMode::kEnabled &&
373 112166501 : V8_UNLIKELY(FLAG_track_retaining_path)) {
374 0 : heap_->AddRetainer(host, object);
375 : }
376 : }
377 : }
378 :
379 : template <FixedArrayVisitationMode fixed_array_mode,
380 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
381 : int MarkingVisitor<fixed_array_mode, retaining_path_mode, MarkingState>::
382 49237 : VisitFixedArrayIncremental(Map map, FixedArray object) {
383 2400799 : MemoryChunk* chunk = MemoryChunk::FromHeapObject(object);
384 2400799 : int size = FixedArray::BodyDescriptor::SizeOf(map, object);
385 2400799 : if (chunk->IsFlagSet(MemoryChunk::HAS_PROGRESS_BAR)) {
386 : DCHECK(FLAG_use_marking_progress_bar);
387 : DCHECK(heap_->IsLargeObject(object));
388 : int start =
389 49489 : Max(FixedArray::BodyDescriptor::kStartOffset, chunk->progress_bar());
390 49489 : int end = Min(size, start + kProgressBarScanningChunk);
391 49489 : if (start < end) {
392 : VisitPointers(object, HeapObject::RawField(object, start),
393 49484 : HeapObject::RawField(object, end));
394 49484 : chunk->set_progress_bar(end);
395 49484 : if (end < size) {
396 : DCHECK(marking_state()->IsBlack(object));
397 : // The object can be pushed back onto the marking worklist only after
398 : // progress bar was updated.
399 49237 : marking_worklist()->Push(object);
400 49237 : heap_->incremental_marking()->NotifyIncompleteScanOfObject(
401 49237 : size - (end - start));
402 : }
403 : }
404 : } else {
405 2351309 : FixedArray::BodyDescriptor::IterateBody(map, object, size, this);
406 : }
407 : return size;
408 : }
409 :
410 : template <FixedArrayVisitationMode fixed_array_mode,
411 : TraceRetainingPathMode retaining_path_mode, typename MarkingState>
412 : void MarkingVisitor<fixed_array_mode, retaining_path_mode, MarkingState>::
413 : VisitDescriptors(DescriptorArray descriptors,
414 : int number_of_own_descriptors) {
415 : // Updating the number of marked descriptor is supported only for black
416 : // descriptor arrays.
417 : DCHECK(marking_state()->IsBlack(descriptors));
418 12304905 : int16_t new_marked = static_cast<int16_t>(number_of_own_descriptors);
419 : int16_t old_marked = descriptors->UpdateNumberOfMarkedDescriptors(
420 16890044 : mark_compact_epoch_, new_marked);
421 16890047 : if (old_marked < new_marked) {
422 : VisitPointers(descriptors,
423 10491141 : MaybeObjectSlot(descriptors->GetDescriptorSlot(old_marked)),
424 31473426 : MaybeObjectSlot(descriptors->GetDescriptorSlot(new_marked)));
425 : }
426 : }
427 :
428 : void MarkCompactCollector::MarkObject(HeapObject host, HeapObject obj) {
429 157164705 : if (marking_state()->WhiteToGrey(obj)) {
430 3347847 : marking_worklist()->Push(obj);
431 3347847 : if (V8_UNLIKELY(FLAG_track_retaining_path)) {
432 0 : heap_->AddRetainer(host, obj);
433 : }
434 : }
435 : }
436 :
437 : void MarkCompactCollector::MarkRootObject(Root root, HeapObject obj) {
438 286485344 : if (marking_state()->WhiteToGrey(obj)) {
439 155258737 : marking_worklist()->Push(obj);
440 155258737 : if (V8_UNLIKELY(FLAG_track_retaining_path)) {
441 0 : heap_->AddRetainingRoot(root, obj);
442 : }
443 : }
444 : }
445 :
446 : #ifdef ENABLE_MINOR_MC
447 :
448 : void MinorMarkCompactCollector::MarkRootObject(HeapObject obj) {
449 0 : if (Heap::InNewSpace(obj) && non_atomic_marking_state_.WhiteToGrey(obj)) {
450 0 : worklist_->Push(kMainThread, obj);
451 : }
452 : }
453 :
454 : #endif
455 :
456 : void MarkCompactCollector::MarkExternallyReferencedObject(HeapObject obj) {
457 5 : if (marking_state()->WhiteToGrey(obj)) {
458 5 : marking_worklist()->Push(obj);
459 5 : if (V8_UNLIKELY(FLAG_track_retaining_path)) {
460 0 : heap_->AddRetainingRoot(Root::kWrapperTracing, obj);
461 : }
462 : }
463 : }
464 :
465 : void MarkCompactCollector::RecordSlot(HeapObject object, ObjectSlot slot,
466 : HeapObject target) {
467 1159265884 : RecordSlot(object, HeapObjectSlot(slot), target);
468 : }
469 :
470 : void MarkCompactCollector::RecordSlot(HeapObject object, HeapObjectSlot slot,
471 : HeapObject target) {
472 6474127078 : Page* target_page = Page::FromHeapObject(target);
473 6478029034 : Page* source_page = Page::FromHeapObject(object);
474 6527878253 : if (target_page->IsEvacuationCandidate<AccessMode::ATOMIC>() &&
475 48043940 : !source_page->ShouldSkipEvacuationSlotRecording<AccessMode::ATOMIC>()) {
476 34039179 : RememberedSet<OLD_TO_OLD>::Insert(source_page, slot.address());
477 : }
478 : }
479 :
480 258785 : void MarkCompactCollector::AddTransitionArray(TransitionArray array) {
481 292163 : weak_objects_.transition_arrays.Push(kMainThread, array);
482 258785 : }
483 :
484 116 : void MarkCompactCollector::AddBytecodeFlushingCandidate(
485 : SharedFunctionInfo flush_candidate) {
486 116 : weak_objects_.bytecode_flushing_candidates.Push(kMainThread, flush_candidate);
487 116 : }
488 :
489 : template <LiveObjectIterationMode mode>
490 4912044 : LiveObjectRange<mode>::iterator::iterator(MemoryChunk* chunk, Bitmap* bitmap,
491 : Address start)
492 : : chunk_(chunk),
493 : one_word_filler_map_(
494 : ReadOnlyRoots(chunk->heap()).one_pointer_filler_map()),
495 : two_word_filler_map_(
496 : ReadOnlyRoots(chunk->heap()).two_pointer_filler_map()),
497 : free_space_map_(ReadOnlyRoots(chunk->heap()).free_space_map()),
498 7981882 : it_(chunk, bitmap) {
499 : it_.Advance(Bitmap::IndexToCell(
500 1227812 : Bitmap::CellAlignIndex(chunk_->AddressToMarkbitIndex(start))));
501 1227812 : if (!it_.Done()) {
502 614045 : cell_base_ = it_.CurrentCellBase();
503 614045 : current_cell_ = *it_.CurrentCell();
504 614045 : AdvanceToNextValidObject();
505 : }
506 1228233 : }
507 :
508 : template <LiveObjectIterationMode mode>
509 : typename LiveObjectRange<mode>::iterator& LiveObjectRange<mode>::iterator::
510 : operator++() {
511 740101629 : AdvanceToNextValidObject();
512 : return *this;
513 : }
514 :
515 : template <LiveObjectIterationMode mode>
516 : typename LiveObjectRange<mode>::iterator LiveObjectRange<mode>::iterator::
517 : operator++(int) {
518 : iterator retval = *this;
519 : ++(*this);
520 : return retval;
521 : }
522 :
523 : template <LiveObjectIterationMode mode>
524 737878382 : void LiveObjectRange<mode>::iterator::AdvanceToNextValidObject() {
525 2037892776 : while (!it_.Done()) {
526 4717473 : HeapObject object;
527 : int size = 0;
528 1304127037 : while (current_cell_ != 0) {
529 : uint32_t trailing_zeros = base::bits::CountTrailingZeros(current_cell_);
530 737879328 : Address addr = cell_base_ + trailing_zeros * kTaggedSize;
531 :
532 : // Clear the first bit of the found object..
533 737879328 : current_cell_ &= ~(1u << trailing_zeros);
534 :
535 : uint32_t second_bit_index = 0;
536 737879328 : if (trailing_zeros >= Bitmap::kBitIndexMask) {
537 : second_bit_index = 0x1;
538 : // The overlapping case; there has to exist a cell after the current
539 : // cell.
540 : // However, if there is a black area at the end of the page, and the
541 : // last word is a one word filler, we are not allowed to advance. In
542 : // that case we can return immediately.
543 2182829839 : if (!it_.Advance()) {
544 : DCHECK(HeapObject::FromAddress(addr)->map() == one_word_filler_map_);
545 0 : current_object_ = HeapObject();
546 : return;
547 : }
548 23135434 : cell_base_ = it_.CurrentCellBase();
549 23135434 : current_cell_ = *it_.CurrentCell();
550 : } else {
551 719652278 : second_bit_index = 1u << (trailing_zeros + 1);
552 : }
553 :
554 : Map map;
555 742787712 : if (current_cell_ & second_bit_index) {
556 : // We found a black object. If the black object is within a black area,
557 : // make sure that we skip all set bits in the black area until the
558 : // object ends.
559 741393833 : HeapObject black_object = HeapObject::FromAddress(addr);
560 : map = Map::cast(ObjectSlot(addr).Acquire_Load());
561 736956850 : size = black_object->SizeFromMap(map);
562 733332933 : Address end = addr + size - kTaggedSize;
563 : // One word filler objects do not borrow the second mark bit. We have
564 : // to jump over the advancing and clearing part.
565 : // Note that we know that we are at a one word filler when
566 : // object_start + object_size - kTaggedSize == object_start.
567 733332933 : if (addr != end) {
568 : DCHECK_EQ(chunk_, MemoryChunk::FromAddress(end));
569 732637048 : uint32_t end_mark_bit_index = chunk_->AddressToMarkbitIndex(end);
570 : unsigned int end_cell_index =
571 732637048 : end_mark_bit_index >> Bitmap::kBitsPerCellLog2;
572 : MarkBit::CellType end_index_mask =
573 732637048 : 1u << Bitmap::IndexInCell(end_mark_bit_index);
574 732637048 : if (it_.Advance(end_cell_index)) {
575 110122245 : cell_base_ = it_.CurrentCellBase();
576 110122245 : current_cell_ = *it_.CurrentCell();
577 : }
578 :
579 : // Clear all bits in current_cell, including the end index.
580 732637048 : current_cell_ &= ~(end_index_mask + end_index_mask - 1);
581 : }
582 :
583 : if (mode == kBlackObjects || mode == kAllLiveObjects) {
584 733332928 : object = black_object;
585 : }
586 : } else if ((mode == kGreyObjects || mode == kAllLiveObjects)) {
587 : map = Map::cast(ObjectSlot(addr).Acquire_Load());
588 5 : object = HeapObject::FromAddress(addr);
589 5 : size = object->SizeFromMap(map);
590 : }
591 :
592 : // We found a live object.
593 734788516 : if (!object.is_null()) {
594 : // Do not use IsFiller() here. This may cause a data race for reading
595 : // out the instance type when a new map concurrently is written into
596 : // this object while iterating over the object.
597 734033695 : if (map == one_word_filler_map_ || map == two_word_filler_map_ ||
598 : map == free_space_map_) {
599 : // There are two reasons why we can get black or grey fillers:
600 : // 1) Black areas together with slack tracking may result in black one
601 : // word filler objects.
602 : // 2) Left trimming may leave black or grey fillers behind because we
603 : // do not clear the old location of the object start.
604 : // We filter these objects out in the iterator.
605 6 : object = HeapObject();
606 : } else {
607 : break;
608 : }
609 : }
610 : }
611 :
612 1296308618 : if (current_cell_ == 0) {
613 621868034 : if (it_.Advance()) {
614 621208471 : cell_base_ = it_.CurrentCellBase();
615 621208471 : current_cell_ = *it_.CurrentCell();
616 : }
617 : }
618 1296308618 : if (!object.is_null()) {
619 734172606 : current_object_ = object;
620 734172606 : current_size_ = size;
621 734172606 : return;
622 : }
623 : }
624 614964 : current_object_ = HeapObject();
625 : }
626 :
627 : template <LiveObjectIterationMode mode>
628 : typename LiveObjectRange<mode>::iterator LiveObjectRange<mode>::begin() {
629 614328 : return iterator(chunk_, bitmap_, start_);
630 : }
631 :
632 : template <LiveObjectIterationMode mode>
633 : typename LiveObjectRange<mode>::iterator LiveObjectRange<mode>::end() {
634 614519 : return iterator(chunk_, bitmap_, end_);
635 : }
636 :
637 1933605 : Isolate* MarkCompactCollectorBase::isolate() { return heap()->isolate(); }
638 :
639 : } // namespace internal
640 : } // namespace v8
641 :
642 : #endif // V8_HEAP_MARK_COMPACT_INL_H_
|
