Line data Source code
1 : // Copyright 2018 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
6 : #define V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
7 :
8 : // Clients of this interface shouldn't depend on lots of heap internals.
9 : // Do not include anything from src/heap here!
10 :
11 : #include "src/heap/heap-write-barrier.h"
12 :
13 : #include "src/globals.h"
14 : // TODO(jkummerow): Get rid of this by moving GetIsolateFromWritableObject
15 : // elsewhere.
16 : #include "src/isolate.h"
17 : #include "src/objects/code.h"
18 : #include "src/objects/compressed-slots-inl.h"
19 : #include "src/objects/fixed-array.h"
20 : #include "src/objects/heap-object.h"
21 : #include "src/objects/maybe-object-inl.h"
22 : #include "src/objects/slots.h"
23 :
24 : namespace v8 {
25 : namespace internal {
26 :
27 : // Defined in heap.cc.
28 : V8_EXPORT_PRIVATE bool Heap_PageFlagsAreConsistent(HeapObject object);
29 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierSlow(HeapObject object,
30 : Address slot,
31 : HeapObject value);
32 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierSlow(HeapObject object, Address slot,
33 : HeapObject value);
34 : V8_EXPORT_PRIVATE void Heap_WriteBarrierForCodeSlow(Code host);
35 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForCodeSlow(Code host,
36 : RelocInfo* rinfo,
37 : HeapObject object);
38 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForCodeSlow(Code host,
39 : RelocInfo* rinfo,
40 : HeapObject object);
41 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForElementsSlow(Heap* heap,
42 : FixedArray array,
43 : int offset,
44 : int length);
45 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForElementsSlow(Heap* heap,
46 : HeapObject object);
47 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForDescriptorArraySlow(
48 : Heap* heap, HeapObject host, HeapObject descriptor_array,
49 : int number_of_own_descriptors);
50 :
51 : // Do not use these internal details anywhere outside of this file. These
52 : // internals are only intended to shortcut write barrier checks.
53 : namespace heap_internals {
54 :
55 : struct Space {
56 : static constexpr uintptr_t kIdOffset = 9 * kSystemPointerSize;
57 : V8_INLINE AllocationSpace identity() {
58 76428 : return *reinterpret_cast<AllocationSpace*>(reinterpret_cast<Address>(this) +
59 76428 : kIdOffset);
60 : }
61 : };
62 :
63 : struct MemoryChunk {
64 : static constexpr uintptr_t kFlagsOffset = sizeof(size_t);
65 : static constexpr uintptr_t kHeapOffset =
66 : kFlagsOffset + kUIntptrSize + 4 * kSystemPointerSize;
67 : static constexpr uintptr_t kOwnerOffset =
68 : kHeapOffset + 2 * kSystemPointerSize;
69 : static constexpr uintptr_t kMarkingBit = uintptr_t{1} << 18;
70 : static constexpr uintptr_t kFromPageBit = uintptr_t{1} << 3;
71 : static constexpr uintptr_t kToPageBit = uintptr_t{1} << 4;
72 :
73 : V8_INLINE static heap_internals::MemoryChunk* FromHeapObject(
74 : HeapObject object) {
75 4611704486 : return reinterpret_cast<MemoryChunk*>(object->ptr() & ~kPageAlignmentMask);
76 : }
77 :
78 1703230204 : V8_INLINE bool IsMarking() const { return GetFlags() & kMarkingBit; }
79 :
80 : V8_INLINE bool InYoungGeneration() const {
81 : constexpr uintptr_t kYoungGenerationMask = kFromPageBit | kToPageBit;
82 1694119020 : return GetFlags() & kYoungGenerationMask;
83 : }
84 :
85 : V8_INLINE uintptr_t GetFlags() const {
86 3363171832 : return *reinterpret_cast<const uintptr_t*>(reinterpret_cast<Address>(this) +
87 3369460701 : kFlagsOffset);
88 : }
89 :
90 : V8_INLINE Heap* GetHeap() {
91 1423255520 : Heap* heap = *reinterpret_cast<Heap**>(reinterpret_cast<Address>(this) +
92 1423261631 : kHeapOffset);
93 : SLOW_DCHECK(heap != nullptr);
94 : return heap;
95 : }
96 :
97 : V8_INLINE Space* GetOwner() {
98 76428 : return *reinterpret_cast<Space**>(reinterpret_cast<Address>(this) +
99 76428 : kOwnerOffset);
100 : }
101 : };
102 :
103 1464456841 : inline void GenerationalBarrierInternal(HeapObject object, Address slot,
104 : HeapObject value) {
105 : DCHECK(Heap_PageFlagsAreConsistent(object));
106 : heap_internals::MemoryChunk* value_chunk =
107 : heap_internals::MemoryChunk::FromHeapObject(value);
108 : heap_internals::MemoryChunk* object_chunk =
109 : heap_internals::MemoryChunk::FromHeapObject(object);
110 :
111 1639237549 : if (!value_chunk->InYoungGeneration() || object_chunk->InYoungGeneration()) {
112 : return;
113 : }
114 :
115 110646333 : Heap_GenerationalBarrierSlow(object, slot, value);
116 : }
117 :
118 3377 : inline void GenerationalEphemeronKeyBarrierInternal(EphemeronHashTable table,
119 : Address slot,
120 : HeapObject value) {
121 : DCHECK(Heap::PageFlagsAreConsistent(table));
122 : heap_internals::MemoryChunk* value_chunk =
123 : heap_internals::MemoryChunk::FromHeapObject(value);
124 : heap_internals::MemoryChunk* table_chunk =
125 : heap_internals::MemoryChunk::FromHeapObject(table);
126 :
127 5689 : if (!value_chunk->InYoungGeneration() || table_chunk->InYoungGeneration()) {
128 : return;
129 : }
130 :
131 : Heap* heap = GetHeapFromWritableObject(table);
132 0 : heap->RecordEphemeronKeyWrite(table, slot);
133 : }
134 :
135 : inline void MarkingBarrierInternal(HeapObject object, Address slot,
136 : HeapObject value) {
137 : DCHECK(Heap_PageFlagsAreConsistent(object));
138 : heap_internals::MemoryChunk* value_chunk =
139 : heap_internals::MemoryChunk::FromHeapObject(value);
140 :
141 1571847219 : if (!value_chunk->IsMarking()) return;
142 :
143 124709268 : Heap_MarkingBarrierSlow(object, slot, value);
144 : }
145 :
146 : } // namespace heap_internals
147 :
148 6288863 : inline void WriteBarrierForCode(Code host, RelocInfo* rinfo, Object value) {
149 : DCHECK(!HasWeakHeapObjectTag(value));
150 6288863 : if (!value->IsHeapObject()) return;
151 : HeapObject object = HeapObject::cast(value);
152 : GenerationalBarrierForCode(host, rinfo, object);
153 : MarkingBarrierForCode(host, rinfo, object);
154 : }
155 :
156 : inline void WriteBarrierForCode(Code host) {
157 34 : Heap_WriteBarrierForCodeSlow(host);
158 : }
159 :
160 9883675 : inline void GenerationalBarrier(HeapObject object, ObjectSlot slot,
161 : Object value) {
162 : DCHECK(!HasWeakHeapObjectTag(*slot));
163 : DCHECK(!HasWeakHeapObjectTag(value));
164 1829451816 : if (!value->IsHeapObject()) return;
165 : heap_internals::GenerationalBarrierInternal(object, slot.address(),
166 1213264439 : HeapObject::cast(value));
167 : }
168 :
169 : inline void GenerationalEphemeronKeyBarrier(EphemeronHashTable table,
170 : ObjectSlot slot, Object value) {
171 : DCHECK(!HasWeakHeapObjectTag(*slot));
172 : DCHECK(!HasWeakHeapObjectTag(value));
173 : DCHECK(value->IsHeapObject());
174 : heap_internals::GenerationalEphemeronKeyBarrierInternal(
175 3377 : table, slot.address(), HeapObject::cast(value));
176 : }
177 :
178 1446792705 : inline void GenerationalBarrier(HeapObject object, MaybeObjectSlot slot,
179 : MaybeObject value) {
180 1446792705 : HeapObject value_heap_object;
181 2642374230 : if (!value->GetHeapObject(&value_heap_object)) return;
182 : heap_internals::GenerationalBarrierInternal(object, slot.address(),
183 251211180 : value_heap_object);
184 : }
185 :
186 5219306 : inline void GenerationalBarrierForElements(Heap* heap, FixedArray array,
187 : int offset, int length) {
188 : heap_internals::MemoryChunk* array_chunk =
189 : heap_internals::MemoryChunk::FromHeapObject(array);
190 5228097 : if (array_chunk->InYoungGeneration()) return;
191 :
192 963 : Heap_GenerationalBarrierForElementsSlow(heap, array, offset, length);
193 : }
194 :
195 5 : inline void GenerationalBarrierForCode(Code host, RelocInfo* rinfo,
196 : HeapObject object) {
197 : heap_internals::MemoryChunk* object_chunk =
198 : heap_internals::MemoryChunk::FromHeapObject(object);
199 6346726 : if (!object_chunk->InYoungGeneration()) return;
200 220056 : Heap_GenerationalBarrierForCodeSlow(host, rinfo, object);
201 : }
202 :
203 1937092396 : inline void MarkingBarrier(HeapObject object, ObjectSlot slot, Object value) {
204 : DCHECK_IMPLIES(slot.address() != kNullAddress, !HasWeakHeapObjectTag(*slot));
205 : DCHECK(!HasWeakHeapObjectTag(value));
206 1937092396 : if (!value->IsHeapObject()) return;
207 : heap_internals::MarkingBarrierInternal(object, slot.address(),
208 : HeapObject::cast(value));
209 : }
210 :
211 1446516164 : inline void MarkingBarrier(HeapObject object, MaybeObjectSlot slot,
212 : MaybeObject value) {
213 : HeapObject value_heap_object;
214 1446516164 : if (!value->GetHeapObject(&value_heap_object)) return;
215 : heap_internals::MarkingBarrierInternal(object, slot.address(),
216 : value_heap_object);
217 : }
218 :
219 5219306 : inline void MarkingBarrierForElements(Heap* heap, HeapObject object) {
220 : heap_internals::MemoryChunk* object_chunk =
221 : heap_internals::MemoryChunk::FromHeapObject(object);
222 5228097 : if (!object_chunk->IsMarking()) return;
223 :
224 459 : Heap_MarkingBarrierForElementsSlow(heap, object);
225 : }
226 :
227 5 : inline void MarkingBarrierForCode(Code host, RelocInfo* rinfo,
228 : HeapObject object) {
229 : DCHECK(!HasWeakHeapObjectTag(object.ptr()));
230 : heap_internals::MemoryChunk* object_chunk =
231 : heap_internals::MemoryChunk::FromHeapObject(object);
232 7651851 : if (!object_chunk->IsMarking()) return;
233 271967 : Heap_MarkingBarrierForCodeSlow(host, rinfo, object);
234 : }
235 :
236 : inline void MarkingBarrierForDescriptorArray(Heap* heap, HeapObject host,
237 : HeapObject descriptor_array,
238 : int number_of_own_descriptors) {
239 : heap_internals::MemoryChunk* chunk =
240 : heap_internals::MemoryChunk::FromHeapObject(descriptor_array);
241 86794512 : if (!chunk->IsMarking()) return;
242 :
243 : Heap_MarkingBarrierForDescriptorArraySlow(heap, host, descriptor_array,
244 8910690 : number_of_own_descriptors);
245 : }
246 :
247 : inline WriteBarrierMode GetWriteBarrierModeForObject(
248 : HeapObject object, const DisallowHeapAllocation* promise) {
249 : DCHECK(Heap_PageFlagsAreConsistent(object));
250 : heap_internals::MemoryChunk* chunk =
251 : heap_internals::MemoryChunk::FromHeapObject(object);
252 31708522 : if (chunk->IsMarking()) return UPDATE_WRITE_BARRIER;
253 27888523 : if (chunk->InYoungGeneration()) return SKIP_WRITE_BARRIER;
254 : return UPDATE_WRITE_BARRIER;
255 : }
256 :
257 : inline bool ObjectInYoungGeneration(const Object object) {
258 15413514 : if (object.IsSmi()) return false;
259 : return heap_internals::MemoryChunk::FromHeapObject(HeapObject::cast(object))
260 : ->InYoungGeneration();
261 : }
262 :
263 : inline Heap* GetHeapFromWritableObject(const HeapObject object) {
264 : heap_internals::MemoryChunk* chunk =
265 : heap_internals::MemoryChunk::FromHeapObject(object);
266 : return chunk->GetHeap();
267 : }
268 :
269 : inline bool GetIsolateFromWritableObject(HeapObject obj, Isolate** isolate) {
270 : heap_internals::MemoryChunk* chunk =
271 : heap_internals::MemoryChunk::FromHeapObject(obj);
272 76428 : if (chunk->GetOwner()->identity() == RO_SPACE) {
273 : *isolate = nullptr;
274 : return false;
275 : }
276 : *isolate = Isolate::FromHeap(chunk->GetHeap());
277 : return true;
278 : }
279 :
280 : } // namespace internal
281 : } // namespace v8
282 :
283 : #endif // V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
|
