Line data Source code
1 : // Copyright 2018 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
6 : #define V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
7 :
8 : // Clients of this interface shouldn't depend on lots of heap internals.
9 : // Do not include anything from src/heap here!
10 :
11 : #include "src/heap/heap-write-barrier.h"
12 :
13 : #include "src/globals.h"
14 : // TODO(jkummerow): Get rid of this by moving GetIsolateFromWritableObject
15 : // elsewhere.
16 : #include "src/isolate.h"
17 : #include "src/objects/code.h"
18 : #include "src/objects/compressed-slots-inl.h"
19 : #include "src/objects/fixed-array.h"
20 : #include "src/objects/heap-object.h"
21 : #include "src/objects/maybe-object-inl.h"
22 : #include "src/objects/slots.h"
23 :
24 : namespace v8 {
25 : namespace internal {
26 :
27 : // Defined in heap.cc.
28 : V8_EXPORT_PRIVATE bool Heap_PageFlagsAreConsistent(HeapObject object);
29 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierSlow(HeapObject object,
30 : Address slot,
31 : HeapObject value);
32 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierSlow(HeapObject object, Address slot,
33 : HeapObject value);
34 : V8_EXPORT_PRIVATE void Heap_WriteBarrierForCodeSlow(Code host);
35 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForCodeSlow(Code host,
36 : RelocInfo* rinfo,
37 : HeapObject object);
38 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForCodeSlow(Code host,
39 : RelocInfo* rinfo,
40 : HeapObject object);
41 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForElementsSlow(Heap* heap,
42 : FixedArray array,
43 : int offset,
44 : int length);
45 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForElementsSlow(Heap* heap,
46 : HeapObject object);
47 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForDescriptorArraySlow(
48 : Heap* heap, HeapObject host, HeapObject descriptor_array,
49 : int number_of_own_descriptors);
50 :
51 : // Do not use these internal details anywhere outside of this file. These
52 : // internals are only intended to shortcut write barrier checks.
53 : namespace heap_internals {
54 :
55 : struct Space {
56 : static constexpr uintptr_t kIdOffset = 9 * kSystemPointerSize;
57 : V8_INLINE AllocationSpace identity() {
58 76352 : return *reinterpret_cast<AllocationSpace*>(reinterpret_cast<Address>(this) +
59 76352 : kIdOffset);
60 : }
61 : };
62 :
63 : struct MemoryChunk {
64 : static constexpr uintptr_t kFlagsOffset = sizeof(size_t);
65 : static constexpr uintptr_t kHeapOffset =
66 : kFlagsOffset + kUIntptrSize + 4 * kSystemPointerSize;
67 : static constexpr uintptr_t kOwnerOffset =
68 : kHeapOffset + 2 * kSystemPointerSize;
69 : static constexpr uintptr_t kMarkingBit = uintptr_t{1} << 18;
70 : static constexpr uintptr_t kFromPageBit = uintptr_t{1} << 3;
71 : static constexpr uintptr_t kToPageBit = uintptr_t{1} << 4;
72 :
73 : V8_INLINE static heap_internals::MemoryChunk* FromHeapObject(
74 : HeapObject object) {
75 4722814740 : return reinterpret_cast<MemoryChunk*>(object->ptr() & ~kPageAlignmentMask);
76 : }
77 :
78 1743615357 : V8_INLINE bool IsMarking() const { return GetFlags() & kMarkingBit; }
79 :
80 : V8_INLINE bool InYoungGeneration() const {
81 0 : constexpr uintptr_t kYoungGenerationMask = kFromPageBit | kToPageBit;
82 1726496550 : return GetFlags() & kYoungGenerationMask;
83 : }
84 :
85 : V8_INLINE uintptr_t GetFlags() const {
86 3435239936 : return *reinterpret_cast<const uintptr_t*>(reinterpret_cast<Address>(this) +
87 3441566187 : kFlagsOffset);
88 : }
89 :
90 : V8_INLINE Heap* GetHeap() {
91 1462576586 : Heap* heap = *reinterpret_cast<Heap**>(reinterpret_cast<Address>(this) +
92 1462582697 : kHeapOffset);
93 : SLOW_DCHECK(heap != nullptr);
94 : return heap;
95 : }
96 :
97 : V8_INLINE Space* GetOwner() {
98 76352 : return *reinterpret_cast<Space**>(reinterpret_cast<Address>(this) +
99 76352 : kOwnerOffset);
100 : }
101 : };
102 :
103 1501437974 : inline void GenerationalBarrierInternal(HeapObject object, Address slot,
104 : HeapObject value) {
105 : DCHECK(Heap_PageFlagsAreConsistent(object));
106 : heap_internals::MemoryChunk* value_chunk =
107 0 : heap_internals::MemoryChunk::FromHeapObject(value);
108 : heap_internals::MemoryChunk* object_chunk =
109 0 : heap_internals::MemoryChunk::FromHeapObject(object);
110 :
111 1676499894 : if (!value_chunk->InYoungGeneration() || object_chunk->InYoungGeneration()) {
112 0 : return;
113 : }
114 :
115 107637943 : Heap_GenerationalBarrierSlow(object, slot, value);
116 : }
117 :
118 5675985 : inline void MarkingBarrierInternal(HeapObject object, Address slot,
119 : HeapObject value) {
120 : DCHECK(Heap_PageFlagsAreConsistent(object));
121 : heap_internals::MemoryChunk* value_chunk =
122 5675984 : heap_internals::MemoryChunk::FromHeapObject(value);
123 :
124 1616478688 : if (!value_chunk->IsMarking()) return;
125 :
126 120258469 : Heap_MarkingBarrierSlow(object, slot, value);
127 : }
128 :
129 : } // namespace heap_internals
130 :
131 6326248 : inline void WriteBarrierForCode(Code host, RelocInfo* rinfo, Object value) {
132 : DCHECK(!HasWeakHeapObjectTag(value));
133 6326248 : if (!value->IsHeapObject()) return;
134 0 : HeapObject object = HeapObject::cast(value);
135 0 : GenerationalBarrierForCode(host, rinfo, object);
136 0 : MarkingBarrierForCode(host, rinfo, object);
137 : }
138 :
139 : inline void WriteBarrierForCode(Code host) {
140 34 : Heap_WriteBarrierForCodeSlow(host);
141 : }
142 :
143 9825360 : inline void GenerationalBarrier(HeapObject object, ObjectSlot slot,
144 : Object value) {
145 : DCHECK(!HasWeakHeapObjectTag(*slot));
146 : DCHECK(!HasWeakHeapObjectTag(value));
147 1808245431 : if (!value->IsHeapObject()) return;
148 234690 : heap_internals::GenerationalBarrierInternal(object, slot.address(),
149 1251720710 : HeapObject::cast(value));
150 : }
151 :
152 1403406238 : inline void GenerationalBarrier(HeapObject object, MaybeObjectSlot slot,
153 : MaybeObject value) {
154 1403406238 : HeapObject value_heap_object;
155 2557060916 : if (!value->GetHeapObject(&value_heap_object)) return;
156 0 : heap_internals::GenerationalBarrierInternal(object, slot.address(),
157 249748648 : value_heap_object);
158 : }
159 :
160 618109 : inline void GenerationalBarrierForElements(Heap* heap, FixedArray array,
161 : int offset, int length) {
162 : heap_internals::MemoryChunk* array_chunk =
163 : heap_internals::MemoryChunk::FromHeapObject(array);
164 626785 : if (array_chunk->InYoungGeneration()) return;
165 :
166 963 : Heap_GenerationalBarrierForElementsSlow(heap, array, offset, length);
167 : }
168 :
169 5 : inline void GenerationalBarrierForCode(Code host, RelocInfo* rinfo,
170 : HeapObject object) {
171 : heap_internals::MemoryChunk* object_chunk =
172 0 : heap_internals::MemoryChunk::FromHeapObject(object);
173 6377849 : if (!object_chunk->InYoungGeneration()) return;
174 220816 : Heap_GenerationalBarrierForCodeSlow(host, rinfo, object);
175 : }
176 :
177 1923262704 : inline void MarkingBarrier(HeapObject object, ObjectSlot slot, Object value) {
178 : DCHECK_IMPLIES(slot.address() != kNullAddress, !HasWeakHeapObjectTag(*slot));
179 : DCHECK(!HasWeakHeapObjectTag(value));
180 1923262704 : if (!value->IsHeapObject()) return;
181 0 : heap_internals::MarkingBarrierInternal(object, slot.address(),
182 0 : HeapObject::cast(value));
183 : }
184 :
185 1403132119 : inline void MarkingBarrier(HeapObject object, MaybeObjectSlot slot,
186 : MaybeObject value) {
187 12454173 : HeapObject value_heap_object;
188 1403132119 : if (!value->GetHeapObject(&value_heap_object)) return;
189 5676005 : heap_internals::MarkingBarrierInternal(object, slot.address(),
190 5675996 : value_heap_object);
191 : }
192 :
193 618109 : inline void MarkingBarrierForElements(Heap* heap, HeapObject object) {
194 : heap_internals::MemoryChunk* object_chunk =
195 : heap_internals::MemoryChunk::FromHeapObject(object);
196 626785 : if (!object_chunk->IsMarking()) return;
197 :
198 230 : Heap_MarkingBarrierForElementsSlow(heap, object);
199 : }
200 :
201 5 : inline void MarkingBarrierForCode(Code host, RelocInfo* rinfo,
202 : HeapObject object) {
203 : DCHECK(!HasWeakHeapObjectTag(object.ptr()));
204 : heap_internals::MemoryChunk* object_chunk =
205 0 : heap_internals::MemoryChunk::FromHeapObject(object);
206 7708856 : if (!object_chunk->IsMarking()) return;
207 276661 : Heap_MarkingBarrierForCodeSlow(host, rinfo, object);
208 : }
209 :
210 : inline void MarkingBarrierForDescriptorArray(Heap* heap, HeapObject host,
211 : HeapObject descriptor_array,
212 : int number_of_own_descriptors) {
213 : heap_internals::MemoryChunk* chunk =
214 : heap_internals::MemoryChunk::FromHeapObject(descriptor_array);
215 86719548 : if (!chunk->IsMarking()) return;
216 :
217 : Heap_MarkingBarrierForDescriptorArraySlow(heap, host, descriptor_array,
218 8137075 : number_of_own_descriptors);
219 : }
220 :
221 : inline WriteBarrierMode GetWriteBarrierModeForObject(
222 : HeapObject object, const DisallowHeapAllocation* promise) {
223 : DCHECK(Heap_PageFlagsAreConsistent(object));
224 : heap_internals::MemoryChunk* chunk =
225 : heap_internals::MemoryChunk::FromHeapObject(object);
226 32081477 : if (chunk->IsMarking()) return UPDATE_WRITE_BARRIER;
227 28545720 : if (chunk->InYoungGeneration()) return SKIP_WRITE_BARRIER;
228 : return UPDATE_WRITE_BARRIER;
229 : }
230 :
231 : inline bool ObjectInYoungGeneration(const Object object) {
232 14447379 : if (object.IsSmi()) return false;
233 : return heap_internals::MemoryChunk::FromHeapObject(HeapObject::cast(object))
234 : ->InYoungGeneration();
235 : }
236 :
237 : inline Heap* GetHeapFromWritableObject(const HeapObject object) {
238 : heap_internals::MemoryChunk* chunk =
239 : heap_internals::MemoryChunk::FromHeapObject(object);
240 : return chunk->GetHeap();
241 : }
242 :
243 : inline bool GetIsolateFromWritableObject(HeapObject obj, Isolate** isolate) {
244 : heap_internals::MemoryChunk* chunk =
245 : heap_internals::MemoryChunk::FromHeapObject(obj);
246 76352 : if (chunk->GetOwner()->identity() == RO_SPACE) {
247 : *isolate = nullptr;
248 : return false;
249 : }
250 : *isolate = Isolate::FromHeap(chunk->GetHeap());
251 : return true;
252 : }
253 :
254 : } // namespace internal
255 : } // namespace v8
256 :
257 : #endif // V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
|
