Line data Source code
1 : // Copyright 2018 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
6 : #define V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
7 :
8 : // Clients of this interface shouldn't depend on lots of heap internals.
9 : // Do not include anything from src/heap here!
10 :
11 : #include "src/heap/heap-write-barrier.h"
12 :
13 : #include "src/globals.h"
14 : // TODO(jkummerow): Get rid of this by moving GetIsolateFromWritableObject
15 : // elsewhere.
16 : #include "src/isolate.h"
17 : #include "src/objects/code.h"
18 : #include "src/objects/fixed-array.h"
19 : #include "src/objects/heap-object.h"
20 : #include "src/objects/maybe-object-inl.h"
21 : #include "src/objects/slots.h"
22 :
23 : namespace v8 {
24 : namespace internal {
25 :
26 : // Defined in heap.cc.
27 : V8_EXPORT_PRIVATE bool Heap_PageFlagsAreConsistent(HeapObject object);
28 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierSlow(HeapObject object,
29 : Address slot,
30 : HeapObject value);
31 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierSlow(HeapObject object, Address slot,
32 : HeapObject value);
33 : V8_EXPORT_PRIVATE void Heap_WriteBarrierForCodeSlow(Code host);
34 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForCodeSlow(Code host,
35 : RelocInfo* rinfo,
36 : HeapObject object);
37 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForCodeSlow(Code host,
38 : RelocInfo* rinfo,
39 : HeapObject object);
40 : V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForElementsSlow(Heap* heap,
41 : FixedArray array,
42 : int offset,
43 : int length);
44 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForElementsSlow(Heap* heap,
45 : HeapObject object);
46 : V8_EXPORT_PRIVATE void Heap_MarkingBarrierForDescriptorArraySlow(
47 : Heap* heap, HeapObject host, HeapObject descriptor_array,
48 : int number_of_own_descriptors);
49 :
50 : // Do not use these internal details anywhere outside of this file. These
51 : // internals are only intended to shortcut write barrier checks.
52 : namespace heap_internals {
53 :
54 : struct Space {
55 : static constexpr uintptr_t kIdOffset = 9 * kSystemPointerSize;
56 : V8_INLINE AllocationSpace identity() {
57 75905 : return *reinterpret_cast<AllocationSpace*>(reinterpret_cast<Address>(this) +
58 75905 : kIdOffset);
59 : }
60 : };
61 :
62 : struct MemoryChunk {
63 : static constexpr uintptr_t kFlagsOffset = sizeof(size_t);
64 : static constexpr uintptr_t kHeapOffset =
65 : kFlagsOffset + kUIntptrSize + 4 * kSystemPointerSize;
66 : static constexpr uintptr_t kOwnerOffset =
67 : kHeapOffset + 2 * kSystemPointerSize;
68 : static constexpr uintptr_t kMarkingBit = uintptr_t{1} << 18;
69 : static constexpr uintptr_t kFromPageBit = uintptr_t{1} << 3;
70 : static constexpr uintptr_t kToPageBit = uintptr_t{1} << 4;
71 :
72 : V8_INLINE static heap_internals::MemoryChunk* FromHeapObject(
73 : HeapObject object) {
74 5582826408 : return reinterpret_cast<MemoryChunk*>(object->ptr() & ~kPageAlignmentMask);
75 : }
76 :
77 2189269652 : V8_INLINE bool IsMarking() const { return GetFlags() & kMarkingBit; }
78 :
79 : V8_INLINE bool InYoungGeneration() const {
80 0 : constexpr uintptr_t kYoungGenerationMask = kFromPageBit | kToPageBit;
81 2194874859 : return GetFlags() & kYoungGenerationMask;
82 : }
83 :
84 : V8_INLINE uintptr_t GetFlags() const {
85 4348134906 : return *reinterpret_cast<const uintptr_t*>(reinterpret_cast<Address>(this) +
86 4348134912 : kFlagsOffset);
87 : }
88 :
89 : V8_INLINE Heap* GetHeap() {
90 1410992095 : Heap* heap = *reinterpret_cast<Heap**>(reinterpret_cast<Address>(this) +
91 1410992095 : kHeapOffset);
92 : SLOW_DCHECK(heap != nullptr);
93 50130860 : return heap;
94 : }
95 :
96 : V8_INLINE Space* GetOwner() {
97 75905 : return *reinterpret_cast<Space**>(reinterpret_cast<Address>(this) +
98 75905 : kOwnerOffset);
99 : }
100 : };
101 :
102 1962039738 : inline void GenerationalBarrierInternal(HeapObject object, Address slot,
103 : HeapObject value) {
104 : DCHECK(Heap_PageFlagsAreConsistent(object));
105 : heap_internals::MemoryChunk* value_chunk =
106 0 : heap_internals::MemoryChunk::FromHeapObject(value);
107 : heap_internals::MemoryChunk* object_chunk =
108 0 : heap_internals::MemoryChunk::FromHeapObject(object);
109 :
110 2138400775 : if (!value_chunk->InYoungGeneration() || object_chunk->InYoungGeneration()) {
111 1962039069 : return;
112 : }
113 :
114 111939479 : Heap_GenerationalBarrierSlow(object, slot, value);
115 : }
116 :
117 2059873600 : inline void MarkingBarrierInternal(HeapObject object, Address slot,
118 : HeapObject value) {
119 : DCHECK(Heap_PageFlagsAreConsistent(object));
120 : heap_internals::MemoryChunk* value_chunk =
121 70 : heap_internals::MemoryChunk::FromHeapObject(value);
122 :
123 4119747107 : if (!value_chunk->IsMarking()) return;
124 :
125 285502283 : Heap_MarkingBarrierSlow(object, slot, value);
126 : }
127 :
128 : } // namespace heap_internals
129 :
130 6249697 : inline void WriteBarrierForCode(Code host, RelocInfo* rinfo, Object value) {
131 : DCHECK(!HasWeakHeapObjectTag(value));
132 6249697 : if (!value->IsHeapObject()) return;
133 6249697 : HeapObject object = HeapObject::cast(value);
134 6249697 : GenerationalBarrierForCode(host, rinfo, object);
135 6249696 : MarkingBarrierForCode(host, rinfo, object);
136 : }
137 :
138 : inline void WriteBarrierForCode(Code host) {
139 40 : Heap_WriteBarrierForCodeSlow(host);
140 : }
141 :
142 2050050267 : inline void GenerationalBarrier(HeapObject object, ObjectSlot slot,
143 : Object value) {
144 : DCHECK(!HasWeakHeapObjectTag(*slot));
145 : DCHECK(!HasWeakHeapObjectTag(value));
146 4100098918 : if (!value->IsHeapObject()) return;
147 : heap_internals::GenerationalBarrierInternal(object, slot.address(),
148 1726148193 : HeapObject::cast(value));
149 : }
150 :
151 1376453665 : inline void GenerationalBarrier(HeapObject object, MaybeObjectSlot slot,
152 : MaybeObject value) {
153 1376453665 : HeapObject value_heap_object;
154 2517167971 : if (!value->GetHeapObject(&value_heap_object)) return;
155 : heap_internals::GenerationalBarrierInternal(object, slot.address(),
156 235891610 : value_heap_object);
157 : }
158 :
159 9234 : inline void GenerationalBarrierForElements(Heap* heap, FixedArray array,
160 : int offset, int length) {
161 : heap_internals::MemoryChunk* array_chunk =
162 : heap_internals::MemoryChunk::FromHeapObject(array);
163 18468 : if (array_chunk->InYoungGeneration()) return;
164 :
165 963 : Heap_GenerationalBarrierForElementsSlow(heap, array, offset, length);
166 : }
167 :
168 6305567 : inline void GenerationalBarrierForCode(Code host, RelocInfo* rinfo,
169 : HeapObject object) {
170 : heap_internals::MemoryChunk* object_chunk =
171 0 : heap_internals::MemoryChunk::FromHeapObject(object);
172 12611134 : if (!object_chunk->InYoungGeneration()) return;
173 217505 : Heap_GenerationalBarrierForCodeSlow(host, rinfo, object);
174 : }
175 :
176 2148154599 : inline void MarkingBarrier(HeapObject object, ObjectSlot slot, Object value) {
177 : DCHECK_IMPLIES(slot.address() != kNullAddress, !HasWeakHeapObjectTag(*slot));
178 : DCHECK(!HasWeakHeapObjectTag(value));
179 4296309422 : if (!value->IsHeapObject()) return;
180 : heap_internals::MarkingBarrierInternal(object, slot.address(),
181 1824251931 : HeapObject::cast(value));
182 : }
183 :
184 1376183794 : inline void MarkingBarrier(HeapObject object, MaybeObjectSlot slot,
185 : MaybeObject value) {
186 1376183794 : HeapObject value_heap_object;
187 2516746339 : if (!value->GetHeapObject(&value_heap_object)) return;
188 : heap_internals::MarkingBarrierInternal(object, slot.address(),
189 235621707 : value_heap_object);
190 : }
191 :
192 9234 : inline void MarkingBarrierForElements(Heap* heap, HeapObject object) {
193 : heap_internals::MemoryChunk* object_chunk =
194 : heap_internals::MemoryChunk::FromHeapObject(object);
195 18468 : if (!object_chunk->IsMarking()) return;
196 :
197 423 : Heap_MarkingBarrierForElementsSlow(heap, object);
198 : }
199 :
200 7638405 : inline void MarkingBarrierForCode(Code host, RelocInfo* rinfo,
201 : HeapObject object) {
202 : DCHECK(!HasWeakHeapObjectTag(object.ptr()));
203 : heap_internals::MemoryChunk* object_chunk =
204 0 : heap_internals::MemoryChunk::FromHeapObject(object);
205 15276808 : if (!object_chunk->IsMarking()) return;
206 291888 : Heap_MarkingBarrierForCodeSlow(host, rinfo, object);
207 : }
208 :
209 81523072 : inline void MarkingBarrierForDescriptorArray(Heap* heap, HeapObject host,
210 : HeapObject descriptor_array,
211 : int number_of_own_descriptors) {
212 : heap_internals::MemoryChunk* chunk =
213 : heap_internals::MemoryChunk::FromHeapObject(descriptor_array);
214 163046145 : if (!chunk->IsMarking()) return;
215 :
216 : Heap_MarkingBarrierForDescriptorArraySlow(heap, host, descriptor_array,
217 9252428 : number_of_own_descriptors);
218 : }
219 :
220 40225374 : inline WriteBarrierMode GetWriteBarrierModeForObject(
221 : HeapObject object, const DisallowHeapAllocation* promise) {
222 : DCHECK(Heap_PageFlagsAreConsistent(object));
223 : heap_internals::MemoryChunk* chunk =
224 : heap_internals::MemoryChunk::FromHeapObject(object);
225 40225387 : if (chunk->IsMarking()) return UPDATE_WRITE_BARRIER;
226 36009599 : if (chunk->InYoungGeneration()) return SKIP_WRITE_BARRIER;
227 : return UPDATE_WRITE_BARRIER;
228 : }
229 :
230 14150756 : inline bool ObjectInYoungGeneration(const Object object) {
231 14150756 : if (object.IsSmi()) return false;
232 : return heap_internals::MemoryChunk::FromHeapObject(HeapObject::cast(object))
233 14149678 : ->InYoungGeneration();
234 : }
235 :
236 1410976599 : inline Heap* GetHeapFromWritableObject(const HeapObject object) {
237 : heap_internals::MemoryChunk* chunk =
238 50130860 : heap_internals::MemoryChunk::FromHeapObject(object);
239 1410976398 : return chunk->GetHeap();
240 : }
241 :
242 75905 : inline bool GetIsolateFromWritableObject(HeapObject obj, Isolate** isolate) {
243 : heap_internals::MemoryChunk* chunk =
244 : heap_internals::MemoryChunk::FromHeapObject(obj);
245 75905 : if (chunk->GetOwner()->identity() == RO_SPACE) {
246 60208 : *isolate = nullptr;
247 60208 : return false;
248 : }
249 15697 : *isolate = Isolate::FromHeap(chunk->GetHeap());
250 15697 : return true;
251 : }
252 :
253 : } // namespace internal
254 : } // namespace v8
255 :
256 : #endif // V8_HEAP_HEAP_WRITE_BARRIER_INL_H_
|
