Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-assembler.h"
6 :
7 : #include <ostream>
8 :
9 : #include "src/code-factory.h"
10 : #include "src/compiler/graph.h"
11 : #include "src/compiler/instruction-selector.h"
12 : #include "src/compiler/linkage.h"
13 : #include "src/compiler/node-matchers.h"
14 : #include "src/compiler/pipeline.h"
15 : #include "src/compiler/raw-machine-assembler.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/frames.h"
18 : #include "src/interface-descriptors.h"
19 : #include "src/interpreter/bytecodes.h"
20 : #include "src/machine-type.h"
21 : #include "src/macro-assembler.h"
22 : #include "src/objects-inl.h"
23 : #include "src/utils.h"
24 : #include "src/zone/zone.h"
25 :
26 : #define REPEAT_1_TO_2(V, T) V(T) V(T, T)
27 : #define REPEAT_1_TO_3(V, T) REPEAT_1_TO_2(V, T) V(T, T, T)
28 : #define REPEAT_1_TO_4(V, T) REPEAT_1_TO_3(V, T) V(T, T, T, T)
29 : #define REPEAT_1_TO_5(V, T) REPEAT_1_TO_4(V, T) V(T, T, T, T, T)
30 : #define REPEAT_1_TO_6(V, T) REPEAT_1_TO_5(V, T) V(T, T, T, T, T, T)
31 : #define REPEAT_1_TO_7(V, T) REPEAT_1_TO_6(V, T) V(T, T, T, T, T, T, T)
32 : #define REPEAT_1_TO_8(V, T) REPEAT_1_TO_7(V, T) V(T, T, T, T, T, T, T, T)
33 : #define REPEAT_1_TO_9(V, T) REPEAT_1_TO_8(V, T) V(T, T, T, T, T, T, T, T, T)
34 : #define REPEAT_1_TO_10(V, T) REPEAT_1_TO_9(V, T) V(T, T, T, T, T, T, T, T, T, T)
35 : #define REPEAT_1_TO_11(V, T) \
36 : REPEAT_1_TO_10(V, T) V(T, T, T, T, T, T, T, T, T, T, T)
37 : #define REPEAT_1_TO_12(V, T) \
38 : REPEAT_1_TO_11(V, T) V(T, T, T, T, T, T, T, T, T, T, T, T)
39 :
40 : namespace v8 {
41 : namespace internal {
42 : namespace compiler {
43 :
44 105565 : CodeAssemblerState::CodeAssemblerState(
45 : Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
46 : Code::Flags flags, const char* name, size_t result_size)
47 : : CodeAssemblerState(
48 : isolate, zone,
49 : Linkage::GetStubCallDescriptor(
50 : isolate, zone, descriptor, descriptor.GetStackParameterCount(),
51 : CallDescriptor::kNoFlags, Operator::kNoProperties,
52 : MachineType::AnyTagged(), result_size),
53 105565 : flags, name) {}
54 :
55 8799 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
56 : int parameter_count, Code::Flags flags,
57 : const char* name)
58 : : CodeAssemblerState(isolate, zone,
59 : Linkage::GetJSCallDescriptor(
60 : zone, false, parameter_count,
61 : Code::ExtractKindFromFlags(flags) == Code::BUILTIN
62 : ? CallDescriptor::kPushArgumentCount
63 : : CallDescriptor::kNoFlags),
64 17598 : flags, name) {}
65 :
66 114364 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
67 : CallDescriptor* call_descriptor,
68 : Code::Flags flags, const char* name)
69 : : raw_assembler_(new RawMachineAssembler(
70 114364 : isolate, new (zone) Graph(zone), call_descriptor,
71 : MachineType::PointerRepresentation(),
72 : InstructionSelector::SupportedMachineOperatorFlags(),
73 114364 : InstructionSelector::AlignmentRequirements())),
74 : flags_(flags),
75 : name_(name),
76 : code_generated_(false),
77 343092 : variables_(zone) {}
78 :
79 228728 : CodeAssemblerState::~CodeAssemblerState() {}
80 :
81 0 : int CodeAssemblerState::parameter_count() const {
82 0 : return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
83 : }
84 :
85 119660 : CodeAssembler::~CodeAssembler() {}
86 :
87 : #if DEBUG
88 : void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
89 : raw_assembler_->PrintCurrentBlock(os);
90 : }
91 : #endif
92 :
93 31304 : void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
94 : const char* file,
95 : int line) {
96 : #if DEBUG
97 : AssemblerDebugInfo debug_info = {msg, file, line};
98 : raw_assembler_->SetInitialDebugInformation(debug_info);
99 : #endif // DEBUG
100 31304 : }
101 :
102 0 : class BreakOnNodeDecorator final : public GraphDecorator {
103 : public:
104 0 : explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}
105 :
106 0 : void Decorate(Node* node) final {
107 0 : if (node->id() == node_id_) {
108 0 : base::OS::DebugBreak();
109 : }
110 0 : }
111 :
112 : private:
113 : NodeId node_id_;
114 : };
115 :
116 0 : void CodeAssembler::BreakOnNode(int node_id) {
117 0 : Graph* graph = raw_assembler()->graph();
118 : Zone* zone = graph->zone();
119 : GraphDecorator* decorator =
120 0 : new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
121 0 : graph->AddDecorator(decorator);
122 0 : }
123 :
124 21028 : void CodeAssembler::RegisterCallGenerationCallbacks(
125 : const CodeAssemblerCallback& call_prologue,
126 : const CodeAssemblerCallback& call_epilogue) {
127 : // The callback can be registered only once.
128 : DCHECK(!state_->call_prologue_);
129 : DCHECK(!state_->call_epilogue_);
130 21028 : state_->call_prologue_ = call_prologue;
131 21028 : state_->call_epilogue_ = call_epilogue;
132 21028 : }
133 :
134 21028 : void CodeAssembler::UnregisterCallGenerationCallbacks() {
135 21028 : state_->call_prologue_ = nullptr;
136 21028 : state_->call_epilogue_ = nullptr;
137 21028 : }
138 :
139 0 : void CodeAssembler::CallPrologue() {
140 382082 : if (state_->call_prologue_) {
141 41035 : state_->call_prologue_();
142 : }
143 0 : }
144 :
145 0 : void CodeAssembler::CallEpilogue() {
146 382082 : if (state_->call_epilogue_) {
147 41035 : state_->call_epilogue_();
148 : }
149 0 : }
150 :
151 : // static
152 111827 : Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state) {
153 : DCHECK(!state->code_generated_);
154 :
155 111827 : RawMachineAssembler* rasm = state->raw_assembler_.get();
156 111827 : Schedule* schedule = rasm->Export();
157 : Handle<Code> code = Pipeline::GenerateCodeForCodeStub(
158 : rasm->isolate(), rasm->call_descriptor(), rasm->graph(), schedule,
159 223654 : state->flags_, state->name_);
160 :
161 111827 : state->code_generated_ = true;
162 111827 : return code;
163 : }
164 :
165 273658 : bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
166 :
167 121 : bool CodeAssembler::IsFloat64RoundUpSupported() const {
168 121 : return raw_assembler()->machine()->Float64RoundUp().IsSupported();
169 : }
170 :
171 121 : bool CodeAssembler::IsFloat64RoundDownSupported() const {
172 121 : return raw_assembler()->machine()->Float64RoundDown().IsSupported();
173 : }
174 :
175 123 : bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
176 123 : return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
177 : }
178 :
179 1512 : bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
180 1512 : return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
181 : }
182 :
183 0 : bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
184 0 : return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
185 : }
186 :
187 0 : bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
188 43 : return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
189 : }
190 :
191 43 : bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
192 : return Is64() ? IsInt64AbsWithOverflowSupported()
193 86 : : IsInt32AbsWithOverflowSupported();
194 : }
195 :
196 311473 : Node* CodeAssembler::Int32Constant(int32_t value) {
197 475591 : return raw_assembler()->Int32Constant(value);
198 : }
199 :
200 14490 : Node* CodeAssembler::Int64Constant(int64_t value) {
201 14490 : return raw_assembler()->Int64Constant(value);
202 : }
203 :
204 3418496 : Node* CodeAssembler::IntPtrConstant(intptr_t value) {
205 3418497 : return raw_assembler()->IntPtrConstant(value);
206 : }
207 :
208 102568 : Node* CodeAssembler::NumberConstant(double value) {
209 102568 : return raw_assembler()->NumberConstant(value);
210 : }
211 :
212 174050 : Node* CodeAssembler::SmiConstant(Smi* value) {
213 174050 : return BitcastWordToTaggedSigned(IntPtrConstant(bit_cast<intptr_t>(value)));
214 : }
215 :
216 141898 : Node* CodeAssembler::SmiConstant(int value) {
217 141898 : return SmiConstant(Smi::FromInt(value));
218 : }
219 :
220 260622 : Node* CodeAssembler::HeapConstant(Handle<HeapObject> object) {
221 762162 : return raw_assembler()->HeapConstant(object);
222 : }
223 :
224 946 : Node* CodeAssembler::CStringConstant(const char* str) {
225 1892 : return HeapConstant(factory()->NewStringFromAsciiChecked(str, TENURED));
226 : }
227 :
228 9808 : Node* CodeAssembler::BooleanConstant(bool value) {
229 19616 : return raw_assembler()->BooleanConstant(value);
230 : }
231 :
232 142930 : Node* CodeAssembler::ExternalConstant(ExternalReference address) {
233 298421 : return raw_assembler()->ExternalConstant(address);
234 : }
235 :
236 5848 : Node* CodeAssembler::Float64Constant(double value) {
237 5848 : return raw_assembler()->Float64Constant(value);
238 : }
239 :
240 43 : Node* CodeAssembler::NaNConstant() {
241 43 : return LoadRoot(Heap::kNanValueRootIndex);
242 : }
243 :
244 215536 : bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
245 : Int64Matcher m(node);
246 306771 : if (m.HasValue() &&
247 : m.IsInRange(std::numeric_limits<int32_t>::min(),
248 : std::numeric_limits<int32_t>::max())) {
249 91221 : out_value = static_cast<int32_t>(m.Value());
250 91221 : return true;
251 : }
252 :
253 : return false;
254 : }
255 :
256 35 : bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
257 : Int64Matcher m(node);
258 35 : if (m.HasValue()) out_value = m.Value();
259 35 : return m.HasValue();
260 : }
261 :
262 83971 : bool CodeAssembler::ToSmiConstant(Node* node, Smi*& out_value) {
263 83971 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
264 : node = node->InputAt(0);
265 : } else {
266 : return false;
267 : }
268 : IntPtrMatcher m(node);
269 0 : if (m.HasValue()) {
270 0 : out_value = Smi::cast(bit_cast<Object*>(m.Value()));
271 0 : return true;
272 : }
273 : return false;
274 : }
275 :
276 2103058 : bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
277 2103058 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
278 : node->opcode() == IrOpcode::kBitcastWordToTagged) {
279 : node = node->InputAt(0);
280 : }
281 : IntPtrMatcher m(node);
282 2103058 : if (m.HasValue()) out_value = m.Value();
283 2103058 : return m.HasValue();
284 : }
285 :
286 494130 : Node* CodeAssembler::Parameter(int value) {
287 988281 : return raw_assembler()->Parameter(value);
288 : }
289 :
290 21 : Node* CodeAssembler::GetJSContextParameter() {
291 21 : CallDescriptor* desc = raw_assembler()->call_descriptor();
292 : DCHECK(desc->IsJSFunctionCall());
293 : return Parameter(Linkage::GetJSCallContextParamIndex(
294 42 : static_cast<int>(desc->JSParameterCount())));
295 : }
296 :
297 60577 : void CodeAssembler::Return(Node* value) {
298 60663 : return raw_assembler()->Return(value);
299 : }
300 :
301 0 : void CodeAssembler::Return(Node* value1, Node* value2) {
302 0 : return raw_assembler()->Return(value1, value2);
303 : }
304 :
305 86 : void CodeAssembler::Return(Node* value1, Node* value2, Node* value3) {
306 86 : return raw_assembler()->Return(value1, value2, value3);
307 : }
308 :
309 917 : void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
310 917 : return raw_assembler()->PopAndReturn(pop, value);
311 : }
312 :
313 86 : void CodeAssembler::ReturnIf(Node* condition, Node* value) {
314 86 : Label if_return(this), if_continue(this);
315 86 : Branch(condition, &if_return, &if_continue);
316 : Bind(&if_return);
317 : Return(value);
318 86 : Bind(&if_continue);
319 86 : }
320 :
321 9130 : void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
322 :
323 9044 : void CodeAssembler::Unreachable() {
324 : DebugBreak();
325 9044 : raw_assembler()->Unreachable();
326 9044 : }
327 :
328 476657 : void CodeAssembler::Comment(const char* format, ...) {
329 953314 : if (!FLAG_code_comments) return;
330 : char buffer[4 * KB];
331 : StringBuilder builder(buffer, arraysize(buffer));
332 : va_list arguments;
333 0 : va_start(arguments, format);
334 0 : builder.AddFormattedList(format, arguments);
335 0 : va_end(arguments);
336 :
337 : // Copy the string before recording it in the assembler to avoid
338 : // issues when the stack allocated buffer goes out of scope.
339 : const int prefix_len = 2;
340 0 : int length = builder.position() + 1;
341 0 : char* copy = reinterpret_cast<char*>(malloc(length + prefix_len));
342 0 : MemCopy(copy + prefix_len, builder.Finalize(), length);
343 0 : copy[0] = ';';
344 0 : copy[1] = ' ';
345 0 : raw_assembler()->Comment(copy);
346 : }
347 :
348 1713870 : void CodeAssembler::Bind(Label* label) { return label->Bind(); }
349 :
350 : #if DEBUG
351 : void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
352 : return label->Bind(debug_info);
353 : }
354 : #endif // DEBUG
355 :
356 851 : Node* CodeAssembler::LoadFramePointer() {
357 851 : return raw_assembler()->LoadFramePointer();
358 : }
359 :
360 54233 : Node* CodeAssembler::LoadParentFramePointer() {
361 54233 : return raw_assembler()->LoadParentFramePointer();
362 : }
363 :
364 43 : Node* CodeAssembler::LoadStackPointer() {
365 43 : return raw_assembler()->LoadStackPointer();
366 : }
367 :
368 : #define DEFINE_CODE_ASSEMBLER_BINARY_OP(name) \
369 : Node* CodeAssembler::name(Node* a, Node* b) { \
370 : return raw_assembler()->name(a, b); \
371 : }
372 3693074 : CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
373 : #undef DEFINE_CODE_ASSEMBLER_BINARY_OP
374 :
375 866299 : Node* CodeAssembler::IntPtrAdd(Node* left, Node* right) {
376 : intptr_t left_constant;
377 866299 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
378 : intptr_t right_constant;
379 866299 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
380 866299 : if (is_left_constant) {
381 322688 : if (is_right_constant) {
382 1806 : return IntPtrConstant(left_constant + right_constant);
383 : }
384 321785 : if (left_constant == 0) {
385 : return right;
386 : }
387 543611 : } else if (is_right_constant) {
388 491055 : if (right_constant == 0) {
389 : return left;
390 : }
391 : }
392 852034 : return raw_assembler()->IntPtrAdd(left, right);
393 : }
394 :
395 37635 : Node* CodeAssembler::IntPtrSub(Node* left, Node* right) {
396 : intptr_t left_constant;
397 37635 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
398 : intptr_t right_constant;
399 37635 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
400 37635 : if (is_left_constant) {
401 880 : if (is_right_constant) {
402 642 : return IntPtrConstant(left_constant - right_constant);
403 : }
404 36755 : } else if (is_right_constant) {
405 30909 : if (right_constant == 0) {
406 : return left;
407 : }
408 : }
409 36728 : return raw_assembler()->IntPtrSub(left, right);
410 : }
411 :
412 23206 : Node* CodeAssembler::WordShl(Node* value, int shift) {
413 23206 : return (shift != 0) ? raw_assembler()->WordShl(value, IntPtrConstant(shift))
414 46412 : : value;
415 : }
416 :
417 20042 : Node* CodeAssembler::WordShr(Node* value, int shift) {
418 17118 : return (shift != 0) ? raw_assembler()->WordShr(value, IntPtrConstant(shift))
419 37160 : : value;
420 : }
421 :
422 15693 : Node* CodeAssembler::Word32Shr(Node* value, int shift) {
423 : return (shift != 0) ? raw_assembler()->Word32Shr(value, Int32Constant(shift))
424 26004 : : value;
425 : }
426 :
427 58304 : Node* CodeAssembler::ChangeUint32ToWord(Node* value) {
428 58304 : if (raw_assembler()->machine()->Is64()) {
429 58304 : value = raw_assembler()->ChangeUint32ToUint64(value);
430 : }
431 58304 : return value;
432 : }
433 :
434 64529 : Node* CodeAssembler::ChangeInt32ToIntPtr(Node* value) {
435 64529 : if (raw_assembler()->machine()->Is64()) {
436 64529 : value = raw_assembler()->ChangeInt32ToInt64(value);
437 : }
438 64529 : return value;
439 : }
440 :
441 129 : Node* CodeAssembler::ChangeFloat64ToUintPtr(Node* value) {
442 129 : if (raw_assembler()->machine()->Is64()) {
443 129 : return raw_assembler()->ChangeFloat64ToUint64(value);
444 : }
445 0 : return raw_assembler()->ChangeFloat64ToUint32(value);
446 : }
447 :
448 709 : Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
449 709 : if (raw_assembler()->machine()->Is64()) {
450 709 : return raw_assembler()->RoundInt64ToFloat64(value);
451 : }
452 0 : return raw_assembler()->ChangeInt32ToFloat64(value);
453 : }
454 :
455 : #define DEFINE_CODE_ASSEMBLER_UNARY_OP(name) \
456 : Node* CodeAssembler::name(Node* a) { return raw_assembler()->name(a); }
457 1116830 : CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
458 : #undef DEFINE_CODE_ASSEMBLER_UNARY_OP
459 :
460 206085 : Node* CodeAssembler::Load(MachineType rep, Node* base) {
461 206085 : return raw_assembler()->Load(rep, base);
462 : }
463 :
464 1286018 : Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset) {
465 1287272 : return raw_assembler()->Load(rep, base, offset);
466 : }
467 :
468 258 : Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
469 258 : return raw_assembler()->AtomicLoad(rep, base, offset);
470 : }
471 :
472 348041 : Node* CodeAssembler::LoadRoot(Heap::RootListIndex root_index) {
473 348041 : if (isolate()->heap()->RootCanBeTreatedAsConstant(root_index)) {
474 : Handle<Object> root = isolate()->heap()->root_handle(root_index);
475 346787 : if (root->IsSmi()) {
476 0 : return SmiConstant(Smi::cast(*root));
477 : } else {
478 346787 : return HeapConstant(Handle<HeapObject>::cast(root));
479 : }
480 : }
481 :
482 : Node* roots_array_start =
483 1254 : ExternalConstant(ExternalReference::roots_array_start(isolate()));
484 : return Load(MachineType::AnyTagged(), roots_array_start,
485 2508 : IntPtrConstant(root_index * kPointerSize));
486 : }
487 :
488 0 : Node* CodeAssembler::Store(Node* base, Node* value) {
489 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
490 0 : kFullWriteBarrier);
491 : }
492 :
493 149483 : Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
494 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
495 149483 : value, kFullWriteBarrier);
496 : }
497 :
498 34887 : Node* CodeAssembler::StoreWithMapWriteBarrier(Node* base, Node* offset,
499 : Node* value) {
500 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
501 34887 : value, kMapWriteBarrier);
502 : }
503 :
504 128829 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
505 : Node* value) {
506 128829 : return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
507 : }
508 :
509 542430 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
510 : Node* offset, Node* value) {
511 542860 : return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
512 : }
513 :
514 129 : Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
515 : Node* offset, Node* value) {
516 129 : return raw_assembler()->AtomicStore(rep, base, offset, value);
517 : }
518 :
519 : #define ATOMIC_FUNCTION(name) \
520 : Node* CodeAssembler::Atomic##name(MachineType type, Node* base, \
521 : Node* offset, Node* value) { \
522 : return raw_assembler()->Atomic##name(type, base, offset, value); \
523 : }
524 516 : ATOMIC_FUNCTION(Exchange);
525 516 : ATOMIC_FUNCTION(Add);
526 516 : ATOMIC_FUNCTION(Sub);
527 516 : ATOMIC_FUNCTION(And);
528 516 : ATOMIC_FUNCTION(Or);
529 516 : ATOMIC_FUNCTION(Xor);
530 : #undef ATOMIC_FUNCTION
531 :
532 258 : Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
533 : Node* offset, Node* old_value,
534 : Node* new_value) {
535 : return raw_assembler()->AtomicCompareExchange(type, base, offset, old_value,
536 258 : new_value);
537 : }
538 :
539 430 : Node* CodeAssembler::StoreRoot(Heap::RootListIndex root_index, Node* value) {
540 : DCHECK(Heap::RootCanBeWrittenAfterInitialization(root_index));
541 : Node* roots_array_start =
542 430 : ExternalConstant(ExternalReference::roots_array_start(isolate()));
543 : return StoreNoWriteBarrier(MachineRepresentation::kTagged, roots_array_start,
544 860 : IntPtrConstant(root_index * kPointerSize), value);
545 : }
546 :
547 0 : Node* CodeAssembler::Retain(Node* value) {
548 0 : return raw_assembler()->Retain(value);
549 : }
550 :
551 5289 : Node* CodeAssembler::Projection(int index, Node* value) {
552 5289 : return raw_assembler()->Projection(index, value);
553 : }
554 :
555 845 : void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
556 : Variable* exception_var) {
557 845 : Label success(this), exception(this, Label::kDeferred);
558 845 : success.MergeVariables();
559 845 : exception.MergeVariables();
560 : DCHECK(!node->op()->HasProperty(Operator::kNoThrow));
561 :
562 1690 : raw_assembler()->Continuations(node, success.label_, exception.label_);
563 :
564 : Bind(&exception);
565 845 : const Operator* op = raw_assembler()->common()->IfException();
566 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
567 845 : if (exception_var != nullptr) {
568 : exception_var->Bind(exception_value);
569 : }
570 845 : Goto(if_exception);
571 :
572 845 : Bind(&success);
573 845 : }
574 :
575 : template <class... TArgs>
576 142026 : Node* CodeAssembler::CallRuntime(Runtime::FunctionId function, Node* context,
577 : TArgs... args) {
578 : int argc = static_cast<int>(sizeof...(args));
579 142026 : CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor(
580 : zone(), function, argc, Operator::kNoProperties,
581 142026 : CallDescriptor::kNoFlags);
582 142026 : int return_count = static_cast<int>(desc->ReturnCount());
583 :
584 : Node* centry =
585 142026 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), return_count));
586 284052 : Node* ref = ExternalConstant(ExternalReference(function, isolate()));
587 : Node* arity = Int32Constant(argc);
588 :
589 142026 : Node* nodes[] = {centry, args..., ref, arity, context};
590 :
591 : CallPrologue();
592 142026 : Node* return_value = raw_assembler()->CallN(desc, arraysize(nodes), nodes);
593 : CallEpilogue();
594 142026 : return return_value;
595 : }
596 :
597 : // Instantiate CallRuntime() for argument counts used by CSA-generated code
598 : #define INSTANTIATE(...) \
599 : template V8_EXPORT_PRIVATE Node* CodeAssembler::CallRuntime( \
600 : Runtime::FunctionId, __VA_ARGS__);
601 : REPEAT_1_TO_7(INSTANTIATE, Node*)
602 : #undef INSTANTIATE
603 :
604 : template <class... TArgs>
605 11781 : Node* CodeAssembler::TailCallRuntime(Runtime::FunctionId function,
606 : Node* context, TArgs... args) {
607 : int argc = static_cast<int>(sizeof...(args));
608 11781 : CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor(
609 : zone(), function, argc, Operator::kNoProperties,
610 11781 : CallDescriptor::kSupportsTailCalls);
611 11781 : int return_count = static_cast<int>(desc->ReturnCount());
612 :
613 : Node* centry =
614 11781 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), return_count));
615 23562 : Node* ref = ExternalConstant(ExternalReference(function, isolate()));
616 : Node* arity = Int32Constant(argc);
617 :
618 11781 : Node* nodes[] = {centry, args..., ref, arity, context};
619 :
620 11781 : return raw_assembler()->TailCallN(desc, arraysize(nodes), nodes);
621 : }
622 :
623 : // Instantiate TailCallRuntime() for argument counts used by CSA-generated code
624 : #define INSTANTIATE(...) \
625 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallRuntime( \
626 : Runtime::FunctionId, __VA_ARGS__);
627 : REPEAT_1_TO_7(INSTANTIATE, Node*)
628 : #undef INSTANTIATE
629 :
630 : template <class... TArgs>
631 46951 : Node* CodeAssembler::CallStubR(const CallInterfaceDescriptor& descriptor,
632 : size_t result_size, Node* target, Node* context,
633 : TArgs... args) {
634 46951 : Node* nodes[] = {target, args..., context};
635 46951 : return CallStubN(descriptor, result_size, arraysize(nodes), nodes);
636 : }
637 :
638 : // Instantiate CallStubR() for argument counts used by CSA-generated code.
639 : #define INSTANTIATE(...) \
640 : template V8_EXPORT_PRIVATE Node* CodeAssembler::CallStubR( \
641 : const CallInterfaceDescriptor& descriptor, size_t, Node*, __VA_ARGS__);
642 : REPEAT_1_TO_8(INSTANTIATE, Node*)
643 : #undef INSTANTIATE
644 :
645 49015 : Node* CodeAssembler::CallStubN(const CallInterfaceDescriptor& descriptor,
646 : size_t result_size, int input_count,
647 : Node* const* inputs) {
648 : // 2 is for target and context.
649 : DCHECK_LE(2, input_count);
650 49015 : int argc = input_count - 2;
651 : DCHECK_LE(descriptor.GetParameterCount(), argc);
652 : // Extra arguments not mentioned in the descriptor are passed on the stack.
653 49015 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
654 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
655 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
656 : isolate(), zone(), descriptor, stack_parameter_count,
657 : CallDescriptor::kNoFlags, Operator::kNoProperties,
658 49015 : MachineType::AnyTagged(), result_size);
659 :
660 : CallPrologue();
661 49015 : Node* return_value = raw_assembler()->CallN(desc, input_count, inputs);
662 : CallEpilogue();
663 49015 : return return_value;
664 : }
665 :
666 : template <class... TArgs>
667 113356 : Node* CodeAssembler::TailCallStub(const CallInterfaceDescriptor& descriptor,
668 : Node* target, Node* context, TArgs... args) {
669 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
670 : size_t result_size = 1;
671 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
672 : isolate(), zone(), descriptor, descriptor.GetStackParameterCount(),
673 : CallDescriptor::kSupportsTailCalls, Operator::kNoProperties,
674 113356 : MachineType::AnyTagged(), result_size);
675 :
676 113356 : Node* nodes[] = {target, args..., context};
677 226712 : CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
678 113356 : return raw_assembler()->TailCallN(desc, arraysize(nodes), nodes);
679 : }
680 :
681 : // Instantiate TailCallStub() for argument counts used by CSA-generated code
682 : #define INSTANTIATE(...) \
683 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallStub( \
684 : const CallInterfaceDescriptor& descriptor, Node*, __VA_ARGS__);
685 : REPEAT_1_TO_12(INSTANTIATE, Node*)
686 : #undef INSTANTIATE
687 :
688 : template <class... TArgs>
689 23736 : Node* CodeAssembler::TailCallBytecodeDispatch(
690 : const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
691 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
692 : CallDescriptor* desc = Linkage::GetBytecodeDispatchCallDescriptor(
693 23736 : isolate(), zone(), descriptor, descriptor.GetStackParameterCount());
694 :
695 23736 : Node* nodes[] = {target, args...};
696 47472 : CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
697 23736 : return raw_assembler()->TailCallN(desc, arraysize(nodes), nodes);
698 : }
699 :
700 : // Instantiate TailCallBytecodeDispatch() for argument counts used by
701 : // CSA-generated code
702 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
703 : const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
704 : Node*, Node*);
705 :
706 21 : Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
707 : int input_count, Node* const* inputs) {
708 21 : CallDescriptor* desc = Linkage::GetSimplifiedCDescriptor(zone(), signature);
709 21 : return raw_assembler()->CallN(desc, input_count, inputs);
710 : }
711 :
712 86 : Node* CodeAssembler::CallCFunction1(MachineType return_type,
713 : MachineType arg0_type, Node* function,
714 : Node* arg0) {
715 : return raw_assembler()->CallCFunction1(return_type, arg0_type, function,
716 86 : arg0);
717 : }
718 :
719 731 : Node* CodeAssembler::CallCFunction2(MachineType return_type,
720 : MachineType arg0_type,
721 : MachineType arg1_type, Node* function,
722 : Node* arg0, Node* arg1) {
723 : return raw_assembler()->CallCFunction2(return_type, arg0_type, arg1_type,
724 731 : function, arg0, arg1);
725 : }
726 :
727 172 : Node* CodeAssembler::CallCFunction3(MachineType return_type,
728 : MachineType arg0_type,
729 : MachineType arg1_type,
730 : MachineType arg2_type, Node* function,
731 : Node* arg0, Node* arg1, Node* arg2) {
732 : return raw_assembler()->CallCFunction3(return_type, arg0_type, arg1_type,
733 172 : arg2_type, function, arg0, arg1, arg2);
734 : }
735 :
736 344 : Node* CodeAssembler::CallCFunction6(
737 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
738 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
739 : MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
740 : Node* arg3, Node* arg4, Node* arg5) {
741 : return raw_assembler()->CallCFunction6(
742 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
743 344 : arg5_type, function, arg0, arg1, arg2, arg3, arg4, arg5);
744 : }
745 :
746 437 : Node* CodeAssembler::CallCFunction9(
747 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
748 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
749 : MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
750 : MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
751 : Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
752 : return raw_assembler()->CallCFunction9(
753 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
754 : arg5_type, arg6_type, arg7_type, arg8_type, function, arg0, arg1, arg2,
755 437 : arg3, arg4, arg5, arg6, arg7, arg8);
756 : }
757 :
758 663919 : void CodeAssembler::Goto(Label* label) {
759 663919 : label->MergeVariables();
760 1327838 : raw_assembler()->Goto(label->label_);
761 663919 : }
762 :
763 333096 : void CodeAssembler::GotoIf(Node* condition, Label* true_label) {
764 : Label false_label(this);
765 333096 : Branch(condition, true_label, &false_label);
766 333096 : Bind(&false_label);
767 333096 : }
768 :
769 233872 : void CodeAssembler::GotoIfNot(Node* condition, Label* false_label) {
770 : Label true_label(this);
771 233872 : Branch(condition, &true_label, false_label);
772 233872 : Bind(&true_label);
773 233872 : }
774 :
775 936327 : void CodeAssembler::Branch(Node* condition, Label* true_label,
776 : Label* false_label) {
777 936327 : true_label->MergeVariables();
778 936326 : false_label->MergeVariables();
779 : return raw_assembler()->Branch(condition, true_label->label_,
780 1872652 : false_label->label_);
781 : }
782 :
783 4880 : void CodeAssembler::Switch(Node* index, Label* default_label,
784 : const int32_t* case_values, Label** case_labels,
785 : size_t case_count) {
786 : RawMachineLabel** labels =
787 4880 : new (zone()->New(sizeof(RawMachineLabel*) * case_count))
788 4880 : RawMachineLabel*[case_count];
789 64424 : for (size_t i = 0; i < case_count; ++i) {
790 59544 : labels[i] = case_labels[i]->label_;
791 59544 : case_labels[i]->MergeVariables();
792 59544 : default_label->MergeVariables();
793 : }
794 : return raw_assembler()->Switch(index, default_label->label_, case_values,
795 9760 : labels, case_count);
796 : }
797 :
798 43 : bool CodeAssembler::UnalignedLoadSupported(const MachineType& machineType,
799 : uint8_t alignment) const {
800 : return raw_assembler()->machine()->UnalignedLoadSupported(machineType,
801 43 : alignment);
802 : }
803 43 : bool CodeAssembler::UnalignedStoreSupported(const MachineType& machineType,
804 : uint8_t alignment) const {
805 : return raw_assembler()->machine()->UnalignedStoreSupported(machineType,
806 43 : alignment);
807 : }
808 :
809 : // RawMachineAssembler delegate helpers:
810 2064113 : Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
811 :
812 5332 : Factory* CodeAssembler::factory() const { return isolate()->factory(); }
813 :
814 117580 : Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }
815 :
816 0 : RawMachineAssembler* CodeAssembler::raw_assembler() const {
817 15778756 : return state_->raw_assembler_.get();
818 : }
819 :
820 : // The core implementation of Variable is stored through an indirection so
821 : // that it can outlive the often block-scoped Variable declarations. This is
822 : // needed to ensure that variable binding and merging through phis can
823 : // properly be verified.
824 : class CodeAssemblerVariable::Impl : public ZoneObject {
825 : public:
826 : explicit Impl(MachineRepresentation rep)
827 : :
828 : #if DEBUG
829 : debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
830 : #endif
831 : value_(nullptr),
832 606617 : rep_(rep) {
833 : }
834 :
835 : #if DEBUG
836 : AssemblerDebugInfo debug_info() const { return debug_info_; }
837 : void set_debug_info(AssemblerDebugInfo debug_info) {
838 : debug_info_ = debug_info;
839 : }
840 :
841 : AssemblerDebugInfo debug_info_;
842 : #endif // DEBUG
843 : Node* value_;
844 : MachineRepresentation rep_;
845 : };
846 :
847 606617 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
848 : MachineRepresentation rep)
849 1213234 : : impl_(new (assembler->zone()) Impl(rep)), state_(assembler->state()) {
850 606617 : state_->variables_.insert(impl_);
851 606617 : }
852 :
853 266343 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
854 : MachineRepresentation rep,
855 : Node* initial_value)
856 266343 : : CodeAssemblerVariable(assembler, rep) {
857 : Bind(initial_value);
858 266343 : }
859 :
860 : #if DEBUG
861 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
862 : AssemblerDebugInfo debug_info,
863 : MachineRepresentation rep)
864 : : impl_(new (assembler->zone()) Impl(rep)), state_(assembler->state()) {
865 : impl_->set_debug_info(debug_info);
866 : state_->variables_.insert(impl_);
867 : }
868 :
869 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
870 : AssemblerDebugInfo debug_info,
871 : MachineRepresentation rep,
872 : Node* initial_value)
873 : : CodeAssemblerVariable(assembler, debug_info, rep) {
874 : impl_->set_debug_info(debug_info);
875 : Bind(initial_value);
876 : }
877 : #endif // DEBUG
878 :
879 606617 : CodeAssemblerVariable::~CodeAssemblerVariable() {
880 606617 : state_->variables_.erase(impl_);
881 606617 : }
882 :
883 1099961 : void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
884 :
885 1149576 : Node* CodeAssemblerVariable::value() const {
886 : #if DEBUG
887 : if (!IsBound()) {
888 : std::stringstream str;
889 : str << "#Use of unbound variable:"
890 : << "#\n Variable: " << *this;
891 : if (state_) {
892 : str << "#\n Current Block: ";
893 : state_->PrintCurrentBlock(str);
894 : }
895 : FATAL(str.str().c_str());
896 : }
897 : #endif // DEBUG
898 1149576 : return impl_->value_;
899 : }
900 :
901 0 : MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
902 :
903 105880 : bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
904 :
905 0 : std::ostream& operator<<(std::ostream& os,
906 : const CodeAssemblerVariable::Impl& impl) {
907 : #if DEBUG
908 : AssemblerDebugInfo info = impl.debug_info();
909 : if (info.name) os << "V" << info;
910 : #endif // DEBUG
911 0 : return os;
912 : }
913 :
914 0 : std::ostream& operator<<(std::ostream& os,
915 : const CodeAssemblerVariable& variable) {
916 : os << *variable.impl_;
917 0 : return os;
918 : }
919 :
920 1725748 : CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
921 : size_t vars_count,
922 : CodeAssemblerVariable* const* vars,
923 : CodeAssemblerLabel::Type type)
924 : : bound_(false),
925 : merge_count_(0),
926 : state_(assembler->state()),
927 3451496 : label_(nullptr) {
928 1725748 : void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
929 : label_ = new (buffer)
930 : RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
931 3451494 : : RawMachineLabel::kNonDeferred);
932 1961723 : for (size_t i = 0; i < vars_count; ++i) {
933 235976 : variable_phis_[vars[i]->impl_] = nullptr;
934 : }
935 1725747 : }
936 :
937 3459752 : CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }
938 :
939 2657350 : void CodeAssemblerLabel::MergeVariables() {
940 2657350 : ++merge_count_;
941 18871535 : for (CodeAssemblerVariable::Impl* var : state_->variables_) {
942 : size_t count = 0;
943 10899486 : Node* node = var->value_;
944 10899486 : if (node != nullptr) {
945 : auto i = variable_merges_.find(var);
946 7986582 : if (i != variable_merges_.end()) {
947 3120204 : i->second.push_back(node);
948 : count = i->second.size();
949 : } else {
950 : count = 1;
951 9732756 : variable_merges_[var] = std::vector<Node*>(1, node);
952 : }
953 : }
954 : // If the following asserts, then you've jumped to a label without a bound
955 : // variable along that path that expects to merge its value into a phi.
956 : DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
957 : count == merge_count_);
958 : USE(count);
959 :
960 : // If the label is already bound, we already know the set of variables to
961 : // merge and phi nodes have already been created.
962 10899486 : if (bound_) {
963 : auto phi = variable_phis_.find(var);
964 525699 : if (phi != variable_phis_.end()) {
965 : DCHECK_NOT_NULL(phi->second);
966 285110 : state_->raw_assembler_->AppendPhiInput(phi->second, node);
967 : } else {
968 : auto i = variable_merges_.find(var);
969 : if (i != variable_merges_.end()) {
970 : // If the following assert fires, then you've declared a variable that
971 : // has the same bound value along all paths up until the point you
972 : // bound this label, but then later merged a path with a new value for
973 : // the variable after the label bind (it's not possible to add phis to
974 : // the bound label after the fact, just make sure to list the variable
975 : // in the label's constructor's list of merged variables).
976 : #if DEBUG
977 : if (find_if(i->second.begin(), i->second.end(),
978 : [node](Node* e) -> bool { return node != e; }) !=
979 : i->second.end()) {
980 : std::stringstream str;
981 : str << "Unmerged variable found when jumping to block. \n"
982 : << "# Variable: " << *var;
983 : if (bound_) {
984 : str << "\n# Target block: " << *label_->block();
985 : }
986 : str << "\n# Current Block: ";
987 : state_->PrintCurrentBlock(str);
988 : FATAL(str.str().c_str());
989 : }
990 : #endif // DEBUG
991 : }
992 : }
993 : }
994 : }
995 2657349 : }
996 :
997 : #if DEBUG
998 : void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
999 : DCHECK(!bound_);
1000 : state_->raw_assembler_->Bind(label_, debug_info);
1001 : UpdateVariablesAfterBind();
1002 : }
1003 : #endif // DEBUG
1004 :
1005 1713869 : void CodeAssemblerLabel::Bind() {
1006 : DCHECK(!bound_);
1007 3427738 : state_->raw_assembler_->Bind(label_);
1008 1713869 : UpdateVariablesAfterBind();
1009 1713870 : }
1010 :
1011 1713869 : void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1012 : // Make sure that all variables that have changed along any path up to this
1013 : // point are marked as merge variables.
1014 11812466 : for (auto var : state_->variables_) {
1015 : Node* shared_value = nullptr;
1016 : auto i = variable_merges_.find(var);
1017 6670858 : if (i != variable_merges_.end()) {
1018 16810127 : for (auto value : i->second) {
1019 : DCHECK(value != nullptr);
1020 7280271 : if (value != shared_value) {
1021 5204370 : if (shared_value == nullptr) {
1022 : shared_value = value;
1023 : } else {
1024 439442 : variable_phis_[var] = nullptr;
1025 : }
1026 : }
1027 : }
1028 : }
1029 : }
1030 :
1031 3829807 : for (auto var : variable_phis_) {
1032 402067 : CodeAssemblerVariable::Impl* var_impl = var.first;
1033 : auto i = variable_merges_.find(var_impl);
1034 : // If the following asserts fire, then a variable that has been marked as
1035 : // being merged at the label--either by explicitly marking it so in the
1036 : // label constructor or by having seen different bound values at branches
1037 : // into the label--doesn't have a bound value along all of the paths that
1038 : // have been merged into the label up to this point.
1039 : DCHECK(i != variable_merges_.end());
1040 : DCHECK_EQ(i->second.size(), merge_count_);
1041 : Node* phi = state_->raw_assembler_->Phi(
1042 804134 : var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
1043 402067 : variable_phis_[var_impl] = phi;
1044 : }
1045 :
1046 : // Bind all variables to a merge phi, the common value along all paths or
1047 : // null.
1048 11812468 : for (auto var : state_->variables_) {
1049 : auto i = variable_phis_.find(var);
1050 6670858 : if (i != variable_phis_.end()) {
1051 402067 : var->value_ = i->second;
1052 : } else {
1053 : auto j = variable_merges_.find(var);
1054 10631652 : if (j != variable_merges_.end() && j->second.size() == merge_count_) {
1055 8482146 : var->value_ = j->second.back();
1056 : } else {
1057 2027718 : var->value_ = nullptr;
1058 : }
1059 : }
1060 : }
1061 :
1062 1713870 : bound_ = true;
1063 1713870 : }
1064 :
1065 : } // namespace compiler
1066 : } // namespace internal
1067 : } // namespace v8
|
