Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-assembler.h"
6 :
7 : #include <ostream>
8 :
9 : #include "src/code-factory.h"
10 : #include "src/compiler/graph.h"
11 : #include "src/compiler/instruction-selector.h"
12 : #include "src/compiler/linkage.h"
13 : #include "src/compiler/node-matchers.h"
14 : #include "src/compiler/pipeline.h"
15 : #include "src/compiler/raw-machine-assembler.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/frames.h"
18 : #include "src/interface-descriptors.h"
19 : #include "src/interpreter/bytecodes.h"
20 : #include "src/machine-type.h"
21 : #include "src/macro-assembler.h"
22 : #include "src/objects-inl.h"
23 : #include "src/utils.h"
24 : #include "src/zone/zone.h"
25 :
26 : #define REPEAT_1_TO_2(V, T) V(T) V(T, T)
27 : #define REPEAT_1_TO_3(V, T) REPEAT_1_TO_2(V, T) V(T, T, T)
28 : #define REPEAT_1_TO_4(V, T) REPEAT_1_TO_3(V, T) V(T, T, T, T)
29 : #define REPEAT_1_TO_5(V, T) REPEAT_1_TO_4(V, T) V(T, T, T, T, T)
30 : #define REPEAT_1_TO_6(V, T) REPEAT_1_TO_5(V, T) V(T, T, T, T, T, T)
31 : #define REPEAT_1_TO_7(V, T) REPEAT_1_TO_6(V, T) V(T, T, T, T, T, T, T)
32 : #define REPEAT_1_TO_8(V, T) REPEAT_1_TO_7(V, T) V(T, T, T, T, T, T, T, T)
33 : #define REPEAT_1_TO_9(V, T) REPEAT_1_TO_8(V, T) V(T, T, T, T, T, T, T, T, T)
34 : #define REPEAT_1_TO_10(V, T) REPEAT_1_TO_9(V, T) V(T, T, T, T, T, T, T, T, T, T)
35 : #define REPEAT_1_TO_11(V, T) \
36 : REPEAT_1_TO_10(V, T) V(T, T, T, T, T, T, T, T, T, T, T)
37 : #define REPEAT_1_TO_12(V, T) \
38 : REPEAT_1_TO_11(V, T) V(T, T, T, T, T, T, T, T, T, T, T, T)
39 :
40 : namespace v8 {
41 : namespace internal {
42 :
43 : constexpr MachineType MachineTypeOf<Smi>::value;
44 : constexpr MachineType MachineTypeOf<Object>::value;
45 :
46 : namespace compiler {
47 :
48 : static_assert(std::is_convertible<TNode<Number>, TNode<Object>>::value,
49 : "test subtyping");
50 : static_assert(std::is_convertible<TNode<UnionT<Smi, HeapNumber>>,
51 : TNode<UnionT<Smi, HeapObject>>>::value,
52 : "test subtyping");
53 : static_assert(
54 : !std::is_convertible<TNode<UnionT<Smi, HeapObject>>, TNode<Number>>::value,
55 : "test subtyping");
56 :
57 44586 : CodeAssemblerState::CodeAssemblerState(
58 : Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
59 : Code::Kind kind, const char* name, size_t result_size)
60 : : CodeAssemblerState(
61 : isolate, zone,
62 : Linkage::GetStubCallDescriptor(
63 : isolate, zone, descriptor, descriptor.GetStackParameterCount(),
64 : CallDescriptor::kNoFlags, Operator::kNoProperties,
65 : MachineType::AnyTagged(), result_size),
66 44586 : kind, name) {}
67 :
68 8688 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
69 : int parameter_count, Code::Kind kind,
70 : const char* name)
71 : : CodeAssemblerState(
72 : isolate, zone,
73 : Linkage::GetJSCallDescriptor(zone, false, parameter_count,
74 : kind == Code::BUILTIN
75 : ? CallDescriptor::kPushArgumentCount
76 : : CallDescriptor::kNoFlags),
77 17376 : kind, name) {}
78 :
79 53292 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
80 : CallDescriptor* call_descriptor,
81 : Code::Kind kind, const char* name)
82 : : raw_assembler_(new RawMachineAssembler(
83 53292 : isolate, new (zone) Graph(zone), call_descriptor,
84 : MachineType::PointerRepresentation(),
85 : InstructionSelector::SupportedMachineOperatorFlags(),
86 53292 : InstructionSelector::AlignmentRequirements())),
87 : kind_(kind),
88 : name_(name),
89 : code_generated_(false),
90 159876 : variables_(zone) {}
91 :
92 106584 : CodeAssemblerState::~CodeAssemblerState() {}
93 :
94 0 : int CodeAssemblerState::parameter_count() const {
95 0 : return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
96 : }
97 :
98 58506 : CodeAssembler::~CodeAssembler() {}
99 :
100 : #if DEBUG
101 : void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
102 : raw_assembler_->PrintCurrentBlock(os);
103 : }
104 : #endif
105 :
106 25854 : void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
107 : const char* file,
108 : int line) {
109 : #if DEBUG
110 : AssemblerDebugInfo debug_info = {msg, file, line};
111 : raw_assembler_->SetInitialDebugInformation(debug_info);
112 : #endif // DEBUG
113 25854 : }
114 :
115 0 : class BreakOnNodeDecorator final : public GraphDecorator {
116 : public:
117 0 : explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}
118 :
119 0 : void Decorate(Node* node) final {
120 0 : if (node->id() == node_id_) {
121 0 : base::OS::DebugBreak();
122 : }
123 0 : }
124 :
125 : private:
126 : NodeId node_id_;
127 : };
128 :
129 0 : void CodeAssembler::BreakOnNode(int node_id) {
130 0 : Graph* graph = raw_assembler()->graph();
131 : Zone* zone = graph->zone();
132 : GraphDecorator* decorator =
133 0 : new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
134 0 : graph->AddDecorator(decorator);
135 0 : }
136 :
137 15786 : void CodeAssembler::RegisterCallGenerationCallbacks(
138 : const CodeAssemblerCallback& call_prologue,
139 : const CodeAssemblerCallback& call_epilogue) {
140 : // The callback can be registered only once.
141 : DCHECK(!state_->call_prologue_);
142 : DCHECK(!state_->call_epilogue_);
143 15786 : state_->call_prologue_ = call_prologue;
144 15786 : state_->call_epilogue_ = call_epilogue;
145 15786 : }
146 :
147 15786 : void CodeAssembler::UnregisterCallGenerationCallbacks() {
148 15786 : state_->call_prologue_ = nullptr;
149 15786 : state_->call_epilogue_ = nullptr;
150 15786 : }
151 :
152 0 : void CodeAssembler::CallPrologue() {
153 245852 : if (state_->call_prologue_) {
154 34805 : state_->call_prologue_();
155 : }
156 0 : }
157 :
158 0 : void CodeAssembler::CallEpilogue() {
159 245852 : if (state_->call_epilogue_) {
160 34805 : state_->call_epilogue_();
161 : }
162 0 : }
163 :
164 : // static
165 51346 : Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state) {
166 : DCHECK(!state->code_generated_);
167 :
168 125838 : RawMachineAssembler* rasm = state->raw_assembler_.get();
169 51346 : Schedule* schedule = rasm->Export();
170 :
171 : JumpOptimizationInfo jump_opt;
172 : bool should_optimize_jumps =
173 51346 : rasm->isolate()->serializer_enabled() && FLAG_turbo_rewrite_far_jumps;
174 :
175 : Handle<Code> code = Pipeline::GenerateCodeForCodeStub(
176 : rasm->isolate(), rasm->call_descriptor(), rasm->graph(), schedule,
177 102692 : state->kind_, state->name_, should_optimize_jumps ? &jump_opt : nullptr);
178 :
179 51346 : if (jump_opt.is_optimizable()) {
180 : jump_opt.set_optimizing();
181 :
182 : // Regenerate machine code
183 : code = Pipeline::GenerateCodeForCodeStub(
184 : rasm->isolate(), rasm->call_descriptor(), rasm->graph(), schedule,
185 46292 : state->kind_, state->name_, &jump_opt);
186 : }
187 :
188 51346 : state->code_generated_ = true;
189 102692 : return code;
190 : }
191 :
192 215630 : bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
193 :
194 101 : bool CodeAssembler::IsFloat64RoundUpSupported() const {
195 101 : return raw_assembler()->machine()->Float64RoundUp().IsSupported();
196 : }
197 :
198 102 : bool CodeAssembler::IsFloat64RoundDownSupported() const {
199 102 : return raw_assembler()->machine()->Float64RoundDown().IsSupported();
200 : }
201 :
202 141 : bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
203 141 : return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
204 : }
205 :
206 1215 : bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
207 1215 : return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
208 : }
209 :
210 0 : bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
211 0 : return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
212 : }
213 :
214 0 : bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
215 31 : return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
216 : }
217 :
218 31 : bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
219 : return Is64() ? IsInt64AbsWithOverflowSupported()
220 62 : : IsInt32AbsWithOverflowSupported();
221 : }
222 :
223 278107 : TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
224 372156 : return UncheckedCast<Int32T>(raw_assembler()->Int32Constant(value));
225 : }
226 :
227 11056 : TNode<Int64T> CodeAssembler::Int64Constant(int64_t value) {
228 11056 : return UncheckedCast<Int64T>(raw_assembler()->Int64Constant(value));
229 : }
230 :
231 1831045 : TNode<IntPtrT> CodeAssembler::IntPtrConstant(intptr_t value) {
232 1831045 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrConstant(value));
233 : }
234 :
235 173 : TNode<Number> CodeAssembler::NumberConstant(double value) {
236 173 : return UncheckedCast<Number>(raw_assembler()->NumberConstant(value));
237 : }
238 :
239 10019 : TNode<Smi> CodeAssembler::SmiConstant(Smi* value) {
240 : return UncheckedCast<Smi>(
241 10019 : BitcastWordToTaggedSigned(IntPtrConstant(bit_cast<intptr_t>(value))));
242 : }
243 :
244 157337 : TNode<Smi> CodeAssembler::SmiConstant(int value) {
245 157337 : return SmiConstant(Smi::FromInt(value));
246 : }
247 :
248 59283 : TNode<HeapObject> CodeAssembler::UntypedHeapConstant(
249 : Handle<HeapObject> object) {
250 350061 : return UncheckedCast<HeapObject>(raw_assembler()->HeapConstant(object));
251 : }
252 :
253 6262 : TNode<String> CodeAssembler::StringConstant(const char* str) {
254 : return UncheckedCast<String>(
255 12524 : HeapConstant(factory()->NewStringFromAsciiChecked(str, TENURED)));
256 : }
257 :
258 9178 : TNode<Oddball> CodeAssembler::BooleanConstant(bool value) {
259 18356 : return UncheckedCast<Oddball>(raw_assembler()->BooleanConstant(value));
260 : }
261 :
262 46839 : TNode<ExternalReference> CodeAssembler::ExternalConstant(
263 : ExternalReference address) {
264 : return UncheckedCast<ExternalReference>(
265 133407 : raw_assembler()->ExternalConstant(address));
266 : }
267 :
268 4857 : TNode<Float64T> CodeAssembler::Float64Constant(double value) {
269 4857 : return UncheckedCast<Float64T>(raw_assembler()->Float64Constant(value));
270 : }
271 :
272 31 : TNode<HeapNumber> CodeAssembler::NaNConstant() {
273 31 : return UncheckedCast<HeapNumber>(LoadRoot(Heap::kNanValueRootIndex));
274 : }
275 :
276 250938 : bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
277 : Int64Matcher m(node);
278 354958 : if (m.HasValue() &&
279 : m.IsInRange(std::numeric_limits<int32_t>::min(),
280 : std::numeric_limits<int32_t>::max())) {
281 104014 : out_value = static_cast<int32_t>(m.Value());
282 104014 : return true;
283 : }
284 :
285 : return false;
286 : }
287 :
288 30 : bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
289 : Int64Matcher m(node);
290 30 : if (m.HasValue()) out_value = m.Value();
291 30 : return m.HasValue();
292 : }
293 :
294 21907 : bool CodeAssembler::ToSmiConstant(Node* node, Smi*& out_value) {
295 21907 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
296 : node = node->InputAt(0);
297 : }
298 : IntPtrMatcher m(node);
299 21907 : if (m.HasValue()) {
300 : intptr_t value = m.Value();
301 : // Make sure that the value is actually a smi
302 5041 : CHECK_EQ(0, value & ((static_cast<intptr_t>(1) << kSmiShiftSize) - 1));
303 5041 : out_value = Smi::cast(bit_cast<Object*>(value));
304 : return true;
305 : }
306 : return false;
307 : }
308 :
309 2091681 : bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
310 2091681 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
311 : node->opcode() == IrOpcode::kBitcastWordToTagged) {
312 : node = node->InputAt(0);
313 : }
314 : IntPtrMatcher m(node);
315 2091681 : if (m.HasValue()) out_value = m.Value();
316 2091681 : return m.HasValue();
317 : }
318 :
319 163279 : Node* CodeAssembler::Parameter(int value) {
320 326558 : return raw_assembler()->Parameter(value);
321 : }
322 :
323 0 : TNode<Context> CodeAssembler::GetJSContextParameter() {
324 0 : CallDescriptor* desc = raw_assembler()->call_descriptor();
325 : DCHECK(desc->IsJSFunctionCall());
326 0 : return CAST(Parameter(Linkage::GetJSCallContextParamIndex(
327 : static_cast<int>(desc->JSParameterCount()))));
328 : }
329 :
330 49799 : void CodeAssembler::Return(SloppyTNode<Object> value) {
331 49861 : return raw_assembler()->Return(value);
332 : }
333 :
334 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
335 : SloppyTNode<Object> value2) {
336 0 : return raw_assembler()->Return(value1, value2);
337 : }
338 :
339 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
340 : SloppyTNode<Object> value2,
341 : SloppyTNode<Object> value3) {
342 0 : return raw_assembler()->Return(value1, value2, value3);
343 : }
344 :
345 6491 : void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
346 6491 : return raw_assembler()->PopAndReturn(pop, value);
347 : }
348 :
349 62 : void CodeAssembler::ReturnIf(Node* condition, Node* value) {
350 62 : Label if_return(this), if_continue(this);
351 62 : Branch(condition, &if_return, &if_continue);
352 : Bind(&if_return);
353 : Return(value);
354 62 : Bind(&if_continue);
355 62 : }
356 :
357 0 : void CodeAssembler::DebugAbort(Node* message) {
358 0 : raw_assembler()->DebugAbort(message);
359 0 : }
360 :
361 11308 : void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
362 :
363 11246 : void CodeAssembler::Unreachable() {
364 : DebugBreak();
365 11246 : raw_assembler()->Unreachable();
366 11246 : }
367 :
368 169754 : void CodeAssembler::Comment(const char* format, ...) {
369 339508 : if (!FLAG_code_comments) return;
370 : char buffer[4 * KB];
371 : StringBuilder builder(buffer, arraysize(buffer));
372 : va_list arguments;
373 0 : va_start(arguments, format);
374 0 : builder.AddFormattedList(format, arguments);
375 0 : va_end(arguments);
376 :
377 : // Copy the string before recording it in the assembler to avoid
378 : // issues when the stack allocated buffer goes out of scope.
379 : const int prefix_len = 2;
380 0 : int length = builder.position() + 1;
381 0 : char* copy = reinterpret_cast<char*>(malloc(length + prefix_len));
382 0 : MemCopy(copy + prefix_len, builder.Finalize(), length);
383 0 : copy[0] = ';';
384 0 : copy[1] = ' ';
385 0 : raw_assembler()->Comment(copy);
386 : }
387 :
388 1038889 : void CodeAssembler::Bind(Label* label) { return label->Bind(); }
389 :
390 : #if DEBUG
391 : void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
392 : return label->Bind(debug_info);
393 : }
394 : #endif // DEBUG
395 :
396 2112 : Node* CodeAssembler::LoadFramePointer() {
397 2112 : return raw_assembler()->LoadFramePointer();
398 : }
399 :
400 15186 : Node* CodeAssembler::LoadParentFramePointer() {
401 15186 : return raw_assembler()->LoadParentFramePointer();
402 : }
403 :
404 31 : Node* CodeAssembler::LoadStackPointer() {
405 31 : return raw_assembler()->LoadStackPointer();
406 : }
407 :
408 : #define DEFINE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
409 : TNode<ResType> CodeAssembler::name(SloppyTNode<Arg1Type> a, \
410 : SloppyTNode<Arg2Type> b) { \
411 : return UncheckedCast<ResType>(raw_assembler()->name(a, b)); \
412 : }
413 1191052 : CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
414 : #undef DEFINE_CODE_ASSEMBLER_BINARY_OP
415 :
416 417004 : TNode<WordT> CodeAssembler::IntPtrAdd(SloppyTNode<WordT> left,
417 : SloppyTNode<WordT> right) {
418 : intptr_t left_constant;
419 417004 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
420 : intptr_t right_constant;
421 417004 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
422 417004 : if (is_left_constant) {
423 132801 : if (is_right_constant) {
424 15996 : return IntPtrConstant(left_constant + right_constant);
425 : }
426 124803 : if (left_constant == 0) {
427 16598 : return right;
428 : }
429 284203 : } else if (is_right_constant) {
430 239222 : if (right_constant == 0) {
431 1644 : return left;
432 : }
433 : }
434 : return UncheckedCast<WordT>(raw_assembler()->IntPtrAdd(left, right));
435 : }
436 :
437 57650 : TNode<WordT> CodeAssembler::IntPtrSub(SloppyTNode<WordT> left,
438 : SloppyTNode<WordT> right) {
439 : intptr_t left_constant;
440 57650 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
441 : intptr_t right_constant;
442 57650 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
443 57650 : if (is_left_constant) {
444 3899 : if (is_right_constant) {
445 6868 : return IntPtrConstant(left_constant - right_constant);
446 : }
447 53751 : } else if (is_right_constant) {
448 49569 : if (right_constant == 0) {
449 3993 : return left;
450 : }
451 : }
452 50223 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrSub(left, right));
453 : }
454 :
455 26377 : TNode<WordT> CodeAssembler::IntPtrMul(SloppyTNode<WordT> left,
456 : SloppyTNode<WordT> right) {
457 : intptr_t left_constant;
458 26377 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
459 : intptr_t right_constant;
460 26377 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
461 26377 : if (is_left_constant) {
462 436 : if (is_right_constant) {
463 870 : return IntPtrConstant(left_constant * right_constant);
464 : }
465 1 : if (left_constant == 1) {
466 1 : return right;
467 : }
468 25941 : } else if (is_right_constant) {
469 25941 : if (right_constant == 1) {
470 3507 : return left;
471 : }
472 : }
473 22434 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrMul(left, right));
474 : }
475 :
476 2464 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> value, int shift) {
477 4710 : return (shift != 0) ? WordShl(value, IntPtrConstant(shift)) : value;
478 : }
479 :
480 15073 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> value, int shift) {
481 27727 : return (shift != 0) ? WordShr(value, IntPtrConstant(shift)) : value;
482 : }
483 :
484 11356 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> value, int shift) {
485 19017 : return (shift != 0) ? Word32Shr(value, Int32Constant(shift)) : value;
486 : }
487 :
488 22075 : TNode<WordT> CodeAssembler::WordOr(SloppyTNode<WordT> left,
489 : SloppyTNode<WordT> right) {
490 : intptr_t left_constant;
491 22075 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
492 : intptr_t right_constant;
493 22075 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
494 22075 : if (is_left_constant) {
495 3381 : if (is_right_constant) {
496 5768 : return IntPtrConstant(left_constant | right_constant);
497 : }
498 497 : if (left_constant == 0) {
499 94 : return right;
500 : }
501 18694 : } else if (is_right_constant) {
502 11596 : if (right_constant == 0) {
503 1 : return left;
504 : }
505 : }
506 19096 : return UncheckedCast<WordT>(raw_assembler()->WordOr(left, right));
507 : }
508 :
509 134126 : TNode<WordT> CodeAssembler::WordAnd(SloppyTNode<WordT> left,
510 : SloppyTNode<WordT> right) {
511 : intptr_t left_constant;
512 134126 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
513 : intptr_t right_constant;
514 134126 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
515 134126 : if (is_left_constant) {
516 466 : if (is_right_constant) {
517 932 : return IntPtrConstant(left_constant & right_constant);
518 : }
519 : }
520 133660 : return UncheckedCast<WordT>(raw_assembler()->WordAnd(left, right));
521 : }
522 :
523 2 : TNode<WordT> CodeAssembler::WordXor(SloppyTNode<WordT> left,
524 : SloppyTNode<WordT> right) {
525 : intptr_t left_constant;
526 2 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
527 : intptr_t right_constant;
528 2 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
529 2 : if (is_left_constant) {
530 1 : if (is_right_constant) {
531 2 : return IntPtrConstant(left_constant ^ right_constant);
532 : }
533 : }
534 1 : return UncheckedCast<WordT>(raw_assembler()->WordXor(left, right));
535 : }
536 :
537 183113 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> left,
538 : SloppyTNode<IntegralT> right) {
539 : intptr_t left_constant;
540 183113 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
541 : intptr_t right_constant;
542 183113 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
543 183113 : if (is_left_constant) {
544 1272 : if (is_right_constant) {
545 2482 : return IntPtrConstant(left_constant << right_constant);
546 : }
547 181841 : } else if (is_right_constant) {
548 181841 : if (right_constant == 0) {
549 0 : return left;
550 : }
551 : }
552 181872 : return UncheckedCast<WordT>(raw_assembler()->WordShl(left, right));
553 : }
554 :
555 28459 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> left,
556 : SloppyTNode<IntegralT> right) {
557 : intptr_t left_constant;
558 28459 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
559 : intptr_t right_constant;
560 28459 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
561 28459 : if (is_left_constant) {
562 2606 : if (is_right_constant) {
563 2606 : return IntPtrConstant(static_cast<uintptr_t>(left_constant) >>
564 5212 : right_constant);
565 : }
566 25853 : } else if (is_right_constant) {
567 25853 : if (right_constant == 0) {
568 0 : return left;
569 : }
570 : }
571 25853 : return UncheckedCast<WordT>(raw_assembler()->WordShr(left, right));
572 : }
573 :
574 62917 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> left,
575 : SloppyTNode<IntegralT> right) {
576 : intptr_t left_constant;
577 62917 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
578 : intptr_t right_constant;
579 62917 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
580 62917 : if (is_left_constant) {
581 2 : if (is_right_constant) {
582 4 : return IntPtrConstant(left_constant >> right_constant);
583 : }
584 62915 : } else if (is_right_constant) {
585 62915 : if (right_constant == 0) {
586 1 : return left;
587 : }
588 : }
589 62914 : return UncheckedCast<WordT>(raw_assembler()->WordSar(left, right));
590 : }
591 :
592 2511 : TNode<Word32T> CodeAssembler::Word32Or(SloppyTNode<Word32T> left,
593 : SloppyTNode<Word32T> right) {
594 : int32_t left_constant;
595 2511 : bool is_left_constant = ToInt32Constant(left, left_constant);
596 : int32_t right_constant;
597 2511 : bool is_right_constant = ToInt32Constant(right, right_constant);
598 2511 : if (is_left_constant) {
599 0 : if (is_right_constant) {
600 0 : return Int32Constant(left_constant | right_constant);
601 : }
602 0 : if (left_constant == 0) {
603 0 : return right;
604 : }
605 2511 : } else if (is_right_constant) {
606 0 : if (right_constant == 0) {
607 0 : return left;
608 : }
609 : }
610 2511 : return UncheckedCast<Word32T>(raw_assembler()->Word32Or(left, right));
611 : }
612 :
613 60268 : TNode<Word32T> CodeAssembler::Word32And(SloppyTNode<Word32T> left,
614 : SloppyTNode<Word32T> right) {
615 : int32_t left_constant;
616 60268 : bool is_left_constant = ToInt32Constant(left, left_constant);
617 : int32_t right_constant;
618 60268 : bool is_right_constant = ToInt32Constant(right, right_constant);
619 60268 : if (is_left_constant) {
620 6 : if (is_right_constant) {
621 12 : return Int32Constant(left_constant & right_constant);
622 : }
623 : }
624 60262 : return UncheckedCast<Word32T>(raw_assembler()->Word32And(left, right));
625 : }
626 :
627 4211 : TNode<Word32T> CodeAssembler::Word32Xor(SloppyTNode<Word32T> left,
628 : SloppyTNode<Word32T> right) {
629 : int32_t left_constant;
630 4211 : bool is_left_constant = ToInt32Constant(left, left_constant);
631 : int32_t right_constant;
632 4211 : bool is_right_constant = ToInt32Constant(right, right_constant);
633 4211 : if (is_left_constant) {
634 6 : if (is_right_constant) {
635 12 : return Int32Constant(left_constant ^ right_constant);
636 : }
637 : }
638 4205 : return UncheckedCast<Word32T>(raw_assembler()->Word32Xor(left, right));
639 : }
640 :
641 2466 : TNode<Word32T> CodeAssembler::Word32Shl(SloppyTNode<Word32T> left,
642 : SloppyTNode<Word32T> right) {
643 : int32_t left_constant;
644 2466 : bool is_left_constant = ToInt32Constant(left, left_constant);
645 : int32_t right_constant;
646 2466 : bool is_right_constant = ToInt32Constant(right, right_constant);
647 2466 : if (is_left_constant) {
648 0 : if (is_right_constant) {
649 0 : return Int32Constant(left_constant << right_constant);
650 : }
651 2466 : } else if (is_right_constant) {
652 2249 : if (right_constant == 0) {
653 0 : return left;
654 : }
655 : }
656 2466 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shl(left, right));
657 : }
658 :
659 14899 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> left,
660 : SloppyTNode<Word32T> right) {
661 : int32_t left_constant;
662 14899 : bool is_left_constant = ToInt32Constant(left, left_constant);
663 : int32_t right_constant;
664 14899 : bool is_right_constant = ToInt32Constant(right, right_constant);
665 14899 : if (is_left_constant) {
666 0 : if (is_right_constant) {
667 0 : return Int32Constant(static_cast<uint32_t>(left_constant) >>
668 0 : right_constant);
669 : }
670 14899 : } else if (is_right_constant) {
671 14217 : if (right_constant == 0) {
672 0 : return left;
673 : }
674 : }
675 14899 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shr(left, right));
676 : }
677 :
678 217 : TNode<Word32T> CodeAssembler::Word32Sar(SloppyTNode<Word32T> left,
679 : SloppyTNode<Word32T> right) {
680 : int32_t left_constant;
681 217 : bool is_left_constant = ToInt32Constant(left, left_constant);
682 : int32_t right_constant;
683 217 : bool is_right_constant = ToInt32Constant(right, right_constant);
684 217 : if (is_left_constant) {
685 0 : if (is_right_constant) {
686 0 : return Int32Constant(left_constant >> right_constant);
687 : }
688 217 : } else if (is_right_constant) {
689 0 : if (right_constant == 0) {
690 0 : return left;
691 : }
692 : }
693 217 : return UncheckedCast<Word32T>(raw_assembler()->Word32Sar(left, right));
694 : }
695 :
696 0 : TNode<Word64T> CodeAssembler::Word64Or(SloppyTNode<Word64T> left,
697 : SloppyTNode<Word64T> right) {
698 : int64_t left_constant;
699 : bool is_left_constant = ToInt64Constant(left, left_constant);
700 : int64_t right_constant;
701 : bool is_right_constant = ToInt64Constant(right, right_constant);
702 0 : if (is_left_constant) {
703 0 : if (is_right_constant) {
704 0 : return Int64Constant(left_constant | right_constant);
705 : }
706 0 : if (left_constant == 0) {
707 0 : return right;
708 : }
709 0 : } else if (is_right_constant) {
710 0 : if (right_constant == 0) {
711 0 : return left;
712 : }
713 : }
714 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Or(left, right));
715 : }
716 :
717 0 : TNode<Word64T> CodeAssembler::Word64And(SloppyTNode<Word64T> left,
718 : SloppyTNode<Word64T> right) {
719 : int64_t left_constant;
720 : bool is_left_constant = ToInt64Constant(left, left_constant);
721 : int64_t right_constant;
722 : bool is_right_constant = ToInt64Constant(right, right_constant);
723 0 : if (is_left_constant) {
724 0 : if (is_right_constant) {
725 0 : return Int64Constant(left_constant & right_constant);
726 : }
727 : }
728 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64And(left, right));
729 : }
730 :
731 0 : TNode<Word64T> CodeAssembler::Word64Xor(SloppyTNode<Word64T> left,
732 : SloppyTNode<Word64T> right) {
733 : int64_t left_constant;
734 : bool is_left_constant = ToInt64Constant(left, left_constant);
735 : int64_t right_constant;
736 : bool is_right_constant = ToInt64Constant(right, right_constant);
737 0 : if (is_left_constant) {
738 0 : if (is_right_constant) {
739 0 : return Int64Constant(left_constant ^ right_constant);
740 : }
741 : }
742 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Xor(left, right));
743 : }
744 :
745 0 : TNode<Word64T> CodeAssembler::Word64Shl(SloppyTNode<Word64T> left,
746 : SloppyTNode<Word64T> right) {
747 : int64_t left_constant;
748 : bool is_left_constant = ToInt64Constant(left, left_constant);
749 : int64_t right_constant;
750 : bool is_right_constant = ToInt64Constant(right, right_constant);
751 0 : if (is_left_constant) {
752 0 : if (is_right_constant) {
753 0 : return Int64Constant(left_constant << right_constant);
754 : }
755 0 : } else if (is_right_constant) {
756 0 : if (right_constant == 0) {
757 0 : return left;
758 : }
759 : }
760 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shl(left, right));
761 : }
762 :
763 0 : TNode<Word64T> CodeAssembler::Word64Shr(SloppyTNode<Word64T> left,
764 : SloppyTNode<Word64T> right) {
765 : int64_t left_constant;
766 : bool is_left_constant = ToInt64Constant(left, left_constant);
767 : int64_t right_constant;
768 : bool is_right_constant = ToInt64Constant(right, right_constant);
769 0 : if (is_left_constant) {
770 0 : if (is_right_constant) {
771 0 : return Int64Constant(static_cast<uint64_t>(left_constant) >>
772 0 : right_constant);
773 : }
774 0 : } else if (is_right_constant) {
775 0 : if (right_constant == 0) {
776 0 : return left;
777 : }
778 : }
779 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shr(left, right));
780 : }
781 :
782 0 : TNode<Word64T> CodeAssembler::Word64Sar(SloppyTNode<Word64T> left,
783 : SloppyTNode<Word64T> right) {
784 : int64_t left_constant;
785 : bool is_left_constant = ToInt64Constant(left, left_constant);
786 : int64_t right_constant;
787 : bool is_right_constant = ToInt64Constant(right, right_constant);
788 0 : if (is_left_constant) {
789 0 : if (is_right_constant) {
790 0 : return Int64Constant(left_constant >> right_constant);
791 : }
792 0 : } else if (is_right_constant) {
793 0 : if (right_constant == 0) {
794 0 : return left;
795 : }
796 : }
797 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Sar(left, right));
798 : }
799 :
800 39975 : TNode<UintPtrT> CodeAssembler::ChangeUint32ToWord(SloppyTNode<Word32T> value) {
801 39975 : if (raw_assembler()->machine()->Is64()) {
802 : return UncheckedCast<UintPtrT>(
803 39975 : raw_assembler()->ChangeUint32ToUint64(value));
804 : }
805 : return ReinterpretCast<UintPtrT>(value);
806 : }
807 :
808 44142 : TNode<IntPtrT> CodeAssembler::ChangeInt32ToIntPtr(SloppyTNode<Word32T> value) {
809 44142 : if (raw_assembler()->machine()->Is64()) {
810 44142 : return ReinterpretCast<IntPtrT>(raw_assembler()->ChangeInt32ToInt64(value));
811 : }
812 : return ReinterpretCast<IntPtrT>(value);
813 : }
814 :
815 93 : TNode<UintPtrT> CodeAssembler::ChangeFloat64ToUintPtr(
816 : SloppyTNode<Float64T> value) {
817 93 : if (raw_assembler()->machine()->Is64()) {
818 : return ReinterpretCast<UintPtrT>(
819 93 : raw_assembler()->ChangeFloat64ToUint64(value));
820 : }
821 : return ReinterpretCast<UintPtrT>(
822 0 : raw_assembler()->ChangeFloat64ToUint32(value));
823 : }
824 :
825 483 : Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
826 483 : if (raw_assembler()->machine()->Is64()) {
827 483 : return raw_assembler()->RoundInt64ToFloat64(value);
828 : }
829 0 : return raw_assembler()->ChangeInt32ToFloat64(value);
830 : }
831 :
832 : #define DEFINE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
833 : TNode<ResType> CodeAssembler::name(SloppyTNode<ArgType> a) { \
834 : return UncheckedCast<ResType>(raw_assembler()->name(a)); \
835 : }
836 997404 : CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
837 : #undef DEFINE_CODE_ASSEMBLER_UNARY_OP
838 :
839 79156 : Node* CodeAssembler::Load(MachineType rep, Node* base) {
840 79156 : return raw_assembler()->Load(rep, base);
841 : }
842 :
843 736725 : Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset) {
844 736917 : return raw_assembler()->Load(rep, base, offset);
845 : }
846 :
847 186 : Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
848 186 : return raw_assembler()->AtomicLoad(rep, base, offset);
849 : }
850 :
851 198332 : TNode<Object> CodeAssembler::LoadRoot(Heap::RootListIndex root_index) {
852 198332 : if (isolate()->heap()->RootCanBeTreatedAsConstant(root_index)) {
853 : Handle<Object> root = isolate()->heap()->root_handle(root_index);
854 198140 : if (root->IsSmi()) {
855 0 : return SmiConstant(Smi::cast(*root));
856 : } else {
857 198140 : return HeapConstant(Handle<HeapObject>::cast(root));
858 : }
859 : }
860 :
861 : Node* roots_array_start =
862 192 : ExternalConstant(ExternalReference::roots_array_start(isolate()));
863 : return UncheckedCast<Object>(Load(MachineType::AnyTagged(), roots_array_start,
864 192 : IntPtrConstant(root_index * kPointerSize)));
865 : }
866 :
867 0 : Node* CodeAssembler::Store(Node* base, Node* value) {
868 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
869 0 : kFullWriteBarrier);
870 : }
871 :
872 26550 : Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
873 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
874 26550 : value, kFullWriteBarrier);
875 : }
876 :
877 3971 : Node* CodeAssembler::StoreWithMapWriteBarrier(Node* base, Node* offset,
878 : Node* value) {
879 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
880 3971 : value, kMapWriteBarrier);
881 : }
882 :
883 38341 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
884 : Node* value) {
885 38341 : return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
886 : }
887 :
888 244133 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
889 : Node* offset, Node* value) {
890 244133 : return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
891 : }
892 :
893 93 : Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
894 : Node* offset, Node* value) {
895 93 : return raw_assembler()->AtomicStore(rep, base, offset, value);
896 : }
897 :
898 : #define ATOMIC_FUNCTION(name) \
899 : Node* CodeAssembler::Atomic##name(MachineType type, Node* base, \
900 : Node* offset, Node* value) { \
901 : return raw_assembler()->Atomic##name(type, base, offset, value); \
902 : }
903 372 : ATOMIC_FUNCTION(Exchange);
904 372 : ATOMIC_FUNCTION(Add);
905 372 : ATOMIC_FUNCTION(Sub);
906 372 : ATOMIC_FUNCTION(And);
907 372 : ATOMIC_FUNCTION(Or);
908 372 : ATOMIC_FUNCTION(Xor);
909 : #undef ATOMIC_FUNCTION
910 :
911 186 : Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
912 : Node* offset, Node* old_value,
913 : Node* new_value) {
914 : return raw_assembler()->AtomicCompareExchange(type, base, offset, old_value,
915 186 : new_value);
916 : }
917 :
918 0 : Node* CodeAssembler::StoreRoot(Heap::RootListIndex root_index, Node* value) {
919 : DCHECK(Heap::RootCanBeWrittenAfterInitialization(root_index));
920 : Node* roots_array_start =
921 0 : ExternalConstant(ExternalReference::roots_array_start(isolate()));
922 : return StoreNoWriteBarrier(MachineRepresentation::kTagged, roots_array_start,
923 0 : IntPtrConstant(root_index * kPointerSize), value);
924 : }
925 :
926 0 : Node* CodeAssembler::Retain(Node* value) {
927 0 : return raw_assembler()->Retain(value);
928 : }
929 :
930 3496 : Node* CodeAssembler::Projection(int index, Node* value) {
931 3496 : return raw_assembler()->Projection(index, value);
932 : }
933 :
934 2287 : void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
935 : Variable* exception_var) {
936 : DCHECK(!node->op()->HasProperty(Operator::kNoThrow));
937 :
938 2287 : if (if_exception == nullptr) {
939 : // If no handler is supplied, don't add continuations
940 496 : return;
941 : }
942 :
943 1791 : Label success(this), exception(this, Label::kDeferred);
944 1791 : success.MergeVariables();
945 1791 : exception.MergeVariables();
946 :
947 3582 : raw_assembler()->Continuations(node, success.label_, exception.label_);
948 :
949 : Bind(&exception);
950 1791 : const Operator* op = raw_assembler()->common()->IfException();
951 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
952 1791 : if (exception_var != nullptr) {
953 : exception_var->Bind(exception_value);
954 : }
955 1791 : Goto(if_exception);
956 :
957 1791 : Bind(&success);
958 : }
959 :
960 : template <class... TArgs>
961 76968 : TNode<Object> CodeAssembler::CallRuntimeImpl(Runtime::FunctionId function,
962 : SloppyTNode<Object> context,
963 : TArgs... args) {
964 : int argc = static_cast<int>(sizeof...(args));
965 76968 : CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor(
966 : zone(), function, argc, Operator::kNoProperties,
967 76968 : CallDescriptor::kNoFlags);
968 76968 : int return_count = static_cast<int>(desc->ReturnCount());
969 :
970 : Node* centry =
971 76968 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), return_count));
972 153936 : Node* ref = ExternalConstant(ExternalReference(function, isolate()));
973 : Node* arity = Int32Constant(argc);
974 :
975 265177 : Node* nodes[] = {centry, args..., ref, arity, context};
976 :
977 : CallPrologue();
978 76968 : Node* return_value = raw_assembler()->CallN(desc, arraysize(nodes), nodes);
979 : CallEpilogue();
980 76968 : return UncheckedCast<Object>(return_value);
981 : }
982 :
983 : // Instantiate CallRuntime() for argument counts used by CSA-generated code
984 : #define INSTANTIATE(...) \
985 : template V8_EXPORT_PRIVATE TNode<Object> CodeAssembler::CallRuntimeImpl( \
986 : Runtime::FunctionId, __VA_ARGS__);
987 : REPEAT_1_TO_7(INSTANTIATE, SloppyTNode<Object>)
988 : #undef INSTANTIATE
989 :
990 : template <class... TArgs>
991 9408 : TNode<Object> CodeAssembler::TailCallRuntimeImpl(Runtime::FunctionId function,
992 : SloppyTNode<Object> context,
993 : TArgs... args) {
994 : int argc = static_cast<int>(sizeof...(args));
995 9408 : CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor(
996 : zone(), function, argc, Operator::kNoProperties,
997 9408 : CallDescriptor::kSupportsTailCalls);
998 9408 : int return_count = static_cast<int>(desc->ReturnCount());
999 :
1000 : Node* centry =
1001 9408 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), return_count));
1002 18816 : Node* ref = ExternalConstant(ExternalReference(function, isolate()));
1003 : Node* arity = Int32Constant(argc);
1004 :
1005 58363 : Node* nodes[] = {centry, args..., ref, arity, context};
1006 :
1007 : return UncheckedCast<Object>(
1008 9408 : raw_assembler()->TailCallN(desc, arraysize(nodes), nodes));
1009 : }
1010 :
1011 : // Instantiate TailCallRuntime() for argument counts used by CSA-generated code
1012 : #define INSTANTIATE(...) \
1013 : template V8_EXPORT_PRIVATE TNode<Object> CodeAssembler::TailCallRuntimeImpl( \
1014 : Runtime::FunctionId, __VA_ARGS__);
1015 : REPEAT_1_TO_7(INSTANTIATE, SloppyTNode<Object>)
1016 : #undef INSTANTIATE
1017 :
1018 : template <class... TArgs>
1019 44935 : Node* CodeAssembler::CallStubR(const CallInterfaceDescriptor& descriptor,
1020 : size_t result_size, Node* target, Node* context,
1021 : TArgs... args) {
1022 44935 : Node* nodes[] = {target, args..., context};
1023 44935 : return CallStubN(descriptor, result_size, arraysize(nodes), nodes);
1024 : }
1025 :
1026 : // Instantiate CallStubR() for argument counts used by CSA-generated code.
1027 : #define INSTANTIATE(...) \
1028 : template V8_EXPORT_PRIVATE Node* CodeAssembler::CallStubR( \
1029 : const CallInterfaceDescriptor& descriptor, size_t, Node*, __VA_ARGS__);
1030 : REPEAT_1_TO_11(INSTANTIATE, Node*)
1031 : #undef INSTANTIATE
1032 :
1033 45958 : Node* CodeAssembler::CallStubN(const CallInterfaceDescriptor& descriptor,
1034 : size_t result_size, int input_count,
1035 : Node* const* inputs) {
1036 : // 2 is for target and context.
1037 : DCHECK_LE(2, input_count);
1038 45958 : int argc = input_count - 2;
1039 : DCHECK_LE(descriptor.GetParameterCount(), argc);
1040 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1041 45958 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1042 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1043 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
1044 : isolate(), zone(), descriptor, stack_parameter_count,
1045 : CallDescriptor::kNoFlags, Operator::kNoProperties,
1046 45958 : MachineType::AnyTagged(), result_size);
1047 :
1048 : CallPrologue();
1049 45958 : Node* return_value = raw_assembler()->CallN(desc, input_count, inputs);
1050 : CallEpilogue();
1051 45958 : return return_value;
1052 : }
1053 :
1054 : template <class... TArgs>
1055 2883 : Node* CodeAssembler::TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
1056 : Node* target, Node* context,
1057 : TArgs... args) {
1058 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1059 : size_t result_size = 1;
1060 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
1061 : isolate(), zone(), descriptor, descriptor.GetStackParameterCount(),
1062 : CallDescriptor::kSupportsTailCalls, Operator::kNoProperties,
1063 2883 : MachineType::AnyTagged(), result_size);
1064 :
1065 2883 : Node* nodes[] = {target, args..., context};
1066 5766 : CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
1067 2883 : return raw_assembler()->TailCallN(desc, arraysize(nodes), nodes);
1068 : }
1069 :
1070 : // Instantiate TailCallStub() for argument counts used by CSA-generated code
1071 : #define INSTANTIATE(...) \
1072 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallStubImpl( \
1073 : const CallInterfaceDescriptor& descriptor, Node*, __VA_ARGS__);
1074 : REPEAT_1_TO_12(INSTANTIATE, Node*)
1075 : #undef INSTANTIATE
1076 :
1077 : template <class... TArgs>
1078 1116 : Node* CodeAssembler::TailCallStubThenBytecodeDispatch(
1079 : const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
1080 : TArgs... args) {
1081 : DCHECK_LE(descriptor.GetParameterCount(), sizeof...(args));
1082 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1083 : int stack_parameter_count =
1084 1116 : sizeof...(args) - descriptor.GetRegisterParameterCount();
1085 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1086 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
1087 : isolate(), zone(), descriptor, stack_parameter_count,
1088 : CallDescriptor::kSupportsTailCalls, Operator::kNoProperties,
1089 1116 : MachineType::AnyTagged(), 0);
1090 :
1091 1116 : Node* nodes[] = {target, args..., context};
1092 1116 : return raw_assembler()->TailCallN(desc, arraysize(nodes), nodes);
1093 : }
1094 :
1095 : // Instantiate TailCallJSAndBytecodeDispatch() for argument counts used by
1096 : // CSA-generated code
1097 : #define INSTANTIATE(...) \
1098 : template V8_EXPORT_PRIVATE Node* \
1099 : CodeAssembler::TailCallStubThenBytecodeDispatch( \
1100 : const CallInterfaceDescriptor&, Node*, Node*, Node*, __VA_ARGS__);
1101 : REPEAT_1_TO_7(INSTANTIATE, Node*)
1102 : #undef INSTANTIATE
1103 :
1104 : template <class... TArgs>
1105 16616 : Node* CodeAssembler::TailCallBytecodeDispatch(
1106 : const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
1107 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1108 : CallDescriptor* desc = Linkage::GetBytecodeDispatchCallDescriptor(
1109 16616 : isolate(), zone(), descriptor, descriptor.GetStackParameterCount());
1110 :
1111 16616 : Node* nodes[] = {target, args...};
1112 33232 : CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
1113 16616 : return raw_assembler()->TailCallN(desc, arraysize(nodes), nodes);
1114 : }
1115 :
1116 : // Instantiate TailCallBytecodeDispatch() for argument counts used by
1117 : // CSA-generated code
1118 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
1119 : const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
1120 : Node*, Node*);
1121 :
1122 0 : Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
1123 : int input_count, Node* const* inputs) {
1124 0 : CallDescriptor* desc = Linkage::GetSimplifiedCDescriptor(zone(), signature);
1125 0 : return raw_assembler()->CallN(desc, input_count, inputs);
1126 : }
1127 :
1128 124 : Node* CodeAssembler::CallCFunction1(MachineType return_type,
1129 : MachineType arg0_type, Node* function,
1130 : Node* arg0) {
1131 : return raw_assembler()->CallCFunction1(return_type, arg0_type, function,
1132 124 : arg0);
1133 : }
1134 :
1135 124 : Node* CodeAssembler::CallCFunction1WithCallerSavedRegisters(
1136 : MachineType return_type, MachineType arg0_type, Node* function, Node* arg0,
1137 : SaveFPRegsMode mode) {
1138 : DCHECK(return_type.LessThanOrEqualPointerSize());
1139 : return raw_assembler()->CallCFunction1WithCallerSavedRegisters(
1140 124 : return_type, arg0_type, function, arg0, mode);
1141 : }
1142 :
1143 1364 : Node* CodeAssembler::CallCFunction2(MachineType return_type,
1144 : MachineType arg0_type,
1145 : MachineType arg1_type, Node* function,
1146 : Node* arg0, Node* arg1) {
1147 : return raw_assembler()->CallCFunction2(return_type, arg0_type, arg1_type,
1148 1364 : function, arg0, arg1);
1149 : }
1150 :
1151 217 : Node* CodeAssembler::CallCFunction3(MachineType return_type,
1152 : MachineType arg0_type,
1153 : MachineType arg1_type,
1154 : MachineType arg2_type, Node* function,
1155 : Node* arg0, Node* arg1, Node* arg2) {
1156 : return raw_assembler()->CallCFunction3(return_type, arg0_type, arg1_type,
1157 217 : arg2_type, function, arg0, arg1, arg2);
1158 : }
1159 :
1160 68 : Node* CodeAssembler::CallCFunction3WithCallerSavedRegisters(
1161 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1162 : MachineType arg2_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1163 : SaveFPRegsMode mode) {
1164 : DCHECK(return_type.LessThanOrEqualPointerSize());
1165 : return raw_assembler()->CallCFunction3WithCallerSavedRegisters(
1166 : return_type, arg0_type, arg1_type, arg2_type, function, arg0, arg1, arg2,
1167 68 : mode);
1168 : }
1169 :
1170 372 : Node* CodeAssembler::CallCFunction6(
1171 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1172 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1173 : MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1174 : Node* arg3, Node* arg4, Node* arg5) {
1175 : return raw_assembler()->CallCFunction6(
1176 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1177 372 : arg5_type, function, arg0, arg1, arg2, arg3, arg4, arg5);
1178 : }
1179 :
1180 316 : Node* CodeAssembler::CallCFunction9(
1181 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1182 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1183 : MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
1184 : MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1185 : Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
1186 : return raw_assembler()->CallCFunction9(
1187 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1188 : arg5_type, arg6_type, arg7_type, arg8_type, function, arg0, arg1, arg2,
1189 316 : arg3, arg4, arg5, arg6, arg7, arg8);
1190 : }
1191 :
1192 410158 : void CodeAssembler::Goto(Label* label) {
1193 410158 : label->MergeVariables();
1194 820316 : raw_assembler()->Goto(label->label_);
1195 410158 : }
1196 :
1197 260407 : void CodeAssembler::GotoIf(SloppyTNode<IntegralT> condition,
1198 : Label* true_label) {
1199 : Label false_label(this);
1200 260407 : Branch(condition, true_label, &false_label);
1201 260407 : Bind(&false_label);
1202 260407 : }
1203 :
1204 99184 : void CodeAssembler::GotoIfNot(SloppyTNode<IntegralT> condition,
1205 : Label* false_label) {
1206 : Label true_label(this);
1207 99184 : Branch(condition, &true_label, false_label);
1208 99184 : Bind(&true_label);
1209 99184 : }
1210 :
1211 576171 : void CodeAssembler::Branch(SloppyTNode<IntegralT> condition, Label* true_label,
1212 : Label* false_label) {
1213 576171 : true_label->MergeVariables();
1214 576171 : false_label->MergeVariables();
1215 : return raw_assembler()->Branch(condition, true_label->label_,
1216 1152342 : false_label->label_);
1217 : }
1218 :
1219 4234 : void CodeAssembler::Switch(Node* index, Label* default_label,
1220 : const int32_t* case_values, Label** case_labels,
1221 : size_t case_count) {
1222 : RawMachineLabel** labels =
1223 4234 : new (zone()->New(sizeof(RawMachineLabel*) * case_count))
1224 4234 : RawMachineLabel*[case_count];
1225 35093 : for (size_t i = 0; i < case_count; ++i) {
1226 30859 : labels[i] = case_labels[i]->label_;
1227 30859 : case_labels[i]->MergeVariables();
1228 : }
1229 4234 : default_label->MergeVariables();
1230 : return raw_assembler()->Switch(index, default_label->label_, case_values,
1231 8468 : labels, case_count);
1232 : }
1233 :
1234 31 : bool CodeAssembler::UnalignedLoadSupported(MachineRepresentation rep) const {
1235 31 : return raw_assembler()->machine()->UnalignedLoadSupported(rep);
1236 : }
1237 31 : bool CodeAssembler::UnalignedStoreSupported(MachineRepresentation rep) const {
1238 31 : return raw_assembler()->machine()->UnalignedStoreSupported(rep);
1239 : }
1240 :
1241 : // RawMachineAssembler delegate helpers:
1242 889780 : Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
1243 :
1244 4278 : Factory* CodeAssembler::factory() const { return isolate()->factory(); }
1245 :
1246 115360 : Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }
1247 :
1248 0 : RawMachineAssembler* CodeAssembler::raw_assembler() const {
1249 9508893 : return state_->raw_assembler_.get();
1250 : }
1251 :
1252 : // The core implementation of Variable is stored through an indirection so
1253 : // that it can outlive the often block-scoped Variable declarations. This is
1254 : // needed to ensure that variable binding and merging through phis can
1255 : // properly be verified.
1256 : class CodeAssemblerVariable::Impl : public ZoneObject {
1257 : public:
1258 : explicit Impl(MachineRepresentation rep)
1259 : :
1260 : #if DEBUG
1261 : debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
1262 : #endif
1263 : value_(nullptr),
1264 376579 : rep_(rep) {
1265 : }
1266 :
1267 : #if DEBUG
1268 : AssemblerDebugInfo debug_info() const { return debug_info_; }
1269 : void set_debug_info(AssemblerDebugInfo debug_info) {
1270 : debug_info_ = debug_info;
1271 : }
1272 :
1273 : AssemblerDebugInfo debug_info_;
1274 : #endif // DEBUG
1275 : Node* value_;
1276 : MachineRepresentation rep_;
1277 : };
1278 :
1279 376579 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1280 : MachineRepresentation rep)
1281 753158 : : impl_(new (assembler->zone()) Impl(rep)), state_(assembler->state()) {
1282 376579 : state_->variables_.insert(impl_);
1283 376579 : }
1284 :
1285 203864 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1286 : MachineRepresentation rep,
1287 : Node* initial_value)
1288 203864 : : CodeAssemblerVariable(assembler, rep) {
1289 : Bind(initial_value);
1290 203864 : }
1291 :
1292 : #if DEBUG
1293 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1294 : AssemblerDebugInfo debug_info,
1295 : MachineRepresentation rep)
1296 : : impl_(new (assembler->zone()) Impl(rep)), state_(assembler->state()) {
1297 : impl_->set_debug_info(debug_info);
1298 : state_->variables_.insert(impl_);
1299 : }
1300 :
1301 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1302 : AssemblerDebugInfo debug_info,
1303 : MachineRepresentation rep,
1304 : Node* initial_value)
1305 : : CodeAssemblerVariable(assembler, debug_info, rep) {
1306 : impl_->set_debug_info(debug_info);
1307 : Bind(initial_value);
1308 : }
1309 : #endif // DEBUG
1310 :
1311 376579 : CodeAssemblerVariable::~CodeAssemblerVariable() {
1312 376579 : state_->variables_.erase(impl_);
1313 376579 : }
1314 :
1315 760510 : void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
1316 :
1317 794162 : Node* CodeAssemblerVariable::value() const {
1318 : #if DEBUG
1319 : if (!IsBound()) {
1320 : std::stringstream str;
1321 : str << "#Use of unbound variable:"
1322 : << "#\n Variable: " << *this;
1323 : if (state_) {
1324 : str << "#\n Current Block: ";
1325 : state_->PrintCurrentBlock(str);
1326 : }
1327 : FATAL(str.str().c_str());
1328 : }
1329 : #endif // DEBUG
1330 794162 : return impl_->value_;
1331 : }
1332 :
1333 0 : MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
1334 :
1335 78563 : bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
1336 :
1337 0 : std::ostream& operator<<(std::ostream& os,
1338 : const CodeAssemblerVariable::Impl& impl) {
1339 : #if DEBUG
1340 : AssemblerDebugInfo info = impl.debug_info();
1341 : if (info.name) os << "V" << info;
1342 : #endif // DEBUG
1343 0 : return os;
1344 : }
1345 :
1346 0 : std::ostream& operator<<(std::ostream& os,
1347 : const CodeAssemblerVariable& variable) {
1348 : os << *variable.impl_;
1349 0 : return os;
1350 : }
1351 :
1352 1052845 : CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
1353 : size_t vars_count,
1354 : CodeAssemblerVariable* const* vars,
1355 : CodeAssemblerLabel::Type type)
1356 : : bound_(false),
1357 : merge_count_(0),
1358 : state_(assembler->state()),
1359 2105690 : label_(nullptr) {
1360 1052845 : void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
1361 : label_ = new (buffer)
1362 : RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
1363 2105690 : : RawMachineLabel::kNonDeferred);
1364 1206511 : for (size_t i = 0; i < vars_count; ++i) {
1365 153666 : variable_phis_[vars[i]->impl_] = nullptr;
1366 : }
1367 1052845 : }
1368 :
1369 2114618 : CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }
1370 :
1371 1601175 : void CodeAssemblerLabel::MergeVariables() {
1372 1601175 : ++merge_count_;
1373 14170978 : for (CodeAssemblerVariable::Impl* var : state_->variables_) {
1374 : size_t count = 0;
1375 9367453 : Node* node = var->value_;
1376 9367453 : if (node != nullptr) {
1377 : auto i = variable_merges_.find(var);
1378 7087903 : if (i != variable_merges_.end()) {
1379 2594425 : i->second.push_back(node);
1380 : count = i->second.size();
1381 : } else {
1382 : count = 1;
1383 8986956 : variable_merges_[var] = std::vector<Node*>(1, node);
1384 : }
1385 : }
1386 : // If the following asserts, then you've jumped to a label without a bound
1387 : // variable along that path that expects to merge its value into a phi.
1388 : DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
1389 : count == merge_count_);
1390 : USE(count);
1391 :
1392 : // If the label is already bound, we already know the set of variables to
1393 : // merge and phi nodes have already been created.
1394 9367453 : if (bound_) {
1395 : auto phi = variable_phis_.find(var);
1396 437271 : if (phi != variable_phis_.end()) {
1397 : DCHECK_NOT_NULL(phi->second);
1398 247948 : state_->raw_assembler_->AppendPhiInput(phi->second, node);
1399 : } else {
1400 : auto i = variable_merges_.find(var);
1401 : if (i != variable_merges_.end()) {
1402 : // If the following assert fires, then you've declared a variable that
1403 : // has the same bound value along all paths up until the point you
1404 : // bound this label, but then later merged a path with a new value for
1405 : // the variable after the label bind (it's not possible to add phis to
1406 : // the bound label after the fact, just make sure to list the variable
1407 : // in the label's constructor's list of merged variables).
1408 : #if DEBUG
1409 : if (find_if(i->second.begin(), i->second.end(),
1410 : [node](Node* e) -> bool { return node != e; }) !=
1411 : i->second.end()) {
1412 : std::stringstream str;
1413 : str << "Unmerged variable found when jumping to block. \n"
1414 : << "# Variable: " << *var;
1415 : if (bound_) {
1416 : str << "\n# Target block: " << *label_->block();
1417 : }
1418 : str << "\n# Current Block: ";
1419 : state_->PrintCurrentBlock(str);
1420 : FATAL(str.str().c_str());
1421 : }
1422 : #endif // DEBUG
1423 : }
1424 : }
1425 : }
1426 : }
1427 1601175 : }
1428 :
1429 : #if DEBUG
1430 : void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
1431 : if (bound_) {
1432 : std::stringstream str;
1433 : str << "Cannot bind the same label twice:"
1434 : << "\n# current: " << debug_info
1435 : << "\n# previous: " << *label_->block();
1436 : FATAL(str.str().c_str());
1437 : }
1438 : state_->raw_assembler_->Bind(label_, debug_info);
1439 : UpdateVariablesAfterBind();
1440 : }
1441 : #endif // DEBUG
1442 :
1443 1038889 : void CodeAssemblerLabel::Bind() {
1444 : DCHECK(!bound_);
1445 2077778 : state_->raw_assembler_->Bind(label_);
1446 1038889 : UpdateVariablesAfterBind();
1447 1038889 : }
1448 :
1449 1038889 : void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1450 : // Make sure that all variables that have changed along any path up to this
1451 : // point are marked as merge variables.
1452 9103429 : for (auto var : state_->variables_) {
1453 : Node* shared_value = nullptr;
1454 : auto i = variable_merges_.find(var);
1455 5986762 : if (i != variable_merges_.end()) {
1456 15235930 : for (auto value : i->second) {
1457 : DCHECK_NOT_NULL(value);
1458 6412850 : if (value != shared_value) {
1459 4772308 : if (shared_value == nullptr) {
1460 : shared_value = value;
1461 : } else {
1462 360768 : variable_phis_[var] = nullptr;
1463 : }
1464 : }
1465 : }
1466 : }
1467 : }
1468 :
1469 2393041 : for (auto var : variable_phis_) {
1470 315263 : CodeAssemblerVariable::Impl* var_impl = var.first;
1471 : auto i = variable_merges_.find(var_impl);
1472 : #if DEBUG
1473 : bool not_found = i == variable_merges_.end();
1474 : if (not_found || i->second.size() != merge_count_) {
1475 : std::stringstream str;
1476 : str << "A variable that has been marked as beeing merged at the label"
1477 : << "\n# doesn't have a bound value along all of the paths that "
1478 : << "\n# have been merged into the label up to this point."
1479 : << "\n#"
1480 : << "\n# This can happen in the following cases:"
1481 : << "\n# - By explicitly marking it so in the label constructor"
1482 : << "\n# - By having seen different bound values at branches"
1483 : << "\n#"
1484 : << "\n# Merge count: expected=" << merge_count_
1485 : << " vs. found=" << (not_found ? 0 : i->second.size())
1486 : << "\n# Variable: " << *var_impl
1487 : << "\n# Current Block: " << *label_->block();
1488 : FATAL(str.str().c_str());
1489 : }
1490 : #endif // DEBUG
1491 : Node* phi = state_->raw_assembler_->Phi(
1492 630526 : var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
1493 315263 : variable_phis_[var_impl] = phi;
1494 : }
1495 :
1496 : // Bind all variables to a merge phi, the common value along all paths or
1497 : // null.
1498 9103429 : for (auto var : state_->variables_) {
1499 : auto i = variable_phis_.find(var);
1500 5986762 : if (i != variable_phis_.end()) {
1501 315263 : var->value_ = i->second;
1502 : } else {
1503 : auto j = variable_merges_.find(var);
1504 9767776 : if (j != variable_merges_.end() && j->second.size() == merge_count_) {
1505 8082754 : var->value_ = j->second.back();
1506 : } else {
1507 1630122 : var->value_ = nullptr;
1508 : }
1509 : }
1510 : }
1511 :
1512 1038889 : bound_ = true;
1513 1038889 : }
1514 :
1515 : } // namespace compiler
1516 :
1517 0 : Smi* CheckObjectType(Object* value, Smi* type, String* location) {
1518 : #ifdef DEBUG
1519 : const char* expected;
1520 : switch (static_cast<ObjectType>(type->value())) {
1521 : #define TYPE_CASE(Name) \
1522 : case ObjectType::k##Name: \
1523 : if (value->Is##Name()) return Smi::FromInt(0); \
1524 : expected = #Name; \
1525 : break;
1526 : #define TYPE_STRUCT_CASE(NAME, Name, name) \
1527 : case ObjectType::k##Name: \
1528 : if (value->Is##Name()) return Smi::FromInt(0); \
1529 : expected = #Name; \
1530 : break;
1531 :
1532 : TYPE_CASE(Object)
1533 : OBJECT_TYPE_LIST(TYPE_CASE)
1534 : HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
1535 : STRUCT_LIST(TYPE_STRUCT_CASE)
1536 : #undef TYPE_CASE
1537 : #undef TYPE_STRUCT_CASE
1538 : }
1539 : std::stringstream value_description;
1540 : value->Print(value_description);
1541 : V8_Fatal(__FILE__, __LINE__,
1542 : "Type cast failed in %s\n"
1543 : " Expected %s but found %s",
1544 : location->ToAsciiArray(), expected, value_description.str().c_str());
1545 : #else
1546 0 : UNREACHABLE();
1547 : #endif
1548 : }
1549 :
1550 : } // namespace internal
1551 : } // namespace v8
|
