Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-assembler.h"
6 :
7 : #include <ostream>
8 :
9 : #include "src/code-factory.h"
10 : #include "src/compiler/backend/instruction-selector.h"
11 : #include "src/compiler/graph.h"
12 : #include "src/compiler/linkage.h"
13 : #include "src/compiler/node-matchers.h"
14 : #include "src/compiler/pipeline.h"
15 : #include "src/compiler/raw-machine-assembler.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/frames.h"
18 : #include "src/interface-descriptors.h"
19 : #include "src/interpreter/bytecodes.h"
20 : #include "src/machine-type.h"
21 : #include "src/macro-assembler.h"
22 : #include "src/memcopy.h"
23 : #include "src/objects-inl.h"
24 : #include "src/objects/smi.h"
25 : #include "src/zone/zone.h"
26 :
27 : namespace v8 {
28 : namespace internal {
29 :
30 : constexpr MachineType MachineTypeOf<Smi>::value;
31 : constexpr MachineType MachineTypeOf<Object>::value;
32 :
33 : namespace compiler {
34 :
35 : static_assert(std::is_convertible<TNode<Number>, TNode<Object>>::value,
36 : "test subtyping");
37 : static_assert(std::is_convertible<TNode<UnionT<Smi, HeapNumber>>,
38 : TNode<UnionT<Smi, HeapObject>>>::value,
39 : "test subtyping");
40 : static_assert(
41 : !std::is_convertible<TNode<UnionT<Smi, HeapObject>>, TNode<Number>>::value,
42 : "test subtyping");
43 :
44 47555 : CodeAssemblerState::CodeAssemblerState(
45 : Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
46 : Code::Kind kind, const char* name, PoisoningMitigationLevel poisoning_level,
47 : int32_t builtin_index)
48 : // TODO(rmcilroy): Should we use Linkage::GetBytecodeDispatchDescriptor for
49 : // bytecode handlers?
50 : : CodeAssemblerState(
51 : isolate, zone,
52 : Linkage::GetStubCallDescriptor(
53 : zone, descriptor, descriptor.GetStackParameterCount(),
54 : CallDescriptor::kNoFlags, Operator::kNoProperties),
55 47555 : kind, name, poisoning_level, builtin_index) {}
56 :
57 20139 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
58 : int parameter_count, Code::Kind kind,
59 : const char* name,
60 : PoisoningMitigationLevel poisoning_level,
61 : int32_t builtin_index)
62 : : CodeAssemblerState(
63 : isolate, zone,
64 : Linkage::GetJSCallDescriptor(
65 : zone, false, parameter_count,
66 : (kind == Code::BUILTIN ? CallDescriptor::kPushArgumentCount
67 : : CallDescriptor::kNoFlags) |
68 : CallDescriptor::kCanUseRoots),
69 40278 : kind, name, poisoning_level, builtin_index) {}
70 :
71 68404 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
72 : CallDescriptor* call_descriptor,
73 : Code::Kind kind, const char* name,
74 : PoisoningMitigationLevel poisoning_level,
75 : int32_t builtin_index)
76 : : raw_assembler_(new RawMachineAssembler(
77 68404 : isolate, new (zone) Graph(zone), call_descriptor,
78 : MachineType::PointerRepresentation(),
79 : InstructionSelector::SupportedMachineOperatorFlags(),
80 68404 : InstructionSelector::AlignmentRequirements(), poisoning_level)),
81 : kind_(kind),
82 : name_(name),
83 : builtin_index_(builtin_index),
84 : code_generated_(false),
85 205212 : variables_(zone) {}
86 :
87 : CodeAssemblerState::~CodeAssemblerState() = default;
88 :
89 0 : int CodeAssemblerState::parameter_count() const {
90 0 : return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
91 : }
92 :
93 : CodeAssembler::~CodeAssembler() = default;
94 :
95 : #if DEBUG
96 : void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
97 : raw_assembler_->PrintCurrentBlock(os);
98 : }
99 : #endif
100 :
101 4139 : bool CodeAssemblerState::InsideBlock() { return raw_assembler_->InsideBlock(); }
102 :
103 63504 : void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
104 : const char* file,
105 : int line) {
106 : #if DEBUG
107 : AssemblerDebugInfo debug_info = {msg, file, line};
108 : raw_assembler_->SetInitialDebugInformation(debug_info);
109 : #endif // DEBUG
110 63504 : }
111 :
112 0 : class BreakOnNodeDecorator final : public GraphDecorator {
113 : public:
114 0 : explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}
115 :
116 0 : void Decorate(Node* node) final {
117 0 : if (node->id() == node_id_) {
118 0 : base::OS::DebugBreak();
119 : }
120 0 : }
121 :
122 : private:
123 : NodeId node_id_;
124 : };
125 :
126 0 : void CodeAssembler::BreakOnNode(int node_id) {
127 0 : Graph* graph = raw_assembler()->graph();
128 : Zone* zone = graph->zone();
129 : GraphDecorator* decorator =
130 0 : new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
131 0 : graph->AddDecorator(decorator);
132 0 : }
133 :
134 27990 : void CodeAssembler::RegisterCallGenerationCallbacks(
135 : const CodeAssemblerCallback& call_prologue,
136 : const CodeAssemblerCallback& call_epilogue) {
137 : // The callback can be registered only once.
138 : DCHECK(!state_->call_prologue_);
139 : DCHECK(!state_->call_epilogue_);
140 27990 : state_->call_prologue_ = call_prologue;
141 27990 : state_->call_epilogue_ = call_epilogue;
142 27990 : }
143 :
144 27990 : void CodeAssembler::UnregisterCallGenerationCallbacks() {
145 27990 : state_->call_prologue_ = nullptr;
146 27990 : state_->call_epilogue_ = nullptr;
147 27990 : }
148 :
149 0 : void CodeAssembler::CallPrologue() {
150 541534 : if (state_->call_prologue_) {
151 60386 : state_->call_prologue_();
152 : }
153 0 : }
154 :
155 0 : void CodeAssembler::CallEpilogue() {
156 541534 : if (state_->call_epilogue_) {
157 60386 : state_->call_epilogue_();
158 : }
159 0 : }
160 :
161 1176 : bool CodeAssembler::Word32ShiftIsSafe() const {
162 1176 : return raw_assembler()->machine()->Word32ShiftIsSafe();
163 : }
164 :
165 5736 : PoisoningMitigationLevel CodeAssembler::poisoning_level() const {
166 5736 : return raw_assembler()->poisoning_level();
167 : }
168 :
169 : // static
170 66915 : Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state,
171 : const AssemblerOptions& options) {
172 : DCHECK(!state->code_generated_);
173 :
174 133830 : RawMachineAssembler* rasm = state->raw_assembler_.get();
175 :
176 : Handle<Code> code;
177 66915 : Graph* graph = rasm->ExportForOptimization();
178 :
179 : code =
180 : Pipeline::GenerateCodeForCodeStub(
181 : rasm->isolate(), rasm->call_descriptor(), graph, state->kind_,
182 66915 : state->name_, state->builtin_index_, rasm->poisoning_level(), options)
183 133830 : .ToHandleChecked();
184 :
185 66915 : state->code_generated_ = true;
186 66915 : return code;
187 : }
188 :
189 246678 : bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
190 :
191 118 : bool CodeAssembler::IsFloat64RoundUpSupported() const {
192 118 : return raw_assembler()->machine()->Float64RoundUp().IsSupported();
193 : }
194 :
195 125 : bool CodeAssembler::IsFloat64RoundDownSupported() const {
196 125 : return raw_assembler()->machine()->Float64RoundDown().IsSupported();
197 : }
198 :
199 392 : bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
200 392 : return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
201 : }
202 :
203 341 : bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
204 341 : return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
205 : }
206 :
207 0 : bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
208 0 : return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
209 : }
210 :
211 0 : bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
212 56 : return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
213 : }
214 :
215 56 : bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
216 : return Is64() ? IsInt64AbsWithOverflowSupported()
217 112 : : IsInt32AbsWithOverflowSupported();
218 : }
219 :
220 : #ifdef DEBUG
221 : void CodeAssembler::GenerateCheckMaybeObjectIsObject(Node* node,
222 : const char* location) {
223 : Label ok(this);
224 : GotoIf(WordNotEqual(WordAnd(BitcastMaybeObjectToWord(node),
225 : IntPtrConstant(kHeapObjectTagMask)),
226 : IntPtrConstant(kWeakHeapObjectTag)),
227 : &ok);
228 : Node* message_node = StringConstant(location);
229 : DebugAbort(message_node);
230 : Unreachable();
231 : Bind(&ok);
232 : }
233 : #endif
234 :
235 686240 : TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
236 866026 : return UncheckedCast<Int32T>(raw_assembler()->Int32Constant(value));
237 : }
238 :
239 13881 : TNode<Int64T> CodeAssembler::Int64Constant(int64_t value) {
240 13881 : return UncheckedCast<Int64T>(raw_assembler()->Int64Constant(value));
241 : }
242 :
243 3234490 : TNode<IntPtrT> CodeAssembler::IntPtrConstant(intptr_t value) {
244 3234490 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrConstant(value));
245 : }
246 :
247 9708 : TNode<Number> CodeAssembler::NumberConstant(double value) {
248 : int smi_value;
249 9708 : if (DoubleToSmiInteger(value, &smi_value)) {
250 : return UncheckedCast<Number>(SmiConstant(smi_value));
251 : } else {
252 : // We allocate the heap number constant eagerly at this point instead of
253 : // deferring allocation to code generation
254 : // (see AllocateAndInstallRequestedHeapObjects) since that makes it easier
255 : // to generate constant lookups for embedded builtins.
256 : return UncheckedCast<Number>(
257 1746 : HeapConstant(isolate()->factory()->NewHeapNumber(value, TENURED)));
258 : }
259 : }
260 :
261 330905 : TNode<Smi> CodeAssembler::SmiConstant(Smi value) {
262 : return UncheckedCast<Smi>(BitcastWordToTaggedSigned(
263 661810 : IntPtrConstant(static_cast<intptr_t>(value.ptr()))));
264 : }
265 :
266 304657 : TNode<Smi> CodeAssembler::SmiConstant(int value) {
267 312619 : return SmiConstant(Smi::FromInt(value));
268 : }
269 :
270 148629 : TNode<HeapObject> CodeAssembler::UntypedHeapConstant(
271 : Handle<HeapObject> object) {
272 748099 : return UncheckedCast<HeapObject>(raw_assembler()->HeapConstant(object));
273 : }
274 :
275 18903 : TNode<String> CodeAssembler::StringConstant(const char* str) {
276 : Handle<String> internalized_string =
277 18903 : factory()->InternalizeOneByteString(OneByteVector(str));
278 18903 : return UncheckedCast<String>(HeapConstant(internalized_string));
279 : }
280 :
281 339 : TNode<Oddball> CodeAssembler::BooleanConstant(bool value) {
282 678 : Handle<Object> object = isolate()->factory()->ToBoolean(value);
283 : return UncheckedCast<Oddball>(
284 678 : raw_assembler()->HeapConstant(Handle<HeapObject>::cast(object)));
285 : }
286 :
287 62990 : TNode<ExternalReference> CodeAssembler::ExternalConstant(
288 : ExternalReference address) {
289 : return UncheckedCast<ExternalReference>(
290 220918 : raw_assembler()->ExternalConstant(address));
291 : }
292 :
293 14009 : TNode<Float64T> CodeAssembler::Float64Constant(double value) {
294 14009 : return UncheckedCast<Float64T>(raw_assembler()->Float64Constant(value));
295 : }
296 :
297 0 : TNode<HeapNumber> CodeAssembler::NaNConstant() {
298 0 : return UncheckedCast<HeapNumber>(LoadRoot(RootIndex::kNanValue));
299 : }
300 :
301 2166941 : bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
302 : {
303 : Int64Matcher m(node);
304 2357167 : if (m.HasValue() && m.IsInRange(std::numeric_limits<int32_t>::min(),
305 : std::numeric_limits<int32_t>::max())) {
306 190221 : out_value = static_cast<int32_t>(m.Value());
307 : return true;
308 : }
309 : }
310 :
311 : {
312 : Int32Matcher m(node);
313 1976720 : if (m.HasValue()) {
314 0 : out_value = m.Value();
315 : return true;
316 : }
317 : }
318 :
319 1976720 : return false;
320 : }
321 :
322 25 : bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
323 : Int64Matcher m(node);
324 5197 : if (m.HasValue()) out_value = m.Value();
325 25 : return m.HasValue();
326 : }
327 :
328 26642 : bool CodeAssembler::ToSmiConstant(Node* node, Smi* out_value) {
329 26642 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
330 : node = node->InputAt(0);
331 : }
332 : IntPtrMatcher m(node);
333 26642 : if (m.HasValue()) {
334 : intptr_t value = m.Value();
335 : // Make sure that the value is actually a smi
336 2081 : CHECK_EQ(0, value & ((static_cast<intptr_t>(1) << kSmiShiftSize) - 1));
337 2081 : *out_value = Smi(static_cast<Address>(value));
338 2081 : return true;
339 : }
340 : return false;
341 : }
342 :
343 4829223 : bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
344 4829223 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
345 : node->opcode() == IrOpcode::kBitcastWordToTagged) {
346 : node = node->InputAt(0);
347 : }
348 : IntPtrMatcher m(node);
349 4829223 : if (m.HasValue()) out_value = m.Value();
350 4829223 : return m.HasValue();
351 : }
352 :
353 1008 : bool CodeAssembler::IsUndefinedConstant(TNode<Object> node) {
354 : compiler::HeapObjectMatcher m(node);
355 1008 : return m.Is(isolate()->factory()->undefined_value());
356 : }
357 :
358 616 : bool CodeAssembler::IsNullConstant(TNode<Object> node) {
359 : compiler::HeapObjectMatcher m(node);
360 616 : return m.Is(isolate()->factory()->null_value());
361 : }
362 :
363 264561 : Node* CodeAssembler::Parameter(int index) {
364 264785 : if (index == kTargetParameterIndex) return raw_assembler()->TargetParameter();
365 528674 : return raw_assembler()->Parameter(index);
366 : }
367 :
368 0 : bool CodeAssembler::IsJSFunctionCall() const {
369 0 : auto call_descriptor = raw_assembler()->call_descriptor();
370 0 : return call_descriptor->IsJSFunctionCall();
371 : }
372 :
373 18984 : TNode<Context> CodeAssembler::GetJSContextParameter() {
374 18984 : auto call_descriptor = raw_assembler()->call_descriptor();
375 : DCHECK(call_descriptor->IsJSFunctionCall());
376 37968 : return CAST(Parameter(Linkage::GetJSCallContextParamIndex(
377 : static_cast<int>(call_descriptor->JSParameterCount()))));
378 : }
379 :
380 61613 : void CodeAssembler::Return(SloppyTNode<Object> value) {
381 61837 : return raw_assembler()->Return(value);
382 : }
383 :
384 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
385 : SloppyTNode<Object> value2) {
386 0 : return raw_assembler()->Return(value1, value2);
387 : }
388 :
389 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
390 : SloppyTNode<Object> value2,
391 : SloppyTNode<Object> value3) {
392 0 : return raw_assembler()->Return(value1, value2, value3);
393 : }
394 :
395 19630 : void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
396 19630 : return raw_assembler()->PopAndReturn(pop, value);
397 : }
398 :
399 224 : void CodeAssembler::ReturnIf(Node* condition, Node* value) {
400 224 : Label if_return(this), if_continue(this);
401 224 : Branch(condition, &if_return, &if_continue);
402 : Bind(&if_return);
403 : Return(value);
404 224 : Bind(&if_continue);
405 224 : }
406 :
407 336 : void CodeAssembler::ReturnRaw(Node* value) {
408 336 : return raw_assembler()->Return(value);
409 : }
410 :
411 525 : void CodeAssembler::DebugAbort(Node* message) {
412 525 : raw_assembler()->DebugAbort(message);
413 525 : }
414 :
415 70627 : void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
416 :
417 64131 : void CodeAssembler::Unreachable() {
418 : DebugBreak();
419 64131 : raw_assembler()->Unreachable();
420 64131 : }
421 :
422 5 : void CodeAssembler::Comment(std::string str) {
423 10 : if (!FLAG_code_comments) return;
424 15 : raw_assembler()->Comment(str);
425 : }
426 :
427 3353679 : void CodeAssembler::Bind(Label* label) { return label->Bind(); }
428 :
429 : #if DEBUG
430 : void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
431 : return label->Bind(debug_info);
432 : }
433 : #endif // DEBUG
434 :
435 7014 : Node* CodeAssembler::LoadFramePointer() {
436 7014 : return raw_assembler()->LoadFramePointer();
437 : }
438 :
439 28496 : Node* CodeAssembler::LoadParentFramePointer() {
440 28496 : return raw_assembler()->LoadParentFramePointer();
441 : }
442 :
443 19152 : Node* CodeAssembler::LoadStackPointer() {
444 19152 : return raw_assembler()->LoadStackPointer();
445 : }
446 :
447 19112 : TNode<Object> CodeAssembler::TaggedPoisonOnSpeculation(
448 : SloppyTNode<Object> value) {
449 : return UncheckedCast<Object>(
450 19112 : raw_assembler()->TaggedPoisonOnSpeculation(value));
451 : }
452 :
453 46536 : TNode<WordT> CodeAssembler::WordPoisonOnSpeculation(SloppyTNode<WordT> value) {
454 46536 : return UncheckedCast<WordT>(raw_assembler()->WordPoisonOnSpeculation(value));
455 : }
456 :
457 : #define DEFINE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
458 : TNode<ResType> CodeAssembler::name(SloppyTNode<Arg1Type> a, \
459 : SloppyTNode<Arg2Type> b) { \
460 : return UncheckedCast<ResType>(raw_assembler()->name(a, b)); \
461 : }
462 666574 : CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
463 : #undef DEFINE_CODE_ASSEMBLER_BINARY_OP
464 :
465 683559 : TNode<WordT> CodeAssembler::IntPtrAdd(SloppyTNode<WordT> left,
466 : SloppyTNode<WordT> right) {
467 : intptr_t left_constant;
468 683559 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
469 : intptr_t right_constant;
470 683559 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
471 683559 : if (is_left_constant) {
472 286788 : if (is_right_constant) {
473 39716 : return IntPtrConstant(left_constant + right_constant);
474 : }
475 266930 : if (left_constant == 0) {
476 41876 : return right;
477 : }
478 396771 : } else if (is_right_constant) {
479 287737 : if (right_constant == 0) {
480 5229 : return left;
481 : }
482 : }
483 : return UncheckedCast<WordT>(raw_assembler()->IntPtrAdd(left, right));
484 : }
485 :
486 509 : TNode<IntPtrT> CodeAssembler::IntPtrDiv(TNode<IntPtrT> left,
487 : TNode<IntPtrT> right) {
488 : intptr_t left_constant;
489 509 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
490 : intptr_t right_constant;
491 509 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
492 509 : if (is_right_constant) {
493 61 : if (is_left_constant) {
494 2 : return IntPtrConstant(left_constant / right_constant);
495 : }
496 118 : if (base::bits::IsPowerOfTwo(right_constant)) {
497 58 : return WordSar(left, WhichPowerOf2(right_constant));
498 : }
499 : }
500 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrDiv(left, right));
501 : }
502 :
503 119829 : TNode<WordT> CodeAssembler::IntPtrSub(SloppyTNode<WordT> left,
504 : SloppyTNode<WordT> right) {
505 : intptr_t left_constant;
506 119829 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
507 : intptr_t right_constant;
508 119829 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
509 119829 : if (is_left_constant) {
510 19299 : if (is_right_constant) {
511 19090 : return IntPtrConstant(left_constant - right_constant);
512 : }
513 100530 : } else if (is_right_constant) {
514 86470 : if (right_constant == 0) {
515 9743 : return left;
516 : }
517 : }
518 100541 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrSub(left, right));
519 : }
520 :
521 55776 : TNode<WordT> CodeAssembler::IntPtrMul(SloppyTNode<WordT> left,
522 : SloppyTNode<WordT> right) {
523 : intptr_t left_constant;
524 55776 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
525 : intptr_t right_constant;
526 55776 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
527 55776 : if (is_left_constant) {
528 2863 : if (is_right_constant) {
529 5722 : return IntPtrConstant(left_constant * right_constant);
530 : }
531 4 : if (base::bits::IsPowerOfTwo(left_constant)) {
532 2 : return WordShl(right, WhichPowerOf2(left_constant));
533 : }
534 52913 : } else if (is_right_constant) {
535 104482 : if (base::bits::IsPowerOfTwo(right_constant)) {
536 18558 : return WordShl(left, WhichPowerOf2(right_constant));
537 : }
538 : }
539 34355 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrMul(left, right));
540 : }
541 :
542 113599 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> value, int shift) {
543 218211 : return (shift != 0) ? WordShl(value, IntPtrConstant(shift)) : value;
544 : }
545 :
546 26516 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> value, int shift) {
547 48439 : return (shift != 0) ? WordShr(value, IntPtrConstant(shift)) : value;
548 : }
549 :
550 2705 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> value, int shift) {
551 5409 : return (shift != 0) ? WordSar(value, IntPtrConstant(shift)) : value;
552 : }
553 :
554 37422 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> value, int shift) {
555 67447 : return (shift != 0) ? Word32Shr(value, Int32Constant(shift)) : value;
556 : }
557 :
558 43261 : TNode<WordT> CodeAssembler::WordOr(SloppyTNode<WordT> left,
559 : SloppyTNode<WordT> right) {
560 : intptr_t left_constant;
561 43261 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
562 : intptr_t right_constant;
563 43261 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
564 43261 : if (is_left_constant) {
565 4930 : if (is_right_constant) {
566 9858 : return IntPtrConstant(left_constant | right_constant);
567 : }
568 1 : if (left_constant == 0) {
569 1 : return right;
570 : }
571 38331 : } else if (is_right_constant) {
572 2522 : if (right_constant == 0) {
573 1 : return left;
574 : }
575 : }
576 38330 : return UncheckedCast<WordT>(raw_assembler()->WordOr(left, right));
577 : }
578 :
579 298851 : TNode<WordT> CodeAssembler::WordAnd(SloppyTNode<WordT> left,
580 : SloppyTNode<WordT> right) {
581 : intptr_t left_constant;
582 298851 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
583 : intptr_t right_constant;
584 298851 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
585 298851 : if (is_left_constant) {
586 393 : if (is_right_constant) {
587 786 : return IntPtrConstant(left_constant & right_constant);
588 : }
589 : }
590 298458 : return UncheckedCast<WordT>(raw_assembler()->WordAnd(left, right));
591 : }
592 :
593 2 : TNode<WordT> CodeAssembler::WordXor(SloppyTNode<WordT> left,
594 : SloppyTNode<WordT> right) {
595 : intptr_t left_constant;
596 2 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
597 : intptr_t right_constant;
598 2 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
599 2 : if (is_left_constant) {
600 1 : if (is_right_constant) {
601 2 : return IntPtrConstant(left_constant ^ right_constant);
602 : }
603 : }
604 1 : return UncheckedCast<WordT>(raw_assembler()->WordXor(left, right));
605 : }
606 :
607 413100 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> left,
608 : SloppyTNode<IntegralT> right) {
609 : intptr_t left_constant;
610 413100 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
611 : intptr_t right_constant;
612 413100 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
613 413100 : if (is_left_constant) {
614 13675 : if (is_right_constant) {
615 27238 : return IntPtrConstant(left_constant << right_constant);
616 : }
617 399425 : } else if (is_right_constant) {
618 399313 : if (right_constant == 0) {
619 0 : return left;
620 : }
621 : }
622 399481 : return UncheckedCast<WordT>(raw_assembler()->WordShl(left, right));
623 : }
624 :
625 27268 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> left,
626 : SloppyTNode<IntegralT> right) {
627 : intptr_t left_constant;
628 27268 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
629 : intptr_t right_constant;
630 27268 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
631 27268 : if (is_left_constant) {
632 5378 : if (is_right_constant) {
633 5378 : return IntPtrConstant(static_cast<uintptr_t>(left_constant) >>
634 10756 : right_constant);
635 : }
636 21890 : } else if (is_right_constant) {
637 21778 : if (right_constant == 0) {
638 0 : return left;
639 : }
640 : }
641 21890 : return UncheckedCast<WordT>(raw_assembler()->WordShr(left, right));
642 : }
643 :
644 157914 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> left,
645 : SloppyTNode<IntegralT> right) {
646 : intptr_t left_constant;
647 157914 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
648 : intptr_t right_constant;
649 157914 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
650 157914 : if (is_left_constant) {
651 2 : if (is_right_constant) {
652 4 : return IntPtrConstant(left_constant >> right_constant);
653 : }
654 157912 : } else if (is_right_constant) {
655 157912 : if (right_constant == 0) {
656 1 : return left;
657 : }
658 : }
659 157911 : return UncheckedCast<WordT>(raw_assembler()->WordSar(left, right));
660 : }
661 :
662 27516 : TNode<Word32T> CodeAssembler::Word32Or(SloppyTNode<Word32T> left,
663 : SloppyTNode<Word32T> right) {
664 : int32_t left_constant;
665 27516 : bool is_left_constant = ToInt32Constant(left, left_constant);
666 : int32_t right_constant;
667 27516 : bool is_right_constant = ToInt32Constant(right, right_constant);
668 27516 : if (is_left_constant) {
669 0 : if (is_right_constant) {
670 0 : return Int32Constant(left_constant | right_constant);
671 : }
672 0 : if (left_constant == 0) {
673 0 : return right;
674 : }
675 27516 : } else if (is_right_constant) {
676 56 : if (right_constant == 0) {
677 0 : return left;
678 : }
679 : }
680 27516 : return UncheckedCast<Word32T>(raw_assembler()->Word32Or(left, right));
681 : }
682 :
683 134177 : TNode<Word32T> CodeAssembler::Word32And(SloppyTNode<Word32T> left,
684 : SloppyTNode<Word32T> right) {
685 : int32_t left_constant;
686 134177 : bool is_left_constant = ToInt32Constant(left, left_constant);
687 : int32_t right_constant;
688 134177 : bool is_right_constant = ToInt32Constant(right, right_constant);
689 134177 : if (is_left_constant) {
690 15 : if (is_right_constant) {
691 10 : return Int32Constant(left_constant & right_constant);
692 : }
693 : }
694 134172 : return UncheckedCast<Word32T>(raw_assembler()->Word32And(left, right));
695 : }
696 :
697 2194 : TNode<Word32T> CodeAssembler::Word32Xor(SloppyTNode<Word32T> left,
698 : SloppyTNode<Word32T> right) {
699 : int32_t left_constant;
700 2194 : bool is_left_constant = ToInt32Constant(left, left_constant);
701 : int32_t right_constant;
702 2194 : bool is_right_constant = ToInt32Constant(right, right_constant);
703 2194 : if (is_left_constant) {
704 5 : if (is_right_constant) {
705 10 : return Int32Constant(left_constant ^ right_constant);
706 : }
707 : }
708 2189 : return UncheckedCast<Word32T>(raw_assembler()->Word32Xor(left, right));
709 : }
710 :
711 9856 : TNode<Word32T> CodeAssembler::Word32Shl(SloppyTNode<Word32T> left,
712 : SloppyTNode<Word32T> right) {
713 : int32_t left_constant;
714 9856 : bool is_left_constant = ToInt32Constant(left, left_constant);
715 : int32_t right_constant;
716 9856 : bool is_right_constant = ToInt32Constant(right, right_constant);
717 9856 : if (is_left_constant) {
718 0 : if (is_right_constant) {
719 0 : return Int32Constant(left_constant << right_constant);
720 : }
721 9856 : } else if (is_right_constant) {
722 6216 : if (right_constant == 0) {
723 0 : return left;
724 : }
725 : }
726 9856 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shl(left, right));
727 : }
728 :
729 40980 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> left,
730 : SloppyTNode<Word32T> right) {
731 : int32_t left_constant;
732 40980 : bool is_left_constant = ToInt32Constant(left, left_constant);
733 : int32_t right_constant;
734 40980 : bool is_right_constant = ToInt32Constant(right, right_constant);
735 40980 : if (is_left_constant) {
736 0 : if (is_right_constant) {
737 0 : return Int32Constant(static_cast<uint32_t>(left_constant) >>
738 0 : right_constant);
739 : }
740 40980 : } else if (is_right_constant) {
741 38012 : if (right_constant == 0) {
742 0 : return left;
743 : }
744 : }
745 40980 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shr(left, right));
746 : }
747 :
748 392 : TNode<Word32T> CodeAssembler::Word32Sar(SloppyTNode<Word32T> left,
749 : SloppyTNode<Word32T> right) {
750 : int32_t left_constant;
751 392 : bool is_left_constant = ToInt32Constant(left, left_constant);
752 : int32_t right_constant;
753 392 : bool is_right_constant = ToInt32Constant(right, right_constant);
754 392 : if (is_left_constant) {
755 0 : if (is_right_constant) {
756 0 : return Int32Constant(left_constant >> right_constant);
757 : }
758 392 : } else if (is_right_constant) {
759 0 : if (right_constant == 0) {
760 0 : return left;
761 : }
762 : }
763 392 : return UncheckedCast<Word32T>(raw_assembler()->Word32Sar(left, right));
764 : }
765 :
766 0 : TNode<Word64T> CodeAssembler::Word64Or(SloppyTNode<Word64T> left,
767 : SloppyTNode<Word64T> right) {
768 : int64_t left_constant;
769 : bool is_left_constant = ToInt64Constant(left, left_constant);
770 : int64_t right_constant;
771 : bool is_right_constant = ToInt64Constant(right, right_constant);
772 0 : if (is_left_constant) {
773 0 : if (is_right_constant) {
774 0 : return Int64Constant(left_constant | right_constant);
775 : }
776 0 : if (left_constant == 0) {
777 0 : return right;
778 : }
779 0 : } else if (is_right_constant) {
780 0 : if (right_constant == 0) {
781 0 : return left;
782 : }
783 : }
784 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Or(left, right));
785 : }
786 :
787 0 : TNode<Word64T> CodeAssembler::Word64And(SloppyTNode<Word64T> left,
788 : SloppyTNode<Word64T> right) {
789 : int64_t left_constant;
790 : bool is_left_constant = ToInt64Constant(left, left_constant);
791 : int64_t right_constant;
792 : bool is_right_constant = ToInt64Constant(right, right_constant);
793 0 : if (is_left_constant) {
794 0 : if (is_right_constant) {
795 0 : return Int64Constant(left_constant & right_constant);
796 : }
797 : }
798 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64And(left, right));
799 : }
800 :
801 0 : TNode<Word64T> CodeAssembler::Word64Xor(SloppyTNode<Word64T> left,
802 : SloppyTNode<Word64T> right) {
803 : int64_t left_constant;
804 : bool is_left_constant = ToInt64Constant(left, left_constant);
805 : int64_t right_constant;
806 : bool is_right_constant = ToInt64Constant(right, right_constant);
807 0 : if (is_left_constant) {
808 0 : if (is_right_constant) {
809 0 : return Int64Constant(left_constant ^ right_constant);
810 : }
811 : }
812 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Xor(left, right));
813 : }
814 :
815 0 : TNode<Word64T> CodeAssembler::Word64Shl(SloppyTNode<Word64T> left,
816 : SloppyTNode<Word64T> right) {
817 : int64_t left_constant;
818 : bool is_left_constant = ToInt64Constant(left, left_constant);
819 : int64_t right_constant;
820 : bool is_right_constant = ToInt64Constant(right, right_constant);
821 0 : if (is_left_constant) {
822 0 : if (is_right_constant) {
823 0 : return Int64Constant(left_constant << right_constant);
824 : }
825 0 : } else if (is_right_constant) {
826 0 : if (right_constant == 0) {
827 0 : return left;
828 : }
829 : }
830 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shl(left, right));
831 : }
832 :
833 0 : TNode<Word64T> CodeAssembler::Word64Shr(SloppyTNode<Word64T> left,
834 : SloppyTNode<Word64T> right) {
835 : int64_t left_constant;
836 : bool is_left_constant = ToInt64Constant(left, left_constant);
837 : int64_t right_constant;
838 : bool is_right_constant = ToInt64Constant(right, right_constant);
839 0 : if (is_left_constant) {
840 0 : if (is_right_constant) {
841 0 : return Int64Constant(static_cast<uint64_t>(left_constant) >>
842 0 : right_constant);
843 : }
844 0 : } else if (is_right_constant) {
845 0 : if (right_constant == 0) {
846 0 : return left;
847 : }
848 : }
849 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shr(left, right));
850 : }
851 :
852 0 : TNode<Word64T> CodeAssembler::Word64Sar(SloppyTNode<Word64T> left,
853 : SloppyTNode<Word64T> right) {
854 : int64_t left_constant;
855 : bool is_left_constant = ToInt64Constant(left, left_constant);
856 : int64_t right_constant;
857 : bool is_right_constant = ToInt64Constant(right, right_constant);
858 0 : if (is_left_constant) {
859 0 : if (is_right_constant) {
860 0 : return Int64Constant(left_constant >> right_constant);
861 : }
862 0 : } else if (is_right_constant) {
863 0 : if (right_constant == 0) {
864 0 : return left;
865 : }
866 : }
867 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Sar(left, right));
868 : }
869 :
870 : #define CODE_ASSEMBLER_COMPARE(Name, ArgT, VarT, ToConstant, op) \
871 : TNode<BoolT> CodeAssembler::Name(SloppyTNode<ArgT> left, \
872 : SloppyTNode<ArgT> right) { \
873 : VarT lhs, rhs; \
874 : if (ToConstant(left, lhs) && ToConstant(right, rhs)) { \
875 : return BoolConstant(lhs op rhs); \
876 : } \
877 : return UncheckedCast<BoolT>(raw_assembler()->Name(left, right)); \
878 : }
879 :
880 7336 : CODE_ASSEMBLER_COMPARE(IntPtrEqual, WordT, intptr_t, ToIntPtrConstant, ==)
881 1197465 : CODE_ASSEMBLER_COMPARE(WordEqual, WordT, intptr_t, ToIntPtrConstant, ==)
882 205877 : CODE_ASSEMBLER_COMPARE(WordNotEqual, WordT, intptr_t, ToIntPtrConstant, !=)
883 408232 : CODE_ASSEMBLER_COMPARE(Word32Equal, Word32T, int32_t, ToInt32Constant, ==)
884 162184 : CODE_ASSEMBLER_COMPARE(Word32NotEqual, Word32T, int32_t, ToInt32Constant, !=)
885 15516 : CODE_ASSEMBLER_COMPARE(Word64Equal, Word64T, int64_t, ToInt64Constant, ==)
886 0 : CODE_ASSEMBLER_COMPARE(Word64NotEqual, Word64T, int64_t, ToInt64Constant, !=)
887 : #undef CODE_ASSEMBLER_COMPARE
888 :
889 114090 : TNode<UintPtrT> CodeAssembler::ChangeUint32ToWord(SloppyTNode<Word32T> value) {
890 114090 : if (raw_assembler()->machine()->Is64()) {
891 : return UncheckedCast<UintPtrT>(
892 114090 : raw_assembler()->ChangeUint32ToUint64(value));
893 : }
894 : return ReinterpretCast<UintPtrT>(value);
895 : }
896 :
897 98420 : TNode<IntPtrT> CodeAssembler::ChangeInt32ToIntPtr(SloppyTNode<Word32T> value) {
898 98420 : if (raw_assembler()->machine()->Is64()) {
899 98420 : return ReinterpretCast<IntPtrT>(raw_assembler()->ChangeInt32ToInt64(value));
900 : }
901 : return ReinterpretCast<IntPtrT>(value);
902 : }
903 :
904 1904 : TNode<UintPtrT> CodeAssembler::ChangeFloat64ToUintPtr(
905 : SloppyTNode<Float64T> value) {
906 1904 : if (raw_assembler()->machine()->Is64()) {
907 : return ReinterpretCast<UintPtrT>(
908 1904 : raw_assembler()->ChangeFloat64ToUint64(value));
909 : }
910 : return ReinterpretCast<UintPtrT>(
911 0 : raw_assembler()->ChangeFloat64ToUint32(value));
912 : }
913 :
914 1736 : TNode<Float64T> CodeAssembler::ChangeUintPtrToFloat64(TNode<UintPtrT> value) {
915 1736 : if (raw_assembler()->machine()->Is64()) {
916 : // TODO(turbofan): Maybe we should introduce a ChangeUint64ToFloat64
917 : // machine operator to TurboFan here?
918 : return ReinterpretCast<Float64T>(
919 1736 : raw_assembler()->RoundUint64ToFloat64(value));
920 : }
921 : return ReinterpretCast<Float64T>(
922 0 : raw_assembler()->ChangeUint32ToFloat64(value));
923 : }
924 :
925 1186 : Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
926 1186 : if (raw_assembler()->machine()->Is64()) {
927 1186 : return raw_assembler()->RoundInt64ToFloat64(value);
928 : }
929 0 : return raw_assembler()->ChangeInt32ToFloat64(value);
930 : }
931 :
932 : #define DEFINE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
933 : TNode<ResType> CodeAssembler::name(SloppyTNode<ArgType> a) { \
934 : return UncheckedCast<ResType>(raw_assembler()->name(a)); \
935 : }
936 2651745 : CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
937 : #undef DEFINE_CODE_ASSEMBLER_UNARY_OP
938 :
939 66767 : Node* CodeAssembler::Load(MachineType rep, Node* base,
940 : LoadSensitivity needs_poisoning) {
941 66767 : return raw_assembler()->Load(rep, base, needs_poisoning);
942 : }
943 :
944 1433135 : Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset,
945 : LoadSensitivity needs_poisoning) {
946 1433476 : return raw_assembler()->Load(rep, base, offset, needs_poisoning);
947 : }
948 :
949 448 : Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
950 448 : return raw_assembler()->AtomicLoad(rep, base, offset);
951 : }
952 :
953 422695 : TNode<Object> CodeAssembler::LoadRoot(RootIndex root_index) {
954 422695 : if (RootsTable::IsImmortalImmovable(root_index)) {
955 : Handle<Object> root = isolate()->root_handle(root_index);
956 844708 : if (root->IsSmi()) {
957 0 : return SmiConstant(Smi::cast(*root));
958 : } else {
959 844708 : return HeapConstant(Handle<HeapObject>::cast(root));
960 : }
961 : }
962 :
963 : // TODO(jgruber): In theory we could generate better code for this by
964 : // letting the macro assembler decide how to load from the roots list. In most
965 : // cases, it would boil down to loading from a fixed kRootRegister offset.
966 : Node* isolate_root =
967 341 : ExternalConstant(ExternalReference::isolate_root(isolate()));
968 : int offset = IsolateData::root_slot_offset(root_index);
969 : return UncheckedCast<Object>(
970 341 : Load(MachineType::AnyTagged(), isolate_root, IntPtrConstant(offset)));
971 : }
972 :
973 0 : Node* CodeAssembler::Store(Node* base, Node* value) {
974 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
975 0 : kFullWriteBarrier);
976 : }
977 :
978 249458 : void CodeAssembler::OptimizedStoreField(MachineRepresentation rep,
979 : TNode<HeapObject> object, int offset,
980 : Node* value,
981 : WriteBarrierKind write_barrier) {
982 : raw_assembler()->OptimizedStoreField(rep, object, offset, value,
983 498916 : write_barrier);
984 249458 : }
985 10715 : void CodeAssembler::OptimizedStoreMap(TNode<HeapObject> object,
986 : TNode<Map> map) {
987 10715 : raw_assembler()->OptimizedStoreMap(object, map);
988 10715 : }
989 :
990 28966 : Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
991 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
992 28966 : value, kFullWriteBarrier);
993 : }
994 :
995 10149 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
996 : Node* value) {
997 10149 : return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
998 : }
999 :
1000 224472 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
1001 : Node* offset, Node* value) {
1002 224584 : return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
1003 : }
1004 :
1005 224 : Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
1006 : Node* offset, Node* value, Node* value_high) {
1007 224 : return raw_assembler()->AtomicStore(rep, base, offset, value, value_high);
1008 : }
1009 :
1010 : #define ATOMIC_FUNCTION(name) \
1011 : Node* CodeAssembler::Atomic##name(MachineType type, Node* base, \
1012 : Node* offset, Node* value, \
1013 : Node* value_high) { \
1014 : return raw_assembler()->Atomic##name(type, base, offset, value, \
1015 : value_high); \
1016 : }
1017 896 : ATOMIC_FUNCTION(Exchange);
1018 896 : ATOMIC_FUNCTION(Add);
1019 896 : ATOMIC_FUNCTION(Sub);
1020 896 : ATOMIC_FUNCTION(And);
1021 896 : ATOMIC_FUNCTION(Or);
1022 896 : ATOMIC_FUNCTION(Xor);
1023 : #undef ATOMIC_FUNCTION
1024 :
1025 448 : Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
1026 : Node* offset, Node* old_value,
1027 : Node* new_value,
1028 : Node* old_value_high,
1029 : Node* new_value_high) {
1030 : return raw_assembler()->AtomicCompareExchange(
1031 448 : type, base, offset, old_value, old_value_high, new_value, new_value_high);
1032 : }
1033 :
1034 112 : Node* CodeAssembler::StoreRoot(RootIndex root_index, Node* value) {
1035 : DCHECK(!RootsTable::IsImmortalImmovable(root_index));
1036 : Node* isolate_root =
1037 112 : ExternalConstant(ExternalReference::isolate_root(isolate()));
1038 : int offset = IsolateData::root_slot_offset(root_index);
1039 : return StoreNoWriteBarrier(MachineRepresentation::kTagged, isolate_root,
1040 224 : IntPtrConstant(offset), value);
1041 : }
1042 :
1043 0 : Node* CodeAssembler::Retain(Node* value) {
1044 0 : return raw_assembler()->Retain(value);
1045 : }
1046 :
1047 19284 : Node* CodeAssembler::Projection(int index, Node* value) {
1048 : DCHECK_LT(index, value->op()->ValueOutputCount());
1049 19284 : return raw_assembler()->Projection(index, value);
1050 : }
1051 :
1052 7692 : void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
1053 : Variable* exception_var) {
1054 7692 : if (if_exception == nullptr) {
1055 : // If no handler is supplied, don't add continuations
1056 3528 : return;
1057 : }
1058 :
1059 : // No catch handlers should be active if we're using catch labels
1060 : DCHECK_EQ(state()->exception_handler_labels_.size(), 0);
1061 : DCHECK(!node->op()->HasProperty(Operator::kNoThrow));
1062 :
1063 4164 : Label success(this), exception(this, Label::kDeferred);
1064 4164 : success.MergeVariables();
1065 4164 : exception.MergeVariables();
1066 :
1067 8328 : raw_assembler()->Continuations(node, success.label_, exception.label_);
1068 :
1069 : Bind(&exception);
1070 4164 : const Operator* op = raw_assembler()->common()->IfException();
1071 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
1072 4164 : if (exception_var != nullptr) {
1073 : exception_var->Bind(exception_value);
1074 : }
1075 4164 : Goto(if_exception);
1076 :
1077 : Bind(&success);
1078 8328 : raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1079 : }
1080 :
1081 63021 : TNode<HeapObject> CodeAssembler::OptimizedAllocate(TNode<IntPtrT> size,
1082 : PretenureFlag pretenure) {
1083 : return UncheckedCast<HeapObject>(
1084 63021 : raw_assembler()->OptimizedAllocate(size, pretenure));
1085 : }
1086 :
1087 278963 : void CodeAssembler::HandleException(Node* node) {
1088 804105 : if (state_->exception_handler_labels_.size() == 0) return;
1089 : CodeAssemblerExceptionHandlerLabel* label =
1090 8196 : state_->exception_handler_labels_.back();
1091 :
1092 8196 : if (node->op()->HasProperty(Operator::kNoThrow)) {
1093 : return;
1094 : }
1095 :
1096 8196 : Label success(this), exception(this, Label::kDeferred);
1097 8196 : success.MergeVariables();
1098 8196 : exception.MergeVariables();
1099 :
1100 16392 : raw_assembler()->Continuations(node, success.label_, exception.label_);
1101 :
1102 : Bind(&exception);
1103 8196 : const Operator* op = raw_assembler()->common()->IfException();
1104 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
1105 8196 : label->AddInputs({UncheckedCast<Object>(exception_value)});
1106 8196 : Goto(label->plain_label());
1107 :
1108 : Bind(&success);
1109 16392 : raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1110 : }
1111 :
1112 : namespace {
1113 : template <size_t kMaxSize>
1114 285785 : class NodeArray {
1115 : public:
1116 : void Add(Node* node) {
1117 : DCHECK_GT(kMaxSize, size());
1118 1292940 : *ptr_++ = node;
1119 : }
1120 :
1121 : Node* const* data() const { return arr_; }
1122 285785 : int size() const { return static_cast<int>(ptr_ - arr_); }
1123 :
1124 : private:
1125 : Node* arr_[kMaxSize];
1126 : Node** ptr_ = arr_;
1127 : };
1128 : } // namespace
1129 :
1130 148729 : TNode<Object> CodeAssembler::CallRuntimeImpl(
1131 : Runtime::FunctionId function, TNode<Object> context,
1132 : std::initializer_list<TNode<Object>> args) {
1133 148729 : int result_size = Runtime::FunctionForId(function)->result_size;
1134 : TNode<Code> centry =
1135 297458 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1136 148729 : return CallRuntimeWithCEntryImpl(function, centry, context, args);
1137 : }
1138 :
1139 148953 : TNode<Object> CodeAssembler::CallRuntimeWithCEntryImpl(
1140 : Runtime::FunctionId function, TNode<Code> centry, TNode<Object> context,
1141 : std::initializer_list<TNode<Object>> args) {
1142 : constexpr size_t kMaxNumArgs = 6;
1143 : DCHECK_GE(kMaxNumArgs, args.size());
1144 148953 : int argc = static_cast<int>(args.size());
1145 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1146 : zone(), function, argc, Operator::kNoProperties,
1147 148953 : CallDescriptor::kNoFlags);
1148 :
1149 148953 : Node* ref = ExternalConstant(ExternalReference::Create(function));
1150 : Node* arity = Int32Constant(argc);
1151 :
1152 : NodeArray<kMaxNumArgs + 4> inputs;
1153 : inputs.Add(centry);
1154 353202 : for (auto arg : args) inputs.Add(arg);
1155 : inputs.Add(ref);
1156 : inputs.Add(arity);
1157 : inputs.Add(context);
1158 :
1159 : CallPrologue();
1160 : Node* return_value =
1161 148953 : raw_assembler()->CallN(call_descriptor, inputs.size(), inputs.data());
1162 148953 : HandleException(return_value);
1163 : CallEpilogue();
1164 148953 : return UncheckedCast<Object>(return_value);
1165 : }
1166 :
1167 7738 : void CodeAssembler::TailCallRuntimeImpl(
1168 : Runtime::FunctionId function, TNode<Int32T> arity, TNode<Object> context,
1169 : std::initializer_list<TNode<Object>> args) {
1170 7738 : int result_size = Runtime::FunctionForId(function)->result_size;
1171 : TNode<Code> centry =
1172 15476 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1173 7738 : return TailCallRuntimeWithCEntryImpl(function, arity, centry, context, args);
1174 : }
1175 :
1176 8522 : void CodeAssembler::TailCallRuntimeWithCEntryImpl(
1177 : Runtime::FunctionId function, TNode<Int32T> arity, TNode<Code> centry,
1178 : TNode<Object> context, std::initializer_list<TNode<Object>> args) {
1179 : constexpr size_t kMaxNumArgs = 6;
1180 : DCHECK_GE(kMaxNumArgs, args.size());
1181 8522 : int argc = static_cast<int>(args.size());
1182 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1183 : zone(), function, argc, Operator::kNoProperties,
1184 8522 : CallDescriptor::kNoFlags);
1185 :
1186 8522 : Node* ref = ExternalConstant(ExternalReference::Create(function));
1187 :
1188 : NodeArray<kMaxNumArgs + 4> inputs;
1189 : inputs.Add(centry);
1190 36537 : for (auto arg : args) inputs.Add(arg);
1191 : inputs.Add(ref);
1192 : inputs.Add(arity);
1193 : inputs.Add(context);
1194 :
1195 8522 : raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1196 8522 : }
1197 :
1198 121814 : Node* CodeAssembler::CallStubN(StubCallMode call_mode,
1199 : const CallInterfaceDescriptor& descriptor,
1200 : size_t result_size, int input_count,
1201 : Node* const* inputs) {
1202 : DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1203 : call_mode == StubCallMode::kCallBuiltinPointer);
1204 :
1205 : // implicit nodes are target and optionally context.
1206 121814 : int implicit_nodes = descriptor.HasContextParameter() ? 2 : 1;
1207 : DCHECK_LE(implicit_nodes, input_count);
1208 121814 : int argc = input_count - implicit_nodes;
1209 : DCHECK_LE(descriptor.GetParameterCount(), argc);
1210 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1211 121814 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1212 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1213 : DCHECK_EQ(result_size, descriptor.GetReturnCount());
1214 :
1215 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1216 : zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1217 121814 : Operator::kNoProperties, call_mode);
1218 :
1219 : CallPrologue();
1220 : Node* return_value =
1221 121814 : raw_assembler()->CallN(call_descriptor, input_count, inputs);
1222 121814 : HandleException(return_value);
1223 : CallEpilogue();
1224 121814 : return return_value;
1225 : }
1226 :
1227 7000 : void CodeAssembler::TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
1228 : TNode<Code> target, TNode<Object> context,
1229 : std::initializer_list<Node*> args) {
1230 : constexpr size_t kMaxNumArgs = 11;
1231 : DCHECK_GE(kMaxNumArgs, args.size());
1232 : DCHECK_EQ(descriptor.GetParameterCount(), args.size());
1233 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1234 : zone(), descriptor, descriptor.GetStackParameterCount(),
1235 7000 : CallDescriptor::kNoFlags, Operator::kNoProperties);
1236 :
1237 : NodeArray<kMaxNumArgs + 2> inputs;
1238 : inputs.Add(target);
1239 29904 : for (auto arg : args) inputs.Add(arg);
1240 7000 : if (descriptor.HasContextParameter()) {
1241 : inputs.Add(context);
1242 : }
1243 :
1244 7000 : raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1245 7000 : }
1246 :
1247 119126 : Node* CodeAssembler::CallStubRImpl(StubCallMode call_mode,
1248 : const CallInterfaceDescriptor& descriptor,
1249 : size_t result_size, Node* target,
1250 : SloppyTNode<Object> context,
1251 : std::initializer_list<Node*> args) {
1252 : DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1253 : call_mode == StubCallMode::kCallBuiltinPointer);
1254 :
1255 : constexpr size_t kMaxNumArgs = 10;
1256 : DCHECK_GE(kMaxNumArgs, args.size());
1257 :
1258 : NodeArray<kMaxNumArgs + 2> inputs;
1259 : inputs.Add(target);
1260 413899 : for (auto arg : args) inputs.Add(arg);
1261 119126 : if (descriptor.HasContextParameter()) {
1262 : inputs.Add(context);
1263 : }
1264 :
1265 : return CallStubN(call_mode, descriptor, result_size, inputs.size(),
1266 119126 : inputs.data());
1267 : }
1268 :
1269 2184 : Node* CodeAssembler::TailCallStubThenBytecodeDispatchImpl(
1270 : const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
1271 : std::initializer_list<Node*> args) {
1272 : constexpr size_t kMaxNumArgs = 6;
1273 : DCHECK_GE(kMaxNumArgs, args.size());
1274 :
1275 : DCHECK_LE(descriptor.GetParameterCount(), args.size());
1276 2184 : int argc = static_cast<int>(args.size());
1277 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1278 2184 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1279 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1280 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1281 : zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1282 2184 : Operator::kNoProperties);
1283 :
1284 : NodeArray<kMaxNumArgs + 2> inputs;
1285 : inputs.Add(target);
1286 9744 : for (auto arg : args) inputs.Add(arg);
1287 : inputs.Add(context);
1288 :
1289 : return raw_assembler()->TailCallN(call_descriptor, inputs.size(),
1290 2184 : inputs.data());
1291 : }
1292 :
1293 : template <class... TArgs>
1294 44352 : Node* CodeAssembler::TailCallBytecodeDispatch(
1295 : const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
1296 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1297 : auto call_descriptor = Linkage::GetBytecodeDispatchCallDescriptor(
1298 44352 : zone(), descriptor, descriptor.GetStackParameterCount());
1299 :
1300 44352 : Node* nodes[] = {target, args...};
1301 88704 : CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
1302 44352 : return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1303 : }
1304 :
1305 : // Instantiate TailCallBytecodeDispatch() for argument counts used by
1306 : // CSA-generated code
1307 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
1308 : const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
1309 : Node*, Node*);
1310 :
1311 504 : TNode<Object> CodeAssembler::TailCallJSCode(TNode<Code> code,
1312 : TNode<Context> context,
1313 : TNode<JSFunction> function,
1314 : TNode<Object> new_target,
1315 : TNode<Int32T> arg_count) {
1316 : JSTrampolineDescriptor descriptor;
1317 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1318 : zone(), descriptor, descriptor.GetStackParameterCount(),
1319 504 : CallDescriptor::kFixedTargetRegister, Operator::kNoProperties);
1320 :
1321 2520 : Node* nodes[] = {code, function, new_target, arg_count, context};
1322 1008 : CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
1323 : return UncheckedCast<Object>(
1324 504 : raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes));
1325 : }
1326 :
1327 0 : Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
1328 : int input_count, Node* const* inputs) {
1329 0 : auto call_descriptor = Linkage::GetSimplifiedCDescriptor(zone(), signature);
1330 0 : return raw_assembler()->CallN(call_descriptor, input_count, inputs);
1331 : }
1332 :
1333 504 : Node* CodeAssembler::CallCFunction1(MachineType return_type,
1334 : MachineType arg0_type, Node* function,
1335 : Node* arg0) {
1336 : return raw_assembler()->CallCFunction1(return_type, arg0_type, function,
1337 504 : arg0);
1338 : }
1339 :
1340 224 : Node* CodeAssembler::CallCFunction1WithCallerSavedRegisters(
1341 : MachineType return_type, MachineType arg0_type, Node* function, Node* arg0,
1342 : SaveFPRegsMode mode) {
1343 : DCHECK(return_type.LessThanOrEqualPointerSize());
1344 : return raw_assembler()->CallCFunction1WithCallerSavedRegisters(
1345 224 : return_type, arg0_type, function, arg0, mode);
1346 : }
1347 :
1348 4495 : Node* CodeAssembler::CallCFunction2(MachineType return_type,
1349 : MachineType arg0_type,
1350 : MachineType arg1_type, Node* function,
1351 : Node* arg0, Node* arg1) {
1352 : return raw_assembler()->CallCFunction2(return_type, arg0_type, arg1_type,
1353 4495 : function, arg0, arg1);
1354 : }
1355 :
1356 6326 : Node* CodeAssembler::CallCFunction3(MachineType return_type,
1357 : MachineType arg0_type,
1358 : MachineType arg1_type,
1359 : MachineType arg2_type, Node* function,
1360 : Node* arg0, Node* arg1, Node* arg2) {
1361 : return raw_assembler()->CallCFunction3(return_type, arg0_type, arg1_type,
1362 6326 : arg2_type, function, arg0, arg1, arg2);
1363 : }
1364 :
1365 117 : Node* CodeAssembler::CallCFunction3WithCallerSavedRegisters(
1366 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1367 : MachineType arg2_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1368 : SaveFPRegsMode mode) {
1369 : DCHECK(return_type.LessThanOrEqualPointerSize());
1370 : return raw_assembler()->CallCFunction3WithCallerSavedRegisters(
1371 : return_type, arg0_type, arg1_type, arg2_type, function, arg0, arg1, arg2,
1372 117 : mode);
1373 : }
1374 :
1375 112 : Node* CodeAssembler::CallCFunction4(
1376 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1377 : MachineType arg2_type, MachineType arg3_type, Node* function, Node* arg0,
1378 : Node* arg1, Node* arg2, Node* arg3) {
1379 : return raw_assembler()->CallCFunction4(return_type, arg0_type, arg1_type,
1380 : arg2_type, arg3_type, function, arg0,
1381 112 : arg1, arg2, arg3);
1382 : }
1383 :
1384 280 : Node* CodeAssembler::CallCFunction5(
1385 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1386 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1387 : Node* function, Node* arg0, Node* arg1, Node* arg2, Node* arg3,
1388 : Node* arg4) {
1389 : return raw_assembler()->CallCFunction5(
1390 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1391 280 : function, arg0, arg1, arg2, arg3, arg4);
1392 : }
1393 :
1394 672 : Node* CodeAssembler::CallCFunction6(
1395 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1396 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1397 : MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1398 : Node* arg3, Node* arg4, Node* arg5) {
1399 : return raw_assembler()->CallCFunction6(
1400 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1401 672 : arg5_type, function, arg0, arg1, arg2, arg3, arg4, arg5);
1402 : }
1403 :
1404 565 : Node* CodeAssembler::CallCFunction9(
1405 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1406 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1407 : MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
1408 : MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1409 : Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
1410 : return raw_assembler()->CallCFunction9(
1411 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1412 : arg5_type, arg6_type, arg7_type, arg8_type, function, arg0, arg1, arg2,
1413 565 : arg3, arg4, arg5, arg6, arg7, arg8);
1414 : }
1415 :
1416 1924734 : void CodeAssembler::Goto(Label* label) {
1417 1924734 : label->MergeVariables();
1418 3849468 : raw_assembler()->Goto(label->label_);
1419 1924734 : }
1420 :
1421 575201 : void CodeAssembler::GotoIf(SloppyTNode<IntegralT> condition,
1422 : Label* true_label) {
1423 : Label false_label(this);
1424 575201 : Branch(condition, true_label, &false_label);
1425 575201 : Bind(&false_label);
1426 575201 : }
1427 :
1428 162521 : void CodeAssembler::GotoIfNot(SloppyTNode<IntegralT> condition,
1429 : Label* false_label) {
1430 : Label true_label(this);
1431 162521 : Branch(condition, &true_label, false_label);
1432 162521 : Bind(&true_label);
1433 162521 : }
1434 :
1435 1265887 : void CodeAssembler::Branch(SloppyTNode<IntegralT> condition, Label* true_label,
1436 0 : Label* false_label) {
1437 : int32_t constant;
1438 1264807 : if (ToInt32Constant(condition, constant)) {
1439 1620 : if ((true_label->is_used() || true_label->is_bound()) &&
1440 0 : (false_label->is_used() || false_label->is_bound())) {
1441 0 : return Goto(constant ? true_label : false_label);
1442 : }
1443 : }
1444 1264807 : true_label->MergeVariables();
1445 1264807 : false_label->MergeVariables();
1446 : return raw_assembler()->Branch(condition, true_label->label_,
1447 2529614 : false_label->label_);
1448 : }
1449 :
1450 26227 : void CodeAssembler::Branch(TNode<BoolT> condition,
1451 : const std::function<void()>& true_body,
1452 : const std::function<void()>& false_body) {
1453 : int32_t constant;
1454 26227 : if (ToInt32Constant(condition, constant)) {
1455 0 : return constant ? true_body() : false_body();
1456 : }
1457 :
1458 26227 : Label vtrue(this), vfalse(this);
1459 26227 : Branch(condition, &vtrue, &vfalse);
1460 :
1461 : Bind(&vtrue);
1462 26227 : true_body();
1463 :
1464 : Bind(&vfalse);
1465 52454 : false_body();
1466 : }
1467 :
1468 982 : void CodeAssembler::Branch(TNode<BoolT> condition, Label* true_label,
1469 : const std::function<void()>& false_body) {
1470 : int32_t constant;
1471 982 : if (ToInt32Constant(condition, constant)) {
1472 1282 : return constant ? Goto(true_label) : false_body();
1473 : }
1474 :
1475 : Label vfalse(this);
1476 341 : Branch(condition, true_label, &vfalse);
1477 : Bind(&vfalse);
1478 341 : false_body();
1479 : }
1480 :
1481 0 : void CodeAssembler::Branch(TNode<BoolT> condition,
1482 : const std::function<void()>& true_body,
1483 : Label* false_label) {
1484 : int32_t constant;
1485 0 : if (ToInt32Constant(condition, constant)) {
1486 0 : return constant ? true_body() : Goto(false_label);
1487 : }
1488 :
1489 : Label vtrue(this);
1490 0 : Branch(condition, &vtrue, false_label);
1491 : Bind(&vtrue);
1492 0 : true_body();
1493 : }
1494 :
1495 9942 : void CodeAssembler::Switch(Node* index, Label* default_label,
1496 : const int32_t* case_values, Label** case_labels,
1497 : size_t case_count) {
1498 : RawMachineLabel** labels =
1499 9942 : new (zone()->New(sizeof(RawMachineLabel*) * case_count))
1500 9942 : RawMachineLabel*[case_count];
1501 90573 : for (size_t i = 0; i < case_count; ++i) {
1502 80631 : labels[i] = case_labels[i]->label_;
1503 80631 : case_labels[i]->MergeVariables();
1504 : }
1505 9942 : default_label->MergeVariables();
1506 : return raw_assembler()->Switch(index, default_label->label_, case_values,
1507 19884 : labels, case_count);
1508 : }
1509 :
1510 56 : bool CodeAssembler::UnalignedLoadSupported(MachineRepresentation rep) const {
1511 56 : return raw_assembler()->machine()->UnalignedLoadSupported(rep);
1512 : }
1513 56 : bool CodeAssembler::UnalignedStoreSupported(MachineRepresentation rep) const {
1514 56 : return raw_assembler()->machine()->UnalignedStoreSupported(rep);
1515 : }
1516 :
1517 : // RawMachineAssembler delegate helpers:
1518 996780 : Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
1519 :
1520 8736 : Factory* CodeAssembler::factory() const { return isolate()->factory(); }
1521 :
1522 157564 : Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }
1523 :
1524 0 : bool CodeAssembler::IsExceptionHandlerActive() const {
1525 0 : return state_->exception_handler_labels_.size() != 0;
1526 : }
1527 :
1528 0 : RawMachineAssembler* CodeAssembler::raw_assembler() const {
1529 22286931 : return state_->raw_assembler_.get();
1530 : }
1531 :
1532 : // The core implementation of Variable is stored through an indirection so
1533 : // that it can outlive the often block-scoped Variable declarations. This is
1534 : // needed to ensure that variable binding and merging through phis can
1535 : // properly be verified.
1536 : class CodeAssemblerVariable::Impl : public ZoneObject {
1537 : public:
1538 : explicit Impl(MachineRepresentation rep, CodeAssemblerState::VariableId id)
1539 : :
1540 : #if DEBUG
1541 : debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
1542 : #endif
1543 : value_(nullptr),
1544 : rep_(rep),
1545 607566 : var_id_(id) {
1546 : }
1547 :
1548 : #if DEBUG
1549 : AssemblerDebugInfo debug_info() const { return debug_info_; }
1550 : void set_debug_info(AssemblerDebugInfo debug_info) {
1551 : debug_info_ = debug_info;
1552 : }
1553 :
1554 : AssemblerDebugInfo debug_info_;
1555 : #endif // DEBUG
1556 : bool operator<(const CodeAssemblerVariable::Impl& other) const {
1557 1216195 : return var_id_ < other.var_id_;
1558 : }
1559 : Node* value_;
1560 : MachineRepresentation rep_;
1561 : CodeAssemblerState::VariableId var_id_;
1562 : };
1563 :
1564 0 : bool CodeAssemblerVariable::ImplComparator::operator()(
1565 150933898 : const CodeAssemblerVariable::Impl* a,
1566 150748786 : const CodeAssemblerVariable::Impl* b) const {
1567 0 : return *a < *b;
1568 : }
1569 :
1570 607566 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1571 : MachineRepresentation rep)
1572 : : impl_(new (assembler->zone())
1573 : Impl(rep, assembler->state()->NextVariableId())),
1574 1215132 : state_(assembler->state()) {
1575 607566 : state_->variables_.insert(impl_);
1576 607566 : }
1577 :
1578 268528 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1579 : MachineRepresentation rep,
1580 : Node* initial_value)
1581 268528 : : CodeAssemblerVariable(assembler, rep) {
1582 : Bind(initial_value);
1583 268528 : }
1584 :
1585 : #if DEBUG
1586 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1587 : AssemblerDebugInfo debug_info,
1588 : MachineRepresentation rep)
1589 : : impl_(new (assembler->zone())
1590 : Impl(rep, assembler->state()->NextVariableId())),
1591 : state_(assembler->state()) {
1592 : impl_->set_debug_info(debug_info);
1593 : state_->variables_.insert(impl_);
1594 : }
1595 :
1596 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1597 : AssemblerDebugInfo debug_info,
1598 : MachineRepresentation rep,
1599 : Node* initial_value)
1600 : : CodeAssemblerVariable(assembler, debug_info, rep) {
1601 : impl_->set_debug_info(debug_info);
1602 : Bind(initial_value);
1603 : }
1604 : #endif // DEBUG
1605 :
1606 607566 : CodeAssemblerVariable::~CodeAssemblerVariable() {
1607 607566 : state_->variables_.erase(impl_);
1608 607566 : }
1609 :
1610 1355840 : void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
1611 :
1612 1539328 : Node* CodeAssemblerVariable::value() const {
1613 : #if DEBUG
1614 : if (!IsBound()) {
1615 : std::stringstream str;
1616 : str << "#Use of unbound variable:"
1617 : << "#\n Variable: " << *this << "#\n Current Block: ";
1618 : state_->PrintCurrentBlock(str);
1619 : FATAL("%s", str.str().c_str());
1620 : }
1621 : if (!state_->InsideBlock()) {
1622 : std::stringstream str;
1623 : str << "#Accessing variable value outside a block:"
1624 : << "#\n Variable: " << *this;
1625 : FATAL("%s", str.str().c_str());
1626 : }
1627 : #endif // DEBUG
1628 1539328 : return impl_->value_;
1629 : }
1630 :
1631 0 : MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
1632 :
1633 147485 : bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
1634 :
1635 0 : std::ostream& operator<<(std::ostream& os,
1636 : const CodeAssemblerVariable::Impl& impl) {
1637 : #if DEBUG
1638 : AssemblerDebugInfo info = impl.debug_info();
1639 : if (info.name) os << "V" << info;
1640 : #endif // DEBUG
1641 0 : return os;
1642 : }
1643 :
1644 0 : std::ostream& operator<<(std::ostream& os,
1645 : const CodeAssemblerVariable& variable) {
1646 : os << *variable.impl_;
1647 0 : return os;
1648 : }
1649 :
1650 3449567 : CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
1651 : size_t vars_count,
1652 : CodeAssemblerVariable* const* vars,
1653 : CodeAssemblerLabel::Type type)
1654 : : bound_(false),
1655 : merge_count_(0),
1656 : state_(assembler->state()),
1657 6899134 : label_(nullptr) {
1658 3449567 : void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
1659 : label_ = new (buffer)
1660 : RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
1661 6899134 : : RawMachineLabel::kNonDeferred);
1662 3694069 : for (size_t i = 0; i < vars_count; ++i) {
1663 244502 : variable_phis_[vars[i]->impl_] = nullptr;
1664 : }
1665 3449567 : }
1666 :
1667 6922430 : CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }
1668 :
1669 4569641 : void CodeAssemblerLabel::MergeVariables() {
1670 4569641 : ++merge_count_;
1671 30733290 : for (CodeAssemblerVariable::Impl* var : state_->variables_) {
1672 : size_t count = 0;
1673 17024367 : Node* node = var->value_;
1674 17024367 : if (node != nullptr) {
1675 : auto i = variable_merges_.find(var);
1676 12547092 : if (i != variable_merges_.end()) {
1677 4432806 : i->second.push_back(node);
1678 : count = i->second.size();
1679 : } else {
1680 : count = 1;
1681 16228572 : variable_merges_[var] = std::vector<Node*>(1, node);
1682 : }
1683 : }
1684 : // If the following asserts, then you've jumped to a label without a bound
1685 : // variable along that path that expects to merge its value into a phi.
1686 : DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
1687 : count == merge_count_);
1688 : USE(count);
1689 :
1690 : // If the label is already bound, we already know the set of variables to
1691 : // merge and phi nodes have already been created.
1692 17024367 : if (bound_) {
1693 : auto phi = variable_phis_.find(var);
1694 791434 : if (phi != variable_phis_.end()) {
1695 : DCHECK_NOT_NULL(phi->second);
1696 474928 : state_->raw_assembler_->AppendPhiInput(phi->second, node);
1697 : } else {
1698 : auto i = variable_merges_.find(var);
1699 : if (i != variable_merges_.end()) {
1700 : // If the following assert fires, then you've declared a variable that
1701 : // has the same bound value along all paths up until the point you
1702 : // bound this label, but then later merged a path with a new value for
1703 : // the variable after the label bind (it's not possible to add phis to
1704 : // the bound label after the fact, just make sure to list the variable
1705 : // in the label's constructor's list of merged variables).
1706 : #if DEBUG
1707 : if (find_if(i->second.begin(), i->second.end(),
1708 : [node](Node* e) -> bool { return node != e; }) !=
1709 : i->second.end()) {
1710 : std::stringstream str;
1711 : str << "Unmerged variable found when jumping to block. \n"
1712 : << "# Variable: " << *var;
1713 : if (bound_) {
1714 : str << "\n# Target block: " << *label_->block();
1715 : }
1716 : str << "\n# Current Block: ";
1717 : state_->PrintCurrentBlock(str);
1718 : FATAL("%s", str.str().c_str());
1719 : }
1720 : #endif // DEBUG
1721 : }
1722 : }
1723 : }
1724 : }
1725 4569641 : }
1726 :
1727 : #if DEBUG
1728 : void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
1729 : if (bound_) {
1730 : std::stringstream str;
1731 : str << "Cannot bind the same label twice:"
1732 : << "\n# current: " << debug_info
1733 : << "\n# previous: " << *label_->block();
1734 : FATAL("%s", str.str().c_str());
1735 : }
1736 : state_->raw_assembler_->Bind(label_, debug_info);
1737 : UpdateVariablesAfterBind();
1738 : }
1739 : #endif // DEBUG
1740 :
1741 3353679 : void CodeAssemblerLabel::Bind() {
1742 : DCHECK(!bound_);
1743 6707358 : state_->raw_assembler_->Bind(label_);
1744 3353679 : UpdateVariablesAfterBind();
1745 3353679 : }
1746 :
1747 3353679 : void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1748 : // Make sure that all variables that have changed along any path up to this
1749 : // point are marked as merge variables.
1750 21190637 : for (auto var : state_->variables_) {
1751 : Node* shared_value = nullptr;
1752 : auto i = variable_merges_.find(var);
1753 11129600 : if (i != variable_merges_.end()) {
1754 27188209 : for (auto value : i->second) {
1755 : DCHECK_NOT_NULL(value);
1756 11334291 : if (value != shared_value) {
1757 8608550 : if (shared_value == nullptr) {
1758 : shared_value = value;
1759 : } else {
1760 681591 : variable_phis_[var] = nullptr;
1761 : }
1762 : }
1763 : }
1764 : }
1765 : }
1766 :
1767 7319395 : for (auto var : variable_phis_) {
1768 612037 : CodeAssemblerVariable::Impl* var_impl = var.first;
1769 : auto i = variable_merges_.find(var_impl);
1770 : #if DEBUG
1771 : bool not_found = i == variable_merges_.end();
1772 : if (not_found || i->second.size() != merge_count_) {
1773 : std::stringstream str;
1774 : str << "A variable that has been marked as beeing merged at the label"
1775 : << "\n# doesn't have a bound value along all of the paths that "
1776 : << "\n# have been merged into the label up to this point."
1777 : << "\n#"
1778 : << "\n# This can happen in the following cases:"
1779 : << "\n# - By explicitly marking it so in the label constructor"
1780 : << "\n# - By having seen different bound values at branches"
1781 : << "\n#"
1782 : << "\n# Merge count: expected=" << merge_count_
1783 : << " vs. found=" << (not_found ? 0 : i->second.size())
1784 : << "\n# Variable: " << *var_impl
1785 : << "\n# Current Block: " << *label_->block();
1786 : FATAL("%s", str.str().c_str());
1787 : }
1788 : #endif // DEBUG
1789 : Node* phi = state_->raw_assembler_->Phi(
1790 1224074 : var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
1791 612037 : variable_phis_[var_impl] = phi;
1792 : }
1793 :
1794 : // Bind all variables to a merge phi, the common value along all paths or
1795 : // null.
1796 21190637 : for (auto var : state_->variables_) {
1797 : auto i = variable_phis_.find(var);
1798 11129600 : if (i != variable_phis_.end()) {
1799 612037 : var->value_ = i->second;
1800 : } else {
1801 : auto j = variable_merges_.find(var);
1802 17832485 : if (j != variable_merges_.end() && j->second.size() == merge_count_) {
1803 14538244 : var->value_ = j->second.back();
1804 : } else {
1805 3248441 : var->value_ = nullptr;
1806 : }
1807 : }
1808 : }
1809 :
1810 3353679 : bound_ = true;
1811 3353679 : }
1812 :
1813 6136346 : void CodeAssemblerParameterizedLabelBase::AddInputs(std::vector<Node*> inputs) {
1814 1238230 : if (!phi_nodes_.empty()) {
1815 : DCHECK_EQ(inputs.size(), phi_nodes_.size());
1816 98461 : for (size_t i = 0; i < inputs.size(); ++i) {
1817 141279 : state_->raw_assembler_->AppendPhiInput(phi_nodes_[i], inputs[i]);
1818 : }
1819 : } else {
1820 : DCHECK_EQ(inputs.size(), phi_inputs_.size());
1821 8600820 : for (size_t i = 0; i < inputs.size(); ++i) {
1822 7413958 : phi_inputs_[i].push_back(inputs[i]);
1823 : }
1824 : }
1825 1191137 : }
1826 :
1827 3402410 : Node* CodeAssemblerParameterizedLabelBase::CreatePhi(
1828 : MachineRepresentation rep, const std::vector<Node*>& inputs) {
1829 10482435 : for (Node* input : inputs) {
1830 : // We use {nullptr} as a sentinel for an uninitialized value. We must not
1831 : // create phi nodes for these.
1832 3703843 : if (input == nullptr) return nullptr;
1833 : }
1834 : return state_->raw_assembler_->Phi(rep, static_cast<int>(inputs.size()),
1835 6752364 : &inputs.front());
1836 : }
1837 :
1838 1116196 : const std::vector<Node*>& CodeAssemblerParameterizedLabelBase::CreatePhis(
1839 3402410 : std::vector<MachineRepresentation> representations) {
1840 : DCHECK(is_used());
1841 : DCHECK(phi_nodes_.empty());
1842 6750998 : phi_nodes_.reserve(phi_inputs_.size());
1843 : DCHECK_EQ(representations.size(), phi_inputs_.size());
1844 9037212 : for (size_t i = 0; i < phi_inputs_.size(); ++i) {
1845 6804820 : phi_nodes_.push_back(CreatePhi(representations[i], phi_inputs_[i]));
1846 : }
1847 1116196 : return phi_nodes_;
1848 : }
1849 :
1850 0 : void CodeAssemblerState::PushExceptionHandler(
1851 : CodeAssemblerExceptionHandlerLabel* label) {
1852 1002 : exception_handler_labels_.push_back(label);
1853 0 : }
1854 :
1855 0 : void CodeAssemblerState::PopExceptionHandler() {
1856 : exception_handler_labels_.pop_back();
1857 0 : }
1858 :
1859 773 : CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
1860 : CodeAssembler* assembler, CodeAssemblerExceptionHandlerLabel* label)
1861 773 : : has_handler_(label != nullptr),
1862 : assembler_(assembler),
1863 : compatibility_label_(nullptr),
1864 773 : exception_(nullptr) {
1865 773 : if (has_handler_) {
1866 : assembler_->state()->PushExceptionHandler(label);
1867 : }
1868 773 : }
1869 :
1870 565 : CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
1871 : CodeAssembler* assembler, CodeAssemblerLabel* label,
1872 : TypedCodeAssemblerVariable<Object>* exception)
1873 565 : : has_handler_(label != nullptr),
1874 : assembler_(assembler),
1875 : compatibility_label_(label),
1876 565 : exception_(exception) {
1877 565 : if (has_handler_) {
1878 458 : label_ = base::make_unique<CodeAssemblerExceptionHandlerLabel>(
1879 : assembler, CodeAssemblerLabel::kDeferred);
1880 229 : assembler_->state()->PushExceptionHandler(label_.get());
1881 : }
1882 565 : }
1883 :
1884 2676 : CodeAssemblerScopedExceptionHandler::~CodeAssemblerScopedExceptionHandler() {
1885 1338 : if (has_handler_) {
1886 1002 : assembler_->state()->PopExceptionHandler();
1887 : }
1888 1567 : if (label_ && label_->is_used()) {
1889 229 : CodeAssembler::Label skip(assembler_);
1890 229 : bool inside_block = assembler_->state()->InsideBlock();
1891 229 : if (inside_block) {
1892 229 : assembler_->Goto(&skip);
1893 : }
1894 : TNode<Object> e;
1895 : assembler_->Bind(label_.get(), &e);
1896 229 : *exception_ = e;
1897 229 : assembler_->Goto(compatibility_label_);
1898 229 : if (inside_block) {
1899 : assembler_->Bind(&skip);
1900 229 : }
1901 : }
1902 1338 : }
1903 :
1904 : } // namespace compiler
1905 :
1906 0 : Address CheckObjectType(Address raw_value, Address raw_type,
1907 : Address raw_location) {
1908 : #ifdef DEBUG
1909 : Object value(raw_value);
1910 : Smi type(raw_type);
1911 : String location = String::cast(Object(raw_location));
1912 : const char* expected;
1913 : switch (static_cast<ObjectType>(type->value())) {
1914 : #define TYPE_CASE(Name) \
1915 : case ObjectType::k##Name: \
1916 : if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
1917 : expected = #Name; \
1918 : break;
1919 : #define TYPE_STRUCT_CASE(NAME, Name, name) \
1920 : case ObjectType::k##Name: \
1921 : if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
1922 : expected = #Name; \
1923 : break;
1924 :
1925 : TYPE_CASE(Object)
1926 : OBJECT_TYPE_LIST(TYPE_CASE)
1927 : HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
1928 : STRUCT_LIST(TYPE_STRUCT_CASE)
1929 : #undef TYPE_CASE
1930 : #undef TYPE_STRUCT_CASE
1931 : }
1932 : std::stringstream value_description;
1933 : value->Print(value_description);
1934 : V8_Fatal(__FILE__, __LINE__,
1935 : "Type cast failed in %s\n"
1936 : " Expected %s but found %s",
1937 : location->ToAsciiArray(), expected, value_description.str().c_str());
1938 : #else
1939 0 : UNREACHABLE();
1940 : #endif
1941 : }
1942 :
1943 : } // namespace internal
1944 183867 : } // namespace v8
|
