Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-assembler.h"
6 :
7 : #include <ostream>
8 :
9 : #include "src/code-factory.h"
10 : #include "src/compiler/backend/instruction-selector.h"
11 : #include "src/compiler/graph.h"
12 : #include "src/compiler/linkage.h"
13 : #include "src/compiler/node-matchers.h"
14 : #include "src/compiler/pipeline.h"
15 : #include "src/compiler/raw-machine-assembler.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/frames.h"
18 : #include "src/interface-descriptors.h"
19 : #include "src/interpreter/bytecodes.h"
20 : #include "src/machine-type.h"
21 : #include "src/macro-assembler.h"
22 : #include "src/memcopy.h"
23 : #include "src/objects-inl.h"
24 : #include "src/objects/smi.h"
25 : #include "src/zone/zone.h"
26 :
27 : namespace v8 {
28 : namespace internal {
29 :
30 : constexpr MachineType MachineTypeOf<Smi>::value;
31 : constexpr MachineType MachineTypeOf<Object>::value;
32 :
33 : namespace compiler {
34 :
35 : static_assert(std::is_convertible<TNode<Number>, TNode<Object>>::value,
36 : "test subtyping");
37 : static_assert(std::is_convertible<TNode<UnionT<Smi, HeapNumber>>,
38 : TNode<UnionT<Smi, HeapObject>>>::value,
39 : "test subtyping");
40 : static_assert(
41 : !std::is_convertible<TNode<UnionT<Smi, HeapObject>>, TNode<Number>>::value,
42 : "test subtyping");
43 :
44 48047 : CodeAssemblerState::CodeAssemblerState(
45 : Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
46 : Code::Kind kind, const char* name, PoisoningMitigationLevel poisoning_level,
47 : int32_t builtin_index)
48 : // TODO(rmcilroy): Should we use Linkage::GetBytecodeDispatchDescriptor for
49 : // bytecode handlers?
50 : : CodeAssemblerState(
51 : isolate, zone,
52 : Linkage::GetStubCallDescriptor(
53 : zone, descriptor, descriptor.GetStackParameterCount(),
54 : CallDescriptor::kNoFlags, Operator::kNoProperties),
55 48047 : kind, name, poisoning_level, builtin_index) {}
56 :
57 20136 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
58 : int parameter_count, Code::Kind kind,
59 : const char* name,
60 : PoisoningMitigationLevel poisoning_level,
61 : int32_t builtin_index)
62 : : CodeAssemblerState(
63 : isolate, zone,
64 : Linkage::GetJSCallDescriptor(
65 : zone, false, parameter_count,
66 : (kind == Code::BUILTIN ? CallDescriptor::kPushArgumentCount
67 : : CallDescriptor::kNoFlags) |
68 : CallDescriptor::kCanUseRoots),
69 40272 : kind, name, poisoning_level, builtin_index) {}
70 :
71 68751 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
72 : CallDescriptor* call_descriptor,
73 : Code::Kind kind, const char* name,
74 : PoisoningMitigationLevel poisoning_level,
75 : int32_t builtin_index)
76 : : raw_assembler_(new RawMachineAssembler(
77 68751 : isolate, new (zone) Graph(zone), call_descriptor,
78 : MachineType::PointerRepresentation(),
79 137502 : InstructionSelector::SupportedMachineOperatorFlags(),
80 206253 : InstructionSelector::AlignmentRequirements(), poisoning_level)),
81 : kind_(kind),
82 : name_(name),
83 : builtin_index_(builtin_index),
84 : code_generated_(false),
85 206253 : variables_(zone) {}
86 :
87 : CodeAssemblerState::~CodeAssemblerState() = default;
88 :
89 0 : int CodeAssemblerState::parameter_count() const {
90 0 : return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
91 : }
92 :
93 : CodeAssembler::~CodeAssembler() = default;
94 :
95 : #if DEBUG
96 : void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
97 : raw_assembler_->PrintCurrentBlock(os);
98 : }
99 : #endif
100 :
101 3388 : bool CodeAssemblerState::InsideBlock() { return raw_assembler_->InsideBlock(); }
102 :
103 64064 : void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
104 : const char* file,
105 : int line) {
106 : #if DEBUG
107 : AssemblerDebugInfo debug_info = {msg, file, line};
108 : raw_assembler_->SetSourcePosition(file, line);
109 : raw_assembler_->SetInitialDebugInformation(debug_info);
110 : #endif // DEBUG
111 64064 : }
112 :
113 0 : class BreakOnNodeDecorator final : public GraphDecorator {
114 : public:
115 0 : explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}
116 :
117 0 : void Decorate(Node* node) final {
118 0 : if (node->id() == node_id_) {
119 0 : base::OS::DebugBreak();
120 : }
121 0 : }
122 :
123 : private:
124 : NodeId node_id_;
125 : };
126 :
127 0 : void CodeAssembler::BreakOnNode(int node_id) {
128 : Graph* graph = raw_assembler()->graph();
129 : Zone* zone = graph->zone();
130 : GraphDecorator* decorator =
131 0 : new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
132 0 : graph->AddDecorator(decorator);
133 0 : }
134 :
135 27990 : void CodeAssembler::RegisterCallGenerationCallbacks(
136 : const CodeAssemblerCallback& call_prologue,
137 : const CodeAssemblerCallback& call_epilogue) {
138 : // The callback can be registered only once.
139 : DCHECK(!state_->call_prologue_);
140 : DCHECK(!state_->call_epilogue_);
141 27990 : state_->call_prologue_ = call_prologue;
142 27990 : state_->call_epilogue_ = call_epilogue;
143 27990 : }
144 :
145 27990 : void CodeAssembler::UnregisterCallGenerationCallbacks() {
146 27990 : state_->call_prologue_ = nullptr;
147 27990 : state_->call_epilogue_ = nullptr;
148 27990 : }
149 :
150 254022 : void CodeAssembler::CallPrologue() {
151 254022 : if (state_->call_prologue_) {
152 : state_->call_prologue_();
153 : }
154 254022 : }
155 :
156 254022 : void CodeAssembler::CallEpilogue() {
157 254022 : if (state_->call_epilogue_) {
158 : state_->call_epilogue_();
159 : }
160 254022 : }
161 :
162 1176 : bool CodeAssembler::Word32ShiftIsSafe() const {
163 1176 : return raw_assembler()->machine()->Word32ShiftIsSafe();
164 : }
165 :
166 5742 : PoisoningMitigationLevel CodeAssembler::poisoning_level() const {
167 5742 : return raw_assembler()->poisoning_level();
168 : }
169 :
170 : // static
171 67268 : Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state,
172 : const AssemblerOptions& options) {
173 : DCHECK(!state->code_generated_);
174 :
175 : RawMachineAssembler* rasm = state->raw_assembler_.get();
176 :
177 : Handle<Code> code;
178 67268 : Graph* graph = rasm->ExportForOptimization();
179 :
180 134536 : code = Pipeline::GenerateCodeForCodeStub(
181 : rasm->isolate(), rasm->call_descriptor(), graph,
182 : rasm->source_positions(), state->kind_, state->name_,
183 67268 : state->builtin_index_, rasm->poisoning_level(), options)
184 : .ToHandleChecked();
185 :
186 67268 : state->code_generated_ = true;
187 67268 : return code;
188 : }
189 :
190 228496 : bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
191 :
192 118 : bool CodeAssembler::IsFloat64RoundUpSupported() const {
193 118 : return raw_assembler()->machine()->Float64RoundUp().IsSupported();
194 : }
195 :
196 125 : bool CodeAssembler::IsFloat64RoundDownSupported() const {
197 125 : return raw_assembler()->machine()->Float64RoundDown().IsSupported();
198 : }
199 :
200 392 : bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
201 392 : return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
202 : }
203 :
204 340 : bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
205 340 : return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
206 : }
207 :
208 0 : bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
209 0 : return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
210 : }
211 :
212 0 : bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
213 56 : return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
214 : }
215 :
216 56 : bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
217 : return Is64() ? IsInt64AbsWithOverflowSupported()
218 112 : : IsInt32AbsWithOverflowSupported();
219 : }
220 :
221 : #ifdef DEBUG
222 : void CodeAssembler::GenerateCheckMaybeObjectIsObject(Node* node,
223 : const char* location) {
224 : Label ok(this);
225 : GotoIf(WordNotEqual(WordAnd(BitcastMaybeObjectToWord(node),
226 : IntPtrConstant(kHeapObjectTagMask)),
227 : IntPtrConstant(kWeakHeapObjectTag)),
228 : &ok);
229 : Node* message_node = StringConstant(location);
230 : DebugAbort(message_node);
231 : Unreachable();
232 : Bind(&ok);
233 : }
234 : #endif
235 :
236 689764 : TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
237 859348 : return UncheckedCast<Int32T>(raw_assembler()->Int32Constant(value));
238 : }
239 :
240 11540 : TNode<Int64T> CodeAssembler::Int64Constant(int64_t value) {
241 11540 : return UncheckedCast<Int64T>(raw_assembler()->Int64Constant(value));
242 : }
243 :
244 3271373 : TNode<IntPtrT> CodeAssembler::IntPtrConstant(intptr_t value) {
245 3271373 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrConstant(value));
246 : }
247 :
248 10660 : TNode<Number> CodeAssembler::NumberConstant(double value) {
249 : int smi_value;
250 10660 : if (DoubleToSmiInteger(value, &smi_value)) {
251 : return UncheckedCast<Number>(SmiConstant(smi_value));
252 : } else {
253 : // We allocate the heap number constant eagerly at this point instead of
254 : // deferring allocation to code generation
255 : // (see AllocateAndInstallRequestedHeapObjects) since that makes it easier
256 : // to generate constant lookups for embedded builtins.
257 : return UncheckedCast<Number>(HeapConstant(
258 1744 : isolate()->factory()->NewHeapNumber(value, AllocationType::kOld)));
259 : }
260 : }
261 :
262 304733 : TNode<Smi> CodeAssembler::SmiConstant(Smi value) {
263 : return UncheckedCast<Smi>(BitcastWordToTaggedSigned(
264 609466 : IntPtrConstant(static_cast<intptr_t>(value.ptr()))));
265 : }
266 :
267 283373 : TNode<Smi> CodeAssembler::SmiConstant(int value) {
268 292289 : return SmiConstant(Smi::FromInt(value));
269 : }
270 :
271 144770 : TNode<HeapObject> CodeAssembler::UntypedHeapConstant(
272 : Handle<HeapObject> object) {
273 758546 : return UncheckedCast<HeapObject>(raw_assembler()->HeapConstant(object));
274 : }
275 :
276 19032 : TNode<String> CodeAssembler::StringConstant(const char* str) {
277 : Handle<String> internalized_string =
278 19032 : factory()->InternalizeOneByteString(OneByteVector(str));
279 19032 : return UncheckedCast<String>(HeapConstant(internalized_string));
280 : }
281 :
282 316 : TNode<Oddball> CodeAssembler::BooleanConstant(bool value) {
283 632 : Handle<Object> object = isolate()->factory()->ToBoolean(value);
284 : return UncheckedCast<Oddball>(
285 316 : raw_assembler()->HeapConstant(Handle<HeapObject>::cast(object)));
286 : }
287 :
288 62938 : TNode<ExternalReference> CodeAssembler::ExternalConstant(
289 : ExternalReference address) {
290 : return UncheckedCast<ExternalReference>(
291 214050 : raw_assembler()->ExternalConstant(address));
292 : }
293 :
294 11052 : TNode<Float64T> CodeAssembler::Float64Constant(double value) {
295 11052 : return UncheckedCast<Float64T>(raw_assembler()->Float64Constant(value));
296 : }
297 :
298 0 : TNode<HeapNumber> CodeAssembler::NaNConstant() {
299 0 : return UncheckedCast<HeapNumber>(LoadRoot(RootIndex::kNanValue));
300 : }
301 :
302 2190260 : bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
303 : {
304 : Int64Matcher m(node);
305 2382397 : if (m.HasValue() && m.IsInRange(std::numeric_limits<int32_t>::min(),
306 : std::numeric_limits<int32_t>::max())) {
307 192133 : out_value = static_cast<int32_t>(m.Value());
308 : return true;
309 : }
310 : }
311 :
312 : {
313 : Int32Matcher m(node);
314 1998127 : if (m.HasValue()) {
315 0 : out_value = m.Value();
316 : return true;
317 : }
318 : }
319 :
320 1998127 : return false;
321 : }
322 :
323 20 : bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
324 : Int64Matcher m(node);
325 5132 : if (m.HasValue()) out_value = m.Value();
326 20 : return m.HasValue();
327 : }
328 :
329 33672 : bool CodeAssembler::ToSmiConstant(Node* node, Smi* out_value) {
330 33672 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
331 : node = node->InputAt(0);
332 : }
333 : IntPtrMatcher m(node);
334 33672 : if (m.HasValue()) {
335 : intptr_t value = m.Value();
336 : // Make sure that the value is actually a smi
337 2400 : CHECK_EQ(0, value & ((static_cast<intptr_t>(1) << kSmiShiftSize) - 1));
338 2400 : *out_value = Smi(static_cast<Address>(value));
339 2400 : return true;
340 : }
341 : return false;
342 : }
343 :
344 4701579 : bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
345 4701579 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
346 : node->opcode() == IrOpcode::kBitcastWordToTagged) {
347 : node = node->InputAt(0);
348 : }
349 : IntPtrMatcher m(node);
350 4701579 : if (m.HasValue()) out_value = m.Value();
351 4701579 : return m.HasValue();
352 : }
353 :
354 1848 : bool CodeAssembler::IsUndefinedConstant(TNode<Object> node) {
355 : compiler::HeapObjectMatcher m(node);
356 1848 : return m.Is(isolate()->factory()->undefined_value());
357 : }
358 :
359 1568 : bool CodeAssembler::IsNullConstant(TNode<Object> node) {
360 : compiler::HeapObjectMatcher m(node);
361 1568 : return m.Is(isolate()->factory()->null_value());
362 : }
363 :
364 263076 : Node* CodeAssembler::Parameter(int index) {
365 263412 : if (index == kTargetParameterIndex) return raw_assembler()->TargetParameter();
366 525480 : return raw_assembler()->Parameter(index);
367 : }
368 :
369 0 : bool CodeAssembler::IsJSFunctionCall() const {
370 : auto call_descriptor = raw_assembler()->call_descriptor();
371 0 : return call_descriptor->IsJSFunctionCall();
372 : }
373 :
374 19208 : TNode<Context> CodeAssembler::GetJSContextParameter() {
375 : auto call_descriptor = raw_assembler()->call_descriptor();
376 : DCHECK(call_descriptor->IsJSFunctionCall());
377 38416 : return CAST(Parameter(Linkage::GetJSCallContextParamIndex(
378 : static_cast<int>(call_descriptor->JSParameterCount()))));
379 : }
380 :
381 61212 : void CodeAssembler::Return(SloppyTNode<Object> value) {
382 61436 : return raw_assembler()->Return(value);
383 : }
384 :
385 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
386 : SloppyTNode<Object> value2) {
387 0 : return raw_assembler()->Return(value1, value2);
388 : }
389 :
390 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
391 : SloppyTNode<Object> value2,
392 : SloppyTNode<Object> value3) {
393 0 : return raw_assembler()->Return(value1, value2, value3);
394 : }
395 :
396 13352 : void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
397 13352 : return raw_assembler()->PopAndReturn(pop, value);
398 : }
399 :
400 224 : void CodeAssembler::ReturnIf(Node* condition, Node* value) {
401 224 : Label if_return(this), if_continue(this);
402 224 : Branch(condition, &if_return, &if_continue);
403 : Bind(&if_return);
404 : Return(value);
405 : Bind(&if_continue);
406 224 : }
407 :
408 336 : void CodeAssembler::ReturnRaw(Node* value) {
409 336 : return raw_assembler()->Return(value);
410 : }
411 :
412 440 : void CodeAssembler::DebugAbort(Node* message) {
413 440 : raw_assembler()->DebugAbort(message);
414 440 : }
415 :
416 206760 : void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
417 :
418 66720 : void CodeAssembler::Unreachable() {
419 : DebugBreak();
420 66720 : raw_assembler()->Unreachable();
421 66720 : }
422 :
423 4 : void CodeAssembler::Comment(std::string str) {
424 4 : if (!FLAG_code_comments) return;
425 4 : raw_assembler()->Comment(str);
426 : }
427 :
428 859504 : void CodeAssembler::SetSourcePosition(const char* file, int line) {
429 859504 : raw_assembler()->SetSourcePosition(file, line);
430 859504 : }
431 :
432 3538240 : void CodeAssembler::Bind(Label* label) { return label->Bind(); }
433 :
434 : #if DEBUG
435 : void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
436 : return label->Bind(debug_info);
437 : }
438 : #endif // DEBUG
439 :
440 8008 : Node* CodeAssembler::LoadFramePointer() {
441 8008 : return raw_assembler()->LoadFramePointer();
442 : }
443 :
444 32696 : Node* CodeAssembler::LoadParentFramePointer() {
445 32696 : return raw_assembler()->LoadParentFramePointer();
446 : }
447 :
448 19432 : Node* CodeAssembler::LoadStackPointer() {
449 19432 : return raw_assembler()->LoadStackPointer();
450 : }
451 :
452 19056 : TNode<Object> CodeAssembler::TaggedPoisonOnSpeculation(
453 : SloppyTNode<Object> value) {
454 : return UncheckedCast<Object>(
455 19056 : raw_assembler()->TaggedPoisonOnSpeculation(value));
456 : }
457 :
458 45864 : TNode<WordT> CodeAssembler::WordPoisonOnSpeculation(SloppyTNode<WordT> value) {
459 45864 : return UncheckedCast<WordT>(raw_assembler()->WordPoisonOnSpeculation(value));
460 : }
461 :
462 : #define DEFINE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
463 : TNode<ResType> CodeAssembler::name(SloppyTNode<Arg1Type> a, \
464 : SloppyTNode<Arg2Type> b) { \
465 : return UncheckedCast<ResType>(raw_assembler()->name(a, b)); \
466 : }
467 798840 : CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
468 : #undef DEFINE_CODE_ASSEMBLER_BINARY_OP
469 :
470 651295 : TNode<WordT> CodeAssembler::IntPtrAdd(SloppyTNode<WordT> left,
471 : SloppyTNode<WordT> right) {
472 : intptr_t left_constant;
473 651295 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
474 : intptr_t right_constant;
475 651295 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
476 651295 : if (is_left_constant) {
477 291818 : if (is_right_constant) {
478 102522 : return IntPtrConstant(left_constant + right_constant);
479 : }
480 240557 : if (left_constant == 0) {
481 38529 : return right;
482 : }
483 359477 : } else if (is_right_constant) {
484 252599 : if (right_constant == 0) {
485 19381 : return left;
486 : }
487 : }
488 : return UncheckedCast<WordT>(raw_assembler()->IntPtrAdd(left, right));
489 : }
490 :
491 509 : TNode<IntPtrT> CodeAssembler::IntPtrDiv(TNode<IntPtrT> left,
492 : TNode<IntPtrT> right) {
493 : intptr_t left_constant;
494 509 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
495 : intptr_t right_constant;
496 509 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
497 509 : if (is_right_constant) {
498 61 : if (is_left_constant) {
499 2 : return IntPtrConstant(left_constant / right_constant);
500 : }
501 118 : if (base::bits::IsPowerOfTwo(right_constant)) {
502 58 : return WordSar(left, WhichPowerOf2(right_constant));
503 : }
504 : }
505 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrDiv(left, right));
506 : }
507 :
508 174529 : TNode<WordT> CodeAssembler::IntPtrSub(SloppyTNode<WordT> left,
509 : SloppyTNode<WordT> right) {
510 : intptr_t left_constant;
511 174529 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
512 : intptr_t right_constant;
513 174529 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
514 174529 : if (is_left_constant) {
515 77353 : if (is_right_constant) {
516 129042 : return IntPtrConstant(left_constant - right_constant);
517 : }
518 97176 : } else if (is_right_constant) {
519 82616 : if (right_constant == 0) {
520 9777 : return left;
521 : }
522 : }
523 100231 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrSub(left, right));
524 : }
525 :
526 30774 : TNode<WordT> CodeAssembler::IntPtrMul(SloppyTNode<WordT> left,
527 : SloppyTNode<WordT> right) {
528 : intptr_t left_constant;
529 30774 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
530 : intptr_t right_constant;
531 30774 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
532 30774 : if (is_left_constant) {
533 2011 : if (is_right_constant) {
534 4018 : return IntPtrConstant(left_constant * right_constant);
535 : }
536 4 : if (base::bits::IsPowerOfTwo(left_constant)) {
537 2 : return WordShl(right, WhichPowerOf2(left_constant));
538 : }
539 28763 : } else if (is_right_constant) {
540 56398 : if (base::bits::IsPowerOfTwo(right_constant)) {
541 10994 : return WordShl(left, WhichPowerOf2(right_constant));
542 : }
543 : }
544 17769 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrMul(left, right));
545 : }
546 :
547 105015 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> value, int shift) {
548 207315 : return (shift != 0) ? WordShl(value, IntPtrConstant(shift)) : value;
549 : }
550 :
551 29032 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> value, int shift) {
552 51847 : return (shift != 0) ? WordShr(value, IntPtrConstant(shift)) : value;
553 : }
554 :
555 3038 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> value, int shift) {
556 6075 : return (shift != 0) ? WordSar(value, IntPtrConstant(shift)) : value;
557 : }
558 :
559 37632 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> value, int shift) {
560 63832 : return (shift != 0) ? Word32Shr(value, Int32Constant(shift)) : value;
561 : }
562 :
563 43480 : TNode<WordT> CodeAssembler::WordOr(SloppyTNode<WordT> left,
564 : SloppyTNode<WordT> right) {
565 : intptr_t left_constant;
566 43480 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
567 : intptr_t right_constant;
568 43480 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
569 43480 : if (is_left_constant) {
570 3530 : if (is_right_constant) {
571 7058 : return IntPtrConstant(left_constant | right_constant);
572 : }
573 1 : if (left_constant == 0) {
574 1 : return right;
575 : }
576 39950 : } else if (is_right_constant) {
577 2522 : if (right_constant == 0) {
578 1 : return left;
579 : }
580 : }
581 39949 : return UncheckedCast<WordT>(raw_assembler()->WordOr(left, right));
582 : }
583 :
584 289074 : TNode<WordT> CodeAssembler::WordAnd(SloppyTNode<WordT> left,
585 : SloppyTNode<WordT> right) {
586 : intptr_t left_constant;
587 289074 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
588 : intptr_t right_constant;
589 289074 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
590 289074 : if (is_left_constant) {
591 393 : if (is_right_constant) {
592 786 : return IntPtrConstant(left_constant & right_constant);
593 : }
594 : }
595 288681 : return UncheckedCast<WordT>(raw_assembler()->WordAnd(left, right));
596 : }
597 :
598 294 : TNode<WordT> CodeAssembler::WordXor(SloppyTNode<WordT> left,
599 : SloppyTNode<WordT> right) {
600 : intptr_t left_constant;
601 294 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
602 : intptr_t right_constant;
603 294 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
604 294 : if (is_left_constant) {
605 1 : if (is_right_constant) {
606 2 : return IntPtrConstant(left_constant ^ right_constant);
607 : }
608 : }
609 293 : return UncheckedCast<WordT>(raw_assembler()->WordXor(left, right));
610 : }
611 :
612 380188 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> left,
613 : SloppyTNode<IntegralT> right) {
614 : intptr_t left_constant;
615 380188 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
616 : intptr_t right_constant;
617 380188 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
618 380188 : if (is_left_constant) {
619 11433 : if (is_right_constant) {
620 22754 : return IntPtrConstant(left_constant << right_constant);
621 : }
622 368755 : } else if (is_right_constant) {
623 368195 : if (right_constant == 0) {
624 0 : return left;
625 : }
626 : }
627 368811 : return UncheckedCast<WordT>(raw_assembler()->WordShl(left, right));
628 : }
629 :
630 28267 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> left,
631 : SloppyTNode<IntegralT> right) {
632 : intptr_t left_constant;
633 28267 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
634 : intptr_t right_constant;
635 28267 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
636 28267 : if (is_left_constant) {
637 3698 : if (is_right_constant) {
638 3698 : return IntPtrConstant(static_cast<uintptr_t>(left_constant) >>
639 7396 : right_constant);
640 : }
641 24569 : } else if (is_right_constant) {
642 24065 : if (right_constant == 0) {
643 0 : return left;
644 : }
645 : }
646 24569 : return UncheckedCast<WordT>(raw_assembler()->WordShr(left, right));
647 : }
648 :
649 158465 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> left,
650 : SloppyTNode<IntegralT> right) {
651 : intptr_t left_constant;
652 158465 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
653 : intptr_t right_constant;
654 158465 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
655 158465 : if (is_left_constant) {
656 2 : if (is_right_constant) {
657 4 : return IntPtrConstant(left_constant >> right_constant);
658 : }
659 158463 : } else if (is_right_constant) {
660 158351 : if (right_constant == 0) {
661 1 : return left;
662 : }
663 : }
664 158462 : return UncheckedCast<WordT>(raw_assembler()->WordSar(left, right));
665 : }
666 :
667 26952 : TNode<Word32T> CodeAssembler::Word32Or(SloppyTNode<Word32T> left,
668 : SloppyTNode<Word32T> right) {
669 : int32_t left_constant;
670 26952 : bool is_left_constant = ToInt32Constant(left, left_constant);
671 : int32_t right_constant;
672 26952 : bool is_right_constant = ToInt32Constant(right, right_constant);
673 26952 : if (is_left_constant) {
674 0 : if (is_right_constant) {
675 0 : return Int32Constant(left_constant | right_constant);
676 : }
677 0 : if (left_constant == 0) {
678 0 : return right;
679 : }
680 26952 : } else if (is_right_constant) {
681 56 : if (right_constant == 0) {
682 0 : return left;
683 : }
684 : }
685 26952 : return UncheckedCast<Word32T>(raw_assembler()->Word32Or(left, right));
686 : }
687 :
688 132116 : TNode<Word32T> CodeAssembler::Word32And(SloppyTNode<Word32T> left,
689 : SloppyTNode<Word32T> right) {
690 : int32_t left_constant;
691 132116 : bool is_left_constant = ToInt32Constant(left, left_constant);
692 : int32_t right_constant;
693 132116 : bool is_right_constant = ToInt32Constant(right, right_constant);
694 132116 : if (is_left_constant) {
695 12 : if (is_right_constant) {
696 8 : return Int32Constant(left_constant & right_constant);
697 : }
698 : }
699 132112 : return UncheckedCast<Word32T>(raw_assembler()->Word32And(left, right));
700 : }
701 :
702 2472 : TNode<Word32T> CodeAssembler::Word32Xor(SloppyTNode<Word32T> left,
703 : SloppyTNode<Word32T> right) {
704 : int32_t left_constant;
705 2472 : bool is_left_constant = ToInt32Constant(left, left_constant);
706 : int32_t right_constant;
707 2472 : bool is_right_constant = ToInt32Constant(right, right_constant);
708 2472 : if (is_left_constant) {
709 4 : if (is_right_constant) {
710 8 : return Int32Constant(left_constant ^ right_constant);
711 : }
712 : }
713 2468 : return UncheckedCast<Word32T>(raw_assembler()->Word32Xor(left, right));
714 : }
715 :
716 8512 : TNode<Word32T> CodeAssembler::Word32Shl(SloppyTNode<Word32T> left,
717 : SloppyTNode<Word32T> right) {
718 : int32_t left_constant;
719 8512 : bool is_left_constant = ToInt32Constant(left, left_constant);
720 : int32_t right_constant;
721 8512 : bool is_right_constant = ToInt32Constant(right, right_constant);
722 8512 : if (is_left_constant) {
723 0 : if (is_right_constant) {
724 0 : return Int32Constant(left_constant << right_constant);
725 : }
726 8512 : } else if (is_right_constant) {
727 4872 : if (right_constant == 0) {
728 0 : return left;
729 : }
730 : }
731 8512 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shl(left, right));
732 : }
733 :
734 36588 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> left,
735 : SloppyTNode<Word32T> right) {
736 : int32_t left_constant;
737 36588 : bool is_left_constant = ToInt32Constant(left, left_constant);
738 : int32_t right_constant;
739 36588 : bool is_right_constant = ToInt32Constant(right, right_constant);
740 36588 : if (is_left_constant) {
741 0 : if (is_right_constant) {
742 0 : return Int32Constant(static_cast<uint32_t>(left_constant) >>
743 0 : right_constant);
744 : }
745 36588 : } else if (is_right_constant) {
746 34572 : if (right_constant == 0) {
747 0 : return left;
748 : }
749 : }
750 36588 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shr(left, right));
751 : }
752 :
753 392 : TNode<Word32T> CodeAssembler::Word32Sar(SloppyTNode<Word32T> left,
754 : SloppyTNode<Word32T> right) {
755 : int32_t left_constant;
756 392 : bool is_left_constant = ToInt32Constant(left, left_constant);
757 : int32_t right_constant;
758 392 : bool is_right_constant = ToInt32Constant(right, right_constant);
759 392 : if (is_left_constant) {
760 0 : if (is_right_constant) {
761 0 : return Int32Constant(left_constant >> right_constant);
762 : }
763 392 : } else if (is_right_constant) {
764 0 : if (right_constant == 0) {
765 0 : return left;
766 : }
767 : }
768 392 : return UncheckedCast<Word32T>(raw_assembler()->Word32Sar(left, right));
769 : }
770 :
771 0 : TNode<Word64T> CodeAssembler::Word64Or(SloppyTNode<Word64T> left,
772 : SloppyTNode<Word64T> right) {
773 : int64_t left_constant;
774 : bool is_left_constant = ToInt64Constant(left, left_constant);
775 : int64_t right_constant;
776 : bool is_right_constant = ToInt64Constant(right, right_constant);
777 0 : if (is_left_constant) {
778 0 : if (is_right_constant) {
779 0 : return Int64Constant(left_constant | right_constant);
780 : }
781 0 : if (left_constant == 0) {
782 0 : return right;
783 : }
784 0 : } else if (is_right_constant) {
785 0 : if (right_constant == 0) {
786 0 : return left;
787 : }
788 : }
789 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Or(left, right));
790 : }
791 :
792 0 : TNode<Word64T> CodeAssembler::Word64And(SloppyTNode<Word64T> left,
793 : SloppyTNode<Word64T> right) {
794 : int64_t left_constant;
795 : bool is_left_constant = ToInt64Constant(left, left_constant);
796 : int64_t right_constant;
797 : bool is_right_constant = ToInt64Constant(right, right_constant);
798 0 : if (is_left_constant) {
799 0 : if (is_right_constant) {
800 0 : return Int64Constant(left_constant & right_constant);
801 : }
802 : }
803 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64And(left, right));
804 : }
805 :
806 0 : TNode<Word64T> CodeAssembler::Word64Xor(SloppyTNode<Word64T> left,
807 : SloppyTNode<Word64T> right) {
808 : int64_t left_constant;
809 : bool is_left_constant = ToInt64Constant(left, left_constant);
810 : int64_t right_constant;
811 : bool is_right_constant = ToInt64Constant(right, right_constant);
812 0 : if (is_left_constant) {
813 0 : if (is_right_constant) {
814 0 : return Int64Constant(left_constant ^ right_constant);
815 : }
816 : }
817 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Xor(left, right));
818 : }
819 :
820 0 : TNode<Word64T> CodeAssembler::Word64Shl(SloppyTNode<Word64T> left,
821 : SloppyTNode<Word64T> right) {
822 : int64_t left_constant;
823 : bool is_left_constant = ToInt64Constant(left, left_constant);
824 : int64_t right_constant;
825 : bool is_right_constant = ToInt64Constant(right, right_constant);
826 0 : if (is_left_constant) {
827 0 : if (is_right_constant) {
828 0 : return Int64Constant(left_constant << right_constant);
829 : }
830 0 : } else if (is_right_constant) {
831 0 : if (right_constant == 0) {
832 0 : return left;
833 : }
834 : }
835 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shl(left, right));
836 : }
837 :
838 0 : TNode<Word64T> CodeAssembler::Word64Shr(SloppyTNode<Word64T> left,
839 : SloppyTNode<Word64T> right) {
840 : int64_t left_constant;
841 : bool is_left_constant = ToInt64Constant(left, left_constant);
842 : int64_t right_constant;
843 : bool is_right_constant = ToInt64Constant(right, right_constant);
844 0 : if (is_left_constant) {
845 0 : if (is_right_constant) {
846 0 : return Int64Constant(static_cast<uint64_t>(left_constant) >>
847 0 : right_constant);
848 : }
849 0 : } else if (is_right_constant) {
850 0 : if (right_constant == 0) {
851 0 : return left;
852 : }
853 : }
854 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shr(left, right));
855 : }
856 :
857 0 : TNode<Word64T> CodeAssembler::Word64Sar(SloppyTNode<Word64T> left,
858 : SloppyTNode<Word64T> right) {
859 : int64_t left_constant;
860 : bool is_left_constant = ToInt64Constant(left, left_constant);
861 : int64_t right_constant;
862 : bool is_right_constant = ToInt64Constant(right, right_constant);
863 0 : if (is_left_constant) {
864 0 : if (is_right_constant) {
865 0 : return Int64Constant(left_constant >> right_constant);
866 : }
867 0 : } else if (is_right_constant) {
868 0 : if (right_constant == 0) {
869 0 : return left;
870 : }
871 : }
872 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Sar(left, right));
873 : }
874 :
875 : #define CODE_ASSEMBLER_COMPARE(Name, ArgT, VarT, ToConstant, op) \
876 : TNode<BoolT> CodeAssembler::Name(SloppyTNode<ArgT> left, \
877 : SloppyTNode<ArgT> right) { \
878 : VarT lhs, rhs; \
879 : if (ToConstant(left, lhs) && ToConstant(right, rhs)) { \
880 : return BoolConstant(lhs op rhs); \
881 : } \
882 : return UncheckedCast<BoolT>(raw_assembler()->Name(left, right)); \
883 : }
884 :
885 7336 : CODE_ASSEMBLER_COMPARE(IntPtrEqual, WordT, intptr_t, ToIntPtrConstant, ==)
886 1119080 : CODE_ASSEMBLER_COMPARE(WordEqual, WordT, intptr_t, ToIntPtrConstant, ==)
887 97416 : CODE_ASSEMBLER_COMPARE(WordNotEqual, WordT, intptr_t, ToIntPtrConstant, !=)
888 432520 : CODE_ASSEMBLER_COMPARE(Word32Equal, Word32T, int32_t, ToInt32Constant, ==)
889 76992 : CODE_ASSEMBLER_COMPARE(Word32NotEqual, Word32T, int32_t, ToInt32Constant, !=)
890 15336 : CODE_ASSEMBLER_COMPARE(Word64Equal, Word64T, int64_t, ToInt64Constant, ==)
891 0 : CODE_ASSEMBLER_COMPARE(Word64NotEqual, Word64T, int64_t, ToInt64Constant, !=)
892 : #undef CODE_ASSEMBLER_COMPARE
893 :
894 115249 : TNode<UintPtrT> CodeAssembler::ChangeUint32ToWord(SloppyTNode<Word32T> value) {
895 115249 : if (raw_assembler()->machine()->Is64()) {
896 : return UncheckedCast<UintPtrT>(
897 115249 : raw_assembler()->ChangeUint32ToUint64(value));
898 : }
899 : return ReinterpretCast<UintPtrT>(value);
900 : }
901 :
902 155300 : TNode<IntPtrT> CodeAssembler::ChangeInt32ToIntPtr(SloppyTNode<Word32T> value) {
903 155300 : if (raw_assembler()->machine()->Is64()) {
904 155300 : return ReinterpretCast<IntPtrT>(raw_assembler()->ChangeInt32ToInt64(value));
905 : }
906 : return ReinterpretCast<IntPtrT>(value);
907 : }
908 :
909 1456 : TNode<UintPtrT> CodeAssembler::ChangeFloat64ToUintPtr(
910 : SloppyTNode<Float64T> value) {
911 1456 : if (raw_assembler()->machine()->Is64()) {
912 : return ReinterpretCast<UintPtrT>(
913 1456 : raw_assembler()->ChangeFloat64ToUint64(value));
914 : }
915 : return ReinterpretCast<UintPtrT>(
916 0 : raw_assembler()->ChangeFloat64ToUint32(value));
917 : }
918 :
919 1736 : TNode<Float64T> CodeAssembler::ChangeUintPtrToFloat64(TNode<UintPtrT> value) {
920 1736 : if (raw_assembler()->machine()->Is64()) {
921 : // TODO(turbofan): Maybe we should introduce a ChangeUint64ToFloat64
922 : // machine operator to TurboFan here?
923 : return ReinterpretCast<Float64T>(
924 1736 : raw_assembler()->RoundUint64ToFloat64(value));
925 : }
926 : return ReinterpretCast<Float64T>(
927 0 : raw_assembler()->ChangeUint32ToFloat64(value));
928 : }
929 :
930 1128 : Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
931 1128 : if (raw_assembler()->machine()->Is64()) {
932 1128 : return raw_assembler()->RoundInt64ToFloat64(value);
933 : }
934 0 : return raw_assembler()->ChangeInt32ToFloat64(value);
935 : }
936 :
937 : #define DEFINE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
938 : TNode<ResType> CodeAssembler::name(SloppyTNode<ArgType> a) { \
939 : return UncheckedCast<ResType>(raw_assembler()->name(a)); \
940 : }
941 2740865 : CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
942 : #undef DEFINE_CODE_ASSEMBLER_UNARY_OP
943 :
944 48868 : Node* CodeAssembler::Load(MachineType rep, Node* base,
945 : LoadSensitivity needs_poisoning) {
946 68188 : return raw_assembler()->Load(rep, base, needs_poisoning);
947 : }
948 :
949 1384003 : Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset,
950 : LoadSensitivity needs_poisoning) {
951 1472243 : return raw_assembler()->Load(rep, base, offset, needs_poisoning);
952 : }
953 :
954 19320 : Node* CodeAssembler::LoadFullTagged(Node* base,
955 : LoadSensitivity needs_poisoning) {
956 : return BitcastWordToTagged(
957 19320 : Load(MachineType::Pointer(), base, needs_poisoning));
958 : }
959 :
960 88240 : Node* CodeAssembler::LoadFullTagged(Node* base, Node* offset,
961 : LoadSensitivity needs_poisoning) {
962 : return BitcastWordToTagged(
963 88240 : Load(MachineType::Pointer(), base, offset, needs_poisoning));
964 : }
965 :
966 448 : Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
967 448 : return raw_assembler()->AtomicLoad(rep, base, offset);
968 : }
969 :
970 443968 : TNode<Object> CodeAssembler::LoadRoot(RootIndex root_index) {
971 443968 : if (RootsTable::IsImmortalImmovable(root_index)) {
972 : Handle<Object> root = isolate()->root_handle(root_index);
973 443628 : if (root->IsSmi()) {
974 0 : return SmiConstant(Smi::cast(*root));
975 : } else {
976 443628 : return HeapConstant(Handle<HeapObject>::cast(root));
977 : }
978 : }
979 :
980 : // TODO(jgruber): In theory we could generate better code for this by
981 : // letting the macro assembler decide how to load from the roots list. In most
982 : // cases, it would boil down to loading from a fixed kRootRegister offset.
983 : Node* isolate_root =
984 340 : ExternalConstant(ExternalReference::isolate_root(isolate()));
985 : int offset = IsolateData::root_slot_offset(root_index);
986 : return UncheckedCast<Object>(
987 680 : LoadFullTagged(isolate_root, IntPtrConstant(offset)));
988 : }
989 :
990 0 : Node* CodeAssembler::Store(Node* base, Node* value) {
991 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
992 0 : kFullWriteBarrier);
993 : }
994 :
995 249856 : void CodeAssembler::OptimizedStoreField(MachineRepresentation rep,
996 : TNode<HeapObject> object, int offset,
997 : Node* value,
998 : WriteBarrierKind write_barrier) {
999 249856 : raw_assembler()->OptimizedStoreField(rep, object, offset, value,
1000 249856 : write_barrier);
1001 249856 : }
1002 10828 : void CodeAssembler::OptimizedStoreMap(TNode<HeapObject> object,
1003 : TNode<Map> map) {
1004 10828 : raw_assembler()->OptimizedStoreMap(object, map);
1005 10828 : }
1006 :
1007 27476 : Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
1008 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
1009 27476 : value, kFullWriteBarrier);
1010 : }
1011 :
1012 56 : Node* CodeAssembler::StoreEphemeronKey(Node* base, Node* offset, Node* value) {
1013 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
1014 56 : value, kEphemeronKeyWriteBarrier);
1015 : }
1016 :
1017 9016 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
1018 : Node* value) {
1019 10080 : return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
1020 : }
1021 :
1022 206878 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
1023 : Node* offset, Node* value) {
1024 212758 : return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
1025 : }
1026 :
1027 1064 : Node* CodeAssembler::StoreFullTaggedNoWriteBarrier(Node* base,
1028 : Node* tagged_value) {
1029 : return StoreNoWriteBarrier(MachineType::PointerRepresentation(), base,
1030 1064 : BitcastTaggedToWord(tagged_value));
1031 : }
1032 :
1033 5880 : Node* CodeAssembler::StoreFullTaggedNoWriteBarrier(Node* base, Node* offset,
1034 : Node* tagged_value) {
1035 : return StoreNoWriteBarrier(MachineType::PointerRepresentation(), base, offset,
1036 5880 : BitcastTaggedToWord(tagged_value));
1037 : }
1038 :
1039 224 : Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
1040 : Node* offset, Node* value, Node* value_high) {
1041 224 : return raw_assembler()->AtomicStore(rep, base, offset, value, value_high);
1042 : }
1043 :
1044 : #define ATOMIC_FUNCTION(name) \
1045 : Node* CodeAssembler::Atomic##name(MachineType type, Node* base, \
1046 : Node* offset, Node* value, \
1047 : Node* value_high) { \
1048 : return raw_assembler()->Atomic##name(type, base, offset, value, \
1049 : value_high); \
1050 : }
1051 896 : ATOMIC_FUNCTION(Exchange)
1052 896 : ATOMIC_FUNCTION(Add)
1053 896 : ATOMIC_FUNCTION(Sub)
1054 896 : ATOMIC_FUNCTION(And)
1055 896 : ATOMIC_FUNCTION(Or)
1056 896 : ATOMIC_FUNCTION(Xor)
1057 : #undef ATOMIC_FUNCTION
1058 :
1059 448 : Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
1060 : Node* offset, Node* old_value,
1061 : Node* new_value,
1062 : Node* old_value_high,
1063 : Node* new_value_high) {
1064 : return raw_assembler()->AtomicCompareExchange(
1065 448 : type, base, offset, old_value, old_value_high, new_value, new_value_high);
1066 : }
1067 :
1068 112 : Node* CodeAssembler::StoreRoot(RootIndex root_index, Node* value) {
1069 : DCHECK(!RootsTable::IsImmortalImmovable(root_index));
1070 : Node* isolate_root =
1071 112 : ExternalConstant(ExternalReference::isolate_root(isolate()));
1072 : int offset = IsolateData::root_slot_offset(root_index);
1073 112 : return StoreFullTaggedNoWriteBarrier(isolate_root, IntPtrConstant(offset),
1074 112 : value);
1075 : }
1076 :
1077 0 : Node* CodeAssembler::Retain(Node* value) {
1078 0 : return raw_assembler()->Retain(value);
1079 : }
1080 :
1081 0 : Node* CodeAssembler::ChangeTaggedToCompressed(Node* tagged) {
1082 0 : return raw_assembler()->ChangeTaggedToCompressed(tagged);
1083 : }
1084 :
1085 0 : Node* CodeAssembler::ChangeCompressedToTagged(Node* compressed) {
1086 0 : return raw_assembler()->ChangeCompressedToTagged(compressed);
1087 : }
1088 :
1089 19056 : Node* CodeAssembler::Projection(int index, Node* value) {
1090 : DCHECK_LT(index, value->op()->ValueOutputCount());
1091 19056 : return raw_assembler()->Projection(index, value);
1092 : }
1093 :
1094 8864 : void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
1095 : Variable* exception_var) {
1096 8864 : if (if_exception == nullptr) {
1097 : // If no handler is supplied, don't add continuations
1098 3528 : return;
1099 : }
1100 :
1101 : // No catch handlers should be active if we're using catch labels
1102 : DCHECK_EQ(state()->exception_handler_labels_.size(), 0);
1103 : DCHECK(!node->op()->HasProperty(Operator::kNoThrow));
1104 :
1105 5336 : Label success(this), exception(this, Label::kDeferred);
1106 5336 : success.MergeVariables();
1107 5336 : exception.MergeVariables();
1108 :
1109 10672 : raw_assembler()->Continuations(node, success.label_, exception.label_);
1110 :
1111 : Bind(&exception);
1112 5336 : const Operator* op = raw_assembler()->common()->IfException();
1113 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
1114 5336 : if (exception_var != nullptr) {
1115 : exception_var->Bind(exception_value);
1116 : }
1117 : Goto(if_exception);
1118 :
1119 : Bind(&success);
1120 5336 : raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1121 : }
1122 :
1123 59212 : TNode<HeapObject> CodeAssembler::OptimizedAllocate(TNode<IntPtrT> size,
1124 : AllocationType allocation) {
1125 : return UncheckedCast<HeapObject>(
1126 59212 : raw_assembler()->OptimizedAllocate(size, allocation));
1127 : }
1128 :
1129 254022 : void CodeAssembler::HandleException(Node* node) {
1130 754546 : if (state_->exception_handler_labels_.size() == 0) return;
1131 : CodeAssemblerExceptionHandlerLabel* label =
1132 7520 : state_->exception_handler_labels_.back();
1133 :
1134 7520 : if (node->op()->HasProperty(Operator::kNoThrow)) {
1135 : return;
1136 : }
1137 :
1138 7520 : Label success(this), exception(this, Label::kDeferred);
1139 7520 : success.MergeVariables();
1140 7520 : exception.MergeVariables();
1141 :
1142 15040 : raw_assembler()->Continuations(node, success.label_, exception.label_);
1143 :
1144 : Bind(&exception);
1145 7520 : const Operator* op = raw_assembler()->common()->IfException();
1146 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
1147 7520 : label->AddInputs({UncheckedCast<Object>(exception_value)});
1148 : Goto(label->plain_label());
1149 :
1150 : Bind(&success);
1151 7520 : raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1152 : }
1153 :
1154 : namespace {
1155 : template <size_t kMaxSize>
1156 270046 : class NodeArray {
1157 : public:
1158 : void Add(Node* node) {
1159 : DCHECK_GT(kMaxSize, size());
1160 1195802 : *ptr_++ = node;
1161 : }
1162 :
1163 : Node* const* data() const { return arr_; }
1164 270046 : int size() const { return static_cast<int>(ptr_ - arr_); }
1165 :
1166 : private:
1167 : Node* arr_[kMaxSize];
1168 : Node** ptr_ = arr_;
1169 : };
1170 : } // namespace
1171 :
1172 141412 : TNode<Object> CodeAssembler::CallRuntimeImpl(
1173 : Runtime::FunctionId function, TNode<Object> context,
1174 : std::initializer_list<TNode<Object>> args) {
1175 141412 : int result_size = Runtime::FunctionForId(function)->result_size;
1176 : TNode<Code> centry =
1177 282824 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1178 141412 : return CallRuntimeWithCEntryImpl(function, centry, context, args);
1179 : }
1180 :
1181 141636 : TNode<Object> CodeAssembler::CallRuntimeWithCEntryImpl(
1182 : Runtime::FunctionId function, TNode<Code> centry, TNode<Object> context,
1183 : std::initializer_list<TNode<Object>> args) {
1184 : constexpr size_t kMaxNumArgs = 6;
1185 : DCHECK_GE(kMaxNumArgs, args.size());
1186 141636 : int argc = static_cast<int>(args.size());
1187 141636 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1188 : zone(), function, argc, Operator::kNoProperties,
1189 141636 : CallDescriptor::kNoFlags);
1190 :
1191 141636 : Node* ref = ExternalConstant(ExternalReference::Create(function));
1192 : Node* arity = Int32Constant(argc);
1193 :
1194 : NodeArray<kMaxNumArgs + 4> inputs;
1195 : inputs.Add(centry);
1196 511148 : for (auto arg : args) inputs.Add(arg);
1197 : inputs.Add(ref);
1198 : inputs.Add(arity);
1199 : inputs.Add(context);
1200 :
1201 141636 : CallPrologue();
1202 : Node* return_value =
1203 141636 : raw_assembler()->CallN(call_descriptor, inputs.size(), inputs.data());
1204 141636 : HandleException(return_value);
1205 141636 : CallEpilogue();
1206 141636 : return UncheckedCast<Object>(return_value);
1207 : }
1208 :
1209 7960 : void CodeAssembler::TailCallRuntimeImpl(
1210 : Runtime::FunctionId function, TNode<Int32T> arity, TNode<Object> context,
1211 : std::initializer_list<TNode<Object>> args) {
1212 7960 : int result_size = Runtime::FunctionForId(function)->result_size;
1213 : TNode<Code> centry =
1214 15920 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1215 7960 : return TailCallRuntimeWithCEntryImpl(function, arity, centry, context, args);
1216 : }
1217 :
1218 9024 : void CodeAssembler::TailCallRuntimeWithCEntryImpl(
1219 : Runtime::FunctionId function, TNode<Int32T> arity, TNode<Code> centry,
1220 : TNode<Object> context, std::initializer_list<TNode<Object>> args) {
1221 : constexpr size_t kMaxNumArgs = 6;
1222 : DCHECK_GE(kMaxNumArgs, args.size());
1223 9024 : int argc = static_cast<int>(args.size());
1224 9024 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1225 : zone(), function, argc, Operator::kNoProperties,
1226 9024 : CallDescriptor::kNoFlags);
1227 :
1228 9024 : Node* ref = ExternalConstant(ExternalReference::Create(function));
1229 :
1230 : NodeArray<kMaxNumArgs + 4> inputs;
1231 : inputs.Add(centry);
1232 65384 : for (auto arg : args) inputs.Add(arg);
1233 : inputs.Add(ref);
1234 : inputs.Add(arity);
1235 : inputs.Add(context);
1236 :
1237 9024 : raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1238 9024 : }
1239 :
1240 112386 : Node* CodeAssembler::CallStubN(StubCallMode call_mode,
1241 : const CallInterfaceDescriptor& descriptor,
1242 : size_t result_size, int input_count,
1243 : Node* const* inputs) {
1244 : DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1245 : call_mode == StubCallMode::kCallBuiltinPointer);
1246 :
1247 : // implicit nodes are target and optionally context.
1248 112386 : int implicit_nodes = descriptor.HasContextParameter() ? 2 : 1;
1249 : DCHECK_LE(implicit_nodes, input_count);
1250 112386 : int argc = input_count - implicit_nodes;
1251 : DCHECK_LE(descriptor.GetParameterCount(), argc);
1252 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1253 112386 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1254 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1255 : DCHECK_EQ(result_size, descriptor.GetReturnCount());
1256 :
1257 112386 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1258 : zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1259 112386 : Operator::kNoProperties, call_mode);
1260 :
1261 112386 : CallPrologue();
1262 : Node* return_value =
1263 112386 : raw_assembler()->CallN(call_descriptor, input_count, inputs);
1264 112386 : HandleException(return_value);
1265 112386 : CallEpilogue();
1266 112386 : return return_value;
1267 : }
1268 :
1269 7504 : void CodeAssembler::TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
1270 : TNode<Code> target, TNode<Object> context,
1271 : std::initializer_list<Node*> args) {
1272 : constexpr size_t kMaxNumArgs = 11;
1273 : DCHECK_GE(kMaxNumArgs, args.size());
1274 : DCHECK_EQ(descriptor.GetParameterCount(), args.size());
1275 7504 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1276 : zone(), descriptor, descriptor.GetStackParameterCount(),
1277 7504 : CallDescriptor::kNoFlags, Operator::kNoProperties);
1278 :
1279 : NodeArray<kMaxNumArgs + 2> inputs;
1280 : inputs.Add(target);
1281 56336 : for (auto arg : args) inputs.Add(arg);
1282 7504 : if (descriptor.HasContextParameter()) {
1283 : inputs.Add(context);
1284 : }
1285 :
1286 7504 : raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1287 7504 : }
1288 :
1289 109698 : Node* CodeAssembler::CallStubRImpl(StubCallMode call_mode,
1290 : const CallInterfaceDescriptor& descriptor,
1291 : size_t result_size, Node* target,
1292 : SloppyTNode<Object> context,
1293 : std::initializer_list<Node*> args) {
1294 : DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1295 : call_mode == StubCallMode::kCallBuiltinPointer);
1296 :
1297 : constexpr size_t kMaxNumArgs = 10;
1298 : DCHECK_GE(kMaxNumArgs, args.size());
1299 :
1300 : NodeArray<kMaxNumArgs + 2> inputs;
1301 : inputs.Add(target);
1302 613270 : for (auto arg : args) inputs.Add(arg);
1303 109698 : if (descriptor.HasContextParameter()) {
1304 : inputs.Add(context);
1305 : }
1306 :
1307 : return CallStubN(call_mode, descriptor, result_size, inputs.size(),
1308 109698 : inputs.data());
1309 : }
1310 :
1311 2184 : Node* CodeAssembler::TailCallStubThenBytecodeDispatchImpl(
1312 : const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
1313 : std::initializer_list<Node*> args) {
1314 : constexpr size_t kMaxNumArgs = 6;
1315 : DCHECK_GE(kMaxNumArgs, args.size());
1316 :
1317 : DCHECK_LE(descriptor.GetParameterCount(), args.size());
1318 2184 : int argc = static_cast<int>(args.size());
1319 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1320 2184 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1321 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1322 2184 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1323 : zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1324 2184 : Operator::kNoProperties);
1325 :
1326 : NodeArray<kMaxNumArgs + 2> inputs;
1327 : inputs.Add(target);
1328 17304 : for (auto arg : args) inputs.Add(arg);
1329 : inputs.Add(context);
1330 :
1331 : return raw_assembler()->TailCallN(call_descriptor, inputs.size(),
1332 2184 : inputs.data());
1333 : }
1334 :
1335 : template <class... TArgs>
1336 43680 : Node* CodeAssembler::TailCallBytecodeDispatch(
1337 : const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
1338 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1339 : auto call_descriptor = Linkage::GetBytecodeDispatchCallDescriptor(
1340 43680 : zone(), descriptor, descriptor.GetStackParameterCount());
1341 :
1342 43680 : Node* nodes[] = {target, args...};
1343 87360 : CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
1344 43680 : return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1345 : }
1346 :
1347 : // Instantiate TailCallBytecodeDispatch() for argument counts used by
1348 : // CSA-generated code
1349 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
1350 : const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
1351 : Node*, Node*);
1352 :
1353 504 : TNode<Object> CodeAssembler::TailCallJSCode(TNode<Code> code,
1354 : TNode<Context> context,
1355 : TNode<JSFunction> function,
1356 : TNode<Object> new_target,
1357 : TNode<Int32T> arg_count) {
1358 : JSTrampolineDescriptor descriptor;
1359 504 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1360 : zone(), descriptor, descriptor.GetStackParameterCount(),
1361 504 : CallDescriptor::kFixedTargetRegister, Operator::kNoProperties);
1362 :
1363 2520 : Node* nodes[] = {code, function, new_target, arg_count, context};
1364 1008 : CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
1365 : return UncheckedCast<Object>(
1366 1008 : raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes));
1367 : }
1368 :
1369 0 : Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
1370 : int input_count, Node* const* inputs) {
1371 0 : auto call_descriptor = Linkage::GetSimplifiedCDescriptor(zone(), signature);
1372 0 : return raw_assembler()->CallN(call_descriptor, input_count, inputs);
1373 : }
1374 :
1375 12536 : Node* CodeAssembler::CallCFunction(
1376 : Node* function, MachineType return_type,
1377 : std::initializer_list<CodeAssembler::CFunctionArg> args) {
1378 12536 : return raw_assembler()->CallCFunction(function, return_type, args);
1379 : }
1380 :
1381 452 : Node* CodeAssembler::CallCFunctionWithCallerSavedRegisters(
1382 : Node* function, MachineType return_type, SaveFPRegsMode mode,
1383 : std::initializer_list<CodeAssembler::CFunctionArg> args) {
1384 : DCHECK(return_type.LessThanOrEqualPointerSize());
1385 : return raw_assembler()->CallCFunctionWithCallerSavedRegisters(
1386 452 : function, return_type, mode, args);
1387 : }
1388 :
1389 2066949 : void CodeAssembler::Goto(Label* label) {
1390 2080269 : label->MergeVariables();
1391 4160538 : raw_assembler()->Goto(label->label_);
1392 2066949 : }
1393 :
1394 532712 : void CodeAssembler::GotoIf(SloppyTNode<IntegralT> condition,
1395 : Label* true_label) {
1396 532712 : Label false_label(this);
1397 532712 : Branch(condition, true_label, &false_label);
1398 : Bind(&false_label);
1399 532712 : }
1400 :
1401 174543 : void CodeAssembler::GotoIfNot(SloppyTNode<IntegralT> condition,
1402 : Label* false_label) {
1403 174543 : Label true_label(this);
1404 174543 : Branch(condition, &true_label, false_label);
1405 : Bind(&true_label);
1406 174543 : }
1407 :
1408 1296551 : void CodeAssembler::Branch(SloppyTNode<IntegralT> condition, Label* true_label,
1409 : Label* false_label) {
1410 : int32_t constant;
1411 1296551 : if (ToInt32Constant(condition, constant)) {
1412 976 : if ((true_label->is_used() || true_label->is_bound()) &&
1413 0 : (false_label->is_used() || false_label->is_bound())) {
1414 0 : return Goto(constant ? true_label : false_label);
1415 : }
1416 : }
1417 1296551 : true_label->MergeVariables();
1418 1296551 : false_label->MergeVariables();
1419 1296551 : return raw_assembler()->Branch(condition, true_label->label_,
1420 1296551 : false_label->label_);
1421 : }
1422 :
1423 27368 : void CodeAssembler::Branch(TNode<BoolT> condition,
1424 : const std::function<void()>& true_body,
1425 : const std::function<void()>& false_body) {
1426 : int32_t constant;
1427 27368 : if (ToInt32Constant(condition, constant)) {
1428 0 : return constant ? true_body() : false_body();
1429 : }
1430 :
1431 27368 : Label vtrue(this), vfalse(this);
1432 27368 : Branch(condition, &vtrue, &vfalse);
1433 :
1434 : Bind(&vtrue);
1435 : true_body();
1436 :
1437 : Bind(&vfalse);
1438 : false_body();
1439 : }
1440 :
1441 1536 : void CodeAssembler::Branch(TNode<BoolT> condition, Label* true_label,
1442 : const std::function<void()>& false_body) {
1443 : int32_t constant;
1444 1536 : if (ToInt32Constant(condition, constant)) {
1445 1272 : return constant ? Goto(true_label) : false_body();
1446 : }
1447 :
1448 900 : Label vfalse(this);
1449 900 : Branch(condition, true_label, &vfalse);
1450 : Bind(&vfalse);
1451 : false_body();
1452 : }
1453 :
1454 0 : void CodeAssembler::Branch(TNode<BoolT> condition,
1455 : const std::function<void()>& true_body,
1456 : Label* false_label) {
1457 : int32_t constant;
1458 0 : if (ToInt32Constant(condition, constant)) {
1459 0 : return constant ? true_body() : Goto(false_label);
1460 : }
1461 :
1462 0 : Label vtrue(this);
1463 0 : Branch(condition, &vtrue, false_label);
1464 : Bind(&vtrue);
1465 : true_body();
1466 : }
1467 :
1468 10888 : void CodeAssembler::Switch(Node* index, Label* default_label,
1469 : const int32_t* case_values, Label** case_labels,
1470 : size_t case_count) {
1471 : RawMachineLabel** labels =
1472 10888 : new (zone()->New(sizeof(RawMachineLabel*) * case_count))
1473 10888 : RawMachineLabel*[case_count];
1474 168480 : for (size_t i = 0; i < case_count; ++i) {
1475 78796 : labels[i] = case_labels[i]->label_;
1476 78796 : case_labels[i]->MergeVariables();
1477 : }
1478 10888 : default_label->MergeVariables();
1479 10888 : return raw_assembler()->Switch(index, default_label->label_, case_values,
1480 10888 : labels, case_count);
1481 : }
1482 :
1483 112 : bool CodeAssembler::UnalignedLoadSupported(MachineRepresentation rep) const {
1484 112 : return raw_assembler()->machine()->UnalignedLoadSupported(rep);
1485 : }
1486 112 : bool CodeAssembler::UnalignedStoreSupported(MachineRepresentation rep) const {
1487 112 : return raw_assembler()->machine()->UnalignedStoreSupported(rep);
1488 : }
1489 :
1490 : // RawMachineAssembler delegate helpers:
1491 375536 : Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
1492 :
1493 9744 : Factory* CodeAssembler::factory() const { return isolate()->factory(); }
1494 :
1495 136992 : Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }
1496 :
1497 0 : bool CodeAssembler::IsExceptionHandlerActive() const {
1498 0 : return state_->exception_handler_labels_.size() != 0;
1499 : }
1500 :
1501 0 : RawMachineAssembler* CodeAssembler::raw_assembler() const {
1502 23262057 : return state_->raw_assembler_.get();
1503 : }
1504 :
1505 : // The core implementation of Variable is stored through an indirection so
1506 : // that it can outlive the often block-scoped Variable declarations. This is
1507 : // needed to ensure that variable binding and merging through phis can
1508 : // properly be verified.
1509 : class CodeAssemblerVariable::Impl : public ZoneObject {
1510 : public:
1511 : explicit Impl(MachineRepresentation rep, CodeAssemblerState::VariableId id)
1512 : :
1513 : #if DEBUG
1514 : debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
1515 : #endif
1516 : value_(nullptr),
1517 : rep_(rep),
1518 615070 : var_id_(id) {
1519 : }
1520 :
1521 : #if DEBUG
1522 : AssemblerDebugInfo debug_info() const { return debug_info_; }
1523 : void set_debug_info(AssemblerDebugInfo debug_info) {
1524 : debug_info_ = debug_info;
1525 : }
1526 :
1527 : AssemblerDebugInfo debug_info_;
1528 : #endif // DEBUG
1529 : bool operator<(const CodeAssemblerVariable::Impl& other) const {
1530 1206269 : return var_id_ < other.var_id_;
1531 : }
1532 : Node* value_;
1533 : MachineRepresentation rep_;
1534 : CodeAssemblerState::VariableId var_id_;
1535 : };
1536 :
1537 0 : bool CodeAssemblerVariable::ImplComparator::operator()(
1538 : const CodeAssemblerVariable::Impl* a,
1539 : const CodeAssemblerVariable::Impl* b) const {
1540 0 : return *a < *b;
1541 : }
1542 :
1543 615070 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1544 : MachineRepresentation rep)
1545 : : impl_(new (assembler->zone())
1546 : Impl(rep, assembler->state()->NextVariableId())),
1547 1230140 : state_(assembler->state()) {
1548 615070 : state_->variables_.insert(impl_);
1549 615070 : }
1550 :
1551 274157 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1552 : MachineRepresentation rep,
1553 : Node* initial_value)
1554 274157 : : CodeAssemblerVariable(assembler, rep) {
1555 : Bind(initial_value);
1556 274157 : }
1557 :
1558 : #if DEBUG
1559 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1560 : AssemblerDebugInfo debug_info,
1561 : MachineRepresentation rep)
1562 : : impl_(new (assembler->zone())
1563 : Impl(rep, assembler->state()->NextVariableId())),
1564 : state_(assembler->state()) {
1565 : impl_->set_debug_info(debug_info);
1566 : state_->variables_.insert(impl_);
1567 : }
1568 :
1569 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1570 : AssemblerDebugInfo debug_info,
1571 : MachineRepresentation rep,
1572 : Node* initial_value)
1573 : : CodeAssemblerVariable(assembler, debug_info, rep) {
1574 : impl_->set_debug_info(debug_info);
1575 : Bind(initial_value);
1576 : }
1577 : #endif // DEBUG
1578 :
1579 1230140 : CodeAssemblerVariable::~CodeAssemblerVariable() {
1580 615070 : state_->variables_.erase(impl_);
1581 615070 : }
1582 :
1583 1350176 : void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
1584 :
1585 1503319 : Node* CodeAssemblerVariable::value() const {
1586 : #if DEBUG
1587 : if (!IsBound()) {
1588 : std::stringstream str;
1589 : str << "#Use of unbound variable:"
1590 : << "#\n Variable: " << *this << "#\n Current Block: ";
1591 : state_->PrintCurrentBlock(str);
1592 : FATAL("%s", str.str().c_str());
1593 : }
1594 : if (!state_->InsideBlock()) {
1595 : std::stringstream str;
1596 : str << "#Accessing variable value outside a block:"
1597 : << "#\n Variable: " << *this;
1598 : FATAL("%s", str.str().c_str());
1599 : }
1600 : #endif // DEBUG
1601 1503319 : return impl_->value_;
1602 : }
1603 :
1604 0 : MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
1605 :
1606 147429 : bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
1607 :
1608 0 : std::ostream& operator<<(std::ostream& os,
1609 : const CodeAssemblerVariable::Impl& impl) {
1610 : #if DEBUG
1611 : AssemblerDebugInfo info = impl.debug_info();
1612 : if (info.name) os << "V" << info;
1613 : #endif // DEBUG
1614 0 : return os;
1615 : }
1616 :
1617 0 : std::ostream& operator<<(std::ostream& os,
1618 : const CodeAssemblerVariable& variable) {
1619 : os << *variable.impl_;
1620 0 : return os;
1621 : }
1622 :
1623 3639440 : CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
1624 : size_t vars_count,
1625 : CodeAssemblerVariable* const* vars,
1626 : CodeAssemblerLabel::Type type)
1627 : : bound_(false),
1628 : merge_count_(0),
1629 : state_(assembler->state()),
1630 7278880 : label_(nullptr) {
1631 : void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
1632 : label_ = new (buffer)
1633 : RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
1634 7278880 : : RawMachineLabel::kNonDeferred);
1635 4075080 : for (size_t i = 0; i < vars_count; ++i) {
1636 217820 : variable_phis_[vars[i]->impl_] = nullptr;
1637 : }
1638 3639440 : }
1639 :
1640 7278880 : CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }
1641 :
1642 4788767 : void CodeAssemblerLabel::MergeVariables() {
1643 4788767 : ++merge_count_;
1644 25689457 : for (CodeAssemblerVariable::Impl* var : state_->variables_) {
1645 : size_t count = 0;
1646 16111923 : Node* node = var->value_;
1647 16111923 : if (node != nullptr) {
1648 : auto i = variable_merges_.find(var);
1649 11632325 : if (i != variable_merges_.end()) {
1650 4047285 : i->second.push_back(node);
1651 : count = i->second.size();
1652 : } else {
1653 : count = 1;
1654 15170080 : variable_merges_[var] = std::vector<Node*>(1, node);
1655 : }
1656 : }
1657 : // If the following asserts, then you've jumped to a label without a bound
1658 : // variable along that path that expects to merge its value into a phi.
1659 : DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
1660 : count == merge_count_);
1661 : USE(count);
1662 :
1663 : // If the label is already bound, we already know the set of variables to
1664 : // merge and phi nodes have already been created.
1665 16111923 : if (bound_) {
1666 : auto phi = variable_phis_.find(var);
1667 754096 : if (phi != variable_phis_.end()) {
1668 : DCHECK_NOT_NULL(phi->second);
1669 435168 : state_->raw_assembler_->AppendPhiInput(phi->second, node);
1670 : } else {
1671 : auto i = variable_merges_.find(var);
1672 : if (i != variable_merges_.end()) {
1673 : // If the following assert fires, then you've declared a variable that
1674 : // has the same bound value along all paths up until the point you
1675 : // bound this label, but then later merged a path with a new value for
1676 : // the variable after the label bind (it's not possible to add phis to
1677 : // the bound label after the fact, just make sure to list the variable
1678 : // in the label's constructor's list of merged variables).
1679 : #if DEBUG
1680 : if (find_if(i->second.begin(), i->second.end(),
1681 : [node](Node* e) -> bool { return node != e; }) !=
1682 : i->second.end()) {
1683 : std::stringstream str;
1684 : str << "Unmerged variable found when jumping to block. \n"
1685 : << "# Variable: " << *var;
1686 : if (bound_) {
1687 : str << "\n# Target block: " << *label_->block();
1688 : }
1689 : str << "\n# Current Block: ";
1690 : state_->PrintCurrentBlock(str);
1691 : FATAL("%s", str.str().c_str());
1692 : }
1693 : #endif // DEBUG
1694 : }
1695 : }
1696 : }
1697 : }
1698 4788767 : }
1699 :
1700 : #if DEBUG
1701 : void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
1702 : if (bound_) {
1703 : std::stringstream str;
1704 : str << "Cannot bind the same label twice:"
1705 : << "\n# current: " << debug_info
1706 : << "\n# previous: " << *label_->block();
1707 : FATAL("%s", str.str().c_str());
1708 : }
1709 : if (FLAG_enable_source_at_csa_bind) {
1710 : state_->raw_assembler_->SetSourcePosition(debug_info.file, debug_info.line);
1711 : }
1712 : state_->raw_assembler_->Bind(label_, debug_info);
1713 : UpdateVariablesAfterBind();
1714 : }
1715 : #endif // DEBUG
1716 :
1717 3538240 : void CodeAssemblerLabel::Bind() {
1718 : DCHECK(!bound_);
1719 7076480 : state_->raw_assembler_->Bind(label_);
1720 3538240 : UpdateVariablesAfterBind();
1721 3538240 : }
1722 :
1723 3538240 : void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1724 : // Make sure that all variables that have changed along any path up to this
1725 : // point are marked as merge variables.
1726 17689607 : for (auto var : state_->variables_) {
1727 : Node* shared_value = nullptr;
1728 : auto i = variable_merges_.find(var);
1729 10613127 : if (i != variable_merges_.end()) {
1730 17919781 : for (auto value : i->second) {
1731 : DCHECK_NOT_NULL(value);
1732 10520637 : if (value != shared_value) {
1733 8046403 : if (shared_value == nullptr) {
1734 : shared_value = value;
1735 : } else {
1736 647259 : variable_phis_[var] = nullptr;
1737 : }
1738 : }
1739 : }
1740 : }
1741 : }
1742 :
1743 4108393 : for (auto var : variable_phis_) {
1744 570153 : CodeAssemblerVariable::Impl* var_impl = var.first;
1745 : auto i = variable_merges_.find(var_impl);
1746 : #if DEBUG
1747 : bool not_found = i == variable_merges_.end();
1748 : if (not_found || i->second.size() != merge_count_) {
1749 : std::stringstream str;
1750 : str << "A variable that has been marked as beeing merged at the label"
1751 : << "\n# doesn't have a bound value along all of the paths that "
1752 : << "\n# have been merged into the label up to this point."
1753 : << "\n#"
1754 : << "\n# This can happen in the following cases:"
1755 : << "\n# - By explicitly marking it so in the label constructor"
1756 : << "\n# - By having seen different bound values at branches"
1757 : << "\n#"
1758 : << "\n# Merge count: expected=" << merge_count_
1759 : << " vs. found=" << (not_found ? 0 : i->second.size())
1760 : << "\n# Variable: " << *var_impl
1761 : << "\n# Current Block: " << *label_->block();
1762 : FATAL("%s", str.str().c_str());
1763 : }
1764 : #endif // DEBUG
1765 570153 : Node* phi = state_->raw_assembler_->Phi(
1766 1140306 : var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
1767 570153 : variable_phis_[var_impl] = phi;
1768 : }
1769 :
1770 : // Bind all variables to a merge phi, the common value along all paths or
1771 : // null.
1772 17689607 : for (auto var : state_->variables_) {
1773 : auto i = variable_phis_.find(var);
1774 10613127 : if (i != variable_phis_.end()) {
1775 570153 : var->value_ = i->second;
1776 : } else {
1777 : auto j = variable_merges_.find(var);
1778 16871965 : if (j != variable_merges_.end() && j->second.size() == merge_count_) {
1779 13568558 : var->value_ = j->second.back();
1780 : } else {
1781 3258695 : var->value_ = nullptr;
1782 : }
1783 : }
1784 : }
1785 :
1786 3538240 : bound_ = true;
1787 3538240 : }
1788 :
1789 1284512 : void CodeAssemblerParameterizedLabelBase::AddInputs(std::vector<Node*> inputs) {
1790 1284512 : if (!phi_nodes_.empty()) {
1791 : DCHECK_EQ(inputs.size(), phi_nodes_.size());
1792 129388 : for (size_t i = 0; i < inputs.size(); ++i) {
1793 : // We use {nullptr} as a sentinel for an uninitialized value.
1794 61916 : if (phi_nodes_[i] == nullptr) continue;
1795 123824 : state_->raw_assembler_->AppendPhiInput(phi_nodes_[i], inputs[i]);
1796 : }
1797 : } else {
1798 : DCHECK_EQ(inputs.size(), phi_inputs_.size());
1799 10159332 : for (size_t i = 0; i < inputs.size(); ++i) {
1800 4440188 : phi_inputs_[i].push_back(inputs[i]);
1801 : }
1802 : }
1803 1284512 : }
1804 :
1805 3996140 : Node* CodeAssemblerParameterizedLabelBase::CreatePhi(
1806 : MachineRepresentation rep, const std::vector<Node*>& inputs) {
1807 8418784 : for (Node* input : inputs) {
1808 : // We use {nullptr} as a sentinel for an uninitialized value. We must not
1809 : // create phi nodes for these.
1810 4439572 : if (input == nullptr) return nullptr;
1811 : }
1812 3979212 : return state_->raw_assembler_->Phi(rep, static_cast<int>(inputs.size()),
1813 3979212 : &inputs.front());
1814 : }
1815 :
1816 1194512 : const std::vector<Node*>& CodeAssemblerParameterizedLabelBase::CreatePhis(
1817 : std::vector<MachineRepresentation> representations) {
1818 : DCHECK(is_used());
1819 : DCHECK(phi_nodes_.empty());
1820 1194512 : phi_nodes_.reserve(phi_inputs_.size());
1821 : DCHECK_EQ(representations.size(), phi_inputs_.size());
1822 9186792 : for (size_t i = 0; i < phi_inputs_.size(); ++i) {
1823 7992280 : phi_nodes_.push_back(CreatePhi(representations[i], phi_inputs_[i]));
1824 : }
1825 1194512 : return phi_nodes_;
1826 : }
1827 :
1828 0 : void CodeAssemblerState::PushExceptionHandler(
1829 : CodeAssemblerExceptionHandlerLabel* label) {
1830 968 : exception_handler_labels_.push_back(label);
1831 0 : }
1832 :
1833 0 : void CodeAssemblerState::PopExceptionHandler() {
1834 : exception_handler_labels_.pop_back();
1835 0 : }
1836 :
1837 740 : CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
1838 : CodeAssembler* assembler, CodeAssemblerExceptionHandlerLabel* label)
1839 740 : : has_handler_(label != nullptr),
1840 : assembler_(assembler),
1841 : compatibility_label_(nullptr),
1842 1480 : exception_(nullptr) {
1843 740 : if (has_handler_) {
1844 : assembler_->state()->PushExceptionHandler(label);
1845 : }
1846 740 : }
1847 :
1848 564 : CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
1849 : CodeAssembler* assembler, CodeAssemblerLabel* label,
1850 : TypedCodeAssemblerVariable<Object>* exception)
1851 564 : : has_handler_(label != nullptr),
1852 : assembler_(assembler),
1853 : compatibility_label_(label),
1854 1128 : exception_(exception) {
1855 564 : if (has_handler_) {
1856 456 : label_ = base::make_unique<CodeAssemblerExceptionHandlerLabel>(
1857 : assembler, CodeAssemblerLabel::kDeferred);
1858 228 : assembler_->state()->PushExceptionHandler(label_.get());
1859 : }
1860 564 : }
1861 :
1862 2608 : CodeAssemblerScopedExceptionHandler::~CodeAssemblerScopedExceptionHandler() {
1863 1304 : if (has_handler_) {
1864 968 : assembler_->state()->PopExceptionHandler();
1865 : }
1866 1304 : if (label_ && label_->is_used()) {
1867 456 : CodeAssembler::Label skip(assembler_);
1868 228 : bool inside_block = assembler_->state()->InsideBlock();
1869 228 : if (inside_block) {
1870 228 : assembler_->Goto(&skip);
1871 : }
1872 : TNode<Object> e;
1873 : assembler_->Bind(label_.get(), &e);
1874 228 : *exception_ = e;
1875 228 : assembler_->Goto(compatibility_label_);
1876 228 : if (inside_block) {
1877 : assembler_->Bind(&skip);
1878 : }
1879 : }
1880 1304 : }
1881 :
1882 : } // namespace compiler
1883 :
1884 0 : Address CheckObjectType(Address raw_value, Address raw_type,
1885 : Address raw_location) {
1886 : #ifdef DEBUG
1887 : Object value(raw_value);
1888 : Smi type(raw_type);
1889 : String location = String::cast(Object(raw_location));
1890 : const char* expected;
1891 : switch (static_cast<ObjectType>(type->value())) {
1892 : #define TYPE_CASE(Name) \
1893 : case ObjectType::k##Name: \
1894 : if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
1895 : expected = #Name; \
1896 : break;
1897 : #define TYPE_STRUCT_CASE(NAME, Name, name) \
1898 : case ObjectType::k##Name: \
1899 : if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
1900 : expected = #Name; \
1901 : break;
1902 :
1903 : TYPE_CASE(Object)
1904 : OBJECT_TYPE_LIST(TYPE_CASE)
1905 : HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
1906 : STRUCT_LIST(TYPE_STRUCT_CASE)
1907 : #undef TYPE_CASE
1908 : #undef TYPE_STRUCT_CASE
1909 : }
1910 : std::stringstream value_description;
1911 : value->Print(value_description);
1912 : V8_Fatal(__FILE__, __LINE__,
1913 : "Type cast failed in %s\n"
1914 : " Expected %s but found %s",
1915 : location->ToAsciiArray(), expected, value_description.str().c_str());
1916 : #else
1917 0 : UNREACHABLE();
1918 : #endif
1919 : }
1920 :
1921 : } // namespace internal
1922 121996 : } // namespace v8
|
