Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-assembler.h"
6 :
7 : #include <ostream>
8 :
9 : #include "src/code-factory.h"
10 : #include "src/compiler/backend/instruction-selector.h"
11 : #include "src/compiler/graph.h"
12 : #include "src/compiler/linkage.h"
13 : #include "src/compiler/node-matchers.h"
14 : #include "src/compiler/pipeline.h"
15 : #include "src/compiler/raw-machine-assembler.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/frames.h"
18 : #include "src/interface-descriptors.h"
19 : #include "src/interpreter/bytecodes.h"
20 : #include "src/machine-type.h"
21 : #include "src/macro-assembler.h"
22 : #include "src/memcopy.h"
23 : #include "src/objects-inl.h"
24 : #include "src/objects/smi.h"
25 : #include "src/zone/zone.h"
26 :
27 : namespace v8 {
28 : namespace internal {
29 :
30 : constexpr MachineType MachineTypeOf<Smi>::value;
31 : constexpr MachineType MachineTypeOf<Object>::value;
32 :
33 : namespace compiler {
34 :
35 : static_assert(std::is_convertible<TNode<Number>, TNode<Object>>::value,
36 : "test subtyping");
37 : static_assert(std::is_convertible<TNode<UnionT<Smi, HeapNumber>>,
38 : TNode<UnionT<Smi, HeapObject>>>::value,
39 : "test subtyping");
40 : static_assert(
41 : !std::is_convertible<TNode<UnionT<Smi, HeapObject>>, TNode<Number>>::value,
42 : "test subtyping");
43 :
44 47967 : CodeAssemblerState::CodeAssemblerState(
45 : Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
46 : Code::Kind kind, const char* name, PoisoningMitigationLevel poisoning_level,
47 : int32_t builtin_index)
48 : // TODO(rmcilroy): Should we use Linkage::GetBytecodeDispatchDescriptor for
49 : // bytecode handlers?
50 : : CodeAssemblerState(
51 : isolate, zone,
52 : Linkage::GetStubCallDescriptor(
53 : zone, descriptor, descriptor.GetStackParameterCount(),
54 : CallDescriptor::kNoFlags, Operator::kNoProperties),
55 47967 : kind, name, poisoning_level, builtin_index) {}
56 :
57 19864 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
58 : int parameter_count, Code::Kind kind,
59 : const char* name,
60 : PoisoningMitigationLevel poisoning_level,
61 : int32_t builtin_index)
62 : : CodeAssemblerState(
63 : isolate, zone,
64 : Linkage::GetJSCallDescriptor(
65 : zone, false, parameter_count,
66 : (kind == Code::BUILTIN ? CallDescriptor::kPushArgumentCount
67 : : CallDescriptor::kNoFlags) |
68 : CallDescriptor::kCanUseRoots),
69 39728 : kind, name, poisoning_level, builtin_index) {}
70 :
71 68399 : CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
72 : CallDescriptor* call_descriptor,
73 : Code::Kind kind, const char* name,
74 : PoisoningMitigationLevel poisoning_level,
75 : int32_t builtin_index)
76 : : raw_assembler_(new RawMachineAssembler(
77 68399 : isolate, new (zone) Graph(zone), call_descriptor,
78 : MachineType::PointerRepresentation(),
79 136798 : InstructionSelector::SupportedMachineOperatorFlags(),
80 205197 : InstructionSelector::AlignmentRequirements(), poisoning_level)),
81 : kind_(kind),
82 : name_(name),
83 : builtin_index_(builtin_index),
84 : code_generated_(false),
85 205197 : variables_(zone) {}
86 :
87 : CodeAssemblerState::~CodeAssemblerState() = default;
88 :
89 0 : int CodeAssemblerState::parameter_count() const {
90 0 : return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
91 : }
92 :
93 : CodeAssembler::~CodeAssembler() = default;
94 :
95 : #if DEBUG
96 : void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
97 : raw_assembler_->PrintCurrentBlock(os);
98 : }
99 : #endif
100 :
101 3132 : bool CodeAssemblerState::InsideBlock() { return raw_assembler_->InsideBlock(); }
102 :
103 63840 : void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
104 : const char* file,
105 : int line) {
106 : #if DEBUG
107 : AssemblerDebugInfo debug_info = {msg, file, line};
108 : raw_assembler_->SetSourcePosition(file, line);
109 : raw_assembler_->SetInitialDebugInformation(debug_info);
110 : #endif // DEBUG
111 63840 : }
112 :
113 0 : class BreakOnNodeDecorator final : public GraphDecorator {
114 : public:
115 0 : explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}
116 :
117 0 : void Decorate(Node* node) final {
118 0 : if (node->id() == node_id_) {
119 0 : base::OS::DebugBreak();
120 : }
121 0 : }
122 :
123 : private:
124 : NodeId node_id_;
125 : };
126 :
127 0 : void CodeAssembler::BreakOnNode(int node_id) {
128 : Graph* graph = raw_assembler()->graph();
129 : Zone* zone = graph->zone();
130 : GraphDecorator* decorator =
131 0 : new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
132 0 : graph->AddDecorator(decorator);
133 0 : }
134 :
135 27990 : void CodeAssembler::RegisterCallGenerationCallbacks(
136 : const CodeAssemblerCallback& call_prologue,
137 : const CodeAssemblerCallback& call_epilogue) {
138 : // The callback can be registered only once.
139 : DCHECK(!state_->call_prologue_);
140 : DCHECK(!state_->call_epilogue_);
141 27990 : state_->call_prologue_ = call_prologue;
142 27990 : state_->call_epilogue_ = call_epilogue;
143 27990 : }
144 :
145 27990 : void CodeAssembler::UnregisterCallGenerationCallbacks() {
146 27990 : state_->call_prologue_ = nullptr;
147 27990 : state_->call_epilogue_ = nullptr;
148 27990 : }
149 :
150 251318 : void CodeAssembler::CallPrologue() {
151 251318 : if (state_->call_prologue_) {
152 : state_->call_prologue_();
153 : }
154 251318 : }
155 :
156 251318 : void CodeAssembler::CallEpilogue() {
157 251318 : if (state_->call_epilogue_) {
158 : state_->call_epilogue_();
159 : }
160 251318 : }
161 :
162 1176 : bool CodeAssembler::Word32ShiftIsSafe() const {
163 1176 : return raw_assembler()->machine()->Word32ShiftIsSafe();
164 : }
165 :
166 5742 : PoisoningMitigationLevel CodeAssembler::poisoning_level() const {
167 5742 : return raw_assembler()->poisoning_level();
168 : }
169 :
170 : // static
171 66916 : Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state,
172 : const AssemblerOptions& options) {
173 : DCHECK(!state->code_generated_);
174 :
175 : RawMachineAssembler* rasm = state->raw_assembler_.get();
176 :
177 : Handle<Code> code;
178 66916 : Graph* graph = rasm->ExportForOptimization();
179 :
180 133832 : code = Pipeline::GenerateCodeForCodeStub(
181 : rasm->isolate(), rasm->call_descriptor(), graph,
182 : rasm->source_positions(), state->kind_, state->name_,
183 66916 : state->builtin_index_, rasm->poisoning_level(), options)
184 : .ToHandleChecked();
185 :
186 66916 : state->code_generated_ = true;
187 66916 : return code;
188 : }
189 :
190 796288 : bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
191 :
192 118 : bool CodeAssembler::IsFloat64RoundUpSupported() const {
193 118 : return raw_assembler()->machine()->Float64RoundUp().IsSupported();
194 : }
195 :
196 125 : bool CodeAssembler::IsFloat64RoundDownSupported() const {
197 125 : return raw_assembler()->machine()->Float64RoundDown().IsSupported();
198 : }
199 :
200 392 : bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
201 392 : return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
202 : }
203 :
204 340 : bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
205 340 : return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
206 : }
207 :
208 0 : bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
209 0 : return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
210 : }
211 :
212 0 : bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
213 56 : return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
214 : }
215 :
216 56 : bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
217 : return Is64() ? IsInt64AbsWithOverflowSupported()
218 112 : : IsInt32AbsWithOverflowSupported();
219 : }
220 :
221 : #ifdef DEBUG
222 : void CodeAssembler::GenerateCheckMaybeObjectIsObject(Node* node,
223 : const char* location) {
224 : Label ok(this);
225 : GotoIf(WordNotEqual(WordAnd(BitcastMaybeObjectToWord(node),
226 : IntPtrConstant(kHeapObjectTagMask)),
227 : IntPtrConstant(kWeakHeapObjectTag)),
228 : &ok);
229 : Node* message_node = StringConstant(location);
230 : DebugAbort(message_node);
231 : Unreachable();
232 : Bind(&ok);
233 : }
234 : #endif
235 :
236 625091 : TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
237 793299 : return UncheckedCast<Int32T>(raw_assembler()->Int32Constant(value));
238 : }
239 :
240 11372 : TNode<Int64T> CodeAssembler::Int64Constant(int64_t value) {
241 11372 : return UncheckedCast<Int64T>(raw_assembler()->Int64Constant(value));
242 : }
243 :
244 3400917 : TNode<IntPtrT> CodeAssembler::IntPtrConstant(intptr_t value) {
245 3400917 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrConstant(value));
246 : }
247 :
248 10644 : TNode<Number> CodeAssembler::NumberConstant(double value) {
249 : int smi_value;
250 10644 : if (DoubleToSmiInteger(value, &smi_value)) {
251 : return UncheckedCast<Number>(SmiConstant(smi_value));
252 : } else {
253 : // We allocate the heap number constant eagerly at this point instead of
254 : // deferring allocation to code generation
255 : // (see AllocateAndInstallRequestedHeapObjects) since that makes it easier
256 : // to generate constant lookups for embedded builtins.
257 : return UncheckedCast<Number>(HeapConstant(
258 1740 : isolate()->factory()->NewHeapNumber(value, AllocationType::kOld)));
259 : }
260 : }
261 :
262 302157 : TNode<Smi> CodeAssembler::SmiConstant(Smi value) {
263 : return UncheckedCast<Smi>(BitcastWordToTaggedSigned(
264 604314 : IntPtrConstant(static_cast<intptr_t>(value.ptr()))));
265 : }
266 :
267 280809 : TNode<Smi> CodeAssembler::SmiConstant(int value) {
268 289713 : return SmiConstant(Smi::FromInt(value));
269 : }
270 :
271 142406 : TNode<HeapObject> CodeAssembler::UntypedHeapConstant(
272 : Handle<HeapObject> object) {
273 752898 : return UncheckedCast<HeapObject>(raw_assembler()->HeapConstant(object));
274 : }
275 :
276 18088 : TNode<String> CodeAssembler::StringConstant(const char* str) {
277 : Handle<String> internalized_string =
278 18088 : factory()->InternalizeOneByteString(OneByteVector(str));
279 18088 : return UncheckedCast<String>(HeapConstant(internalized_string));
280 : }
281 :
282 316 : TNode<Oddball> CodeAssembler::BooleanConstant(bool value) {
283 632 : Handle<Object> object = isolate()->factory()->ToBoolean(value);
284 : return UncheckedCast<Oddball>(
285 316 : raw_assembler()->HeapConstant(Handle<HeapObject>::cast(object)));
286 : }
287 :
288 62202 : TNode<ExternalReference> CodeAssembler::ExternalConstant(
289 : ExternalReference address) {
290 : return UncheckedCast<ExternalReference>(
291 211998 : raw_assembler()->ExternalConstant(address));
292 : }
293 :
294 10924 : TNode<Float64T> CodeAssembler::Float64Constant(double value) {
295 10924 : return UncheckedCast<Float64T>(raw_assembler()->Float64Constant(value));
296 : }
297 :
298 0 : TNode<HeapNumber> CodeAssembler::NaNConstant() {
299 0 : return UncheckedCast<HeapNumber>(LoadRoot(RootIndex::kNanValue));
300 : }
301 :
302 2368401 : bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
303 : {
304 : Int64Matcher m(node);
305 2554306 : if (m.HasValue() && m.IsInRange(std::numeric_limits<int32_t>::min(),
306 : std::numeric_limits<int32_t>::max())) {
307 185901 : out_value = static_cast<int32_t>(m.Value());
308 : return true;
309 : }
310 : }
311 :
312 : {
313 : Int32Matcher m(node);
314 2182500 : if (m.HasValue()) {
315 0 : out_value = m.Value();
316 : return true;
317 : }
318 : }
319 :
320 2182500 : return false;
321 : }
322 :
323 20 : bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
324 : Int64Matcher m(node);
325 5132 : if (m.HasValue()) out_value = m.Value();
326 20 : return m.HasValue();
327 : }
328 :
329 33272 : bool CodeAssembler::ToSmiConstant(Node* node, Smi* out_value) {
330 33272 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
331 : node = node->InputAt(0);
332 : }
333 : IntPtrMatcher m(node);
334 33272 : if (m.HasValue()) {
335 : intptr_t value = m.Value();
336 : // Make sure that the value is actually a smi
337 : CHECK_EQ(0, value & ((static_cast<intptr_t>(1) << kSmiShiftSize) - 1));
338 2344 : *out_value = Smi(static_cast<Address>(value));
339 2344 : return true;
340 : }
341 : return false;
342 : }
343 :
344 5158657 : bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
345 5158657 : if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
346 : node->opcode() == IrOpcode::kBitcastWordToTagged) {
347 : node = node->InputAt(0);
348 : }
349 : IntPtrMatcher m(node);
350 5158657 : if (m.HasValue()) out_value = m.Value();
351 5158657 : return m.HasValue();
352 : }
353 :
354 1848 : bool CodeAssembler::IsUndefinedConstant(TNode<Object> node) {
355 : compiler::HeapObjectMatcher m(node);
356 1848 : return m.Is(isolate()->factory()->undefined_value());
357 : }
358 :
359 1568 : bool CodeAssembler::IsNullConstant(TNode<Object> node) {
360 : compiler::HeapObjectMatcher m(node);
361 1568 : return m.Is(isolate()->factory()->null_value());
362 : }
363 :
364 261832 : Node* CodeAssembler::Parameter(int index) {
365 262056 : if (index == kTargetParameterIndex) return raw_assembler()->TargetParameter();
366 523216 : return raw_assembler()->Parameter(index);
367 : }
368 :
369 0 : bool CodeAssembler::IsJSFunctionCall() const {
370 : auto call_descriptor = raw_assembler()->call_descriptor();
371 0 : return call_descriptor->IsJSFunctionCall();
372 : }
373 :
374 19040 : TNode<Context> CodeAssembler::GetJSContextParameter() {
375 : auto call_descriptor = raw_assembler()->call_descriptor();
376 : DCHECK(call_descriptor->IsJSFunctionCall());
377 38080 : return CAST(Parameter(Linkage::GetJSCallContextParamIndex(
378 : static_cast<int>(call_descriptor->JSParameterCount()))));
379 : }
380 :
381 60636 : void CodeAssembler::Return(SloppyTNode<Object> value) {
382 60860 : return raw_assembler()->Return(value);
383 : }
384 :
385 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
386 : SloppyTNode<Object> value2) {
387 0 : return raw_assembler()->Return(value1, value2);
388 : }
389 :
390 0 : void CodeAssembler::Return(SloppyTNode<Object> value1,
391 : SloppyTNode<Object> value2,
392 : SloppyTNode<Object> value3) {
393 0 : return raw_assembler()->Return(value1, value2, value3);
394 : }
395 :
396 13352 : void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
397 13352 : return raw_assembler()->PopAndReturn(pop, value);
398 : }
399 :
400 224 : void CodeAssembler::ReturnIf(Node* condition, Node* value) {
401 224 : Label if_return(this), if_continue(this);
402 224 : Branch(condition, &if_return, &if_continue);
403 : Bind(&if_return);
404 : Return(value);
405 : Bind(&if_continue);
406 224 : }
407 :
408 336 : void CodeAssembler::ReturnRaw(Node* value) {
409 336 : return raw_assembler()->Return(value);
410 : }
411 :
412 0 : void CodeAssembler::DebugAbort(Node* message) {
413 0 : raw_assembler()->DebugAbort(message);
414 0 : }
415 :
416 217464 : void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
417 :
418 65512 : void CodeAssembler::Unreachable() {
419 : DebugBreak();
420 65512 : raw_assembler()->Unreachable();
421 65512 : }
422 :
423 4 : void CodeAssembler::Comment(std::string str) {
424 4 : if (!FLAG_code_comments) return;
425 4 : raw_assembler()->Comment(str);
426 : }
427 :
428 863532 : void CodeAssembler::SetSourcePosition(const char* file, int line) {
429 863532 : raw_assembler()->SetSourcePosition(file, line);
430 863532 : }
431 :
432 3631636 : void CodeAssembler::Bind(Label* label) { return label->Bind(); }
433 :
434 : #if DEBUG
435 : void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
436 : return label->Bind(debug_info);
437 : }
438 : #endif // DEBUG
439 :
440 8004 : Node* CodeAssembler::LoadFramePointer() {
441 8004 : return raw_assembler()->LoadFramePointer();
442 : }
443 :
444 28664 : Node* CodeAssembler::LoadParentFramePointer() {
445 28664 : return raw_assembler()->LoadParentFramePointer();
446 : }
447 :
448 19264 : Node* CodeAssembler::LoadStackPointer() {
449 19264 : return raw_assembler()->LoadStackPointer();
450 : }
451 :
452 19056 : TNode<Object> CodeAssembler::TaggedPoisonOnSpeculation(
453 : SloppyTNode<Object> value) {
454 : return UncheckedCast<Object>(
455 19056 : raw_assembler()->TaggedPoisonOnSpeculation(value));
456 : }
457 :
458 45864 : TNode<WordT> CodeAssembler::WordPoisonOnSpeculation(SloppyTNode<WordT> value) {
459 45864 : return UncheckedCast<WordT>(raw_assembler()->WordPoisonOnSpeculation(value));
460 : }
461 :
462 : #define DEFINE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
463 : TNode<ResType> CodeAssembler::name(SloppyTNode<Arg1Type> a, \
464 : SloppyTNode<Arg2Type> b) { \
465 : return UncheckedCast<ResType>(raw_assembler()->name(a, b)); \
466 : }
467 870720 : CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
468 : #undef DEFINE_CODE_ASSEMBLER_BINARY_OP
469 :
470 707023 : TNode<WordT> CodeAssembler::IntPtrAdd(SloppyTNode<WordT> left,
471 : SloppyTNode<WordT> right) {
472 : intptr_t left_constant;
473 707023 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
474 : intptr_t right_constant;
475 707023 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
476 707023 : if (is_left_constant) {
477 313610 : if (is_right_constant) {
478 101266 : return IntPtrConstant(left_constant + right_constant);
479 : }
480 262977 : if (left_constant == 0) {
481 38301 : return right;
482 : }
483 393413 : } else if (is_right_constant) {
484 304259 : if (right_constant == 0) {
485 25621 : return left;
486 : }
487 : }
488 : return UncheckedCast<WordT>(raw_assembler()->IntPtrAdd(left, right));
489 : }
490 :
491 509 : TNode<IntPtrT> CodeAssembler::IntPtrDiv(TNode<IntPtrT> left,
492 : TNode<IntPtrT> right) {
493 : intptr_t left_constant;
494 509 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
495 : intptr_t right_constant;
496 509 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
497 509 : if (is_right_constant) {
498 61 : if (is_left_constant) {
499 2 : return IntPtrConstant(left_constant / right_constant);
500 : }
501 118 : if (base::bits::IsPowerOfTwo(right_constant)) {
502 58 : return WordSar(left, WhichPowerOf2(right_constant));
503 : }
504 : }
505 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrDiv(left, right));
506 : }
507 :
508 108325 : TNode<WordT> CodeAssembler::IntPtrSub(SloppyTNode<WordT> left,
509 : SloppyTNode<WordT> right) {
510 : intptr_t left_constant;
511 108325 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
512 : intptr_t right_constant;
513 108325 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
514 108325 : if (is_left_constant) {
515 19561 : if (is_right_constant) {
516 13682 : return IntPtrConstant(left_constant - right_constant);
517 : }
518 88764 : } else if (is_right_constant) {
519 82600 : if (right_constant == 0) {
520 8601 : return left;
521 : }
522 : }
523 92883 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrSub(left, right));
524 : }
525 :
526 54774 : TNode<WordT> CodeAssembler::IntPtrMul(SloppyTNode<WordT> left,
527 : SloppyTNode<WordT> right) {
528 : intptr_t left_constant;
529 54774 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
530 : intptr_t right_constant;
531 54774 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
532 54774 : if (is_left_constant) {
533 2011 : if (is_right_constant) {
534 4018 : return IntPtrConstant(left_constant * right_constant);
535 : }
536 4 : if (base::bits::IsPowerOfTwo(left_constant)) {
537 2 : return WordShl(right, WhichPowerOf2(left_constant));
538 : }
539 52763 : } else if (is_right_constant) {
540 104406 : if (base::bits::IsPowerOfTwo(right_constant)) {
541 17926 : return WordShl(left, WhichPowerOf2(right_constant));
542 : }
543 : }
544 34837 : return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrMul(left, right));
545 : }
546 :
547 112851 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> value, int shift) {
548 217343 : return (shift != 0) ? WordShl(value, IntPtrConstant(shift)) : value;
549 : }
550 :
551 28808 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> value, int shift) {
552 51399 : return (shift != 0) ? WordShr(value, IntPtrConstant(shift)) : value;
553 : }
554 :
555 562 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> value, int shift) {
556 1123 : return (shift != 0) ? WordSar(value, IntPtrConstant(shift)) : value;
557 : }
558 :
559 37632 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> value, int shift) {
560 63832 : return (shift != 0) ? Word32Shr(value, Int32Constant(shift)) : value;
561 : }
562 :
563 7752 : TNode<WordT> CodeAssembler::WordOr(SloppyTNode<WordT> left,
564 : SloppyTNode<WordT> right) {
565 : intptr_t left_constant;
566 7752 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
567 : intptr_t right_constant;
568 7752 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
569 7752 : if (is_left_constant) {
570 3530 : if (is_right_constant) {
571 7058 : return IntPtrConstant(left_constant | right_constant);
572 : }
573 1 : if (left_constant == 0) {
574 1 : return right;
575 : }
576 4222 : } else if (is_right_constant) {
577 2522 : if (right_constant == 0) {
578 1 : return left;
579 : }
580 : }
581 4221 : return UncheckedCast<WordT>(raw_assembler()->WordOr(left, right));
582 : }
583 :
584 307482 : TNode<WordT> CodeAssembler::WordAnd(SloppyTNode<WordT> left,
585 : SloppyTNode<WordT> right) {
586 : intptr_t left_constant;
587 307482 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
588 : intptr_t right_constant;
589 307482 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
590 307482 : if (is_left_constant) {
591 393 : if (is_right_constant) {
592 786 : return IntPtrConstant(left_constant & right_constant);
593 : }
594 : }
595 307089 : return UncheckedCast<WordT>(raw_assembler()->WordAnd(left, right));
596 : }
597 :
598 294 : TNode<WordT> CodeAssembler::WordXor(SloppyTNode<WordT> left,
599 : SloppyTNode<WordT> right) {
600 : intptr_t left_constant;
601 294 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
602 : intptr_t right_constant;
603 294 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
604 294 : if (is_left_constant) {
605 1 : if (is_right_constant) {
606 2 : return IntPtrConstant(left_constant ^ right_constant);
607 : }
608 : }
609 293 : return UncheckedCast<WordT>(raw_assembler()->WordXor(left, right));
610 : }
611 :
612 470037 : TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> left,
613 : SloppyTNode<IntegralT> right) {
614 : intptr_t left_constant;
615 470037 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
616 : intptr_t right_constant;
617 470037 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
618 470037 : if (is_left_constant) {
619 11433 : if (is_right_constant) {
620 22754 : return IntPtrConstant(left_constant << right_constant);
621 : }
622 458604 : } else if (is_right_constant) {
623 458044 : if (right_constant == 0) {
624 0 : return left;
625 : }
626 : }
627 458660 : return UncheckedCast<WordT>(raw_assembler()->WordShl(left, right));
628 : }
629 :
630 28043 : TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> left,
631 : SloppyTNode<IntegralT> right) {
632 : intptr_t left_constant;
633 28043 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
634 : intptr_t right_constant;
635 28043 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
636 28043 : if (is_left_constant) {
637 3698 : if (is_right_constant) {
638 3698 : return IntPtrConstant(static_cast<uintptr_t>(left_constant) >>
639 7396 : right_constant);
640 : }
641 24345 : } else if (is_right_constant) {
642 23841 : if (right_constant == 0) {
643 0 : return left;
644 : }
645 : }
646 24345 : return UncheckedCast<WordT>(raw_assembler()->WordShr(left, right));
647 : }
648 :
649 230893 : TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> left,
650 : SloppyTNode<IntegralT> right) {
651 : intptr_t left_constant;
652 230893 : bool is_left_constant = ToIntPtrConstant(left, left_constant);
653 : intptr_t right_constant;
654 230893 : bool is_right_constant = ToIntPtrConstant(right, right_constant);
655 230893 : if (is_left_constant) {
656 2 : if (is_right_constant) {
657 4 : return IntPtrConstant(left_constant >> right_constant);
658 : }
659 230891 : } else if (is_right_constant) {
660 230779 : if (right_constant == 0) {
661 1 : return left;
662 : }
663 : }
664 230890 : return UncheckedCast<WordT>(raw_assembler()->WordSar(left, right));
665 : }
666 :
667 62568 : TNode<Word32T> CodeAssembler::Word32Or(SloppyTNode<Word32T> left,
668 : SloppyTNode<Word32T> right) {
669 : int32_t left_constant;
670 62568 : bool is_left_constant = ToInt32Constant(left, left_constant);
671 : int32_t right_constant;
672 62568 : bool is_right_constant = ToInt32Constant(right, right_constant);
673 62568 : if (is_left_constant) {
674 0 : if (is_right_constant) {
675 0 : return Int32Constant(left_constant | right_constant);
676 : }
677 0 : if (left_constant == 0) {
678 0 : return right;
679 : }
680 62568 : } else if (is_right_constant) {
681 56 : if (right_constant == 0) {
682 0 : return left;
683 : }
684 : }
685 62568 : return UncheckedCast<Word32T>(raw_assembler()->Word32Or(left, right));
686 : }
687 :
688 133008 : TNode<Word32T> CodeAssembler::Word32And(SloppyTNode<Word32T> left,
689 : SloppyTNode<Word32T> right) {
690 : int32_t left_constant;
691 133008 : bool is_left_constant = ToInt32Constant(left, left_constant);
692 : int32_t right_constant;
693 133008 : bool is_right_constant = ToInt32Constant(right, right_constant);
694 133008 : if (is_left_constant) {
695 12 : if (is_right_constant) {
696 8 : return Int32Constant(left_constant & right_constant);
697 : }
698 : }
699 133004 : return UncheckedCast<Word32T>(raw_assembler()->Word32And(left, right));
700 : }
701 :
702 2472 : TNode<Word32T> CodeAssembler::Word32Xor(SloppyTNode<Word32T> left,
703 : SloppyTNode<Word32T> right) {
704 : int32_t left_constant;
705 2472 : bool is_left_constant = ToInt32Constant(left, left_constant);
706 : int32_t right_constant;
707 2472 : bool is_right_constant = ToInt32Constant(right, right_constant);
708 2472 : if (is_left_constant) {
709 4 : if (is_right_constant) {
710 8 : return Int32Constant(left_constant ^ right_constant);
711 : }
712 : }
713 2468 : return UncheckedCast<Word32T>(raw_assembler()->Word32Xor(left, right));
714 : }
715 :
716 8512 : TNode<Word32T> CodeAssembler::Word32Shl(SloppyTNode<Word32T> left,
717 : SloppyTNode<Word32T> right) {
718 : int32_t left_constant;
719 8512 : bool is_left_constant = ToInt32Constant(left, left_constant);
720 : int32_t right_constant;
721 8512 : bool is_right_constant = ToInt32Constant(right, right_constant);
722 8512 : if (is_left_constant) {
723 0 : if (is_right_constant) {
724 0 : return Int32Constant(left_constant << right_constant);
725 : }
726 8512 : } else if (is_right_constant) {
727 4872 : if (right_constant == 0) {
728 0 : return left;
729 : }
730 : }
731 8512 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shl(left, right));
732 : }
733 :
734 36588 : TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> left,
735 : SloppyTNode<Word32T> right) {
736 : int32_t left_constant;
737 36588 : bool is_left_constant = ToInt32Constant(left, left_constant);
738 : int32_t right_constant;
739 36588 : bool is_right_constant = ToInt32Constant(right, right_constant);
740 36588 : if (is_left_constant) {
741 0 : if (is_right_constant) {
742 0 : return Int32Constant(static_cast<uint32_t>(left_constant) >>
743 0 : right_constant);
744 : }
745 36588 : } else if (is_right_constant) {
746 34572 : if (right_constant == 0) {
747 0 : return left;
748 : }
749 : }
750 36588 : return UncheckedCast<Word32T>(raw_assembler()->Word32Shr(left, right));
751 : }
752 :
753 392 : TNode<Word32T> CodeAssembler::Word32Sar(SloppyTNode<Word32T> left,
754 : SloppyTNode<Word32T> right) {
755 : int32_t left_constant;
756 392 : bool is_left_constant = ToInt32Constant(left, left_constant);
757 : int32_t right_constant;
758 392 : bool is_right_constant = ToInt32Constant(right, right_constant);
759 392 : if (is_left_constant) {
760 0 : if (is_right_constant) {
761 0 : return Int32Constant(left_constant >> right_constant);
762 : }
763 392 : } else if (is_right_constant) {
764 0 : if (right_constant == 0) {
765 0 : return left;
766 : }
767 : }
768 392 : return UncheckedCast<Word32T>(raw_assembler()->Word32Sar(left, right));
769 : }
770 :
771 0 : TNode<Word64T> CodeAssembler::Word64Or(SloppyTNode<Word64T> left,
772 : SloppyTNode<Word64T> right) {
773 : int64_t left_constant;
774 : bool is_left_constant = ToInt64Constant(left, left_constant);
775 : int64_t right_constant;
776 : bool is_right_constant = ToInt64Constant(right, right_constant);
777 0 : if (is_left_constant) {
778 0 : if (is_right_constant) {
779 0 : return Int64Constant(left_constant | right_constant);
780 : }
781 0 : if (left_constant == 0) {
782 0 : return right;
783 : }
784 0 : } else if (is_right_constant) {
785 0 : if (right_constant == 0) {
786 0 : return left;
787 : }
788 : }
789 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Or(left, right));
790 : }
791 :
792 0 : TNode<Word64T> CodeAssembler::Word64And(SloppyTNode<Word64T> left,
793 : SloppyTNode<Word64T> right) {
794 : int64_t left_constant;
795 : bool is_left_constant = ToInt64Constant(left, left_constant);
796 : int64_t right_constant;
797 : bool is_right_constant = ToInt64Constant(right, right_constant);
798 0 : if (is_left_constant) {
799 0 : if (is_right_constant) {
800 0 : return Int64Constant(left_constant & right_constant);
801 : }
802 : }
803 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64And(left, right));
804 : }
805 :
806 0 : TNode<Word64T> CodeAssembler::Word64Xor(SloppyTNode<Word64T> left,
807 : SloppyTNode<Word64T> right) {
808 : int64_t left_constant;
809 : bool is_left_constant = ToInt64Constant(left, left_constant);
810 : int64_t right_constant;
811 : bool is_right_constant = ToInt64Constant(right, right_constant);
812 0 : if (is_left_constant) {
813 0 : if (is_right_constant) {
814 0 : return Int64Constant(left_constant ^ right_constant);
815 : }
816 : }
817 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Xor(left, right));
818 : }
819 :
820 0 : TNode<Word64T> CodeAssembler::Word64Shl(SloppyTNode<Word64T> left,
821 : SloppyTNode<Word64T> right) {
822 : int64_t left_constant;
823 : bool is_left_constant = ToInt64Constant(left, left_constant);
824 : int64_t right_constant;
825 : bool is_right_constant = ToInt64Constant(right, right_constant);
826 0 : if (is_left_constant) {
827 0 : if (is_right_constant) {
828 0 : return Int64Constant(left_constant << right_constant);
829 : }
830 0 : } else if (is_right_constant) {
831 0 : if (right_constant == 0) {
832 0 : return left;
833 : }
834 : }
835 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shl(left, right));
836 : }
837 :
838 0 : TNode<Word64T> CodeAssembler::Word64Shr(SloppyTNode<Word64T> left,
839 : SloppyTNode<Word64T> right) {
840 : int64_t left_constant;
841 : bool is_left_constant = ToInt64Constant(left, left_constant);
842 : int64_t right_constant;
843 : bool is_right_constant = ToInt64Constant(right, right_constant);
844 0 : if (is_left_constant) {
845 0 : if (is_right_constant) {
846 0 : return Int64Constant(static_cast<uint64_t>(left_constant) >>
847 0 : right_constant);
848 : }
849 0 : } else if (is_right_constant) {
850 0 : if (right_constant == 0) {
851 0 : return left;
852 : }
853 : }
854 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Shr(left, right));
855 : }
856 :
857 0 : TNode<Word64T> CodeAssembler::Word64Sar(SloppyTNode<Word64T> left,
858 : SloppyTNode<Word64T> right) {
859 : int64_t left_constant;
860 : bool is_left_constant = ToInt64Constant(left, left_constant);
861 : int64_t right_constant;
862 : bool is_right_constant = ToInt64Constant(right, right_constant);
863 0 : if (is_left_constant) {
864 0 : if (is_right_constant) {
865 0 : return Int64Constant(left_constant >> right_constant);
866 : }
867 0 : } else if (is_right_constant) {
868 0 : if (right_constant == 0) {
869 0 : return left;
870 : }
871 : }
872 0 : return UncheckedCast<Word64T>(raw_assembler()->Word64Sar(left, right));
873 : }
874 :
875 : #define CODE_ASSEMBLER_COMPARE(Name, ArgT, VarT, ToConstant, op) \
876 : TNode<BoolT> CodeAssembler::Name(SloppyTNode<ArgT> left, \
877 : SloppyTNode<ArgT> right) { \
878 : VarT lhs, rhs; \
879 : if (ToConstant(left, lhs) && ToConstant(right, rhs)) { \
880 : return BoolConstant(lhs op rhs); \
881 : } \
882 : return UncheckedCast<BoolT>(raw_assembler()->Name(left, right)); \
883 : }
884 :
885 7224 : CODE_ASSEMBLER_COMPARE(IntPtrEqual, WordT, intptr_t, ToIntPtrConstant, ==)
886 1145956 : CODE_ASSEMBLER_COMPARE(WordEqual, WordT, intptr_t, ToIntPtrConstant, ==)
887 96440 : CODE_ASSEMBLER_COMPARE(WordNotEqual, WordT, intptr_t, ToIntPtrConstant, !=)
888 476272 : CODE_ASSEMBLER_COMPARE(Word32Equal, Word32T, int32_t, ToInt32Constant, ==)
889 77272 : CODE_ASSEMBLER_COMPARE(Word32NotEqual, Word32T, int32_t, ToInt32Constant, !=)
890 15336 : CODE_ASSEMBLER_COMPARE(Word64Equal, Word64T, int64_t, ToInt64Constant, ==)
891 0 : CODE_ASSEMBLER_COMPARE(Word64NotEqual, Word64T, int64_t, ToInt64Constant, !=)
892 : #undef CODE_ASSEMBLER_COMPARE
893 :
894 115249 : TNode<UintPtrT> CodeAssembler::ChangeUint32ToWord(SloppyTNode<Word32T> value) {
895 115249 : if (raw_assembler()->machine()->Is64()) {
896 : return UncheckedCast<UintPtrT>(
897 115249 : raw_assembler()->ChangeUint32ToUint64(value));
898 : }
899 : return ReinterpretCast<UintPtrT>(value);
900 : }
901 :
902 151756 : TNode<IntPtrT> CodeAssembler::ChangeInt32ToIntPtr(SloppyTNode<Word32T> value) {
903 151756 : if (raw_assembler()->machine()->Is64()) {
904 151756 : return ReinterpretCast<IntPtrT>(raw_assembler()->ChangeInt32ToInt64(value));
905 : }
906 : return ReinterpretCast<IntPtrT>(value);
907 : }
908 :
909 1456 : TNode<UintPtrT> CodeAssembler::ChangeFloat64ToUintPtr(
910 : SloppyTNode<Float64T> value) {
911 1456 : if (raw_assembler()->machine()->Is64()) {
912 : return ReinterpretCast<UintPtrT>(
913 1456 : raw_assembler()->ChangeFloat64ToUint64(value));
914 : }
915 : return ReinterpretCast<UintPtrT>(
916 0 : raw_assembler()->ChangeFloat64ToUint32(value));
917 : }
918 :
919 1736 : TNode<Float64T> CodeAssembler::ChangeUintPtrToFloat64(TNode<UintPtrT> value) {
920 1736 : if (raw_assembler()->machine()->Is64()) {
921 : // TODO(turbofan): Maybe we should introduce a ChangeUint64ToFloat64
922 : // machine operator to TurboFan here?
923 : return ReinterpretCast<Float64T>(
924 1736 : raw_assembler()->RoundUint64ToFloat64(value));
925 : }
926 : return ReinterpretCast<Float64T>(
927 0 : raw_assembler()->ChangeUint32ToFloat64(value));
928 : }
929 :
930 1128 : Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
931 1128 : if (raw_assembler()->machine()->Is64()) {
932 1128 : return raw_assembler()->RoundInt64ToFloat64(value);
933 : }
934 0 : return raw_assembler()->ChangeInt32ToFloat64(value);
935 : }
936 :
937 : #define DEFINE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
938 : TNode<ResType> CodeAssembler::name(SloppyTNode<ArgType> a) { \
939 : return UncheckedCast<ResType>(raw_assembler()->name(a)); \
940 : }
941 3474521 : CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
942 : #undef DEFINE_CODE_ASSEMBLER_UNARY_OP
943 :
944 48188 : Node* CodeAssembler::Load(MachineType rep, Node* base,
945 : LoadSensitivity needs_poisoning) {
946 67508 : return raw_assembler()->Load(rep, base, needs_poisoning);
947 : }
948 :
949 1401383 : Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset,
950 : LoadSensitivity needs_poisoning) {
951 1485759 : return raw_assembler()->Load(rep, base, offset, needs_poisoning);
952 : }
953 :
954 19320 : Node* CodeAssembler::LoadFullTagged(Node* base,
955 : LoadSensitivity needs_poisoning) {
956 : return BitcastWordToTagged(
957 19320 : Load(MachineType::Pointer(), base, needs_poisoning));
958 : }
959 :
960 84376 : Node* CodeAssembler::LoadFullTagged(Node* base, Node* offset,
961 : LoadSensitivity needs_poisoning) {
962 : return BitcastWordToTagged(
963 84376 : Load(MachineType::Pointer(), base, offset, needs_poisoning));
964 : }
965 :
966 448 : Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
967 448 : return raw_assembler()->AtomicLoad(rep, base, offset);
968 : }
969 :
970 442948 : TNode<Object> CodeAssembler::LoadRoot(RootIndex root_index) {
971 442948 : if (RootsTable::IsImmortalImmovable(root_index)) {
972 : Handle<Object> root = isolate()->root_handle(root_index);
973 442608 : if (root->IsSmi()) {
974 0 : return SmiConstant(Smi::cast(*root));
975 : } else {
976 442608 : return HeapConstant(Handle<HeapObject>::cast(root));
977 : }
978 : }
979 :
980 : // TODO(jgruber): In theory we could generate better code for this by
981 : // letting the macro assembler decide how to load from the roots list. In most
982 : // cases, it would boil down to loading from a fixed kRootRegister offset.
983 : Node* isolate_root =
984 340 : ExternalConstant(ExternalReference::isolate_root(isolate()));
985 : int offset = IsolateData::root_slot_offset(root_index);
986 : return UncheckedCast<Object>(
987 680 : LoadFullTagged(isolate_root, IntPtrConstant(offset)));
988 : }
989 :
990 0 : Node* CodeAssembler::Store(Node* base, Node* value) {
991 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
992 0 : kFullWriteBarrier);
993 : }
994 :
995 249356 : void CodeAssembler::OptimizedStoreField(MachineRepresentation rep,
996 : TNode<HeapObject> object, int offset,
997 : Node* value,
998 : WriteBarrierKind write_barrier) {
999 249356 : raw_assembler()->OptimizedStoreField(rep, object, offset, value,
1000 249356 : write_barrier);
1001 249356 : }
1002 10764 : void CodeAssembler::OptimizedStoreMap(TNode<HeapObject> object,
1003 : TNode<Map> map) {
1004 10764 : raw_assembler()->OptimizedStoreMap(object, map);
1005 10764 : }
1006 :
1007 27920 : Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
1008 : return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
1009 27920 : value, kFullWriteBarrier);
1010 : }
1011 :
1012 8844 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
1013 : Node* value) {
1014 9908 : return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
1015 : }
1016 :
1017 153989 : Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
1018 : Node* offset, Node* value) {
1019 159869 : return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
1020 : }
1021 :
1022 1064 : Node* CodeAssembler::StoreFullTaggedNoWriteBarrier(Node* base,
1023 : Node* tagged_value) {
1024 : return StoreNoWriteBarrier(MachineType::PointerRepresentation(), base,
1025 1064 : BitcastTaggedToWord(tagged_value));
1026 : }
1027 :
1028 5880 : Node* CodeAssembler::StoreFullTaggedNoWriteBarrier(Node* base, Node* offset,
1029 : Node* tagged_value) {
1030 : return StoreNoWriteBarrier(MachineType::PointerRepresentation(), base, offset,
1031 5880 : BitcastTaggedToWord(tagged_value));
1032 : }
1033 :
1034 224 : Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
1035 : Node* offset, Node* value, Node* value_high) {
1036 224 : return raw_assembler()->AtomicStore(rep, base, offset, value, value_high);
1037 : }
1038 :
1039 : #define ATOMIC_FUNCTION(name) \
1040 : Node* CodeAssembler::Atomic##name(MachineType type, Node* base, \
1041 : Node* offset, Node* value, \
1042 : Node* value_high) { \
1043 : return raw_assembler()->Atomic##name(type, base, offset, value, \
1044 : value_high); \
1045 : }
1046 896 : ATOMIC_FUNCTION(Exchange)
1047 896 : ATOMIC_FUNCTION(Add)
1048 896 : ATOMIC_FUNCTION(Sub)
1049 896 : ATOMIC_FUNCTION(And)
1050 896 : ATOMIC_FUNCTION(Or)
1051 896 : ATOMIC_FUNCTION(Xor)
1052 : #undef ATOMIC_FUNCTION
1053 :
1054 448 : Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
1055 : Node* offset, Node* old_value,
1056 : Node* new_value,
1057 : Node* old_value_high,
1058 : Node* new_value_high) {
1059 : return raw_assembler()->AtomicCompareExchange(
1060 448 : type, base, offset, old_value, old_value_high, new_value, new_value_high);
1061 : }
1062 :
1063 112 : Node* CodeAssembler::StoreRoot(RootIndex root_index, Node* value) {
1064 : DCHECK(!RootsTable::IsImmortalImmovable(root_index));
1065 : Node* isolate_root =
1066 112 : ExternalConstant(ExternalReference::isolate_root(isolate()));
1067 : int offset = IsolateData::root_slot_offset(root_index);
1068 112 : return StoreFullTaggedNoWriteBarrier(isolate_root, IntPtrConstant(offset),
1069 112 : value);
1070 : }
1071 :
1072 0 : Node* CodeAssembler::Retain(Node* value) {
1073 0 : return raw_assembler()->Retain(value);
1074 : }
1075 :
1076 40368 : Node* CodeAssembler::Projection(int index, Node* value) {
1077 : DCHECK_LT(index, value->op()->ValueOutputCount());
1078 40368 : return raw_assembler()->Projection(index, value);
1079 : }
1080 :
1081 7856 : void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
1082 : Variable* exception_var) {
1083 7856 : if (if_exception == nullptr) {
1084 : // If no handler is supplied, don't add continuations
1085 3528 : return;
1086 : }
1087 :
1088 : // No catch handlers should be active if we're using catch labels
1089 : DCHECK_EQ(state()->exception_handler_labels_.size(), 0);
1090 : DCHECK(!node->op()->HasProperty(Operator::kNoThrow));
1091 :
1092 4328 : Label success(this), exception(this, Label::kDeferred);
1093 4328 : success.MergeVariables();
1094 4328 : exception.MergeVariables();
1095 :
1096 8656 : raw_assembler()->Continuations(node, success.label_, exception.label_);
1097 :
1098 : Bind(&exception);
1099 4328 : const Operator* op = raw_assembler()->common()->IfException();
1100 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
1101 4328 : if (exception_var != nullptr) {
1102 : exception_var->Bind(exception_value);
1103 : }
1104 : Goto(if_exception);
1105 :
1106 : Bind(&success);
1107 4328 : raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1108 : }
1109 :
1110 64444 : TNode<HeapObject> CodeAssembler::OptimizedAllocate(TNode<IntPtrT> size,
1111 : AllocationType allocation) {
1112 : return UncheckedCast<HeapObject>(
1113 64444 : raw_assembler()->OptimizedAllocate(size, allocation));
1114 : }
1115 :
1116 251318 : void CodeAssembler::HandleException(Node* node) {
1117 746446 : if (state_->exception_handler_labels_.size() == 0) return;
1118 : CodeAssemblerExceptionHandlerLabel* label =
1119 7508 : state_->exception_handler_labels_.back();
1120 :
1121 7508 : if (node->op()->HasProperty(Operator::kNoThrow)) {
1122 : return;
1123 : }
1124 :
1125 7508 : Label success(this), exception(this, Label::kDeferred);
1126 7508 : success.MergeVariables();
1127 7508 : exception.MergeVariables();
1128 :
1129 15016 : raw_assembler()->Continuations(node, success.label_, exception.label_);
1130 :
1131 : Bind(&exception);
1132 7508 : const Operator* op = raw_assembler()->common()->IfException();
1133 : Node* exception_value = raw_assembler()->AddNode(op, node, node);
1134 7508 : label->AddInputs({UncheckedCast<Object>(exception_value)});
1135 : Goto(label->plain_label());
1136 :
1137 : Bind(&success);
1138 7508 : raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1139 : }
1140 :
1141 : namespace {
1142 : template <size_t kMaxSize>
1143 267342 : class NodeArray {
1144 : public:
1145 : void Add(Node* node) {
1146 : DCHECK_GT(kMaxSize, size());
1147 1183694 : *ptr_++ = node;
1148 : }
1149 :
1150 : Node* const* data() const { return arr_; }
1151 267342 : int size() const { return static_cast<int>(ptr_ - arr_); }
1152 :
1153 : private:
1154 : Node* arr_[kMaxSize];
1155 : Node** ptr_ = arr_;
1156 : };
1157 : } // namespace
1158 :
1159 140096 : TNode<Object> CodeAssembler::CallRuntimeImpl(
1160 : Runtime::FunctionId function, TNode<Object> context,
1161 : std::initializer_list<TNode<Object>> args) {
1162 140096 : int result_size = Runtime::FunctionForId(function)->result_size;
1163 : TNode<Code> centry =
1164 280192 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1165 140096 : return CallRuntimeWithCEntryImpl(function, centry, context, args);
1166 : }
1167 :
1168 140320 : TNode<Object> CodeAssembler::CallRuntimeWithCEntryImpl(
1169 : Runtime::FunctionId function, TNode<Code> centry, TNode<Object> context,
1170 : std::initializer_list<TNode<Object>> args) {
1171 : constexpr size_t kMaxNumArgs = 6;
1172 : DCHECK_GE(kMaxNumArgs, args.size());
1173 140320 : int argc = static_cast<int>(args.size());
1174 140320 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1175 : zone(), function, argc, Operator::kNoProperties,
1176 140320 : CallDescriptor::kNoFlags);
1177 :
1178 140320 : Node* ref = ExternalConstant(ExternalReference::Create(function));
1179 : Node* arity = Int32Constant(argc);
1180 :
1181 : NodeArray<kMaxNumArgs + 4> inputs;
1182 : inputs.Add(centry);
1183 506992 : for (auto arg : args) inputs.Add(arg);
1184 : inputs.Add(ref);
1185 : inputs.Add(arity);
1186 : inputs.Add(context);
1187 :
1188 140320 : CallPrologue();
1189 : Node* return_value =
1190 140320 : raw_assembler()->CallN(call_descriptor, inputs.size(), inputs.data());
1191 140320 : HandleException(return_value);
1192 140320 : CallEpilogue();
1193 140320 : return UncheckedCast<Object>(return_value);
1194 : }
1195 :
1196 7960 : void CodeAssembler::TailCallRuntimeImpl(
1197 : Runtime::FunctionId function, TNode<Int32T> arity, TNode<Object> context,
1198 : std::initializer_list<TNode<Object>> args) {
1199 7960 : int result_size = Runtime::FunctionForId(function)->result_size;
1200 : TNode<Code> centry =
1201 15920 : HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1202 7960 : return TailCallRuntimeWithCEntryImpl(function, arity, centry, context, args);
1203 : }
1204 :
1205 9024 : void CodeAssembler::TailCallRuntimeWithCEntryImpl(
1206 : Runtime::FunctionId function, TNode<Int32T> arity, TNode<Code> centry,
1207 : TNode<Object> context, std::initializer_list<TNode<Object>> args) {
1208 : constexpr size_t kMaxNumArgs = 6;
1209 : DCHECK_GE(kMaxNumArgs, args.size());
1210 9024 : int argc = static_cast<int>(args.size());
1211 9024 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1212 : zone(), function, argc, Operator::kNoProperties,
1213 9024 : CallDescriptor::kNoFlags);
1214 :
1215 9024 : Node* ref = ExternalConstant(ExternalReference::Create(function));
1216 :
1217 : NodeArray<kMaxNumArgs + 4> inputs;
1218 : inputs.Add(centry);
1219 65384 : for (auto arg : args) inputs.Add(arg);
1220 : inputs.Add(ref);
1221 : inputs.Add(arity);
1222 : inputs.Add(context);
1223 :
1224 9024 : raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1225 9024 : }
1226 :
1227 110998 : Node* CodeAssembler::CallStubN(StubCallMode call_mode,
1228 : const CallInterfaceDescriptor& descriptor,
1229 : size_t result_size, int input_count,
1230 : Node* const* inputs) {
1231 : DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1232 : call_mode == StubCallMode::kCallBuiltinPointer);
1233 :
1234 : // implicit nodes are target and optionally context.
1235 110998 : int implicit_nodes = descriptor.HasContextParameter() ? 2 : 1;
1236 : DCHECK_LE(implicit_nodes, input_count);
1237 110998 : int argc = input_count - implicit_nodes;
1238 : DCHECK_LE(descriptor.GetParameterCount(), argc);
1239 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1240 110998 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1241 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1242 : DCHECK_EQ(result_size, descriptor.GetReturnCount());
1243 :
1244 110998 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1245 : zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1246 110998 : Operator::kNoProperties, call_mode);
1247 :
1248 110998 : CallPrologue();
1249 : Node* return_value =
1250 110998 : raw_assembler()->CallN(call_descriptor, input_count, inputs);
1251 110998 : HandleException(return_value);
1252 110998 : CallEpilogue();
1253 110998 : return return_value;
1254 : }
1255 :
1256 7504 : void CodeAssembler::TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
1257 : TNode<Code> target, TNode<Object> context,
1258 : std::initializer_list<Node*> args) {
1259 : constexpr size_t kMaxNumArgs = 11;
1260 : DCHECK_GE(kMaxNumArgs, args.size());
1261 : DCHECK_EQ(descriptor.GetParameterCount(), args.size());
1262 7504 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1263 : zone(), descriptor, descriptor.GetStackParameterCount(),
1264 7504 : CallDescriptor::kNoFlags, Operator::kNoProperties);
1265 :
1266 : NodeArray<kMaxNumArgs + 2> inputs;
1267 : inputs.Add(target);
1268 56336 : for (auto arg : args) inputs.Add(arg);
1269 7504 : if (descriptor.HasContextParameter()) {
1270 : inputs.Add(context);
1271 : }
1272 :
1273 7504 : raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1274 7504 : }
1275 :
1276 108310 : Node* CodeAssembler::CallStubRImpl(StubCallMode call_mode,
1277 : const CallInterfaceDescriptor& descriptor,
1278 : size_t result_size, Node* target,
1279 : SloppyTNode<Object> context,
1280 : std::initializer_list<Node*> args) {
1281 : DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1282 : call_mode == StubCallMode::kCallBuiltinPointer);
1283 :
1284 : constexpr size_t kMaxNumArgs = 10;
1285 : DCHECK_GE(kMaxNumArgs, args.size());
1286 :
1287 : NodeArray<kMaxNumArgs + 2> inputs;
1288 : inputs.Add(target);
1289 603954 : for (auto arg : args) inputs.Add(arg);
1290 108310 : if (descriptor.HasContextParameter()) {
1291 : inputs.Add(context);
1292 : }
1293 :
1294 : return CallStubN(call_mode, descriptor, result_size, inputs.size(),
1295 108310 : inputs.data());
1296 : }
1297 :
1298 2184 : Node* CodeAssembler::TailCallStubThenBytecodeDispatchImpl(
1299 : const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
1300 : std::initializer_list<Node*> args) {
1301 : constexpr size_t kMaxNumArgs = 6;
1302 : DCHECK_GE(kMaxNumArgs, args.size());
1303 :
1304 : DCHECK_LE(descriptor.GetParameterCount(), args.size());
1305 2184 : int argc = static_cast<int>(args.size());
1306 : // Extra arguments not mentioned in the descriptor are passed on the stack.
1307 2184 : int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1308 : DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1309 2184 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1310 : zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1311 2184 : Operator::kNoProperties);
1312 :
1313 : NodeArray<kMaxNumArgs + 2> inputs;
1314 : inputs.Add(target);
1315 17304 : for (auto arg : args) inputs.Add(arg);
1316 : inputs.Add(context);
1317 :
1318 : return raw_assembler()->TailCallN(call_descriptor, inputs.size(),
1319 2184 : inputs.data());
1320 : }
1321 :
1322 : template <class... TArgs>
1323 43680 : Node* CodeAssembler::TailCallBytecodeDispatch(
1324 : const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
1325 : DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1326 : auto call_descriptor = Linkage::GetBytecodeDispatchCallDescriptor(
1327 43680 : zone(), descriptor, descriptor.GetStackParameterCount());
1328 :
1329 43680 : Node* nodes[] = {target, args...};
1330 87360 : CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
1331 43680 : return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1332 : }
1333 :
1334 : // Instantiate TailCallBytecodeDispatch() for argument counts used by
1335 : // CSA-generated code
1336 : template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
1337 : const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
1338 : Node*, Node*);
1339 :
1340 504 : TNode<Object> CodeAssembler::TailCallJSCode(TNode<Code> code,
1341 : TNode<Context> context,
1342 : TNode<JSFunction> function,
1343 : TNode<Object> new_target,
1344 : TNode<Int32T> arg_count) {
1345 : JSTrampolineDescriptor descriptor;
1346 504 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1347 : zone(), descriptor, descriptor.GetStackParameterCount(),
1348 504 : CallDescriptor::kFixedTargetRegister, Operator::kNoProperties);
1349 :
1350 2520 : Node* nodes[] = {code, function, new_target, arg_count, context};
1351 1008 : CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
1352 : return UncheckedCast<Object>(
1353 1008 : raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes));
1354 : }
1355 :
1356 0 : Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
1357 : int input_count, Node* const* inputs) {
1358 0 : auto call_descriptor = Linkage::GetSimplifiedCDescriptor(zone(), signature);
1359 0 : return raw_assembler()->CallN(call_descriptor, input_count, inputs);
1360 : }
1361 :
1362 504 : Node* CodeAssembler::CallCFunction1(MachineType return_type,
1363 : MachineType arg0_type, Node* function,
1364 : Node* arg0) {
1365 : return raw_assembler()->CallCFunction1(return_type, arg0_type, function,
1366 504 : arg0);
1367 : }
1368 :
1369 224 : Node* CodeAssembler::CallCFunction1WithCallerSavedRegisters(
1370 : MachineType return_type, MachineType arg0_type, Node* function, Node* arg0,
1371 : SaveFPRegsMode mode) {
1372 : DCHECK(return_type.LessThanOrEqualPointerSize());
1373 : return raw_assembler()->CallCFunction1WithCallerSavedRegisters(
1374 224 : return_type, arg0_type, function, arg0, mode);
1375 : }
1376 :
1377 4492 : Node* CodeAssembler::CallCFunction2(MachineType return_type,
1378 : MachineType arg0_type,
1379 : MachineType arg1_type, Node* function,
1380 : Node* arg0, Node* arg1) {
1381 : return raw_assembler()->CallCFunction2(return_type, arg0_type, arg1_type,
1382 4492 : function, arg0, arg1);
1383 : }
1384 :
1385 5908 : Node* CodeAssembler::CallCFunction3(MachineType return_type,
1386 : MachineType arg0_type,
1387 : MachineType arg1_type,
1388 : MachineType arg2_type, Node* function,
1389 : Node* arg0, Node* arg1, Node* arg2) {
1390 : return raw_assembler()->CallCFunction3(return_type, arg0_type, arg1_type,
1391 5908 : arg2_type, function, arg0, arg1, arg2);
1392 : }
1393 :
1394 116 : Node* CodeAssembler::CallCFunction3WithCallerSavedRegisters(
1395 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1396 : MachineType arg2_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1397 : SaveFPRegsMode mode) {
1398 : DCHECK(return_type.LessThanOrEqualPointerSize());
1399 : return raw_assembler()->CallCFunction3WithCallerSavedRegisters(
1400 : return_type, arg0_type, arg1_type, arg2_type, function, arg0, arg1, arg2,
1401 116 : mode);
1402 : }
1403 :
1404 112 : Node* CodeAssembler::CallCFunction4(
1405 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1406 : MachineType arg2_type, MachineType arg3_type, Node* function, Node* arg0,
1407 : Node* arg1, Node* arg2, Node* arg3) {
1408 : return raw_assembler()->CallCFunction4(return_type, arg0_type, arg1_type,
1409 : arg2_type, arg3_type, function, arg0,
1410 112 : arg1, arg2, arg3);
1411 : }
1412 :
1413 280 : Node* CodeAssembler::CallCFunction5(
1414 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1415 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1416 : Node* function, Node* arg0, Node* arg1, Node* arg2, Node* arg3,
1417 : Node* arg4) {
1418 : return raw_assembler()->CallCFunction5(
1419 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1420 280 : function, arg0, arg1, arg2, arg3, arg4);
1421 : }
1422 :
1423 672 : Node* CodeAssembler::CallCFunction6(
1424 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1425 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1426 : MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1427 : Node* arg3, Node* arg4, Node* arg5) {
1428 : return raw_assembler()->CallCFunction6(
1429 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1430 672 : arg5_type, function, arg0, arg1, arg2, arg3, arg4, arg5);
1431 : }
1432 :
1433 564 : Node* CodeAssembler::CallCFunction9(
1434 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1435 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
1436 : MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
1437 : MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
1438 : Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
1439 : return raw_assembler()->CallCFunction9(
1440 : return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
1441 : arg5_type, arg6_type, arg7_type, arg8_type, function, arg0, arg1, arg2,
1442 564 : arg3, arg4, arg5, arg6, arg7, arg8);
1443 : }
1444 :
1445 2120193 : void CodeAssembler::Goto(Label* label) {
1446 2132493 : label->MergeVariables();
1447 4264986 : raw_assembler()->Goto(label->label_);
1448 2120193 : }
1449 :
1450 578668 : void CodeAssembler::GotoIf(SloppyTNode<IntegralT> condition,
1451 : Label* true_label) {
1452 578668 : Label false_label(this);
1453 578668 : Branch(condition, true_label, &false_label);
1454 : Bind(&false_label);
1455 578668 : }
1456 :
1457 172855 : void CodeAssembler::GotoIfNot(SloppyTNode<IntegralT> condition,
1458 : Label* false_label) {
1459 172855 : Label true_label(this);
1460 172855 : Branch(condition, &true_label, false_label);
1461 : Bind(&true_label);
1462 172855 : }
1463 :
1464 1341127 : void CodeAssembler::Branch(SloppyTNode<IntegralT> condition, Label* true_label,
1465 : Label* false_label) {
1466 : int32_t constant;
1467 1341127 : if (ToInt32Constant(condition, constant)) {
1468 976 : if ((true_label->is_used() || true_label->is_bound()) &&
1469 0 : (false_label->is_used() || false_label->is_bound())) {
1470 0 : return Goto(constant ? true_label : false_label);
1471 : }
1472 : }
1473 1341127 : true_label->MergeVariables();
1474 1341127 : false_label->MergeVariables();
1475 1341127 : return raw_assembler()->Branch(condition, true_label->label_,
1476 1341127 : false_label->label_);
1477 : }
1478 :
1479 26772 : void CodeAssembler::Branch(TNode<BoolT> condition,
1480 : const std::function<void()>& true_body,
1481 : const std::function<void()>& false_body) {
1482 : int32_t constant;
1483 26772 : if (ToInt32Constant(condition, constant)) {
1484 0 : return constant ? true_body() : false_body();
1485 : }
1486 :
1487 26772 : Label vtrue(this), vfalse(this);
1488 26772 : Branch(condition, &vtrue, &vfalse);
1489 :
1490 : Bind(&vtrue);
1491 : true_body();
1492 :
1493 : Bind(&vfalse);
1494 : false_body();
1495 : }
1496 :
1497 976 : void CodeAssembler::Branch(TNode<BoolT> condition, Label* true_label,
1498 : const std::function<void()>& false_body) {
1499 : int32_t constant;
1500 976 : if (ToInt32Constant(condition, constant)) {
1501 1272 : return constant ? Goto(true_label) : false_body();
1502 : }
1503 :
1504 340 : Label vfalse(this);
1505 340 : Branch(condition, true_label, &vfalse);
1506 : Bind(&vfalse);
1507 : false_body();
1508 : }
1509 :
1510 0 : void CodeAssembler::Branch(TNode<BoolT> condition,
1511 : const std::function<void()>& true_body,
1512 : Label* false_label) {
1513 : int32_t constant;
1514 0 : if (ToInt32Constant(condition, constant)) {
1515 0 : return constant ? true_body() : Goto(false_label);
1516 : }
1517 :
1518 0 : Label vtrue(this);
1519 0 : Branch(condition, &vtrue, false_label);
1520 : Bind(&vtrue);
1521 : true_body();
1522 : }
1523 :
1524 9936 : void CodeAssembler::Switch(Node* index, Label* default_label,
1525 : const int32_t* case_values, Label** case_labels,
1526 : size_t case_count) {
1527 : RawMachineLabel** labels =
1528 9936 : new (zone()->New(sizeof(RawMachineLabel*) * case_count))
1529 9936 : RawMachineLabel*[case_count];
1530 159800 : for (size_t i = 0; i < case_count; ++i) {
1531 74932 : labels[i] = case_labels[i]->label_;
1532 74932 : case_labels[i]->MergeVariables();
1533 : }
1534 9936 : default_label->MergeVariables();
1535 9936 : return raw_assembler()->Switch(index, default_label->label_, case_values,
1536 9936 : labels, case_count);
1537 : }
1538 :
1539 112 : bool CodeAssembler::UnalignedLoadSupported(MachineRepresentation rep) const {
1540 112 : return raw_assembler()->machine()->UnalignedLoadSupported(rep);
1541 : }
1542 112 : bool CodeAssembler::UnalignedStoreSupported(MachineRepresentation rep) const {
1543 112 : return raw_assembler()->machine()->UnalignedStoreSupported(rep);
1544 : }
1545 :
1546 : // RawMachineAssembler delegate helpers:
1547 370288 : Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
1548 :
1549 8736 : Factory* CodeAssembler::factory() const { return isolate()->factory(); }
1550 :
1551 137296 : Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }
1552 :
1553 0 : bool CodeAssembler::IsExceptionHandlerActive() const {
1554 0 : return state_->exception_handler_labels_.size() != 0;
1555 : }
1556 :
1557 0 : RawMachineAssembler* CodeAssembler::raw_assembler() const {
1558 24362316 : return state_->raw_assembler_.get();
1559 : }
1560 :
1561 : // The core implementation of Variable is stored through an indirection so
1562 : // that it can outlive the often block-scoped Variable declarations. This is
1563 : // needed to ensure that variable binding and merging through phis can
1564 : // properly be verified.
1565 : class CodeAssemblerVariable::Impl : public ZoneObject {
1566 : public:
1567 : explicit Impl(MachineRepresentation rep, CodeAssemblerState::VariableId id)
1568 : :
1569 : #if DEBUG
1570 : debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
1571 : #endif
1572 : value_(nullptr),
1573 : rep_(rep),
1574 616302 : var_id_(id) {
1575 : }
1576 :
1577 : #if DEBUG
1578 : AssemblerDebugInfo debug_info() const { return debug_info_; }
1579 : void set_debug_info(AssemblerDebugInfo debug_info) {
1580 : debug_info_ = debug_info;
1581 : }
1582 :
1583 : AssemblerDebugInfo debug_info_;
1584 : #endif // DEBUG
1585 : bool operator<(const CodeAssemblerVariable::Impl& other) const {
1586 1214417 : return var_id_ < other.var_id_;
1587 : }
1588 : Node* value_;
1589 : MachineRepresentation rep_;
1590 : CodeAssemblerState::VariableId var_id_;
1591 : };
1592 :
1593 0 : bool CodeAssemblerVariable::ImplComparator::operator()(
1594 : const CodeAssemblerVariable::Impl* a,
1595 : const CodeAssemblerVariable::Impl* b) const {
1596 0 : return *a < *b;
1597 : }
1598 :
1599 616302 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1600 : MachineRepresentation rep)
1601 : : impl_(new (assembler->zone())
1602 : Impl(rep, assembler->state()->NextVariableId())),
1603 1232604 : state_(assembler->state()) {
1604 616302 : state_->variables_.insert(impl_);
1605 616302 : }
1606 :
1607 273021 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1608 : MachineRepresentation rep,
1609 : Node* initial_value)
1610 273021 : : CodeAssemblerVariable(assembler, rep) {
1611 : Bind(initial_value);
1612 273021 : }
1613 :
1614 : #if DEBUG
1615 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1616 : AssemblerDebugInfo debug_info,
1617 : MachineRepresentation rep)
1618 : : impl_(new (assembler->zone())
1619 : Impl(rep, assembler->state()->NextVariableId())),
1620 : state_(assembler->state()) {
1621 : impl_->set_debug_info(debug_info);
1622 : state_->variables_.insert(impl_);
1623 : }
1624 :
1625 : CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
1626 : AssemblerDebugInfo debug_info,
1627 : MachineRepresentation rep,
1628 : Node* initial_value)
1629 : : CodeAssemblerVariable(assembler, debug_info, rep) {
1630 : impl_->set_debug_info(debug_info);
1631 : Bind(initial_value);
1632 : }
1633 : #endif // DEBUG
1634 :
1635 1232604 : CodeAssemblerVariable::~CodeAssemblerVariable() {
1636 616302 : state_->variables_.erase(impl_);
1637 616302 : }
1638 :
1639 1363416 : void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
1640 :
1641 1506379 : Node* CodeAssemblerVariable::value() const {
1642 : #if DEBUG
1643 : if (!IsBound()) {
1644 : std::stringstream str;
1645 : str << "#Use of unbound variable:"
1646 : << "#\n Variable: " << *this << "#\n Current Block: ";
1647 : state_->PrintCurrentBlock(str);
1648 : FATAL("%s", str.str().c_str());
1649 : }
1650 : if (!state_->InsideBlock()) {
1651 : std::stringstream str;
1652 : str << "#Accessing variable value outside a block:"
1653 : << "#\n Variable: " << *this;
1654 : FATAL("%s", str.str().c_str());
1655 : }
1656 : #endif // DEBUG
1657 1506379 : return impl_->value_;
1658 : }
1659 :
1660 0 : MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
1661 :
1662 143565 : bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
1663 :
1664 0 : std::ostream& operator<<(std::ostream& os,
1665 : const CodeAssemblerVariable::Impl& impl) {
1666 : #if DEBUG
1667 : AssemblerDebugInfo info = impl.debug_info();
1668 : if (info.name) os << "V" << info;
1669 : #endif // DEBUG
1670 0 : return os;
1671 : }
1672 :
1673 0 : std::ostream& operator<<(std::ostream& os,
1674 : const CodeAssemblerVariable& variable) {
1675 : os << *variable.impl_;
1676 0 : return os;
1677 : }
1678 :
1679 3722508 : CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
1680 : size_t vars_count,
1681 : CodeAssemblerVariable* const* vars,
1682 : CodeAssemblerLabel::Type type)
1683 : : bound_(false),
1684 : merge_count_(0),
1685 : state_(assembler->state()),
1686 7445016 : label_(nullptr) {
1687 : void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
1688 : label_ = new (buffer)
1689 : RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
1690 7445016 : : RawMachineLabel::kNonDeferred);
1691 4155444 : for (size_t i = 0; i < vars_count; ++i) {
1692 216468 : variable_phis_[vars[i]->impl_] = nullptr;
1693 : }
1694 3722508 : }
1695 :
1696 7445016 : CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }
1697 :
1698 4923287 : void CodeAssemblerLabel::MergeVariables() {
1699 4923287 : ++merge_count_;
1700 26779149 : for (CodeAssemblerVariable::Impl* var : state_->variables_) {
1701 : size_t count = 0;
1702 16932575 : Node* node = var->value_;
1703 16932575 : if (node != nullptr) {
1704 : auto i = variable_merges_.find(var);
1705 12248917 : if (i != variable_merges_.end()) {
1706 4326177 : i->second.push_back(node);
1707 : count = i->second.size();
1708 : } else {
1709 : count = 1;
1710 15845480 : variable_merges_[var] = std::vector<Node*>(1, node);
1711 : }
1712 : }
1713 : // If the following asserts, then you've jumped to a label without a bound
1714 : // variable along that path that expects to merge its value into a phi.
1715 : DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
1716 : count == merge_count_);
1717 : USE(count);
1718 :
1719 : // If the label is already bound, we already know the set of variables to
1720 : // merge and phi nodes have already been created.
1721 16932575 : if (bound_) {
1722 : auto phi = variable_phis_.find(var);
1723 749108 : if (phi != variable_phis_.end()) {
1724 : DCHECK_NOT_NULL(phi->second);
1725 431912 : state_->raw_assembler_->AppendPhiInput(phi->second, node);
1726 : } else {
1727 : auto i = variable_merges_.find(var);
1728 : if (i != variable_merges_.end()) {
1729 : // If the following assert fires, then you've declared a variable that
1730 : // has the same bound value along all paths up until the point you
1731 : // bound this label, but then later merged a path with a new value for
1732 : // the variable after the label bind (it's not possible to add phis to
1733 : // the bound label after the fact, just make sure to list the variable
1734 : // in the label's constructor's list of merged variables).
1735 : #if DEBUG
1736 : if (find_if(i->second.begin(), i->second.end(),
1737 : [node](Node* e) -> bool { return node != e; }) !=
1738 : i->second.end()) {
1739 : std::stringstream str;
1740 : str << "Unmerged variable found when jumping to block. \n"
1741 : << "# Variable: " << *var;
1742 : if (bound_) {
1743 : str << "\n# Target block: " << *label_->block();
1744 : }
1745 : str << "\n# Current Block: ";
1746 : state_->PrintCurrentBlock(str);
1747 : FATAL("%s", str.str().c_str());
1748 : }
1749 : #endif // DEBUG
1750 : }
1751 : }
1752 : }
1753 : }
1754 4923287 : }
1755 :
1756 : #if DEBUG
1757 : void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
1758 : if (bound_) {
1759 : std::stringstream str;
1760 : str << "Cannot bind the same label twice:"
1761 : << "\n# current: " << debug_info
1762 : << "\n# previous: " << *label_->block();
1763 : FATAL("%s", str.str().c_str());
1764 : }
1765 : if (FLAG_enable_source_at_csa_bind) {
1766 : state_->raw_assembler_->SetSourcePosition(debug_info.file, debug_info.line);
1767 : }
1768 : state_->raw_assembler_->Bind(label_, debug_info);
1769 : UpdateVariablesAfterBind();
1770 : }
1771 : #endif // DEBUG
1772 :
1773 3631636 : void CodeAssemblerLabel::Bind() {
1774 : DCHECK(!bound_);
1775 7263272 : state_->raw_assembler_->Bind(label_);
1776 3631636 : UpdateVariablesAfterBind();
1777 3631636 : }
1778 :
1779 3631636 : void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1780 : // Make sure that all variables that have changed along any path up to this
1781 : // point are marked as merge variables.
1782 18337031 : for (auto var : state_->variables_) {
1783 : Node* shared_value = nullptr;
1784 : auto i = variable_merges_.find(var);
1785 11073759 : if (i != variable_merges_.end()) {
1786 18830645 : for (auto value : i->second) {
1787 : DCHECK_NOT_NULL(value);
1788 11090377 : if (value != shared_value) {
1789 8404339 : if (shared_value == nullptr) {
1790 : shared_value = value;
1791 : } else {
1792 664071 : variable_phis_[var] = nullptr;
1793 : }
1794 : }
1795 : }
1796 : }
1797 : }
1798 :
1799 4204989 : for (auto var : variable_phis_) {
1800 573353 : CodeAssemblerVariable::Impl* var_impl = var.first;
1801 : auto i = variable_merges_.find(var_impl);
1802 : #if DEBUG
1803 : bool not_found = i == variable_merges_.end();
1804 : if (not_found || i->second.size() != merge_count_) {
1805 : std::stringstream str;
1806 : str << "A variable that has been marked as beeing merged at the label"
1807 : << "\n# doesn't have a bound value along all of the paths that "
1808 : << "\n# have been merged into the label up to this point."
1809 : << "\n#"
1810 : << "\n# This can happen in the following cases:"
1811 : << "\n# - By explicitly marking it so in the label constructor"
1812 : << "\n# - By having seen different bound values at branches"
1813 : << "\n#"
1814 : << "\n# Merge count: expected=" << merge_count_
1815 : << " vs. found=" << (not_found ? 0 : i->second.size())
1816 : << "\n# Variable: " << *var_impl
1817 : << "\n# Current Block: " << *label_->block();
1818 : FATAL("%s", str.str().c_str());
1819 : }
1820 : #endif // DEBUG
1821 573353 : Node* phi = state_->raw_assembler_->Phi(
1822 1146706 : var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
1823 573353 : variable_phis_[var_impl] = phi;
1824 : }
1825 :
1826 : // Bind all variables to a merge phi, the common value along all paths or
1827 : // null.
1828 18337031 : for (auto var : state_->variables_) {
1829 : auto i = variable_phis_.find(var);
1830 11073759 : if (i != variable_phis_.end()) {
1831 573353 : var->value_ = i->second;
1832 : } else {
1833 : auto j = variable_merges_.find(var);
1834 17667321 : if (j != variable_merges_.end() && j->second.size() == merge_count_) {
1835 14245422 : var->value_ = j->second.back();
1836 : } else {
1837 3377695 : var->value_ = nullptr;
1838 : }
1839 : }
1840 : }
1841 :
1842 3631636 : bound_ = true;
1843 3631636 : }
1844 :
1845 1332280 : void CodeAssemblerParameterizedLabelBase::AddInputs(std::vector<Node*> inputs) {
1846 1332280 : if (!phi_nodes_.empty()) {
1847 : DCHECK_EQ(inputs.size(), phi_nodes_.size());
1848 123592 : for (size_t i = 0; i < inputs.size(); ++i) {
1849 : // We use {nullptr} as a sentinel for an uninitialized value.
1850 59080 : if (phi_nodes_[i] == nullptr) continue;
1851 118160 : state_->raw_assembler_->AppendPhiInput(phi_nodes_[i], inputs[i]);
1852 : }
1853 : } else {
1854 : DCHECK_EQ(inputs.size(), phi_inputs_.size());
1855 9930256 : for (size_t i = 0; i < inputs.size(); ++i) {
1856 4301704 : phi_inputs_[i].push_back(inputs[i]);
1857 : }
1858 : }
1859 1332280 : }
1860 :
1861 3891504 : Node* CodeAssemblerParameterizedLabelBase::CreatePhi(
1862 : MachineRepresentation rep, const std::vector<Node*>& inputs) {
1863 8175680 : for (Node* input : inputs) {
1864 : // We use {nullptr} as a sentinel for an uninitialized value. We must not
1865 : // create phi nodes for these.
1866 4301088 : if (input == nullptr) return nullptr;
1867 : }
1868 3874592 : return state_->raw_assembler_->Phi(rep, static_cast<int>(inputs.size()),
1869 3874592 : &inputs.front());
1870 : }
1871 :
1872 1245032 : const std::vector<Node*>& CodeAssemblerParameterizedLabelBase::CreatePhis(
1873 : std::vector<MachineRepresentation> representations) {
1874 : DCHECK(is_used());
1875 : DCHECK(phi_nodes_.empty());
1876 1245032 : phi_nodes_.reserve(phi_inputs_.size());
1877 : DCHECK_EQ(representations.size(), phi_inputs_.size());
1878 9028040 : for (size_t i = 0; i < phi_inputs_.size(); ++i) {
1879 7783008 : phi_nodes_.push_back(CreatePhi(representations[i], phi_inputs_[i]));
1880 : }
1881 1245032 : return phi_nodes_;
1882 : }
1883 :
1884 0 : void CodeAssemblerState::PushExceptionHandler(
1885 : CodeAssemblerExceptionHandlerLabel* label) {
1886 956 : exception_handler_labels_.push_back(label);
1887 0 : }
1888 :
1889 0 : void CodeAssemblerState::PopExceptionHandler() {
1890 : exception_handler_labels_.pop_back();
1891 0 : }
1892 :
1893 728 : CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
1894 : CodeAssembler* assembler, CodeAssemblerExceptionHandlerLabel* label)
1895 728 : : has_handler_(label != nullptr),
1896 : assembler_(assembler),
1897 : compatibility_label_(nullptr),
1898 1456 : exception_(nullptr) {
1899 728 : if (has_handler_) {
1900 : assembler_->state()->PushExceptionHandler(label);
1901 : }
1902 728 : }
1903 :
1904 564 : CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
1905 : CodeAssembler* assembler, CodeAssemblerLabel* label,
1906 : TypedCodeAssemblerVariable<Object>* exception)
1907 564 : : has_handler_(label != nullptr),
1908 : assembler_(assembler),
1909 : compatibility_label_(label),
1910 1128 : exception_(exception) {
1911 564 : if (has_handler_) {
1912 456 : label_ = base::make_unique<CodeAssemblerExceptionHandlerLabel>(
1913 : assembler, CodeAssemblerLabel::kDeferred);
1914 228 : assembler_->state()->PushExceptionHandler(label_.get());
1915 : }
1916 564 : }
1917 :
1918 2584 : CodeAssemblerScopedExceptionHandler::~CodeAssemblerScopedExceptionHandler() {
1919 1292 : if (has_handler_) {
1920 956 : assembler_->state()->PopExceptionHandler();
1921 : }
1922 1292 : if (label_ && label_->is_used()) {
1923 456 : CodeAssembler::Label skip(assembler_);
1924 228 : bool inside_block = assembler_->state()->InsideBlock();
1925 228 : if (inside_block) {
1926 228 : assembler_->Goto(&skip);
1927 : }
1928 : TNode<Object> e;
1929 : assembler_->Bind(label_.get(), &e);
1930 228 : *exception_ = e;
1931 228 : assembler_->Goto(compatibility_label_);
1932 228 : if (inside_block) {
1933 : assembler_->Bind(&skip);
1934 : }
1935 : }
1936 1292 : }
1937 :
1938 : } // namespace compiler
1939 :
1940 0 : Address CheckObjectType(Address raw_value, Address raw_type,
1941 : Address raw_location) {
1942 : #ifdef DEBUG
1943 : Object value(raw_value);
1944 : Smi type(raw_type);
1945 : String location = String::cast(Object(raw_location));
1946 : const char* expected;
1947 : switch (static_cast<ObjectType>(type->value())) {
1948 : #define TYPE_CASE(Name) \
1949 : case ObjectType::k##Name: \
1950 : if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
1951 : expected = #Name; \
1952 : break;
1953 : #define TYPE_STRUCT_CASE(NAME, Name, name) \
1954 : case ObjectType::k##Name: \
1955 : if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
1956 : expected = #Name; \
1957 : break;
1958 :
1959 : TYPE_CASE(Object)
1960 : OBJECT_TYPE_LIST(TYPE_CASE)
1961 : HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
1962 : STRUCT_LIST(TYPE_STRUCT_CASE)
1963 : #undef TYPE_CASE
1964 : #undef TYPE_STRUCT_CASE
1965 : }
1966 : std::stringstream value_description;
1967 : value->Print(value_description);
1968 : V8_Fatal(__FILE__, __LINE__,
1969 : "Type cast failed in %s\n"
1970 : " Expected %s but found %s",
1971 : location->ToAsciiArray(), expected, value_description.str().c_str());
1972 : #else
1973 0 : UNREACHABLE();
1974 : #endif
1975 : }
1976 :
1977 : } // namespace internal
1978 120216 : } // namespace v8
|
