Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/backend/instruction-selector.h"
6 :
7 : #include <limits>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/base/adapters.h"
11 : #include "src/compiler/backend/instruction-selector-impl.h"
12 : #include "src/compiler/compiler-source-position-table.h"
13 : #include "src/compiler/node-matchers.h"
14 : #include "src/compiler/pipeline.h"
15 : #include "src/compiler/schedule.h"
16 : #include "src/compiler/state-values-utils.h"
17 : #include "src/deoptimizer.h"
18 :
19 : namespace v8 {
20 : namespace internal {
21 : namespace compiler {
22 :
23 2640017 : InstructionSelector::InstructionSelector(
24 : Zone* zone, size_t node_count, Linkage* linkage,
25 : InstructionSequence* sequence, Schedule* schedule,
26 : SourcePositionTable* source_positions, Frame* frame,
27 : EnableSwitchJumpTable enable_switch_jump_table,
28 : SourcePositionMode source_position_mode, Features features,
29 : EnableScheduling enable_scheduling,
30 : EnableRootsRelativeAddressing enable_roots_relative_addressing,
31 : PoisoningMitigationLevel poisoning_level, EnableTraceTurboJson trace_turbo)
32 : : zone_(zone),
33 : linkage_(linkage),
34 : sequence_(sequence),
35 : source_positions_(source_positions),
36 : source_position_mode_(source_position_mode),
37 : features_(features),
38 : schedule_(schedule),
39 : current_block_(nullptr),
40 : instructions_(zone),
41 : continuation_inputs_(sequence->zone()),
42 : continuation_outputs_(sequence->zone()),
43 : defined_(node_count, false, zone),
44 : used_(node_count, false, zone),
45 : effect_level_(node_count, 0, zone),
46 : virtual_registers_(node_count,
47 : InstructionOperand::kInvalidVirtualRegister, zone),
48 : virtual_register_rename_(zone),
49 : scheduler_(nullptr),
50 : enable_scheduling_(enable_scheduling),
51 : enable_roots_relative_addressing_(enable_roots_relative_addressing),
52 : enable_switch_jump_table_(enable_switch_jump_table),
53 : poisoning_level_(poisoning_level),
54 : frame_(frame),
55 : instruction_selection_failed_(false),
56 : instr_origins_(sequence->zone()),
57 7925301 : trace_turbo_(trace_turbo) {
58 2642642 : instructions_.reserve(node_count);
59 2641825 : continuation_inputs_.reserve(5);
60 2642552 : continuation_outputs_.reserve(2);
61 :
62 2642934 : if (trace_turbo_ == kEnableTraceTurboJson) {
63 18 : instr_origins_.assign(node_count, {-1, 0});
64 : }
65 2642934 : }
66 :
67 2641535 : bool InstructionSelector::SelectInstructions() {
68 : // Mark the inputs of all phis in loop headers as used.
69 : BasicBlockVector* blocks = schedule()->rpo_order();
70 23112122 : for (auto const block : *blocks) {
71 20470587 : if (!block->IsLoopHeader()) continue;
72 : DCHECK_LE(2u, block->PredecessorCount());
73 2155857 : for (Node* const phi : *block) {
74 1924232 : if (phi->opcode() != IrOpcode::kPhi) continue;
75 :
76 : // Mark all inputs as used.
77 1425572 : for (Node* const input : phi->inputs()) {
78 : MarkAsUsed(input);
79 : }
80 : }
81 : }
82 :
83 : // Visit each basic block in post order.
84 23116468 : for (auto i = blocks->rbegin(); i != blocks->rend(); ++i) {
85 20473533 : VisitBlock(*i);
86 20474835 : if (instruction_selection_failed()) return false;
87 : }
88 :
89 : // Schedule the selected instructions.
90 2642935 : if (UseInstructionScheduling()) {
91 2196 : scheduler_ = new (zone()) InstructionScheduler(zone(), sequence());
92 : }
93 :
94 23120757 : for (auto const block : *blocks) {
95 : InstructionBlock* instruction_block =
96 20477458 : sequence()->InstructionBlockAt(RpoNumber::FromInt(block->rpo_number()));
97 24654142 : for (size_t i = 0; i < instruction_block->phis().size(); i++) {
98 2088203 : UpdateRenamesInPhi(instruction_block->PhiAt(i));
99 : }
100 20477738 : size_t end = instruction_block->code_end();
101 20477738 : size_t start = instruction_block->code_start();
102 : DCHECK_LE(end, start);
103 20477738 : StartBlock(RpoNumber::FromInt(block->rpo_number()));
104 20476683 : if (end != start) {
105 117303820 : while (start-- > end + 1) {
106 48413932 : UpdateRenames(instructions_[start]);
107 48414371 : AddInstruction(instructions_[start]);
108 : }
109 20476955 : UpdateRenames(instructions_[end]);
110 20477827 : AddTerminator(instructions_[end]);
111 : }
112 20476058 : EndBlock(RpoNumber::FromInt(block->rpo_number()));
113 : }
114 : #if DEBUG
115 : sequence()->ValidateSSA();
116 : #endif
117 : return true;
118 : }
119 :
120 20477229 : void InstructionSelector::StartBlock(RpoNumber rpo) {
121 20477229 : if (UseInstructionScheduling()) {
122 : DCHECK_NOT_NULL(scheduler_);
123 130913 : scheduler_->StartBlock(rpo);
124 : } else {
125 20346316 : sequence()->StartBlock(rpo);
126 : }
127 20476710 : }
128 :
129 20477487 : void InstructionSelector::EndBlock(RpoNumber rpo) {
130 20477487 : if (UseInstructionScheduling()) {
131 : DCHECK_NOT_NULL(scheduler_);
132 130913 : scheduler_->EndBlock(rpo);
133 : } else {
134 20346574 : sequence()->EndBlock(rpo);
135 : }
136 20477815 : }
137 :
138 20477412 : void InstructionSelector::AddTerminator(Instruction* instr) {
139 20477412 : if (UseInstructionScheduling()) {
140 : DCHECK_NOT_NULL(scheduler_);
141 130913 : scheduler_->AddTerminator(instr);
142 : } else {
143 20346499 : sequence()->AddInstruction(instr);
144 : }
145 20477572 : }
146 :
147 48414328 : void InstructionSelector::AddInstruction(Instruction* instr) {
148 48414328 : if (UseInstructionScheduling()) {
149 : DCHECK_NOT_NULL(scheduler_);
150 194488 : scheduler_->AddInstruction(instr);
151 : } else {
152 48219840 : sequence()->AddInstruction(instr);
153 : }
154 48413747 : }
155 :
156 22 : Instruction* InstructionSelector::Emit(InstructionCode opcode,
157 : InstructionOperand output,
158 : size_t temp_count,
159 : InstructionOperand* temps) {
160 18413003 : size_t output_count = output.IsInvalid() ? 0 : 1;
161 19025972 : return Emit(opcode, output_count, &output, 0, nullptr, temp_count, temps);
162 : }
163 :
164 4963190 : Instruction* InstructionSelector::Emit(InstructionCode opcode,
165 : InstructionOperand output,
166 : InstructionOperand a, size_t temp_count,
167 : InstructionOperand* temps) {
168 5690317 : size_t output_count = output.IsInvalid() ? 0 : 1;
169 14728971 : return Emit(opcode, output_count, &output, 1, &a, temp_count, temps);
170 : }
171 :
172 1335730 : Instruction* InstructionSelector::Emit(InstructionCode opcode,
173 : InstructionOperand output,
174 : InstructionOperand a,
175 : InstructionOperand b, size_t temp_count,
176 : InstructionOperand* temps) {
177 1335730 : size_t output_count = output.IsInvalid() ? 0 : 1;
178 1335730 : InstructionOperand inputs[] = {a, b};
179 : size_t input_count = arraysize(inputs);
180 : return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
181 1335730 : temps);
182 : }
183 :
184 1980 : Instruction* InstructionSelector::Emit(InstructionCode opcode,
185 : InstructionOperand output,
186 : InstructionOperand a,
187 : InstructionOperand b,
188 : InstructionOperand c, size_t temp_count,
189 : InstructionOperand* temps) {
190 1980 : size_t output_count = output.IsInvalid() ? 0 : 1;
191 1980 : InstructionOperand inputs[] = {a, b, c};
192 : size_t input_count = arraysize(inputs);
193 : return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
194 1980 : temps);
195 : }
196 :
197 0 : Instruction* InstructionSelector::Emit(
198 : InstructionCode opcode, InstructionOperand output, InstructionOperand a,
199 : InstructionOperand b, InstructionOperand c, InstructionOperand d,
200 : size_t temp_count, InstructionOperand* temps) {
201 0 : size_t output_count = output.IsInvalid() ? 0 : 1;
202 0 : InstructionOperand inputs[] = {a, b, c, d};
203 : size_t input_count = arraysize(inputs);
204 : return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
205 0 : temps);
206 : }
207 :
208 0 : Instruction* InstructionSelector::Emit(
209 : InstructionCode opcode, InstructionOperand output, InstructionOperand a,
210 : InstructionOperand b, InstructionOperand c, InstructionOperand d,
211 : InstructionOperand e, size_t temp_count, InstructionOperand* temps) {
212 0 : size_t output_count = output.IsInvalid() ? 0 : 1;
213 0 : InstructionOperand inputs[] = {a, b, c, d, e};
214 : size_t input_count = arraysize(inputs);
215 : return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
216 0 : temps);
217 : }
218 :
219 0 : Instruction* InstructionSelector::Emit(
220 : InstructionCode opcode, InstructionOperand output, InstructionOperand a,
221 : InstructionOperand b, InstructionOperand c, InstructionOperand d,
222 : InstructionOperand e, InstructionOperand f, size_t temp_count,
223 : InstructionOperand* temps) {
224 0 : size_t output_count = output.IsInvalid() ? 0 : 1;
225 0 : InstructionOperand inputs[] = {a, b, c, d, e, f};
226 : size_t input_count = arraysize(inputs);
227 : return Emit(opcode, output_count, &output, input_count, inputs, temp_count,
228 0 : temps);
229 : }
230 :
231 66237741 : Instruction* InstructionSelector::Emit(
232 : InstructionCode opcode, size_t output_count, InstructionOperand* outputs,
233 : size_t input_count, InstructionOperand* inputs, size_t temp_count,
234 : InstructionOperand* temps) {
235 132475482 : if (output_count >= Instruction::kMaxOutputCount ||
236 132473607 : input_count >= Instruction::kMaxInputCount ||
237 : temp_count >= Instruction::kMaxTempCount) {
238 : set_instruction_selection_failed();
239 9 : return nullptr;
240 : }
241 :
242 : Instruction* instr =
243 : Instruction::New(instruction_zone(), opcode, output_count, outputs,
244 66237732 : input_count, inputs, temp_count, temps);
245 66234025 : return Emit(instr);
246 : }
247 :
248 0 : Instruction* InstructionSelector::Emit(Instruction* instr) {
249 68876363 : instructions_.push_back(instr);
250 66234025 : return instr;
251 : }
252 :
253 12590031 : bool InstructionSelector::CanCover(Node* user, Node* node) const {
254 : // 1. Both {user} and {node} must be in the same basic block.
255 12590031 : if (schedule()->block(node) != schedule()->block(user)) {
256 : return false;
257 : }
258 : // 2. Pure {node}s must be owned by the {user}.
259 11621392 : if (node->op()->HasProperty(Operator::kPure)) {
260 9016162 : return node->OwnedBy(user);
261 : }
262 : // 3. Impure {node}s must match the effect level of {user}.
263 2605230 : if (GetEffectLevel(node) != GetEffectLevel(user)) {
264 : return false;
265 : }
266 : // 4. Only {node} must have value edges pointing to {user}.
267 16014372 : for (Edge const edge : node->use_edges()) {
268 6820610 : if (edge.from() != user && NodeProperties::IsValueEdge(edge)) {
269 168642 : return false;
270 : }
271 : }
272 2373152 : return true;
273 : }
274 :
275 470011 : bool InstructionSelector::CanCoverTransitively(Node* user, Node* node,
276 : Node* node_input) const {
277 470011 : if (CanCover(user, node) && CanCover(node, node_input)) {
278 : // If {node} is pure, transitivity might not hold.
279 269321 : if (node->op()->HasProperty(Operator::kPure)) {
280 : // If {node_input} is pure, the effect levels do not matter.
281 269320 : if (node_input->op()->HasProperty(Operator::kPure)) return true;
282 : // Otherwise, {user} and {node_input} must have the same effect level.
283 264983 : return GetEffectLevel(user) == GetEffectLevel(node_input);
284 : }
285 : return true;
286 : }
287 : return false;
288 : }
289 :
290 148070 : bool InstructionSelector::IsOnlyUserOfNodeInSameBlock(Node* user,
291 : Node* node) const {
292 148070 : BasicBlock* bb_user = schedule()->block(user);
293 148071 : BasicBlock* bb_node = schedule()->block(node);
294 148074 : if (bb_user != bb_node) return false;
295 305984 : for (Edge const edge : node->use_edges()) {
296 : Node* from = edge.from();
297 170754 : if ((from != user) && (schedule()->block(from) == bb_user)) {
298 : return false;
299 : }
300 : }
301 135230 : return true;
302 : }
303 :
304 68890474 : void InstructionSelector::UpdateRenames(Instruction* instruction) {
305 341335618 : for (size_t i = 0; i < instruction->InputCount(); i++) {
306 136224424 : TryRename(instruction->InputAt(i));
307 : }
308 68888622 : }
309 :
310 2088192 : void InstructionSelector::UpdateRenamesInPhi(PhiInstruction* phi) {
311 12196278 : for (size_t i = 0; i < phi->operands().size(); i++) {
312 5054033 : int vreg = phi->operands()[i];
313 : int renamed = GetRename(vreg);
314 5054033 : if (vreg != renamed) {
315 291849 : phi->RenameInput(i, renamed);
316 : }
317 : }
318 2088202 : }
319 :
320 0 : int InstructionSelector::GetRename(int virtual_register) {
321 : int rename = virtual_register;
322 : while (true) {
323 158902928 : if (static_cast<size_t>(rename) >= virtual_register_rename_.size()) break;
324 44931055 : int next = virtual_register_rename_[rename];
325 44931055 : if (next == InstructionOperand::kInvalidVirtualRegister) {
326 : break;
327 : }
328 : rename = next;
329 : }
330 0 : return rename;
331 : }
332 :
333 136224719 : void InstructionSelector::TryRename(InstructionOperand* op) {
334 136224719 : if (!op->IsUnallocated()) return;
335 : UnallocatedOperand* unalloc = UnallocatedOperand::cast(op);
336 : int vreg = unalloc->virtual_register();
337 : int rename = GetRename(vreg);
338 71687588 : if (rename != vreg) {
339 2414074 : *unalloc = UnallocatedOperand(*unalloc, rename);
340 : }
341 : }
342 :
343 2420014 : void InstructionSelector::SetRename(const Node* node, const Node* rename) {
344 2420014 : int vreg = GetVirtualRegister(node);
345 4840036 : if (static_cast<size_t>(vreg) >= virtual_register_rename_.size()) {
346 2126764 : int invalid = InstructionOperand::kInvalidVirtualRegister;
347 2126764 : virtual_register_rename_.resize(vreg + 1, invalid);
348 : }
349 2420018 : virtual_register_rename_[vreg] = GetVirtualRegister(rename);
350 2420026 : }
351 :
352 157260977 : int InstructionSelector::GetVirtualRegister(const Node* node) {
353 : DCHECK_NOT_NULL(node);
354 157260977 : size_t const id = node->id();
355 : DCHECK_LT(id, virtual_registers_.size());
356 157260977 : int virtual_register = virtual_registers_[id];
357 157260977 : if (virtual_register == InstructionOperand::kInvalidVirtualRegister) {
358 44336618 : virtual_register = sequence()->NextVirtualRegister();
359 44336457 : virtual_registers_[id] = virtual_register;
360 : }
361 157260816 : return virtual_register;
362 : }
363 :
364 249 : const std::map<NodeId, int> InstructionSelector::GetVirtualRegistersForTesting()
365 : const {
366 : std::map<NodeId, int> virtual_registers;
367 4509 : for (size_t n = 0; n < virtual_registers_.size(); ++n) {
368 2130 : if (virtual_registers_[n] != InstructionOperand::kInvalidVirtualRegister) {
369 727 : NodeId const id = static_cast<NodeId>(n);
370 1454 : virtual_registers.insert(std::make_pair(id, virtual_registers_[n]));
371 : }
372 : }
373 249 : return virtual_registers;
374 : }
375 :
376 940620 : bool InstructionSelector::IsDefined(Node* node) const {
377 : DCHECK_NOT_NULL(node);
378 52796656 : size_t const id = node->id();
379 : DCHECK_LT(id, defined_.size());
380 940620 : return defined_[id];
381 : }
382 :
383 13694537 : void InstructionSelector::MarkAsDefined(Node* node) {
384 : DCHECK_NOT_NULL(node);
385 39245640 : size_t const id = node->id();
386 : DCHECK_LT(id, defined_.size());
387 : defined_[id] = true;
388 13694537 : }
389 :
390 855602 : bool InstructionSelector::IsUsed(Node* node) const {
391 : DCHECK_NOT_NULL(node);
392 : // TODO(bmeurer): This is a terrible monster hack, but we have to make sure
393 : // that the Retain is actually emitted, otherwise the GC will mess up.
394 115463596 : if (node->opcode() == IrOpcode::kRetain) return true;
395 115453777 : if (!node->op()->HasProperty(Operator::kEliminatable)) return true;
396 101488822 : size_t const id = node->id();
397 : DCHECK_LT(id, used_.size());
398 853162 : return used_[id];
399 : }
400 :
401 35535269 : void InstructionSelector::MarkAsUsed(Node* node) {
402 : DCHECK_NOT_NULL(node);
403 84451426 : size_t const id = node->id();
404 : DCHECK_LT(id, used_.size());
405 : used_[id] = true;
406 35535269 : }
407 :
408 15142438 : int InstructionSelector::GetEffectLevel(Node* node) const {
409 : DCHECK_NOT_NULL(node);
410 20882864 : size_t const id = node->id();
411 : DCHECK_LT(id, effect_level_.size());
412 20882864 : return effect_level_[id];
413 : }
414 :
415 0 : void InstructionSelector::SetEffectLevel(Node* node, int effect_level) {
416 : DCHECK_NOT_NULL(node);
417 123622869 : size_t const id = node->id();
418 : DCHECK_LT(id, effect_level_.size());
419 123622869 : effect_level_[id] = effect_level;
420 0 : }
421 :
422 13523584 : bool InstructionSelector::CanAddressRelativeToRootsRegister() const {
423 20726926 : return enable_roots_relative_addressing_ == kEnableRootsRelativeAddressing &&
424 13523584 : CanUseRootsRegister();
425 : }
426 :
427 3011427 : bool InstructionSelector::CanUseRootsRegister() const {
428 : return linkage()->GetIncomingDescriptor()->flags() &
429 3011427 : CallDescriptor::kCanUseRoots;
430 : }
431 :
432 0 : void InstructionSelector::MarkAsRepresentation(MachineRepresentation rep,
433 : const InstructionOperand& op) {
434 : UnallocatedOperand unalloc = UnallocatedOperand::cast(op);
435 6054687 : sequence()->MarkAsRepresentation(rep, unalloc.virtual_register());
436 0 : }
437 :
438 27257582 : void InstructionSelector::MarkAsRepresentation(MachineRepresentation rep,
439 : Node* node) {
440 27257582 : sequence()->MarkAsRepresentation(rep, GetVirtualRegister(node));
441 27259851 : }
442 :
443 : namespace {
444 :
445 22530159 : InstructionOperand OperandForDeopt(Isolate* isolate, OperandGenerator* g,
446 : Node* input, FrameStateInputKind kind,
447 : MachineRepresentation rep) {
448 22530159 : if (rep == MachineRepresentation::kNone) {
449 0 : return g->TempImmediate(FrameStateDescriptor::kImpossibleValue);
450 : }
451 :
452 22530159 : switch (input->opcode()) {
453 : case IrOpcode::kInt32Constant:
454 : case IrOpcode::kInt64Constant:
455 : case IrOpcode::kNumberConstant:
456 : case IrOpcode::kFloat32Constant:
457 : case IrOpcode::kFloat64Constant:
458 : case IrOpcode::kDelayedStringConstant:
459 667670 : return g->UseImmediate(input);
460 : case IrOpcode::kHeapConstant: {
461 5656890 : if (!CanBeTaggedPointer(rep)) {
462 : // If we have inconsistent static and dynamic types, e.g. if we
463 : // smi-check a string, we can get here with a heap object that
464 : // says it is a smi. In that case, we return an invalid instruction
465 : // operand, which will be interpreted as an optimized-out value.
466 :
467 : // TODO(jarin) Ideally, we should turn the current instruction
468 : // into an abort (we should never execute it).
469 0 : return InstructionOperand();
470 : }
471 :
472 5656890 : Handle<HeapObject> constant = HeapConstantOf(input->op());
473 : RootIndex root_index;
474 5656892 : if (isolate->roots_table().IsRootHandle(constant, &root_index) &&
475 : root_index == RootIndex::kOptimizedOut) {
476 : // For an optimized-out object we return an invalid instruction
477 : // operand, so that we take the fast path for optimized-out values.
478 341 : return InstructionOperand();
479 : }
480 :
481 5656551 : return g->UseImmediate(input);
482 : }
483 : case IrOpcode::kArgumentsElementsState:
484 : case IrOpcode::kArgumentsLengthState:
485 : case IrOpcode::kObjectState:
486 : case IrOpcode::kTypedObjectState:
487 0 : UNREACHABLE();
488 : break;
489 : default:
490 16205599 : switch (kind) {
491 : case FrameStateInputKind::kStackSlot:
492 13904666 : return g->UseUniqueSlot(input);
493 : case FrameStateInputKind::kAny:
494 : // Currently deopts "wrap" other operations, so the deopt's inputs
495 : // are potentially needed until the end of the deoptimising code.
496 2300933 : return g->UseAnyAtEnd(input);
497 : }
498 : }
499 0 : UNREACHABLE();
500 : }
501 :
502 : } // namespace
503 :
504 3330637 : class StateObjectDeduplicator {
505 : public:
506 : explicit StateObjectDeduplicator(Zone* zone) : objects_(zone) {}
507 : static const size_t kNotDuplicated = SIZE_MAX;
508 :
509 108013 : size_t GetObjectId(Node* node) {
510 : DCHECK(node->opcode() == IrOpcode::kTypedObjectState ||
511 : node->opcode() == IrOpcode::kObjectId ||
512 : node->opcode() == IrOpcode::kArgumentsElementsState);
513 268461 : for (size_t i = 0; i < objects_.size(); ++i) {
514 99097 : if (objects_[i] == node) return i;
515 : // ObjectId nodes are the Turbofan way to express objects with the same
516 : // identity in the deopt info. So they should always be mapped to
517 : // previously appearing TypedObjectState nodes.
518 295665 : if (HasObjectId(objects_[i]) && HasObjectId(node) &&
519 98284 : ObjectIdOf(objects_[i]->op()) == ObjectIdOf(node->op())) {
520 : return i;
521 : }
522 : }
523 : DCHECK(node->opcode() == IrOpcode::kTypedObjectState ||
524 : node->opcode() == IrOpcode::kArgumentsElementsState);
525 : return kNotDuplicated;
526 : }
527 :
528 : size_t InsertObject(Node* node) {
529 : DCHECK(node->opcode() == IrOpcode::kTypedObjectState ||
530 : node->opcode() == IrOpcode::kObjectId ||
531 : node->opcode() == IrOpcode::kArgumentsElementsState);
532 : size_t id = objects_.size();
533 113993 : objects_.push_back(node);
534 : return id;
535 : }
536 :
537 : private:
538 : static bool HasObjectId(Node* node) {
539 197381 : return node->opcode() == IrOpcode::kTypedObjectState ||
540 : node->opcode() == IrOpcode::kObjectId;
541 : }
542 :
543 : ZoneVector<Node*> objects_;
544 : };
545 :
546 : // Returns the number of instruction operands added to inputs.
547 50328360 : size_t InstructionSelector::AddOperandToStateValueDescriptor(
548 : StateValueList* values, InstructionOperandVector* inputs,
549 : OperandGenerator* g, StateObjectDeduplicator* deduplicator, Node* input,
550 : MachineType type, FrameStateInputKind kind, Zone* zone) {
551 50328360 : if (input == nullptr) {
552 27678086 : values->PushOptimizedOut();
553 27678271 : return 0;
554 : }
555 :
556 22650274 : switch (input->opcode()) {
557 : case IrOpcode::kArgumentsElementsState: {
558 5980 : values->PushArgumentsElements(ArgumentsStateTypeOf(input->op()));
559 : // The elements backing store of an arguments object participates in the
560 : // duplicate object counting, but can itself never appear duplicated.
561 : DCHECK_EQ(StateObjectDeduplicator::kNotDuplicated,
562 : deduplicator->GetObjectId(input));
563 : deduplicator->InsertObject(input);
564 5980 : return 0;
565 : }
566 : case IrOpcode::kArgumentsLengthState: {
567 6288 : values->PushArgumentsLength(ArgumentsStateTypeOf(input->op()));
568 6288 : return 0;
569 : }
570 : case IrOpcode::kObjectState: {
571 0 : UNREACHABLE();
572 : }
573 : case IrOpcode::kTypedObjectState:
574 : case IrOpcode::kObjectId: {
575 108013 : size_t id = deduplicator->GetObjectId(input);
576 108013 : if (id == StateObjectDeduplicator::kNotDuplicated) {
577 : DCHECK_EQ(IrOpcode::kTypedObjectState, input->opcode());
578 : size_t entries = 0;
579 : id = deduplicator->InsertObject(input);
580 89140 : StateValueList* nested = values->PushRecursiveField(zone, id);
581 : int const input_count = input->op()->ValueInputCount();
582 89140 : ZoneVector<MachineType> const* types = MachineTypesOf(input->op());
583 1150446 : for (int i = 0; i < input_count; ++i) {
584 : entries += AddOperandToStateValueDescriptor(
585 530653 : nested, inputs, g, deduplicator, input->InputAt(i), types->at(i),
586 530653 : kind, zone);
587 : }
588 : return entries;
589 : } else {
590 : // Deoptimizer counts duplicate objects for the running id, so we have
591 : // to push the input again.
592 : deduplicator->InsertObject(input);
593 18873 : values->PushDuplicate(id);
594 18873 : return 0;
595 : }
596 : }
597 : default: {
598 : InstructionOperand op =
599 22529993 : OperandForDeopt(isolate(), g, input, kind, type.representation());
600 22530032 : if (op.kind() == InstructionOperand::INVALID) {
601 : // Invalid operand means the value is impossible or optimized-out.
602 341 : values->PushOptimizedOut();
603 347 : return 0;
604 : } else {
605 22529691 : inputs->push_back(op);
606 : values->PushPlain(type);
607 22529891 : return 1;
608 : }
609 : }
610 : }
611 : }
612 :
613 : // Returns the number of instruction operands added to inputs.
614 3683977 : size_t InstructionSelector::AddInputsToFrameStateDescriptor(
615 : FrameStateDescriptor* descriptor, Node* state, OperandGenerator* g,
616 : StateObjectDeduplicator* deduplicator, InstructionOperandVector* inputs,
617 : FrameStateInputKind kind, Zone* zone) {
618 : DCHECK_EQ(IrOpcode::kFrameState, state->op()->opcode());
619 :
620 : size_t entries = 0;
621 : size_t initial_size = inputs->size();
622 : USE(initial_size); // initial_size is only used for debug.
623 :
624 3683977 : if (descriptor->outer_state()) {
625 : entries += AddInputsToFrameStateDescriptor(
626 : descriptor->outer_state(), state->InputAt(kFrameStateOuterStateInput),
627 353349 : g, deduplicator, inputs, kind, zone);
628 : }
629 :
630 : Node* parameters = state->InputAt(kFrameStateParametersInput);
631 : Node* locals = state->InputAt(kFrameStateLocalsInput);
632 : Node* stack = state->InputAt(kFrameStateStackInput);
633 : Node* context = state->InputAt(kFrameStateContextInput);
634 : Node* function = state->InputAt(kFrameStateFunctionInput);
635 :
636 : DCHECK_EQ(descriptor->parameters_count(),
637 : StateValuesAccess(parameters).size());
638 : DCHECK_EQ(descriptor->locals_count(), StateValuesAccess(locals).size());
639 : DCHECK_EQ(descriptor->stack_count(), StateValuesAccess(stack).size());
640 :
641 : StateValueList* values_descriptor = descriptor->GetStateValueDescriptors();
642 :
643 : DCHECK_EQ(values_descriptor->size(), 0u);
644 3683977 : values_descriptor->ReserveSize(descriptor->GetSize());
645 :
646 : entries += AddOperandToStateValueDescriptor(
647 : values_descriptor, inputs, g, deduplicator, function,
648 3683985 : MachineType::AnyTagged(), FrameStateInputKind::kStackSlot, zone);
649 22585114 : for (StateValuesAccess::TypedNode input_node :
650 9450553 : StateValuesAccess(parameters)) {
651 9450518 : entries += AddOperandToStateValueDescriptor(values_descriptor, inputs, g,
652 : deduplicator, input_node.node,
653 9450518 : input_node.type, kind, zone);
654 : }
655 3683986 : if (descriptor->HasContext()) {
656 : entries += AddOperandToStateValueDescriptor(
657 : values_descriptor, inputs, g, deduplicator, context,
658 3599599 : MachineType::AnyTagged(), FrameStateInputKind::kStackSlot, zone);
659 : }
660 62695438 : for (StateValuesAccess::TypedNode input_node : StateValuesAccess(locals)) {
661 29505469 : entries += AddOperandToStateValueDescriptor(values_descriptor, inputs, g,
662 : deduplicator, input_node.node,
663 29505469 : input_node.type, kind, zone);
664 : }
665 10800371 : for (StateValuesAccess::TypedNode input_node : StateValuesAccess(stack)) {
666 3558187 : entries += AddOperandToStateValueDescriptor(values_descriptor, inputs, g,
667 : deduplicator, input_node.node,
668 3558187 : input_node.type, kind, zone);
669 : }
670 : DCHECK_EQ(initial_size + entries, inputs->size());
671 3683990 : return entries;
672 : }
673 :
674 562861 : Instruction* InstructionSelector::EmitWithContinuation(
675 : InstructionCode opcode, FlagsContinuation* cont) {
676 562861 : return EmitWithContinuation(opcode, 0, nullptr, 0, nullptr, cont);
677 : }
678 :
679 0 : Instruction* InstructionSelector::EmitWithContinuation(
680 : InstructionCode opcode, InstructionOperand a, FlagsContinuation* cont) {
681 0 : return EmitWithContinuation(opcode, 0, nullptr, 1, &a, cont);
682 : }
683 :
684 4255318 : Instruction* InstructionSelector::EmitWithContinuation(
685 : InstructionCode opcode, InstructionOperand a, InstructionOperand b,
686 : FlagsContinuation* cont) {
687 4255318 : InstructionOperand inputs[] = {a, b};
688 : return EmitWithContinuation(opcode, 0, nullptr, arraysize(inputs), inputs,
689 4255318 : cont);
690 : }
691 :
692 0 : Instruction* InstructionSelector::EmitWithContinuation(
693 : InstructionCode opcode, InstructionOperand a, InstructionOperand b,
694 : InstructionOperand c, FlagsContinuation* cont) {
695 0 : InstructionOperand inputs[] = {a, b, c};
696 : return EmitWithContinuation(opcode, 0, nullptr, arraysize(inputs), inputs,
697 0 : cont);
698 : }
699 :
700 6799575 : Instruction* InstructionSelector::EmitWithContinuation(
701 : InstructionCode opcode, size_t output_count, InstructionOperand* outputs,
702 : size_t input_count, InstructionOperand* inputs, FlagsContinuation* cont) {
703 : OperandGenerator g(this);
704 :
705 : opcode = cont->Encode(opcode);
706 :
707 6799575 : continuation_inputs_.resize(0);
708 33665524 : for (size_t i = 0; i < input_count; i++) {
709 13432690 : continuation_inputs_.push_back(inputs[i]);
710 : }
711 :
712 6799991 : continuation_outputs_.resize(0);
713 8595598 : for (size_t i = 0; i < output_count; i++) {
714 897799 : continuation_outputs_.push_back(outputs[i]);
715 : }
716 :
717 6799977 : if (cont->IsBranch()) {
718 10737890 : continuation_inputs_.push_back(g.Label(cont->true_block()));
719 10737995 : continuation_inputs_.push_back(g.Label(cont->false_block()));
720 1431116 : } else if (cont->IsDeoptimize()) {
721 330903 : opcode |= MiscField::encode(static_cast<int>(input_count));
722 330903 : AppendDeoptimizeArguments(&continuation_inputs_, cont->kind(),
723 : cont->reason(), cont->feedback(),
724 330903 : cont->frame_state());
725 1100213 : } else if (cont->IsSet()) {
726 752929 : continuation_outputs_.push_back(g.DefineAsRegister(cont->result()));
727 723744 : } else if (cont->IsTrap()) {
728 141988 : int trap_id = static_cast<int>(cont->trap_id());
729 284016 : continuation_inputs_.push_back(g.UseImmediate(trap_id));
730 : } else {
731 : DCHECK(cont->IsNone());
732 : }
733 :
734 : size_t const emit_inputs_size = continuation_inputs_.size();
735 : auto* emit_inputs =
736 6800097 : emit_inputs_size ? &continuation_inputs_.front() : nullptr;
737 : size_t const emit_outputs_size = continuation_outputs_.size();
738 : auto* emit_outputs =
739 6800097 : emit_outputs_size ? &continuation_outputs_.front() : nullptr;
740 : return Emit(opcode, emit_outputs_size, emit_outputs, emit_inputs_size,
741 6800097 : emit_inputs, 0, nullptr);
742 : }
743 :
744 375517 : void InstructionSelector::AppendDeoptimizeArguments(
745 : InstructionOperandVector* args, DeoptimizeKind kind,
746 : DeoptimizeReason reason, VectorSlotPair const& feedback,
747 : Node* frame_state) {
748 : OperandGenerator g(this);
749 375517 : FrameStateDescriptor* const descriptor = GetFrameStateDescriptor(frame_state);
750 : DCHECK_NE(DeoptimizeKind::kLazy, kind);
751 : int const state_id =
752 375517 : sequence()->AddDeoptimizationEntry(descriptor, kind, reason, feedback);
753 751039 : args->push_back(g.TempImmediate(state_id));
754 : StateObjectDeduplicator deduplicator(instruction_zone());
755 : AddInputsToFrameStateDescriptor(descriptor, frame_state, &g, &deduplicator,
756 : args, FrameStateInputKind::kAny,
757 375518 : instruction_zone());
758 375521 : }
759 :
760 44616 : Instruction* InstructionSelector::EmitDeoptimize(
761 : InstructionCode opcode, size_t output_count, InstructionOperand* outputs,
762 : size_t input_count, InstructionOperand* inputs, DeoptimizeKind kind,
763 : DeoptimizeReason reason, VectorSlotPair const& feedback,
764 : Node* frame_state) {
765 : InstructionOperandVector args(instruction_zone());
766 44616 : for (size_t i = 0; i < input_count; ++i) {
767 0 : args.push_back(inputs[i]);
768 : }
769 44616 : opcode |= MiscField::encode(static_cast<int>(input_count));
770 44616 : AppendDeoptimizeArguments(&args, kind, reason, feedback, frame_state);
771 : return Emit(opcode, output_count, outputs, args.size(), &args.front(), 0,
772 89232 : nullptr);
773 : }
774 :
775 : // An internal helper class for generating the operands to calls.
776 : // TODO(bmeurer): Get rid of the CallBuffer business and make
777 : // InstructionSelector::VisitCall platform independent instead.
778 6183281 : struct CallBuffer {
779 6183326 : CallBuffer(Zone* zone, const CallDescriptor* call_descriptor,
780 : FrameStateDescriptor* frame_state)
781 : : descriptor(call_descriptor),
782 : frame_state_descriptor(frame_state),
783 : output_nodes(zone),
784 : outputs(zone),
785 : instruction_args(zone),
786 6183326 : pushed_nodes(zone) {
787 6183326 : output_nodes.reserve(call_descriptor->ReturnCount());
788 6183468 : outputs.reserve(call_descriptor->ReturnCount());
789 6183524 : pushed_nodes.reserve(input_count());
790 6183587 : instruction_args.reserve(input_count() + frame_state_value_count());
791 6183643 : }
792 :
793 : const CallDescriptor* descriptor;
794 : FrameStateDescriptor* frame_state_descriptor;
795 : ZoneVector<PushParameter> output_nodes;
796 : InstructionOperandVector outputs;
797 : InstructionOperandVector instruction_args;
798 : ZoneVector<PushParameter> pushed_nodes;
799 :
800 : size_t input_count() const { return descriptor->InputCount(); }
801 :
802 : size_t frame_state_count() const { return descriptor->FrameStateCount(); }
803 :
804 : size_t frame_state_value_count() const {
805 : return (frame_state_descriptor == nullptr)
806 : ? 0
807 2955120 : : (frame_state_descriptor->GetTotalSize() +
808 9138707 : 1); // Include deopt id.
809 : }
810 : };
811 :
812 : // TODO(bmeurer): Get rid of the CallBuffer business and make
813 : // InstructionSelector::VisitCall platform independent instead.
814 6183506 : void InstructionSelector::InitializeCallBuffer(Node* call, CallBuffer* buffer,
815 : CallBufferFlags flags,
816 : bool is_tail_call,
817 : int stack_param_delta) {
818 : OperandGenerator g(this);
819 6183506 : size_t ret_count = buffer->descriptor->ReturnCount();
820 : DCHECK_LE(call->op()->ValueOutputCount(), ret_count);
821 : DCHECK_EQ(
822 : call->op()->ValueInputCount(),
823 : static_cast<int>(buffer->input_count() + buffer->frame_state_count()));
824 :
825 6183506 : if (ret_count > 0) {
826 : // Collect the projections that represent multiple outputs from this call.
827 6045488 : if (ret_count == 1) {
828 : PushParameter result = {call, buffer->descriptor->GetReturnLocation(0)};
829 6040233 : buffer->output_nodes.push_back(result);
830 : } else {
831 5255 : buffer->output_nodes.resize(ret_count);
832 : int stack_count = 0;
833 55677 : for (size_t i = 0; i < ret_count; ++i) {
834 25211 : LinkageLocation location = buffer->descriptor->GetReturnLocation(i);
835 25211 : buffer->output_nodes[i] = PushParameter(nullptr, location);
836 25211 : if (location.IsCallerFrameSlot()) {
837 14699 : stack_count += location.GetSizeInPointers();
838 : }
839 : }
840 78803 : for (Edge const edge : call->use_edges()) {
841 36774 : if (!NodeProperties::IsValueEdge(edge)) continue;
842 : Node* node = edge.from();
843 : DCHECK_EQ(IrOpcode::kProjection, node->opcode());
844 14417 : size_t const index = ProjectionIndexOf(node->op());
845 :
846 : DCHECK_LT(index, buffer->output_nodes.size());
847 : DCHECK(!buffer->output_nodes[index].node);
848 14417 : buffer->output_nodes[index].node = node;
849 : }
850 5255 : frame_->EnsureReturnSlots(stack_count);
851 : }
852 :
853 : // Filter out the outputs that aren't live because no projection uses them.
854 : size_t outputs_needed_by_framestate =
855 6045563 : buffer->frame_state_descriptor == nullptr
856 : ? 0
857 : : buffer->frame_state_descriptor->state_combine()
858 6045563 : .ConsumedOutputCount();
859 18176375 : for (size_t i = 0; i < buffer->output_nodes.size(); i++) {
860 6065559 : bool output_is_live = buffer->output_nodes[i].node != nullptr ||
861 : i < outputs_needed_by_framestate;
862 6065559 : if (output_is_live) {
863 6054748 : LinkageLocation location = buffer->output_nodes[i].location;
864 : MachineRepresentation rep = location.GetType().representation();
865 :
866 : Node* output = buffer->output_nodes[i].node;
867 : InstructionOperand op = output == nullptr
868 : ? g.TempLocation(location)
869 6054748 : : g.DefineAsLocation(output, location);
870 : MarkAsRepresentation(rep, op);
871 :
872 6054578 : if (!UnallocatedOperand::cast(op).HasFixedSlotPolicy()) {
873 6049507 : buffer->outputs.push_back(op);
874 6049524 : buffer->output_nodes[i].node = nullptr;
875 : }
876 : }
877 : }
878 : }
879 :
880 : // The first argument is always the callee code.
881 : Node* callee = call->InputAt(0);
882 : bool call_code_immediate = (flags & kCallCodeImmediate) != 0;
883 : bool call_address_immediate = (flags & kCallAddressImmediate) != 0;
884 : bool call_use_fixed_target_reg = (flags & kCallFixedTargetRegister) != 0;
885 : bool call_through_slot = (flags & kAllowCallThroughSlot) != 0;
886 6183428 : switch (buffer->descriptor->kind()) {
887 : case CallDescriptor::kCallCodeObject:
888 : // TODO(jgruber, v8:7449): The below is a hack to support tail-calls from
889 : // JS-linkage callers with a register code target. The problem is that the
890 : // code target register may be clobbered before the final jmp by
891 : // AssemblePopArgumentsAdaptorFrame. As a more permanent fix we could
892 : // entirely remove support for tail-calls from JS-linkage callers.
893 14212652 : buffer->instruction_args.push_back(
894 4864918 : (call_code_immediate && callee->opcode() == IrOpcode::kHeapConstant)
895 : ? g.UseImmediate(callee)
896 : : call_use_fixed_target_reg
897 : ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
898 : : is_tail_call ? g.UseUniqueRegister(callee)
899 : : call_through_slot ? g.UseUniqueSlot(callee)
900 : : g.UseRegister(callee));
901 4864923 : break;
902 : case CallDescriptor::kCallAddress:
903 242038 : buffer->instruction_args.push_back(
904 108648 : (call_address_immediate &&
905 : callee->opcode() == IrOpcode::kExternalConstant)
906 : ? g.UseImmediate(callee)
907 : : call_use_fixed_target_reg
908 : ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
909 : : g.UseRegister(callee));
910 108650 : break;
911 : case CallDescriptor::kCallWasmFunction:
912 : case CallDescriptor::kCallWasmImportWrapper:
913 2557271 : buffer->instruction_args.push_back(
914 1182289 : (call_address_immediate &&
915 989572 : (callee->opcode() == IrOpcode::kRelocatableInt64Constant ||
916 : callee->opcode() == IrOpcode::kRelocatableInt32Constant))
917 : ? g.UseImmediate(callee)
918 : : call_use_fixed_target_reg
919 : ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
920 : : g.UseRegister(callee));
921 1182206 : break;
922 : case CallDescriptor::kCallBuiltinPointer:
923 : // The common case for builtin pointers is to have the target in a
924 : // register. If we have a constant, we use a register anyway to simplify
925 : // related code.
926 7216 : buffer->instruction_args.push_back(
927 : call_use_fixed_target_reg
928 : ? g.UseFixed(callee, kJavaScriptCallCodeStartRegister)
929 : : g.UseRegister(callee));
930 3608 : break;
931 : case CallDescriptor::kCallJSFunction:
932 23884 : buffer->instruction_args.push_back(
933 47768 : g.UseLocation(callee, buffer->descriptor->GetInputLocation(0)));
934 23884 : break;
935 : }
936 : DCHECK_EQ(1u, buffer->instruction_args.size());
937 :
938 : // Argument 1 is used for poison-alias index (encoded in a word-sized
939 : // immediate. This an index of the operand that aliases with poison register
940 : // or -1 if there is no aliasing.
941 12367029 : buffer->instruction_args.push_back(g.TempImmediate(-1));
942 : const size_t poison_alias_index = 1;
943 : DCHECK_EQ(buffer->instruction_args.size() - 1, poison_alias_index);
944 :
945 : // If the call needs a frame state, we insert the state information as
946 : // follows (n is the number of value inputs to the frame state):
947 : // arg 2 : deoptimization id.
948 : // arg 3 - arg (n + 2) : value inputs to the frame state.
949 : size_t frame_state_entries = 0;
950 : USE(frame_state_entries); // frame_state_entries is only used for debug.
951 6183692 : if (buffer->frame_state_descriptor != nullptr) {
952 : Node* frame_state =
953 5910238 : call->InputAt(static_cast<int>(buffer->descriptor->InputCount()));
954 :
955 : // If it was a syntactic tail call we need to drop the current frame and
956 : // all the frames on top of it that are either an arguments adaptor frame
957 : // or a tail caller frame.
958 2955119 : if (is_tail_call) {
959 0 : frame_state = NodeProperties::GetFrameStateInput(frame_state);
960 : buffer->frame_state_descriptor =
961 0 : buffer->frame_state_descriptor->outer_state();
962 0 : while (buffer->frame_state_descriptor != nullptr &&
963 : buffer->frame_state_descriptor->type() ==
964 : FrameStateType::kArgumentsAdaptor) {
965 0 : frame_state = NodeProperties::GetFrameStateInput(frame_state);
966 : buffer->frame_state_descriptor =
967 0 : buffer->frame_state_descriptor->outer_state();
968 : }
969 : }
970 :
971 5910237 : int const state_id = sequence()->AddDeoptimizationEntry(
972 : buffer->frame_state_descriptor, DeoptimizeKind::kLazy,
973 2955114 : DeoptimizeReason::kUnknown, VectorSlotPair());
974 5910234 : buffer->instruction_args.push_back(g.TempImmediate(state_id));
975 :
976 : StateObjectDeduplicator deduplicator(instruction_zone());
977 :
978 : frame_state_entries =
979 2955116 : 1 + AddInputsToFrameStateDescriptor(
980 : buffer->frame_state_descriptor, frame_state, &g, &deduplicator,
981 : &buffer->instruction_args, FrameStateInputKind::kStackSlot,
982 : instruction_zone());
983 :
984 : DCHECK_EQ(2 + frame_state_entries, buffer->instruction_args.size());
985 : }
986 :
987 : size_t input_count = static_cast<size_t>(buffer->input_count());
988 :
989 : // Split the arguments into pushed_nodes and instruction_args. Pushed
990 : // arguments require an explicit push instruction before the call and do
991 : // not appear as arguments to the call. Everything else ends up
992 : // as an InstructionOperand argument to the call.
993 : auto iter(call->inputs().begin());
994 : size_t pushed_count = 0;
995 6183689 : bool call_tail = (flags & kCallTail) != 0;
996 59368937 : for (size_t index = 0; index < input_count; ++iter, ++index) {
997 : DCHECK(iter != call->inputs().end());
998 : DCHECK_NE(IrOpcode::kFrameState, (*iter)->op()->opcode());
999 32776073 : if (index == 0) continue; // The first argument (callee) is already done.
1000 :
1001 40819178 : LinkageLocation location = buffer->descriptor->GetInputLocation(index);
1002 20409589 : if (call_tail) {
1003 : location = LinkageLocation::ConvertToTailCallerLocation(
1004 : location, stack_param_delta);
1005 : }
1006 20409589 : InstructionOperand op = g.UseLocation(*iter, location);
1007 : UnallocatedOperand unallocated = UnallocatedOperand::cast(op);
1008 20409887 : if (unallocated.HasFixedSlotPolicy() && !call_tail) {
1009 3546863 : int stack_index = -unallocated.fixed_slot_index() - 1;
1010 7093726 : if (static_cast<size_t>(stack_index) >= buffer->pushed_nodes.size()) {
1011 1743872 : buffer->pushed_nodes.resize(stack_index + 1);
1012 : }
1013 : PushParameter param = {*iter, location};
1014 3546866 : buffer->pushed_nodes[stack_index] = param;
1015 : pushed_count++;
1016 : } else {
1017 : // If we do load poisoning and the linkage uses the poisoning register,
1018 : // then we request the input in memory location, and during code
1019 : // generation, we move the input to the register.
1020 16863031 : if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison &&
1021 : unallocated.HasFixedRegisterPolicy()) {
1022 : int reg = unallocated.fixed_register_index();
1023 7 : if (Register::from_code(reg) == kSpeculationPoisonRegister) {
1024 : buffer->instruction_args[poison_alias_index] = g.TempImmediate(
1025 0 : static_cast<int32_t>(buffer->instruction_args.size()));
1026 0 : op = g.UseRegisterOrSlotOrConstant(*iter);
1027 : }
1028 : }
1029 16863024 : buffer->instruction_args.push_back(op);
1030 : }
1031 : }
1032 : DCHECK_EQ(input_count, buffer->instruction_args.size() + pushed_count -
1033 : frame_state_entries - 1);
1034 6183482 : if (V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK && call_tail &&
1035 6183482 : stack_param_delta != 0) {
1036 : // For tail calls that change the size of their parameter list and keep
1037 : // their return address on the stack, move the return address to just above
1038 : // the parameters.
1039 : LinkageLocation saved_return_location =
1040 19444 : LinkageLocation::ForSavedCallerReturnAddress();
1041 : InstructionOperand return_address =
1042 : g.UsePointerLocation(LinkageLocation::ConvertToTailCallerLocation(
1043 : saved_return_location, stack_param_delta),
1044 19444 : saved_return_location);
1045 19444 : buffer->instruction_args.push_back(return_address);
1046 : }
1047 6183482 : }
1048 :
1049 0 : bool InstructionSelector::IsSourcePositionUsed(Node* node) {
1050 37784256 : return (source_position_mode_ == kAllSourcePositions ||
1051 14611116 : node->opcode() == IrOpcode::kCall ||
1052 14611213 : node->opcode() == IrOpcode::kCallWithCallerSavedRegisters ||
1053 14605028 : node->opcode() == IrOpcode::kTrapIf ||
1054 14469241 : node->opcode() == IrOpcode::kTrapUnless ||
1055 33499284 : node->opcode() == IrOpcode::kProtectedLoad ||
1056 0 : node->opcode() == IrOpcode::kProtectedStore);
1057 : }
1058 :
1059 20475116 : void InstructionSelector::VisitBlock(BasicBlock* block) {
1060 : DCHECK(!current_block_);
1061 20475116 : current_block_ = block;
1062 : auto current_num_instructions = [&] {
1063 : DCHECK_GE(kMaxInt, instructions_.size());
1064 475714446 : return static_cast<int>(instructions_.size());
1065 20475116 : };
1066 : int current_block_end = current_num_instructions();
1067 :
1068 : int effect_level = 0;
1069 135064304 : for (Node* const node : *block) {
1070 : SetEffectLevel(node, effect_level);
1071 224302788 : if (node->opcode() == IrOpcode::kStore ||
1072 109713531 : node->opcode() == IrOpcode::kUnalignedStore ||
1073 103877483 : node->opcode() == IrOpcode::kCall ||
1074 103873798 : node->opcode() == IrOpcode::kCallWithCallerSavedRegisters ||
1075 218368307 : node->opcode() == IrOpcode::kProtectedLoad ||
1076 : node->opcode() == IrOpcode::kProtectedStore) {
1077 10956162 : ++effect_level;
1078 : }
1079 : }
1080 :
1081 : // We visit the control first, then the nodes in the block, so the block's
1082 : // control input should be on the same effect level as the last node.
1083 20475116 : if (block->control_input() != nullptr) {
1084 : SetEffectLevel(block->control_input(), effect_level);
1085 : }
1086 :
1087 72060967 : auto FinishEmittedInstructions = [&](Node* node, int instruction_start) {
1088 149573511 : if (instruction_selection_failed()) return false;
1089 144124192 : if (current_num_instructions() == instruction_start) return true;
1090 : std::reverse(instructions_.begin() + instruction_start,
1091 : instructions_.end());
1092 62285285 : if (!node) return true;
1093 53485077 : if (!source_positions_) return true;
1094 39873507 : SourcePosition source_position = source_positions_->GetSourcePosition(node);
1095 59000660 : if (source_position.IsKnown() && IsSourcePositionUsed(node)) {
1096 4900211 : sequence()->SetSourcePosition(instructions_[instruction_start],
1097 4900211 : source_position);
1098 : }
1099 : return true;
1100 20475116 : };
1101 :
1102 : // Generate code for the block control "top down", but schedule the code
1103 : // "bottom up".
1104 20475116 : VisitControl(block);
1105 20471930 : if (!FinishEmittedInstructions(block->control_input(), current_block_end))
1106 9 : return;
1107 :
1108 : // Visit code in reverse control flow order, because architecture-specific
1109 : // matching may cover more than one node at a time.
1110 135082646 : for (auto node : base::Reversed(*block)) {
1111 : int current_node_end = current_num_instructions();
1112 : // Skip nodes that are unused or already defined.
1113 166464030 : if (IsUsed(node) && !IsDefined(node)) {
1114 : // Generate code for this node "top down", but schedule the code "bottom
1115 : // up".
1116 51600068 : VisitNode(node);
1117 51600070 : if (!FinishEmittedInstructions(node, current_node_end)) return;
1118 : }
1119 114612567 : if (trace_turbo_ == kEnableTraceTurboJson) {
1120 131 : instr_origins_[node->id()] = {current_num_instructions(),
1121 : current_node_end};
1122 : }
1123 : }
1124 :
1125 : // We're done with the block.
1126 : InstructionBlock* instruction_block =
1127 20474652 : sequence()->InstructionBlockAt(RpoNumber::FromInt(block->rpo_number()));
1128 20474622 : if (current_num_instructions() == current_block_end) {
1129 : // Avoid empty block: insert a {kArchNop} instruction.
1130 : Emit(Instruction::New(sequence()->zone(), kArchNop));
1131 : }
1132 : instruction_block->set_code_start(current_num_instructions());
1133 : instruction_block->set_code_end(current_block_end);
1134 20474822 : current_block_ = nullptr;
1135 : }
1136 :
1137 20474879 : void InstructionSelector::VisitControl(BasicBlock* block) {
1138 : #ifdef DEBUG
1139 : // SSA deconstruction requires targets of branches not to have phis.
1140 : // Edge split form guarantees this property, but is more strict.
1141 : if (block->SuccessorCount() > 1) {
1142 : for (BasicBlock* const successor : block->successors()) {
1143 : for (Node* const node : *successor) {
1144 : if (IrOpcode::IsPhiOpcode(node->opcode())) {
1145 : std::ostringstream str;
1146 : str << "You might have specified merged variables for a label with "
1147 : << "only one predecessor." << std::endl
1148 : << "# Current Block: " << *successor << std::endl
1149 : << "# Node: " << *node;
1150 : FATAL("%s", str.str().c_str());
1151 : }
1152 : }
1153 : }
1154 : }
1155 : #endif
1156 :
1157 : Node* input = block->control_input();
1158 20474879 : int instruction_end = static_cast<int>(instructions_.size());
1159 20474879 : switch (block->control()) {
1160 : case BasicBlock::kGoto:
1161 8802931 : VisitGoto(block->SuccessorAt(0));
1162 8802807 : break;
1163 : case BasicBlock::kCall: {
1164 : DCHECK_EQ(IrOpcode::kCall, input->opcode());
1165 : BasicBlock* success = block->SuccessorAt(0);
1166 : BasicBlock* exception = block->SuccessorAt(1);
1167 226559 : VisitCall(input, exception);
1168 226559 : VisitGoto(success);
1169 226559 : break;
1170 : }
1171 : case BasicBlock::kTailCall: {
1172 : DCHECK_EQ(IrOpcode::kTailCall, input->opcode());
1173 119827 : VisitTailCall(input);
1174 119826 : break;
1175 : }
1176 : case BasicBlock::kBranch: {
1177 : DCHECK_EQ(IrOpcode::kBranch, input->opcode());
1178 : BasicBlock* tbranch = block->SuccessorAt(0);
1179 : BasicBlock* fbranch = block->SuccessorAt(1);
1180 5368938 : if (tbranch == fbranch) {
1181 0 : VisitGoto(tbranch);
1182 : } else {
1183 5368938 : VisitBranch(input, tbranch, fbranch);
1184 : }
1185 : break;
1186 : }
1187 : case BasicBlock::kSwitch: {
1188 : DCHECK_EQ(IrOpcode::kSwitch, input->opcode());
1189 : // Last successor must be {IfDefault}.
1190 34729 : BasicBlock* default_branch = block->successors().back();
1191 : DCHECK_EQ(IrOpcode::kIfDefault, default_branch->front()->opcode());
1192 : // All other successors must be {IfValue}s.
1193 : int32_t min_value = std::numeric_limits<int32_t>::max();
1194 : int32_t max_value = std::numeric_limits<int32_t>::min();
1195 34729 : size_t case_count = block->SuccessorCount() - 1;
1196 : ZoneVector<CaseInfo> cases(case_count, zone());
1197 820633 : for (size_t i = 0; i < case_count; ++i) {
1198 : BasicBlock* branch = block->SuccessorAt(i);
1199 392952 : const IfValueParameters& p = IfValueParametersOf(branch->front()->op());
1200 392952 : cases[i] = CaseInfo{p.value(), p.comparison_order(), branch};
1201 392952 : if (min_value > p.value()) min_value = p.value();
1202 392952 : if (max_value < p.value()) max_value = p.value();
1203 : }
1204 : SwitchInfo sw(cases, min_value, max_value, default_branch);
1205 34729 : VisitSwitch(input, sw);
1206 : break;
1207 : }
1208 : case BasicBlock::kReturn: {
1209 : DCHECK_EQ(IrOpcode::kReturn, input->opcode());
1210 3002145 : VisitReturn(input);
1211 2999683 : break;
1212 : }
1213 : case BasicBlock::kDeoptimize: {
1214 44616 : DeoptimizeParameters p = DeoptimizeParametersOf(input->op());
1215 : Node* value = input->InputAt(0);
1216 : VisitDeoptimize(p.kind(), p.reason(), p.feedback(), value);
1217 : break;
1218 : }
1219 : case BasicBlock::kThrow:
1220 : DCHECK_EQ(IrOpcode::kThrow, input->opcode());
1221 : VisitThrow(input);
1222 : break;
1223 : case BasicBlock::kNone: {
1224 : // Exit block doesn't have control.
1225 : DCHECK_NULL(input);
1226 : break;
1227 : }
1228 : default:
1229 0 : UNREACHABLE();
1230 : break;
1231 : }
1232 20471708 : if (trace_turbo_ == kEnableTraceTurboJson && input) {
1233 10 : int instruction_start = static_cast<int>(instructions_.size());
1234 10 : instr_origins_[input->id()] = {instruction_start, instruction_end};
1235 : }
1236 20471708 : }
1237 :
1238 0 : void InstructionSelector::MarkPairProjectionsAsWord32(Node* node) {
1239 0 : Node* projection0 = NodeProperties::FindProjection(node, 0);
1240 0 : if (projection0) {
1241 : MarkAsWord32(projection0);
1242 : }
1243 0 : Node* projection1 = NodeProperties::FindProjection(node, 1);
1244 0 : if (projection1) {
1245 : MarkAsWord32(projection1);
1246 : }
1247 0 : }
1248 :
1249 51599568 : void InstructionSelector::VisitNode(Node* node) {
1250 : DCHECK_NOT_NULL(schedule()->block(node)); // should only use scheduled nodes.
1251 51599568 : switch (node->opcode()) {
1252 : case IrOpcode::kStart:
1253 : case IrOpcode::kLoop:
1254 : case IrOpcode::kEnd:
1255 : case IrOpcode::kBranch:
1256 : case IrOpcode::kIfTrue:
1257 : case IrOpcode::kIfFalse:
1258 : case IrOpcode::kIfSuccess:
1259 : case IrOpcode::kSwitch:
1260 : case IrOpcode::kIfValue:
1261 : case IrOpcode::kIfDefault:
1262 : case IrOpcode::kEffectPhi:
1263 : case IrOpcode::kMerge:
1264 : case IrOpcode::kTerminate:
1265 : case IrOpcode::kBeginRegion:
1266 : // No code needed for these graph artifacts.
1267 : return;
1268 : case IrOpcode::kIfException:
1269 204202 : return MarkAsReference(node), VisitIfException(node);
1270 : case IrOpcode::kFinishRegion:
1271 : return MarkAsReference(node), VisitFinishRegion(node);
1272 : case IrOpcode::kParameter: {
1273 : MachineType type =
1274 3739166 : linkage()->GetParameterType(ParameterIndexOf(node->op()));
1275 3738693 : MarkAsRepresentation(type.representation(), node);
1276 3739538 : return VisitParameter(node);
1277 : }
1278 : case IrOpcode::kOsrValue:
1279 21392 : return MarkAsReference(node), VisitOsrValue(node);
1280 : case IrOpcode::kPhi: {
1281 2088193 : MachineRepresentation rep = PhiRepresentationOf(node->op());
1282 2088181 : if (rep == MachineRepresentation::kNone) return;
1283 2088187 : MarkAsRepresentation(rep, node);
1284 2088198 : return VisitPhi(node);
1285 : }
1286 : case IrOpcode::kProjection:
1287 292963 : return VisitProjection(node);
1288 : case IrOpcode::kInt32Constant:
1289 : case IrOpcode::kInt64Constant:
1290 : case IrOpcode::kExternalConstant:
1291 : case IrOpcode::kRelocatableInt32Constant:
1292 : case IrOpcode::kRelocatableInt64Constant:
1293 7659749 : return VisitConstant(node);
1294 : case IrOpcode::kFloat32Constant:
1295 112594 : return MarkAsFloat32(node), VisitConstant(node);
1296 : case IrOpcode::kFloat64Constant:
1297 301778 : return MarkAsFloat64(node), VisitConstant(node);
1298 : case IrOpcode::kHeapConstant:
1299 4877069 : return MarkAsReference(node), VisitConstant(node);
1300 : case IrOpcode::kNumberConstant: {
1301 1383981 : double value = OpParameter<double>(node->op());
1302 1383981 : if (!IsSmiDouble(value)) MarkAsReference(node);
1303 1383981 : return VisitConstant(node);
1304 : }
1305 : case IrOpcode::kDelayedStringConstant:
1306 1667 : return MarkAsReference(node), VisitConstant(node);
1307 : case IrOpcode::kCall:
1308 5836081 : return VisitCall(node);
1309 : case IrOpcode::kCallWithCallerSavedRegisters:
1310 900 : return VisitCallWithCallerSavedRegisters(node);
1311 : case IrOpcode::kDeoptimizeIf:
1312 111369 : return VisitDeoptimizeIf(node);
1313 : case IrOpcode::kDeoptimizeUnless:
1314 219534 : return VisitDeoptimizeUnless(node);
1315 : case IrOpcode::kTrapIf:
1316 6255 : return VisitTrapIf(node, TrapIdOf(node->op()));
1317 : case IrOpcode::kTrapUnless:
1318 135802 : return VisitTrapUnless(node, TrapIdOf(node->op()));
1319 : case IrOpcode::kFrameState:
1320 : case IrOpcode::kStateValues:
1321 : case IrOpcode::kObjectState:
1322 : return;
1323 : case IrOpcode::kDebugAbort:
1324 160 : VisitDebugAbort(node);
1325 160 : return;
1326 : case IrOpcode::kDebugBreak:
1327 : VisitDebugBreak(node);
1328 : return;
1329 : case IrOpcode::kUnreachable:
1330 : VisitUnreachable(node);
1331 : return;
1332 : case IrOpcode::kDeadValue:
1333 0 : VisitDeadValue(node);
1334 0 : return;
1335 : case IrOpcode::kComment:
1336 4 : VisitComment(node);
1337 4 : return;
1338 : case IrOpcode::kRetain:
1339 9316 : VisitRetain(node);
1340 9316 : return;
1341 : case IrOpcode::kLoad: {
1342 7275009 : LoadRepresentation type = LoadRepresentationOf(node->op());
1343 7274865 : MarkAsRepresentation(type.representation(), node);
1344 7275297 : return VisitLoad(node);
1345 : }
1346 : case IrOpcode::kPoisonedLoad: {
1347 0 : LoadRepresentation type = LoadRepresentationOf(node->op());
1348 0 : MarkAsRepresentation(type.representation(), node);
1349 0 : return VisitPoisonedLoad(node);
1350 : }
1351 : case IrOpcode::kStore:
1352 4878491 : return VisitStore(node);
1353 : case IrOpcode::kProtectedStore:
1354 145206 : return VisitProtectedStore(node);
1355 : case IrOpcode::kWord32And:
1356 186375 : return MarkAsWord32(node), VisitWord32And(node);
1357 : case IrOpcode::kWord32Or:
1358 60775 : return MarkAsWord32(node), VisitWord32Or(node);
1359 : case IrOpcode::kWord32Xor:
1360 22650 : return MarkAsWord32(node), VisitWord32Xor(node);
1361 : case IrOpcode::kWord32Shl:
1362 38736 : return MarkAsWord32(node), VisitWord32Shl(node);
1363 : case IrOpcode::kWord32Shr:
1364 87436 : return MarkAsWord32(node), VisitWord32Shr(node);
1365 : case IrOpcode::kWord32Sar:
1366 26205 : return MarkAsWord32(node), VisitWord32Sar(node);
1367 : case IrOpcode::kWord32Ror:
1368 27725 : return MarkAsWord32(node), VisitWord32Ror(node);
1369 : case IrOpcode::kWord32Equal:
1370 108136 : return VisitWord32Equal(node);
1371 : case IrOpcode::kWord32Clz:
1372 665 : return MarkAsWord32(node), VisitWord32Clz(node);
1373 : case IrOpcode::kWord32Ctz:
1374 332 : return MarkAsWord32(node), VisitWord32Ctz(node);
1375 : case IrOpcode::kWord32ReverseBits:
1376 0 : return MarkAsWord32(node), VisitWord32ReverseBits(node);
1377 : case IrOpcode::kWord32ReverseBytes:
1378 44 : return MarkAsWord32(node), VisitWord32ReverseBytes(node);
1379 : case IrOpcode::kInt32AbsWithOverflow:
1380 0 : return MarkAsWord32(node), VisitInt32AbsWithOverflow(node);
1381 : case IrOpcode::kWord32Popcnt:
1382 80 : return MarkAsWord32(node), VisitWord32Popcnt(node);
1383 : case IrOpcode::kWord64Popcnt:
1384 42 : return MarkAsWord32(node), VisitWord64Popcnt(node);
1385 : case IrOpcode::kWord64And:
1386 149572 : return MarkAsWord64(node), VisitWord64And(node);
1387 : case IrOpcode::kWord64Or:
1388 92689 : return MarkAsWord64(node), VisitWord64Or(node);
1389 : case IrOpcode::kWord64Xor:
1390 877 : return MarkAsWord64(node), VisitWord64Xor(node);
1391 : case IrOpcode::kWord64Shl:
1392 381490 : return MarkAsWord64(node), VisitWord64Shl(node);
1393 : case IrOpcode::kWord64Shr:
1394 49303 : return MarkAsWord64(node), VisitWord64Shr(node);
1395 : case IrOpcode::kWord64Sar:
1396 193321 : return MarkAsWord64(node), VisitWord64Sar(node);
1397 : case IrOpcode::kWord64Ror:
1398 102 : return MarkAsWord64(node), VisitWord64Ror(node);
1399 : case IrOpcode::kWord64Clz:
1400 34 : return MarkAsWord64(node), VisitWord64Clz(node);
1401 : case IrOpcode::kWord64Ctz:
1402 34 : return MarkAsWord64(node), VisitWord64Ctz(node);
1403 : case IrOpcode::kWord64ReverseBits:
1404 0 : return MarkAsWord64(node), VisitWord64ReverseBits(node);
1405 : case IrOpcode::kWord64ReverseBytes:
1406 12 : return MarkAsWord64(node), VisitWord64ReverseBytes(node);
1407 : case IrOpcode::kInt64AbsWithOverflow:
1408 0 : return MarkAsWord64(node), VisitInt64AbsWithOverflow(node);
1409 : case IrOpcode::kWord64Equal:
1410 38264 : return VisitWord64Equal(node);
1411 : case IrOpcode::kInt32Add:
1412 276259 : return MarkAsWord32(node), VisitInt32Add(node);
1413 : case IrOpcode::kInt32AddWithOverflow:
1414 13924 : return MarkAsWord32(node), VisitInt32AddWithOverflow(node);
1415 : case IrOpcode::kInt32Sub:
1416 50461 : return MarkAsWord32(node), VisitInt32Sub(node);
1417 : case IrOpcode::kInt32SubWithOverflow:
1418 13924 : return VisitInt32SubWithOverflow(node);
1419 : case IrOpcode::kInt32Mul:
1420 57320 : return MarkAsWord32(node), VisitInt32Mul(node);
1421 : case IrOpcode::kInt32MulWithOverflow:
1422 13924 : return MarkAsWord32(node), VisitInt32MulWithOverflow(node);
1423 : case IrOpcode::kInt32MulHigh:
1424 4198 : return VisitInt32MulHigh(node);
1425 : case IrOpcode::kInt32Div:
1426 15712 : return MarkAsWord32(node), VisitInt32Div(node);
1427 : case IrOpcode::kInt32Mod:
1428 16290 : return MarkAsWord32(node), VisitInt32Mod(node);
1429 : case IrOpcode::kInt32LessThan:
1430 28119 : return VisitInt32LessThan(node);
1431 : case IrOpcode::kInt32LessThanOrEqual:
1432 27760 : return VisitInt32LessThanOrEqual(node);
1433 : case IrOpcode::kUint32Div:
1434 14503 : return MarkAsWord32(node), VisitUint32Div(node);
1435 : case IrOpcode::kUint32LessThan:
1436 33215 : return VisitUint32LessThan(node);
1437 : case IrOpcode::kUint32LessThanOrEqual:
1438 27927 : return VisitUint32LessThanOrEqual(node);
1439 : case IrOpcode::kUint32Mod:
1440 14618 : return MarkAsWord32(node), VisitUint32Mod(node);
1441 : case IrOpcode::kUint32MulHigh:
1442 1643 : return VisitUint32MulHigh(node);
1443 : case IrOpcode::kInt64Add:
1444 1946341 : return MarkAsWord64(node), VisitInt64Add(node);
1445 : case IrOpcode::kInt64AddWithOverflow:
1446 26896 : return MarkAsWord64(node), VisitInt64AddWithOverflow(node);
1447 : case IrOpcode::kInt64Sub:
1448 33195 : return MarkAsWord64(node), VisitInt64Sub(node);
1449 : case IrOpcode::kInt64SubWithOverflow:
1450 26896 : return MarkAsWord64(node), VisitInt64SubWithOverflow(node);
1451 : case IrOpcode::kInt64Mul:
1452 20556 : return MarkAsWord64(node), VisitInt64Mul(node);
1453 : case IrOpcode::kInt64Div:
1454 1852 : return MarkAsWord64(node), VisitInt64Div(node);
1455 : case IrOpcode::kInt64Mod:
1456 880 : return MarkAsWord64(node), VisitInt64Mod(node);
1457 : case IrOpcode::kInt64LessThan:
1458 1365 : return VisitInt64LessThan(node);
1459 : case IrOpcode::kInt64LessThanOrEqual:
1460 827 : return VisitInt64LessThanOrEqual(node);
1461 : case IrOpcode::kUint64Div:
1462 896 : return MarkAsWord64(node), VisitUint64Div(node);
1463 : case IrOpcode::kUint64LessThan:
1464 5727 : return VisitUint64LessThan(node);
1465 : case IrOpcode::kUint64LessThanOrEqual:
1466 408 : return VisitUint64LessThanOrEqual(node);
1467 : case IrOpcode::kUint64Mod:
1468 872 : return MarkAsWord64(node), VisitUint64Mod(node);
1469 : case IrOpcode::kBitcastTaggedToWord:
1470 1313453 : return MarkAsRepresentation(MachineType::PointerRepresentation(), node),
1471 : VisitBitcastTaggedToWord(node);
1472 : case IrOpcode::kBitcastWordToTagged:
1473 545303 : return MarkAsReference(node), VisitBitcastWordToTagged(node);
1474 : case IrOpcode::kBitcastWordToTaggedSigned:
1475 698932 : return MarkAsRepresentation(MachineRepresentation::kTaggedSigned, node),
1476 698932 : EmitIdentity(node);
1477 : case IrOpcode::kChangeFloat32ToFloat64:
1478 20453 : return MarkAsFloat64(node), VisitChangeFloat32ToFloat64(node);
1479 : case IrOpcode::kChangeInt32ToFloat64:
1480 378595 : return MarkAsFloat64(node), VisitChangeInt32ToFloat64(node);
1481 : case IrOpcode::kChangeInt64ToFloat64:
1482 243 : return MarkAsFloat64(node), VisitChangeInt64ToFloat64(node);
1483 : case IrOpcode::kChangeUint32ToFloat64:
1484 11333 : return MarkAsFloat64(node), VisitChangeUint32ToFloat64(node);
1485 : case IrOpcode::kChangeFloat64ToInt32:
1486 1778 : return MarkAsWord32(node), VisitChangeFloat64ToInt32(node);
1487 : case IrOpcode::kChangeFloat64ToInt64:
1488 24 : return MarkAsWord64(node), VisitChangeFloat64ToInt64(node);
1489 : case IrOpcode::kChangeFloat64ToUint32:
1490 642 : return MarkAsWord32(node), VisitChangeFloat64ToUint32(node);
1491 : case IrOpcode::kChangeFloat64ToUint64:
1492 2912 : return MarkAsWord64(node), VisitChangeFloat64ToUint64(node);
1493 : case IrOpcode::kFloat64SilenceNaN:
1494 : MarkAsFloat64(node);
1495 6658 : if (CanProduceSignalingNaN(node->InputAt(0))) {
1496 6464 : return VisitFloat64SilenceNaN(node);
1497 : } else {
1498 194 : return EmitIdentity(node);
1499 : }
1500 : case IrOpcode::kTruncateFloat64ToInt64:
1501 103 : return MarkAsWord64(node), VisitTruncateFloat64ToInt64(node);
1502 : case IrOpcode::kTruncateFloat64ToUint32:
1503 60 : return MarkAsWord32(node), VisitTruncateFloat64ToUint32(node);
1504 : case IrOpcode::kTruncateFloat32ToInt32:
1505 348 : return MarkAsWord32(node), VisitTruncateFloat32ToInt32(node);
1506 : case IrOpcode::kTruncateFloat32ToUint32:
1507 56 : return MarkAsWord32(node), VisitTruncateFloat32ToUint32(node);
1508 : case IrOpcode::kTryTruncateFloat32ToInt64:
1509 52 : return MarkAsWord64(node), VisitTryTruncateFloat32ToInt64(node);
1510 : case IrOpcode::kTryTruncateFloat64ToInt64:
1511 1014 : return MarkAsWord64(node), VisitTryTruncateFloat64ToInt64(node);
1512 : case IrOpcode::kTryTruncateFloat32ToUint64:
1513 52 : return MarkAsWord64(node), VisitTryTruncateFloat32ToUint64(node);
1514 : case IrOpcode::kTryTruncateFloat64ToUint64:
1515 60 : return MarkAsWord64(node), VisitTryTruncateFloat64ToUint64(node);
1516 : case IrOpcode::kChangeInt32ToInt64:
1517 253573 : return MarkAsWord64(node), VisitChangeInt32ToInt64(node);
1518 : case IrOpcode::kChangeUint32ToUint64:
1519 281934 : return MarkAsWord64(node), VisitChangeUint32ToUint64(node);
1520 : // TODO(mips-team): Support compress pointers.
1521 : #ifdef V8_COMPRESS_POINTERS
1522 : case IrOpcode::kChangeTaggedToCompressed:
1523 : return MarkAsCompressed(node), VisitChangeTaggedToCompressed(node);
1524 : case IrOpcode::kChangeTaggedPointerToCompressedPointer:
1525 : return MarkAsCompressed(node),
1526 : VisitChangeTaggedPointerToCompressedPointer(node);
1527 : case IrOpcode::kChangeTaggedSignedToCompressedSigned:
1528 : return MarkAsWord32(node),
1529 : VisitChangeTaggedSignedToCompressedSigned(node);
1530 : case IrOpcode::kChangeCompressedToTagged:
1531 : return MarkAsReference(node), VisitChangeCompressedToTagged(node);
1532 : case IrOpcode::kChangeCompressedPointerToTaggedPointer:
1533 : return MarkAsReference(node),
1534 : VisitChangeCompressedPointerToTaggedPointer(node);
1535 : case IrOpcode::kChangeCompressedSignedToTaggedSigned:
1536 : return MarkAsWord64(node),
1537 : VisitChangeCompressedSignedToTaggedSigned(node);
1538 : #endif
1539 : case IrOpcode::kTruncateFloat64ToFloat32:
1540 18009 : return MarkAsFloat32(node), VisitTruncateFloat64ToFloat32(node);
1541 : case IrOpcode::kTruncateFloat64ToWord32:
1542 53474 : return MarkAsWord32(node), VisitTruncateFloat64ToWord32(node);
1543 : case IrOpcode::kTruncateInt64ToInt32:
1544 495551 : return MarkAsWord32(node), VisitTruncateInt64ToInt32(node);
1545 : case IrOpcode::kRoundFloat64ToInt32:
1546 126314 : return MarkAsWord32(node), VisitRoundFloat64ToInt32(node);
1547 : case IrOpcode::kRoundInt64ToFloat32:
1548 48 : return MarkAsFloat32(node), VisitRoundInt64ToFloat32(node);
1549 : case IrOpcode::kRoundInt32ToFloat32:
1550 984 : return MarkAsFloat32(node), VisitRoundInt32ToFloat32(node);
1551 : case IrOpcode::kRoundInt64ToFloat64:
1552 3242 : return MarkAsFloat64(node), VisitRoundInt64ToFloat64(node);
1553 : case IrOpcode::kBitcastFloat32ToInt32:
1554 51074 : return MarkAsWord32(node), VisitBitcastFloat32ToInt32(node);
1555 : case IrOpcode::kRoundUint32ToFloat32:
1556 88 : return MarkAsFloat32(node), VisitRoundUint32ToFloat32(node);
1557 : case IrOpcode::kRoundUint64ToFloat32:
1558 31 : return MarkAsFloat64(node), VisitRoundUint64ToFloat32(node);
1559 : case IrOpcode::kRoundUint64ToFloat64:
1560 3608 : return MarkAsFloat64(node), VisitRoundUint64ToFloat64(node);
1561 : case IrOpcode::kBitcastFloat64ToInt64:
1562 50772 : return MarkAsWord64(node), VisitBitcastFloat64ToInt64(node);
1563 : case IrOpcode::kBitcastInt32ToFloat32:
1564 327 : return MarkAsFloat32(node), VisitBitcastInt32ToFloat32(node);
1565 : case IrOpcode::kBitcastInt64ToFloat64:
1566 215 : return MarkAsFloat64(node), VisitBitcastInt64ToFloat64(node);
1567 : case IrOpcode::kFloat32Add:
1568 1761 : return MarkAsFloat32(node), VisitFloat32Add(node);
1569 : case IrOpcode::kFloat32Sub:
1570 2609 : return MarkAsFloat32(node), VisitFloat32Sub(node);
1571 : case IrOpcode::kFloat32Neg:
1572 176 : return MarkAsFloat32(node), VisitFloat32Neg(node);
1573 : case IrOpcode::kFloat32Mul:
1574 890 : return MarkAsFloat32(node), VisitFloat32Mul(node);
1575 : case IrOpcode::kFloat32Div:
1576 385 : return MarkAsFloat32(node), VisitFloat32Div(node);
1577 : case IrOpcode::kFloat32Abs:
1578 68 : return MarkAsFloat32(node), VisitFloat32Abs(node);
1579 : case IrOpcode::kFloat32Sqrt:
1580 182 : return MarkAsFloat32(node), VisitFloat32Sqrt(node);
1581 : case IrOpcode::kFloat32Equal:
1582 114 : return VisitFloat32Equal(node);
1583 : case IrOpcode::kFloat32LessThan:
1584 106 : return VisitFloat32LessThan(node);
1585 : case IrOpcode::kFloat32LessThanOrEqual:
1586 111 : return VisitFloat32LessThanOrEqual(node);
1587 : case IrOpcode::kFloat32Max:
1588 66 : return MarkAsFloat32(node), VisitFloat32Max(node);
1589 : case IrOpcode::kFloat32Min:
1590 66 : return MarkAsFloat32(node), VisitFloat32Min(node);
1591 : case IrOpcode::kFloat64Add:
1592 79583 : return MarkAsFloat64(node), VisitFloat64Add(node);
1593 : case IrOpcode::kFloat64Sub:
1594 15820 : return MarkAsFloat64(node), VisitFloat64Sub(node);
1595 : case IrOpcode::kFloat64Neg:
1596 9726 : return MarkAsFloat64(node), VisitFloat64Neg(node);
1597 : case IrOpcode::kFloat64Mul:
1598 11904 : return MarkAsFloat64(node), VisitFloat64Mul(node);
1599 : case IrOpcode::kFloat64Div:
1600 12023 : return MarkAsFloat64(node), VisitFloat64Div(node);
1601 : case IrOpcode::kFloat64Mod:
1602 1576 : return MarkAsFloat64(node), VisitFloat64Mod(node);
1603 : case IrOpcode::kFloat64Min:
1604 339 : return MarkAsFloat64(node), VisitFloat64Min(node);
1605 : case IrOpcode::kFloat64Max:
1606 252 : return MarkAsFloat64(node), VisitFloat64Max(node);
1607 : case IrOpcode::kFloat64Abs:
1608 629 : return MarkAsFloat64(node), VisitFloat64Abs(node);
1609 : case IrOpcode::kFloat64Acos:
1610 : return MarkAsFloat64(node), VisitFloat64Acos(node);
1611 : case IrOpcode::kFloat64Acosh:
1612 : return MarkAsFloat64(node), VisitFloat64Acosh(node);
1613 : case IrOpcode::kFloat64Asin:
1614 : return MarkAsFloat64(node), VisitFloat64Asin(node);
1615 : case IrOpcode::kFloat64Asinh:
1616 : return MarkAsFloat64(node), VisitFloat64Asinh(node);
1617 : case IrOpcode::kFloat64Atan:
1618 : return MarkAsFloat64(node), VisitFloat64Atan(node);
1619 : case IrOpcode::kFloat64Atanh:
1620 : return MarkAsFloat64(node), VisitFloat64Atanh(node);
1621 : case IrOpcode::kFloat64Atan2:
1622 : return MarkAsFloat64(node), VisitFloat64Atan2(node);
1623 : case IrOpcode::kFloat64Cbrt:
1624 : return MarkAsFloat64(node), VisitFloat64Cbrt(node);
1625 : case IrOpcode::kFloat64Cos:
1626 : return MarkAsFloat64(node), VisitFloat64Cos(node);
1627 : case IrOpcode::kFloat64Cosh:
1628 : return MarkAsFloat64(node), VisitFloat64Cosh(node);
1629 : case IrOpcode::kFloat64Exp:
1630 : return MarkAsFloat64(node), VisitFloat64Exp(node);
1631 : case IrOpcode::kFloat64Expm1:
1632 : return MarkAsFloat64(node), VisitFloat64Expm1(node);
1633 : case IrOpcode::kFloat64Log:
1634 : return MarkAsFloat64(node), VisitFloat64Log(node);
1635 : case IrOpcode::kFloat64Log1p:
1636 : return MarkAsFloat64(node), VisitFloat64Log1p(node);
1637 : case IrOpcode::kFloat64Log10:
1638 : return MarkAsFloat64(node), VisitFloat64Log10(node);
1639 : case IrOpcode::kFloat64Log2:
1640 : return MarkAsFloat64(node), VisitFloat64Log2(node);
1641 : case IrOpcode::kFloat64Pow:
1642 : return MarkAsFloat64(node), VisitFloat64Pow(node);
1643 : case IrOpcode::kFloat64Sin:
1644 : return MarkAsFloat64(node), VisitFloat64Sin(node);
1645 : case IrOpcode::kFloat64Sinh:
1646 : return MarkAsFloat64(node), VisitFloat64Sinh(node);
1647 : case IrOpcode::kFloat64Sqrt:
1648 415 : return MarkAsFloat64(node), VisitFloat64Sqrt(node);
1649 : case IrOpcode::kFloat64Tan:
1650 : return MarkAsFloat64(node), VisitFloat64Tan(node);
1651 : case IrOpcode::kFloat64Tanh:
1652 : return MarkAsFloat64(node), VisitFloat64Tanh(node);
1653 : case IrOpcode::kFloat64Equal:
1654 2773 : return VisitFloat64Equal(node);
1655 : case IrOpcode::kFloat64LessThan:
1656 4907 : return VisitFloat64LessThan(node);
1657 : case IrOpcode::kFloat64LessThanOrEqual:
1658 1148 : return VisitFloat64LessThanOrEqual(node);
1659 : case IrOpcode::kFloat32RoundDown:
1660 51 : return MarkAsFloat32(node), VisitFloat32RoundDown(node);
1661 : case IrOpcode::kFloat64RoundDown:
1662 27107 : return MarkAsFloat64(node), VisitFloat64RoundDown(node);
1663 : case IrOpcode::kFloat32RoundUp:
1664 50 : return MarkAsFloat32(node), VisitFloat32RoundUp(node);
1665 : case IrOpcode::kFloat64RoundUp:
1666 8507 : return MarkAsFloat64(node), VisitFloat64RoundUp(node);
1667 : case IrOpcode::kFloat32RoundTruncate:
1668 196 : return MarkAsFloat32(node), VisitFloat32RoundTruncate(node);
1669 : case IrOpcode::kFloat64RoundTruncate:
1670 7927 : return MarkAsFloat64(node), VisitFloat64RoundTruncate(node);
1671 : case IrOpcode::kFloat64RoundTiesAway:
1672 0 : return MarkAsFloat64(node), VisitFloat64RoundTiesAway(node);
1673 : case IrOpcode::kFloat32RoundTiesEven:
1674 24 : return MarkAsFloat32(node), VisitFloat32RoundTiesEven(node);
1675 : case IrOpcode::kFloat64RoundTiesEven:
1676 1060 : return MarkAsFloat64(node), VisitFloat64RoundTiesEven(node);
1677 : case IrOpcode::kFloat64ExtractLowWord32:
1678 116 : return MarkAsWord32(node), VisitFloat64ExtractLowWord32(node);
1679 : case IrOpcode::kFloat64ExtractHighWord32:
1680 101393 : return MarkAsWord32(node), VisitFloat64ExtractHighWord32(node);
1681 : case IrOpcode::kFloat64InsertLowWord32:
1682 116 : return MarkAsFloat64(node), VisitFloat64InsertLowWord32(node);
1683 : case IrOpcode::kFloat64InsertHighWord32:
1684 116 : return MarkAsFloat64(node), VisitFloat64InsertHighWord32(node);
1685 : case IrOpcode::kTaggedPoisonOnSpeculation:
1686 : return MarkAsReference(node), VisitTaggedPoisonOnSpeculation(node);
1687 : case IrOpcode::kWord32PoisonOnSpeculation:
1688 : return MarkAsWord32(node), VisitWord32PoisonOnSpeculation(node);
1689 : case IrOpcode::kWord64PoisonOnSpeculation:
1690 : return MarkAsWord64(node), VisitWord64PoisonOnSpeculation(node);
1691 : case IrOpcode::kStackSlot:
1692 367597 : return VisitStackSlot(node);
1693 : case IrOpcode::kLoadStackPointer:
1694 1 : return VisitLoadStackPointer(node);
1695 : case IrOpcode::kLoadFramePointer:
1696 32802 : return VisitLoadFramePointer(node);
1697 : case IrOpcode::kLoadParentFramePointer:
1698 58132 : return VisitLoadParentFramePointer(node);
1699 : case IrOpcode::kUnalignedLoad: {
1700 0 : LoadRepresentation type = LoadRepresentationOf(node->op());
1701 0 : MarkAsRepresentation(type.representation(), node);
1702 0 : return VisitUnalignedLoad(node);
1703 : }
1704 : case IrOpcode::kUnalignedStore:
1705 0 : return VisitUnalignedStore(node);
1706 : case IrOpcode::kInt32PairAdd:
1707 : MarkAsWord32(node);
1708 0 : MarkPairProjectionsAsWord32(node);
1709 0 : return VisitInt32PairAdd(node);
1710 : case IrOpcode::kInt32PairSub:
1711 : MarkAsWord32(node);
1712 0 : MarkPairProjectionsAsWord32(node);
1713 0 : return VisitInt32PairSub(node);
1714 : case IrOpcode::kInt32PairMul:
1715 : MarkAsWord32(node);
1716 0 : MarkPairProjectionsAsWord32(node);
1717 0 : return VisitInt32PairMul(node);
1718 : case IrOpcode::kWord32PairShl:
1719 : MarkAsWord32(node);
1720 0 : MarkPairProjectionsAsWord32(node);
1721 0 : return VisitWord32PairShl(node);
1722 : case IrOpcode::kWord32PairShr:
1723 : MarkAsWord32(node);
1724 0 : MarkPairProjectionsAsWord32(node);
1725 0 : return VisitWord32PairShr(node);
1726 : case IrOpcode::kWord32PairSar:
1727 : MarkAsWord32(node);
1728 0 : MarkPairProjectionsAsWord32(node);
1729 0 : return VisitWord32PairSar(node);
1730 : case IrOpcode::kWord32AtomicLoad: {
1731 1050 : LoadRepresentation type = LoadRepresentationOf(node->op());
1732 1049 : MarkAsRepresentation(type.representation(), node);
1733 1051 : return VisitWord32AtomicLoad(node);
1734 : }
1735 : case IrOpcode::kWord64AtomicLoad: {
1736 580 : LoadRepresentation type = LoadRepresentationOf(node->op());
1737 580 : MarkAsRepresentation(type.representation(), node);
1738 580 : return VisitWord64AtomicLoad(node);
1739 : }
1740 : case IrOpcode::kWord32AtomicStore:
1741 2078 : return VisitWord32AtomicStore(node);
1742 : case IrOpcode::kWord64AtomicStore:
1743 2183 : return VisitWord64AtomicStore(node);
1744 : case IrOpcode::kWord32AtomicPairStore:
1745 0 : return VisitWord32AtomicPairStore(node);
1746 : case IrOpcode::kWord32AtomicPairLoad: {
1747 : MarkAsWord32(node);
1748 0 : MarkPairProjectionsAsWord32(node);
1749 0 : return VisitWord32AtomicPairLoad(node);
1750 : }
1751 : #define ATOMIC_CASE(name, rep) \
1752 : case IrOpcode::k##rep##Atomic##name: { \
1753 : MachineType type = AtomicOpType(node->op()); \
1754 : MarkAsRepresentation(type.representation(), node); \
1755 : return Visit##rep##Atomic##name(node); \
1756 : }
1757 2842 : ATOMIC_CASE(Add, Word32)
1758 2489 : ATOMIC_CASE(Add, Word64)
1759 2677 : ATOMIC_CASE(Sub, Word32)
1760 2128 : ATOMIC_CASE(Sub, Word64)
1761 2651 : ATOMIC_CASE(And, Word32)
1762 2378 : ATOMIC_CASE(And, Word64)
1763 2444 : ATOMIC_CASE(Or, Word32)
1764 2592 : ATOMIC_CASE(Or, Word64)
1765 2714 : ATOMIC_CASE(Xor, Word32)
1766 2154 : ATOMIC_CASE(Xor, Word64)
1767 2282 : ATOMIC_CASE(Exchange, Word32)
1768 2225 : ATOMIC_CASE(Exchange, Word64)
1769 754 : ATOMIC_CASE(CompareExchange, Word32)
1770 355 : ATOMIC_CASE(CompareExchange, Word64)
1771 : #undef ATOMIC_CASE
1772 : #define ATOMIC_CASE(name) \
1773 : case IrOpcode::kWord32AtomicPair##name: { \
1774 : MarkAsWord32(node); \
1775 : MarkPairProjectionsAsWord32(node); \
1776 : return VisitWord32AtomicPair##name(node); \
1777 : }
1778 0 : ATOMIC_CASE(Add)
1779 0 : ATOMIC_CASE(Sub)
1780 0 : ATOMIC_CASE(And)
1781 0 : ATOMIC_CASE(Or)
1782 0 : ATOMIC_CASE(Xor)
1783 0 : ATOMIC_CASE(Exchange)
1784 0 : ATOMIC_CASE(CompareExchange)
1785 : #undef ATOMIC_CASE
1786 : case IrOpcode::kProtectedLoad: {
1787 96945 : LoadRepresentation type = LoadRepresentationOf(node->op());
1788 96941 : MarkAsRepresentation(type.representation(), node);
1789 96906 : return VisitProtectedLoad(node);
1790 : }
1791 : case IrOpcode::kSignExtendWord8ToInt32:
1792 12 : return MarkAsWord32(node), VisitSignExtendWord8ToInt32(node);
1793 : case IrOpcode::kSignExtendWord16ToInt32:
1794 4 : return MarkAsWord32(node), VisitSignExtendWord16ToInt32(node);
1795 : case IrOpcode::kSignExtendWord8ToInt64:
1796 12 : return MarkAsWord64(node), VisitSignExtendWord8ToInt64(node);
1797 : case IrOpcode::kSignExtendWord16ToInt64:
1798 4 : return MarkAsWord64(node), VisitSignExtendWord16ToInt64(node);
1799 : case IrOpcode::kSignExtendWord32ToInt64:
1800 4 : return MarkAsWord64(node), VisitSignExtendWord32ToInt64(node);
1801 : case IrOpcode::kUnsafePointerAdd:
1802 6746 : MarkAsRepresentation(MachineType::PointerRepresentation(), node);
1803 : return VisitUnsafePointerAdd(node);
1804 : case IrOpcode::kF32x4Splat:
1805 140 : return MarkAsSimd128(node), VisitF32x4Splat(node);
1806 : case IrOpcode::kF32x4ExtractLane:
1807 32 : return MarkAsFloat32(node), VisitF32x4ExtractLane(node);
1808 : case IrOpcode::kF32x4ReplaceLane:
1809 32 : return MarkAsSimd128(node), VisitF32x4ReplaceLane(node);
1810 : case IrOpcode::kF32x4SConvertI32x4:
1811 4 : return MarkAsSimd128(node), VisitF32x4SConvertI32x4(node);
1812 : case IrOpcode::kF32x4UConvertI32x4:
1813 4 : return MarkAsSimd128(node), VisitF32x4UConvertI32x4(node);
1814 : case IrOpcode::kF32x4Abs:
1815 4 : return MarkAsSimd128(node), VisitF32x4Abs(node);
1816 : case IrOpcode::kF32x4Neg:
1817 4 : return MarkAsSimd128(node), VisitF32x4Neg(node);
1818 : case IrOpcode::kF32x4RecipApprox:
1819 4 : return MarkAsSimd128(node), VisitF32x4RecipApprox(node);
1820 : case IrOpcode::kF32x4RecipSqrtApprox:
1821 4 : return MarkAsSimd128(node), VisitF32x4RecipSqrtApprox(node);
1822 : case IrOpcode::kF32x4Add:
1823 12 : return MarkAsSimd128(node), VisitF32x4Add(node);
1824 : case IrOpcode::kF32x4AddHoriz:
1825 4 : return MarkAsSimd128(node), VisitF32x4AddHoriz(node);
1826 : case IrOpcode::kF32x4Sub:
1827 4 : return MarkAsSimd128(node), VisitF32x4Sub(node);
1828 : case IrOpcode::kF32x4Mul:
1829 4 : return MarkAsSimd128(node), VisitF32x4Mul(node);
1830 : case IrOpcode::kF32x4Min:
1831 4 : return MarkAsSimd128(node), VisitF32x4Min(node);
1832 : case IrOpcode::kF32x4Max:
1833 4 : return MarkAsSimd128(node), VisitF32x4Max(node);
1834 : case IrOpcode::kF32x4Eq:
1835 4 : return MarkAsSimd128(node), VisitF32x4Eq(node);
1836 : case IrOpcode::kF32x4Ne:
1837 4 : return MarkAsSimd128(node), VisitF32x4Ne(node);
1838 : case IrOpcode::kF32x4Lt:
1839 8 : return MarkAsSimd128(node), VisitF32x4Lt(node);
1840 : case IrOpcode::kF32x4Le:
1841 8 : return MarkAsSimd128(node), VisitF32x4Le(node);
1842 : case IrOpcode::kI32x4Splat:
1843 1088 : return MarkAsSimd128(node), VisitI32x4Splat(node);
1844 : case IrOpcode::kI32x4ExtractLane:
1845 1908 : return MarkAsWord32(node), VisitI32x4ExtractLane(node);
1846 : case IrOpcode::kI32x4ReplaceLane:
1847 1784 : return MarkAsSimd128(node), VisitI32x4ReplaceLane(node);
1848 : case IrOpcode::kI32x4SConvertF32x4:
1849 4 : return MarkAsSimd128(node), VisitI32x4SConvertF32x4(node);
1850 : case IrOpcode::kI32x4SConvertI16x8Low:
1851 4 : return MarkAsSimd128(node), VisitI32x4SConvertI16x8Low(node);
1852 : case IrOpcode::kI32x4SConvertI16x8High:
1853 4 : return MarkAsSimd128(node), VisitI32x4SConvertI16x8High(node);
1854 : case IrOpcode::kI32x4Neg:
1855 4 : return MarkAsSimd128(node), VisitI32x4Neg(node);
1856 : case IrOpcode::kI32x4Shl:
1857 124 : return MarkAsSimd128(node), VisitI32x4Shl(node);
1858 : case IrOpcode::kI32x4ShrS:
1859 124 : return MarkAsSimd128(node), VisitI32x4ShrS(node);
1860 : case IrOpcode::kI32x4Add:
1861 12 : return MarkAsSimd128(node), VisitI32x4Add(node);
1862 : case IrOpcode::kI32x4AddHoriz:
1863 4 : return MarkAsSimd128(node), VisitI32x4AddHoriz(node);
1864 : case IrOpcode::kI32x4Sub:
1865 4 : return MarkAsSimd128(node), VisitI32x4Sub(node);
1866 : case IrOpcode::kI32x4Mul:
1867 4 : return MarkAsSimd128(node), VisitI32x4Mul(node);
1868 : case IrOpcode::kI32x4MinS:
1869 4 : return MarkAsSimd128(node), VisitI32x4MinS(node);
1870 : case IrOpcode::kI32x4MaxS:
1871 4 : return MarkAsSimd128(node), VisitI32x4MaxS(node);
1872 : case IrOpcode::kI32x4Eq:
1873 12 : return MarkAsSimd128(node), VisitI32x4Eq(node);
1874 : case IrOpcode::kI32x4Ne:
1875 16 : return MarkAsSimd128(node), VisitI32x4Ne(node);
1876 : case IrOpcode::kI32x4GtS:
1877 8 : return MarkAsSimd128(node), VisitI32x4GtS(node);
1878 : case IrOpcode::kI32x4GeS:
1879 8 : return MarkAsSimd128(node), VisitI32x4GeS(node);
1880 : case IrOpcode::kI32x4UConvertF32x4:
1881 4 : return MarkAsSimd128(node), VisitI32x4UConvertF32x4(node);
1882 : case IrOpcode::kI32x4UConvertI16x8Low:
1883 4 : return MarkAsSimd128(node), VisitI32x4UConvertI16x8Low(node);
1884 : case IrOpcode::kI32x4UConvertI16x8High:
1885 4 : return MarkAsSimd128(node), VisitI32x4UConvertI16x8High(node);
1886 : case IrOpcode::kI32x4ShrU:
1887 124 : return MarkAsSimd128(node), VisitI32x4ShrU(node);
1888 : case IrOpcode::kI32x4MinU:
1889 4 : return MarkAsSimd128(node), VisitI32x4MinU(node);
1890 : case IrOpcode::kI32x4MaxU:
1891 4 : return MarkAsSimd128(node), VisitI32x4MaxU(node);
1892 : case IrOpcode::kI32x4GtU:
1893 8 : return MarkAsSimd128(node), VisitI32x4GtU(node);
1894 : case IrOpcode::kI32x4GeU:
1895 8 : return MarkAsSimd128(node), VisitI32x4GeU(node);
1896 : case IrOpcode::kI16x8Splat:
1897 436 : return MarkAsSimd128(node), VisitI16x8Splat(node);
1898 : case IrOpcode::kI16x8ExtractLane:
1899 48 : return MarkAsWord32(node), VisitI16x8ExtractLane(node);
1900 : case IrOpcode::kI16x8ReplaceLane:
1901 52 : return MarkAsSimd128(node), VisitI16x8ReplaceLane(node);
1902 : case IrOpcode::kI16x8SConvertI8x16Low:
1903 4 : return MarkAsSimd128(node), VisitI16x8SConvertI8x16Low(node);
1904 : case IrOpcode::kI16x8SConvertI8x16High:
1905 4 : return MarkAsSimd128(node), VisitI16x8SConvertI8x16High(node);
1906 : case IrOpcode::kI16x8Neg:
1907 4 : return MarkAsSimd128(node), VisitI16x8Neg(node);
1908 : case IrOpcode::kI16x8Shl:
1909 60 : return MarkAsSimd128(node), VisitI16x8Shl(node);
1910 : case IrOpcode::kI16x8ShrS:
1911 60 : return MarkAsSimd128(node), VisitI16x8ShrS(node);
1912 : case IrOpcode::kI16x8SConvertI32x4:
1913 4 : return MarkAsSimd128(node), VisitI16x8SConvertI32x4(node);
1914 : case IrOpcode::kI16x8Add:
1915 4 : return MarkAsSimd128(node), VisitI16x8Add(node);
1916 : case IrOpcode::kI16x8AddSaturateS:
1917 4 : return MarkAsSimd128(node), VisitI16x8AddSaturateS(node);
1918 : case IrOpcode::kI16x8AddHoriz:
1919 4 : return MarkAsSimd128(node), VisitI16x8AddHoriz(node);
1920 : case IrOpcode::kI16x8Sub:
1921 4 : return MarkAsSimd128(node), VisitI16x8Sub(node);
1922 : case IrOpcode::kI16x8SubSaturateS:
1923 4 : return MarkAsSimd128(node), VisitI16x8SubSaturateS(node);
1924 : case IrOpcode::kI16x8Mul:
1925 4 : return MarkAsSimd128(node), VisitI16x8Mul(node);
1926 : case IrOpcode::kI16x8MinS:
1927 4 : return MarkAsSimd128(node), VisitI16x8MinS(node);
1928 : case IrOpcode::kI16x8MaxS:
1929 4 : return MarkAsSimd128(node), VisitI16x8MaxS(node);
1930 : case IrOpcode::kI16x8Eq:
1931 12 : return MarkAsSimd128(node), VisitI16x8Eq(node);
1932 : case IrOpcode::kI16x8Ne:
1933 16 : return MarkAsSimd128(node), VisitI16x8Ne(node);
1934 : case IrOpcode::kI16x8GtS:
1935 8 : return MarkAsSimd128(node), VisitI16x8GtS(node);
1936 : case IrOpcode::kI16x8GeS:
1937 8 : return MarkAsSimd128(node), VisitI16x8GeS(node);
1938 : case IrOpcode::kI16x8UConvertI8x16Low:
1939 4 : return MarkAsSimd128(node), VisitI16x8UConvertI8x16Low(node);
1940 : case IrOpcode::kI16x8UConvertI8x16High:
1941 4 : return MarkAsSimd128(node), VisitI16x8UConvertI8x16High(node);
1942 : case IrOpcode::kI16x8ShrU:
1943 60 : return MarkAsSimd128(node), VisitI16x8ShrU(node);
1944 : case IrOpcode::kI16x8UConvertI32x4:
1945 4 : return MarkAsSimd128(node), VisitI16x8UConvertI32x4(node);
1946 : case IrOpcode::kI16x8AddSaturateU:
1947 4 : return MarkAsSimd128(node), VisitI16x8AddSaturateU(node);
1948 : case IrOpcode::kI16x8SubSaturateU:
1949 4 : return MarkAsSimd128(node), VisitI16x8SubSaturateU(node);
1950 : case IrOpcode::kI16x8MinU:
1951 4 : return MarkAsSimd128(node), VisitI16x8MinU(node);
1952 : case IrOpcode::kI16x8MaxU:
1953 4 : return MarkAsSimd128(node), VisitI16x8MaxU(node);
1954 : case IrOpcode::kI16x8GtU:
1955 16 : return MarkAsSimd128(node), VisitI16x8GtU(node);
1956 : case IrOpcode::kI16x8GeU:
1957 16 : return MarkAsSimd128(node), VisitI16x8GeU(node);
1958 : case IrOpcode::kI8x16Splat:
1959 320 : return MarkAsSimd128(node), VisitI8x16Splat(node);
1960 : case IrOpcode::kI8x16ExtractLane:
1961 48 : return MarkAsWord32(node), VisitI8x16ExtractLane(node);
1962 : case IrOpcode::kI8x16ReplaceLane:
1963 84 : return MarkAsSimd128(node), VisitI8x16ReplaceLane(node);
1964 : case IrOpcode::kI8x16Neg:
1965 4 : return MarkAsSimd128(node), VisitI8x16Neg(node);
1966 : case IrOpcode::kI8x16Shl:
1967 28 : return MarkAsSimd128(node), VisitI8x16Shl(node);
1968 : case IrOpcode::kI8x16ShrS:
1969 28 : return MarkAsSimd128(node), VisitI8x16ShrS(node);
1970 : case IrOpcode::kI8x16SConvertI16x8:
1971 4 : return MarkAsSimd128(node), VisitI8x16SConvertI16x8(node);
1972 : case IrOpcode::kI8x16Add:
1973 4 : return MarkAsSimd128(node), VisitI8x16Add(node);
1974 : case IrOpcode::kI8x16AddSaturateS:
1975 4 : return MarkAsSimd128(node), VisitI8x16AddSaturateS(node);
1976 : case IrOpcode::kI8x16Sub:
1977 4 : return MarkAsSimd128(node), VisitI8x16Sub(node);
1978 : case IrOpcode::kI8x16SubSaturateS:
1979 4 : return MarkAsSimd128(node), VisitI8x16SubSaturateS(node);
1980 : case IrOpcode::kI8x16Mul:
1981 4 : return MarkAsSimd128(node), VisitI8x16Mul(node);
1982 : case IrOpcode::kI8x16MinS:
1983 4 : return MarkAsSimd128(node), VisitI8x16MinS(node);
1984 : case IrOpcode::kI8x16MaxS:
1985 4 : return MarkAsSimd128(node), VisitI8x16MaxS(node);
1986 : case IrOpcode::kI8x16Eq:
1987 12 : return MarkAsSimd128(node), VisitI8x16Eq(node);
1988 : case IrOpcode::kI8x16Ne:
1989 16 : return MarkAsSimd128(node), VisitI8x16Ne(node);
1990 : case IrOpcode::kI8x16GtS:
1991 8 : return MarkAsSimd128(node), VisitI8x16GtS(node);
1992 : case IrOpcode::kI8x16GeS:
1993 8 : return MarkAsSimd128(node), VisitI8x16GeS(node);
1994 : case IrOpcode::kI8x16ShrU:
1995 28 : return MarkAsSimd128(node), VisitI8x16ShrU(node);
1996 : case IrOpcode::kI8x16UConvertI16x8:
1997 4 : return MarkAsSimd128(node), VisitI8x16UConvertI16x8(node);
1998 : case IrOpcode::kI8x16AddSaturateU:
1999 4 : return MarkAsSimd128(node), VisitI8x16AddSaturateU(node);
2000 : case IrOpcode::kI8x16SubSaturateU:
2001 4 : return MarkAsSimd128(node), VisitI8x16SubSaturateU(node);
2002 : case IrOpcode::kI8x16MinU:
2003 4 : return MarkAsSimd128(node), VisitI8x16MinU(node);
2004 : case IrOpcode::kI8x16MaxU:
2005 4 : return MarkAsSimd128(node), VisitI8x16MaxU(node);
2006 : case IrOpcode::kI8x16GtU:
2007 16 : return MarkAsSimd128(node), VisitI8x16GtU(node);
2008 : case IrOpcode::kI8x16GeU:
2009 16 : return MarkAsSimd128(node), VisitI8x16GeU(node);
2010 : case IrOpcode::kS128Zero:
2011 22 : return MarkAsSimd128(node), VisitS128Zero(node);
2012 : case IrOpcode::kS128And:
2013 4 : return MarkAsSimd128(node), VisitS128And(node);
2014 : case IrOpcode::kS128Or:
2015 4 : return MarkAsSimd128(node), VisitS128Or(node);
2016 : case IrOpcode::kS128Xor:
2017 4 : return MarkAsSimd128(node), VisitS128Xor(node);
2018 : case IrOpcode::kS128Not:
2019 4 : return MarkAsSimd128(node), VisitS128Not(node);
2020 : case IrOpcode::kS128Select:
2021 28 : return MarkAsSimd128(node), VisitS128Select(node);
2022 : case IrOpcode::kS8x16Shuffle:
2023 5008 : return MarkAsSimd128(node), VisitS8x16Shuffle(node);
2024 : case IrOpcode::kS1x4AnyTrue:
2025 20 : return MarkAsWord32(node), VisitS1x4AnyTrue(node);
2026 : case IrOpcode::kS1x4AllTrue:
2027 20 : return MarkAsWord32(node), VisitS1x4AllTrue(node);
2028 : case IrOpcode::kS1x8AnyTrue:
2029 20 : return MarkAsWord32(node), VisitS1x8AnyTrue(node);
2030 : case IrOpcode::kS1x8AllTrue:
2031 20 : return MarkAsWord32(node), VisitS1x8AllTrue(node);
2032 : case IrOpcode::kS1x16AnyTrue:
2033 20 : return MarkAsWord32(node), VisitS1x16AnyTrue(node);
2034 : case IrOpcode::kS1x16AllTrue:
2035 20 : return MarkAsWord32(node), VisitS1x16AllTrue(node);
2036 : default:
2037 0 : FATAL("Unexpected operator #%d:%s @ node #%d", node->opcode(),
2038 0 : node->op()->mnemonic(), node->id());
2039 : break;
2040 : }
2041 : }
2042 :
2043 0 : void InstructionSelector::EmitWordPoisonOnSpeculation(Node* node) {
2044 0 : if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison) {
2045 : OperandGenerator g(this);
2046 0 : Node* input_node = NodeProperties::GetValueInput(node, 0);
2047 0 : InstructionOperand input = g.UseRegister(input_node);
2048 0 : InstructionOperand output = g.DefineSameAsFirst(node);
2049 : Emit(kArchWordPoisonOnSpeculation, output, input);
2050 : } else {
2051 0 : EmitIdentity(node);
2052 : }
2053 0 : }
2054 :
2055 0 : void InstructionSelector::VisitWord32PoisonOnSpeculation(Node* node) {
2056 0 : EmitWordPoisonOnSpeculation(node);
2057 0 : }
2058 :
2059 0 : void InstructionSelector::VisitWord64PoisonOnSpeculation(Node* node) {
2060 0 : EmitWordPoisonOnSpeculation(node);
2061 0 : }
2062 :
2063 0 : void InstructionSelector::VisitTaggedPoisonOnSpeculation(Node* node) {
2064 0 : EmitWordPoisonOnSpeculation(node);
2065 0 : }
2066 :
2067 1 : void InstructionSelector::VisitLoadStackPointer(Node* node) {
2068 : OperandGenerator g(this);
2069 1 : Emit(kArchStackPointer, g.DefineAsRegister(node));
2070 1 : }
2071 :
2072 32802 : void InstructionSelector::VisitLoadFramePointer(Node* node) {
2073 : OperandGenerator g(this);
2074 32802 : Emit(kArchFramePointer, g.DefineAsRegister(node));
2075 32802 : }
2076 :
2077 58132 : void InstructionSelector::VisitLoadParentFramePointer(Node* node) {
2078 : OperandGenerator g(this);
2079 58132 : Emit(kArchParentFramePointer, g.DefineAsRegister(node));
2080 58132 : }
2081 :
2082 0 : void InstructionSelector::VisitFloat64Acos(Node* node) {
2083 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Acos);
2084 0 : }
2085 :
2086 0 : void InstructionSelector::VisitFloat64Acosh(Node* node) {
2087 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Acosh);
2088 0 : }
2089 :
2090 0 : void InstructionSelector::VisitFloat64Asin(Node* node) {
2091 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Asin);
2092 0 : }
2093 :
2094 0 : void InstructionSelector::VisitFloat64Asinh(Node* node) {
2095 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Asinh);
2096 0 : }
2097 :
2098 0 : void InstructionSelector::VisitFloat64Atan(Node* node) {
2099 133 : VisitFloat64Ieee754Unop(node, kIeee754Float64Atan);
2100 0 : }
2101 :
2102 0 : void InstructionSelector::VisitFloat64Atanh(Node* node) {
2103 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Atanh);
2104 0 : }
2105 :
2106 0 : void InstructionSelector::VisitFloat64Atan2(Node* node) {
2107 129 : VisitFloat64Ieee754Binop(node, kIeee754Float64Atan2);
2108 0 : }
2109 :
2110 0 : void InstructionSelector::VisitFloat64Cbrt(Node* node) {
2111 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Cbrt);
2112 0 : }
2113 :
2114 0 : void InstructionSelector::VisitFloat64Cos(Node* node) {
2115 271 : VisitFloat64Ieee754Unop(node, kIeee754Float64Cos);
2116 0 : }
2117 :
2118 0 : void InstructionSelector::VisitFloat64Cosh(Node* node) {
2119 123 : VisitFloat64Ieee754Unop(node, kIeee754Float64Cosh);
2120 0 : }
2121 :
2122 0 : void InstructionSelector::VisitFloat64Exp(Node* node) {
2123 147 : VisitFloat64Ieee754Unop(node, kIeee754Float64Exp);
2124 0 : }
2125 :
2126 0 : void InstructionSelector::VisitFloat64Expm1(Node* node) {
2127 123 : VisitFloat64Ieee754Unop(node, kIeee754Float64Expm1);
2128 0 : }
2129 :
2130 0 : void InstructionSelector::VisitFloat64Log(Node* node) {
2131 252 : VisitFloat64Ieee754Unop(node, kIeee754Float64Log);
2132 0 : }
2133 :
2134 0 : void InstructionSelector::VisitFloat64Log1p(Node* node) {
2135 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Log1p);
2136 0 : }
2137 :
2138 0 : void InstructionSelector::VisitFloat64Log2(Node* node) {
2139 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Log2);
2140 0 : }
2141 :
2142 0 : void InstructionSelector::VisitFloat64Log10(Node* node) {
2143 116 : VisitFloat64Ieee754Unop(node, kIeee754Float64Log10);
2144 0 : }
2145 :
2146 0 : void InstructionSelector::VisitFloat64Pow(Node* node) {
2147 334 : VisitFloat64Ieee754Binop(node, kIeee754Float64Pow);
2148 0 : }
2149 :
2150 0 : void InstructionSelector::VisitFloat64Sin(Node* node) {
2151 268 : VisitFloat64Ieee754Unop(node, kIeee754Float64Sin);
2152 0 : }
2153 :
2154 0 : void InstructionSelector::VisitFloat64Sinh(Node* node) {
2155 123 : VisitFloat64Ieee754Unop(node, kIeee754Float64Sinh);
2156 0 : }
2157 :
2158 0 : void InstructionSelector::VisitFloat64Tan(Node* node) {
2159 168 : VisitFloat64Ieee754Unop(node, kIeee754Float64Tan);
2160 0 : }
2161 :
2162 0 : void InstructionSelector::VisitFloat64Tanh(Node* node) {
2163 123 : VisitFloat64Ieee754Unop(node, kIeee754Float64Tanh);
2164 0 : }
2165 :
2166 344 : void InstructionSelector::EmitTableSwitch(const SwitchInfo& sw,
2167 : InstructionOperand& index_operand) {
2168 : OperandGenerator g(this);
2169 344 : size_t input_count = 2 + sw.value_range();
2170 : DCHECK_LE(sw.value_range(), std::numeric_limits<size_t>::max() - 2);
2171 : auto* inputs = zone()->NewArray<InstructionOperand>(input_count);
2172 344 : inputs[0] = index_operand;
2173 : InstructionOperand default_operand = g.Label(sw.default_branch());
2174 344 : std::fill(&inputs[1], &inputs[input_count], default_operand);
2175 200159 : for (const CaseInfo& c : sw.CasesUnsorted()) {
2176 199815 : size_t value = c.value - sw.min_value();
2177 : DCHECK_LE(0u, value);
2178 : DCHECK_LT(value + 2, input_count);
2179 399630 : inputs[value + 2] = g.Label(c.branch);
2180 : }
2181 344 : Emit(kArchTableSwitch, 0, nullptr, input_count, inputs, 0, nullptr);
2182 344 : }
2183 :
2184 0 : void InstructionSelector::EmitLookupSwitch(const SwitchInfo& sw,
2185 : InstructionOperand& value_operand) {
2186 : OperandGenerator g(this);
2187 0 : std::vector<CaseInfo> cases = sw.CasesSortedByOriginalOrder();
2188 0 : size_t input_count = 2 + sw.case_count() * 2;
2189 : DCHECK_LE(sw.case_count(), (std::numeric_limits<size_t>::max() - 2) / 2);
2190 : auto* inputs = zone()->NewArray<InstructionOperand>(input_count);
2191 0 : inputs[0] = value_operand;
2192 0 : inputs[1] = g.Label(sw.default_branch());
2193 0 : for (size_t index = 0; index < cases.size(); ++index) {
2194 : const CaseInfo& c = cases[index];
2195 0 : inputs[index * 2 + 2 + 0] = g.TempImmediate(c.value);
2196 0 : inputs[index * 2 + 2 + 1] = g.Label(c.branch);
2197 : }
2198 0 : Emit(kArchLookupSwitch, 0, nullptr, input_count, inputs, 0, nullptr);
2199 0 : }
2200 :
2201 34385 : void InstructionSelector::EmitBinarySearchSwitch(
2202 : const SwitchInfo& sw, InstructionOperand& value_operand) {
2203 : OperandGenerator g(this);
2204 34385 : size_t input_count = 2 + sw.case_count() * 2;
2205 : DCHECK_LE(sw.case_count(), (std::numeric_limits<size_t>::max() - 2) / 2);
2206 : auto* inputs = zone()->NewArray<InstructionOperand>(input_count);
2207 34385 : inputs[0] = value_operand;
2208 34385 : inputs[1] = g.Label(sw.default_branch());
2209 34385 : std::vector<CaseInfo> cases = sw.CasesSortedByValue();
2210 : std::stable_sort(cases.begin(), cases.end(),
2211 : [](CaseInfo a, CaseInfo b) { return a.value < b.value; });
2212 420659 : for (size_t index = 0; index < cases.size(); ++index) {
2213 : const CaseInfo& c = cases[index];
2214 193137 : inputs[index * 2 + 2 + 0] = g.TempImmediate(c.value);
2215 386274 : inputs[index * 2 + 2 + 1] = g.Label(c.branch);
2216 : }
2217 34385 : Emit(kArchBinarySearchSwitch, 0, nullptr, input_count, inputs, 0, nullptr);
2218 34385 : }
2219 :
2220 0 : void InstructionSelector::VisitBitcastTaggedToWord(Node* node) {
2221 1313453 : EmitIdentity(node);
2222 0 : }
2223 :
2224 545302 : void InstructionSelector::VisitBitcastWordToTagged(Node* node) {
2225 : OperandGenerator g(this);
2226 545302 : Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(node->InputAt(0)));
2227 545304 : }
2228 :
2229 : // 32 bit targets do not implement the following instructions.
2230 : #if V8_TARGET_ARCH_32_BIT
2231 :
2232 : void InstructionSelector::VisitWord64And(Node* node) { UNIMPLEMENTED(); }
2233 :
2234 : void InstructionSelector::VisitWord64Or(Node* node) { UNIMPLEMENTED(); }
2235 :
2236 : void InstructionSelector::VisitWord64Xor(Node* node) { UNIMPLEMENTED(); }
2237 :
2238 : void InstructionSelector::VisitWord64Shl(Node* node) { UNIMPLEMENTED(); }
2239 :
2240 : void InstructionSelector::VisitWord64Shr(Node* node) { UNIMPLEMENTED(); }
2241 :
2242 : void InstructionSelector::VisitWord64Sar(Node* node) { UNIMPLEMENTED(); }
2243 :
2244 : void InstructionSelector::VisitWord64Ror(Node* node) { UNIMPLEMENTED(); }
2245 :
2246 : void InstructionSelector::VisitWord64Clz(Node* node) { UNIMPLEMENTED(); }
2247 :
2248 : void InstructionSelector::VisitWord64Ctz(Node* node) { UNIMPLEMENTED(); }
2249 :
2250 : void InstructionSelector::VisitWord64ReverseBits(Node* node) {
2251 : UNIMPLEMENTED();
2252 : }
2253 :
2254 : void InstructionSelector::VisitWord64Popcnt(Node* node) { UNIMPLEMENTED(); }
2255 :
2256 : void InstructionSelector::VisitWord64Equal(Node* node) { UNIMPLEMENTED(); }
2257 :
2258 : void InstructionSelector::VisitInt64Add(Node* node) { UNIMPLEMENTED(); }
2259 :
2260 : void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
2261 : UNIMPLEMENTED();
2262 : }
2263 :
2264 : void InstructionSelector::VisitInt64Sub(Node* node) { UNIMPLEMENTED(); }
2265 :
2266 : void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
2267 : UNIMPLEMENTED();
2268 : }
2269 :
2270 : void InstructionSelector::VisitInt64Mul(Node* node) { UNIMPLEMENTED(); }
2271 :
2272 : void InstructionSelector::VisitInt64Div(Node* node) { UNIMPLEMENTED(); }
2273 :
2274 : void InstructionSelector::VisitInt64LessThan(Node* node) { UNIMPLEMENTED(); }
2275 :
2276 : void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2277 : UNIMPLEMENTED();
2278 : }
2279 :
2280 : void InstructionSelector::VisitUint64Div(Node* node) { UNIMPLEMENTED(); }
2281 :
2282 : void InstructionSelector::VisitInt64Mod(Node* node) { UNIMPLEMENTED(); }
2283 :
2284 : void InstructionSelector::VisitUint64LessThan(Node* node) { UNIMPLEMENTED(); }
2285 :
2286 : void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2287 : UNIMPLEMENTED();
2288 : }
2289 :
2290 : void InstructionSelector::VisitUint64Mod(Node* node) { UNIMPLEMENTED(); }
2291 :
2292 : void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
2293 : UNIMPLEMENTED();
2294 : }
2295 :
2296 : void InstructionSelector::VisitChangeInt64ToFloat64(Node* node) {
2297 : UNIMPLEMENTED();
2298 : }
2299 :
2300 : void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
2301 : UNIMPLEMENTED();
2302 : }
2303 :
2304 : // TODO(mips-team): Support compress pointers.
2305 : #ifdef V8_COMPRESS_POINTERS
2306 : void InstructionSelector::VisitChangeTaggedToCompressed(Node* node) {
2307 : UNIMPLEMENTED();
2308 : }
2309 :
2310 : void InstructionSelector::VisitChangeTaggedPointerToCompressedPointer(
2311 : Node* node) {
2312 : UNIMPLEMENTED();
2313 : }
2314 :
2315 : void InstructionSelector::VisitChangeTaggedSignedToCompressedSigned(
2316 : Node* node) {
2317 : UNIMPLEMENTED();
2318 : }
2319 :
2320 : void InstructionSelector::VisitChangeCompressedToTagged(Node* node) {
2321 : UNIMPLEMENTED();
2322 : }
2323 :
2324 : void InstructionSelector::VisitChangeCompressedPointerToTaggedPointer(
2325 : Node* node) {
2326 : UNIMPLEMENTED();
2327 : }
2328 :
2329 : void InstructionSelector::VisitChangeCompressedSignedToTaggedSigned(
2330 : Node* node) {
2331 : UNIMPLEMENTED();
2332 : }
2333 : #endif
2334 :
2335 : void InstructionSelector::VisitChangeFloat64ToInt64(Node* node) {
2336 : UNIMPLEMENTED();
2337 : }
2338 :
2339 : void InstructionSelector::VisitChangeFloat64ToUint64(Node* node) {
2340 : UNIMPLEMENTED();
2341 : }
2342 :
2343 : void InstructionSelector::VisitTruncateFloat64ToInt64(Node* node) {
2344 : UNIMPLEMENTED();
2345 : }
2346 :
2347 : void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
2348 : UNIMPLEMENTED();
2349 : }
2350 :
2351 : void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
2352 : UNIMPLEMENTED();
2353 : }
2354 :
2355 : void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
2356 : UNIMPLEMENTED();
2357 : }
2358 :
2359 : void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
2360 : UNIMPLEMENTED();
2361 : }
2362 :
2363 : void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
2364 : UNIMPLEMENTED();
2365 : }
2366 :
2367 : void InstructionSelector::VisitRoundInt64ToFloat32(Node* node) {
2368 : UNIMPLEMENTED();
2369 : }
2370 :
2371 : void InstructionSelector::VisitRoundInt64ToFloat64(Node* node) {
2372 : UNIMPLEMENTED();
2373 : }
2374 :
2375 : void InstructionSelector::VisitRoundUint64ToFloat32(Node* node) {
2376 : UNIMPLEMENTED();
2377 : }
2378 :
2379 : void InstructionSelector::VisitRoundUint64ToFloat64(Node* node) {
2380 : UNIMPLEMENTED();
2381 : }
2382 :
2383 : void InstructionSelector::VisitBitcastFloat64ToInt64(Node* node) {
2384 : UNIMPLEMENTED();
2385 : }
2386 :
2387 : void InstructionSelector::VisitBitcastInt64ToFloat64(Node* node) {
2388 : UNIMPLEMENTED();
2389 : }
2390 :
2391 : void InstructionSelector::VisitSignExtendWord8ToInt64(Node* node) {
2392 : UNIMPLEMENTED();
2393 : }
2394 :
2395 : void InstructionSelector::VisitSignExtendWord16ToInt64(Node* node) {
2396 : UNIMPLEMENTED();
2397 : }
2398 :
2399 : void InstructionSelector::VisitSignExtendWord32ToInt64(Node* node) {
2400 : UNIMPLEMENTED();
2401 : }
2402 : #endif // V8_TARGET_ARCH_32_BIT
2403 :
2404 : // 64 bit targets do not implement the following instructions.
2405 : #if V8_TARGET_ARCH_64_BIT
2406 0 : void InstructionSelector::VisitInt32PairAdd(Node* node) { UNIMPLEMENTED(); }
2407 :
2408 0 : void InstructionSelector::VisitInt32PairSub(Node* node) { UNIMPLEMENTED(); }
2409 :
2410 0 : void InstructionSelector::VisitInt32PairMul(Node* node) { UNIMPLEMENTED(); }
2411 :
2412 0 : void InstructionSelector::VisitWord32PairShl(Node* node) { UNIMPLEMENTED(); }
2413 :
2414 0 : void InstructionSelector::VisitWord32PairShr(Node* node) { UNIMPLEMENTED(); }
2415 :
2416 0 : void InstructionSelector::VisitWord32PairSar(Node* node) { UNIMPLEMENTED(); }
2417 : #endif // V8_TARGET_ARCH_64_BIT
2418 :
2419 : #if !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_MIPS
2420 0 : void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
2421 0 : UNIMPLEMENTED();
2422 : }
2423 :
2424 0 : void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
2425 0 : UNIMPLEMENTED();
2426 : }
2427 :
2428 0 : void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
2429 0 : UNIMPLEMENTED();
2430 : }
2431 :
2432 0 : void InstructionSelector::VisitWord32AtomicPairSub(Node* node) {
2433 0 : UNIMPLEMENTED();
2434 : }
2435 :
2436 0 : void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) {
2437 0 : UNIMPLEMENTED();
2438 : }
2439 :
2440 0 : void InstructionSelector::VisitWord32AtomicPairOr(Node* node) {
2441 0 : UNIMPLEMENTED();
2442 : }
2443 :
2444 0 : void InstructionSelector::VisitWord32AtomicPairXor(Node* node) {
2445 0 : UNIMPLEMENTED();
2446 : }
2447 :
2448 0 : void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
2449 0 : UNIMPLEMENTED();
2450 : }
2451 :
2452 0 : void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
2453 0 : UNIMPLEMENTED();
2454 : }
2455 : #endif // !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_MIPS
2456 :
2457 : #if !V8_TARGET_ARCH_X64 && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_MIPS64 && \
2458 : !V8_TARGET_ARCH_S390 && !V8_TARGET_ARCH_PPC
2459 : void InstructionSelector::VisitWord64AtomicLoad(Node* node) { UNIMPLEMENTED(); }
2460 :
2461 : void InstructionSelector::VisitWord64AtomicStore(Node* node) {
2462 : UNIMPLEMENTED();
2463 : }
2464 :
2465 : void InstructionSelector::VisitWord64AtomicAdd(Node* node) { UNIMPLEMENTED(); }
2466 :
2467 : void InstructionSelector::VisitWord64AtomicSub(Node* node) { UNIMPLEMENTED(); }
2468 :
2469 : void InstructionSelector::VisitWord64AtomicAnd(Node* node) { UNIMPLEMENTED(); }
2470 :
2471 : void InstructionSelector::VisitWord64AtomicOr(Node* node) { UNIMPLEMENTED(); }
2472 :
2473 : void InstructionSelector::VisitWord64AtomicXor(Node* node) { UNIMPLEMENTED(); }
2474 :
2475 : void InstructionSelector::VisitWord64AtomicExchange(Node* node) {
2476 : UNIMPLEMENTED();
2477 : }
2478 :
2479 : void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) {
2480 : UNIMPLEMENTED();
2481 : }
2482 : #endif // !V8_TARGET_ARCH_X64 && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_PPC
2483 : // !V8_TARGET_ARCH_MIPS64 && !V8_TARGET_ARCH_S390
2484 :
2485 1 : void InstructionSelector::VisitFinishRegion(Node* node) { EmitIdentity(node); }
2486 :
2487 3738636 : void InstructionSelector::VisitParameter(Node* node) {
2488 : OperandGenerator g(this);
2489 3738636 : int index = ParameterIndexOf(node->op());
2490 : InstructionOperand op =
2491 3738839 : linkage()->ParameterHasSecondaryLocation(index)
2492 : ? g.DefineAsDualLocation(
2493 : node, linkage()->GetParameterLocation(index),
2494 554712 : linkage()->GetParameterSecondaryLocation(index))
2495 4294063 : : g.DefineAsLocation(node, linkage()->GetParameterLocation(index));
2496 :
2497 : Emit(kArchNop, op);
2498 3739662 : }
2499 :
2500 : namespace {
2501 : LinkageLocation ExceptionLocation() {
2502 : return LinkageLocation::ForRegister(kReturnRegister0.code(),
2503 : MachineType::IntPtr());
2504 : }
2505 : } // namespace
2506 :
2507 204201 : void InstructionSelector::VisitIfException(Node* node) {
2508 : OperandGenerator g(this);
2509 : DCHECK_EQ(IrOpcode::kCall, node->InputAt(1)->opcode());
2510 204201 : Emit(kArchNop, g.DefineAsLocation(node, ExceptionLocation()));
2511 204205 : }
2512 :
2513 21392 : void InstructionSelector::VisitOsrValue(Node* node) {
2514 : OperandGenerator g(this);
2515 21392 : int index = OsrValueIndexOf(node->op());
2516 21392 : Emit(kArchNop,
2517 : g.DefineAsLocation(node, linkage()->GetOsrValueLocation(index)));
2518 21392 : }
2519 :
2520 2088182 : void InstructionSelector::VisitPhi(Node* node) {
2521 : const int input_count = node->op()->ValueInputCount();
2522 : DCHECK_EQ(input_count, current_block_->PredecessorCount());
2523 : PhiInstruction* phi = new (instruction_zone())
2524 : PhiInstruction(instruction_zone(), GetVirtualRegister(node),
2525 4176384 : static_cast<size_t>(input_count));
2526 : sequence()
2527 2088184 : ->InstructionBlockAt(RpoNumber::FromInt(current_block_->rpo_number()))
2528 2088184 : ->AddPhi(phi);
2529 12196284 : for (int i = 0; i < input_count; ++i) {
2530 : Node* const input = node->InputAt(i);
2531 : MarkAsUsed(input);
2532 5054026 : phi->SetInput(static_cast<size_t>(i), GetVirtualRegister(input));
2533 : }
2534 2088211 : }
2535 :
2536 292961 : void InstructionSelector::VisitProjection(Node* node) {
2537 : OperandGenerator g(this);
2538 : Node* value = node->InputAt(0);
2539 292961 : switch (value->opcode()) {
2540 : case IrOpcode::kInt32AddWithOverflow:
2541 : case IrOpcode::kInt32SubWithOverflow:
2542 : case IrOpcode::kInt32MulWithOverflow:
2543 : case IrOpcode::kInt64AddWithOverflow:
2544 : case IrOpcode::kInt64SubWithOverflow:
2545 : case IrOpcode::kTryTruncateFloat32ToInt64:
2546 : case IrOpcode::kTryTruncateFloat64ToInt64:
2547 : case IrOpcode::kTryTruncateFloat32ToUint64:
2548 : case IrOpcode::kTryTruncateFloat64ToUint64:
2549 : case IrOpcode::kInt32PairAdd:
2550 : case IrOpcode::kInt32PairSub:
2551 : case IrOpcode::kInt32PairMul:
2552 : case IrOpcode::kWord32PairShl:
2553 : case IrOpcode::kWord32PairShr:
2554 : case IrOpcode::kWord32PairSar:
2555 : case IrOpcode::kInt32AbsWithOverflow:
2556 : case IrOpcode::kInt64AbsWithOverflow:
2557 278551 : if (ProjectionIndexOf(node->op()) == 0u) {
2558 181815 : Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(value));
2559 : } else {
2560 : DCHECK_EQ(1u, ProjectionIndexOf(node->op()));
2561 : MarkAsUsed(value);
2562 : }
2563 : break;
2564 : default:
2565 : break;
2566 : }
2567 292967 : }
2568 :
2569 14336006 : void InstructionSelector::VisitConstant(Node* node) {
2570 : // We must emit a NOP here because every live range needs a defining
2571 : // instruction in the register allocator.
2572 : OperandGenerator g(this);
2573 14336006 : Emit(kArchNop, g.DefineAsConstant(node));
2574 14337677 : }
2575 :
2576 6063437 : void InstructionSelector::VisitCall(Node* node, BasicBlock* handler) {
2577 : OperandGenerator g(this);
2578 6063437 : auto call_descriptor = CallDescriptorOf(node->op());
2579 :
2580 : FrameStateDescriptor* frame_state_descriptor = nullptr;
2581 6063457 : if (call_descriptor->NeedsFrameState()) {
2582 2955116 : frame_state_descriptor = GetFrameStateDescriptor(
2583 2955116 : node->InputAt(static_cast<int>(call_descriptor->InputCount())));
2584 : }
2585 :
2586 6063453 : CallBuffer buffer(zone(), call_descriptor, frame_state_descriptor);
2587 : CallDescriptor::Flags flags = call_descriptor->flags();
2588 :
2589 : // Compute InstructionOperands for inputs and outputs.
2590 : // TODO(turbofan): on some architectures it's probably better to use
2591 : // the code object in a register if there are multiple uses of it.
2592 : // Improve constant pool and the heuristics in the register allocator
2593 : // for where to emit constants.
2594 : CallBufferFlags call_buffer_flags(kCallCodeImmediate | kCallAddressImmediate);
2595 6063816 : if (flags & CallDescriptor::kAllowCallThroughSlot) {
2596 : // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative call
2597 : // instead once builtins are embedded in every build configuration.
2598 : call_buffer_flags |= kAllowCallThroughSlot;
2599 : #ifndef V8_TARGET_ARCH_32_BIT
2600 : // kAllowCallThroughSlot is only supported on ia32.
2601 0 : UNREACHABLE();
2602 : #endif
2603 : }
2604 6063816 : InitializeCallBuffer(node, &buffer, call_buffer_flags, false);
2605 :
2606 6063655 : EmitPrepareArguments(&(buffer.pushed_nodes), call_descriptor, node);
2607 :
2608 : // Pass label of exception handler block.
2609 6063472 : if (handler) {
2610 : DCHECK_EQ(IrOpcode::kIfException, handler->front()->opcode());
2611 : flags |= CallDescriptor::kHasExceptionHandler;
2612 453120 : buffer.instruction_args.push_back(g.Label(handler));
2613 : }
2614 :
2615 : // Select the appropriate opcode based on the call type.
2616 : InstructionCode opcode = kArchNop;
2617 6063474 : switch (call_descriptor->kind()) {
2618 : case CallDescriptor::kCallAddress:
2619 25882 : opcode = kArchCallCFunction | MiscField::encode(static_cast<int>(
2620 25882 : call_descriptor->ParameterCount()));
2621 25882 : break;
2622 : case CallDescriptor::kCallCodeObject:
2623 4828078 : opcode = kArchCallCodeObject | MiscField::encode(flags);
2624 4828078 : break;
2625 : case CallDescriptor::kCallJSFunction:
2626 23884 : opcode = kArchCallJSFunction | MiscField::encode(flags);
2627 23884 : break;
2628 : case CallDescriptor::kCallWasmFunction:
2629 : case CallDescriptor::kCallWasmImportWrapper:
2630 1182019 : opcode = kArchCallWasmFunction | MiscField::encode(flags);
2631 1182019 : break;
2632 : case CallDescriptor::kCallBuiltinPointer:
2633 3608 : opcode = kArchCallBuiltinPointer | MiscField::encode(flags);
2634 3608 : break;
2635 : }
2636 :
2637 : // Emit the call instruction.
2638 : size_t const output_count = buffer.outputs.size();
2639 6063474 : auto* outputs = output_count ? &buffer.outputs.front() : nullptr;
2640 : Instruction* call_instr =
2641 : Emit(opcode, output_count, outputs, buffer.instruction_args.size(),
2642 6063474 : &buffer.instruction_args.front());
2643 6063592 : if (instruction_selection_failed()) return;
2644 : call_instr->MarkAsCall();
2645 :
2646 6063591 : EmitPrepareResults(&(buffer.output_nodes), call_descriptor, node);
2647 : }
2648 :
2649 900 : void InstructionSelector::VisitCallWithCallerSavedRegisters(
2650 : Node* node, BasicBlock* handler) {
2651 : OperandGenerator g(this);
2652 900 : const auto fp_mode = CallDescriptorOf(node->op())->get_save_fp_mode();
2653 900 : Emit(kArchSaveCallerRegisters | MiscField::encode(static_cast<int>(fp_mode)),
2654 : g.NoOutput());
2655 900 : VisitCall(node, handler);
2656 900 : Emit(kArchRestoreCallerRegisters |
2657 : MiscField::encode(static_cast<int>(fp_mode)),
2658 : g.NoOutput());
2659 900 : }
2660 :
2661 119827 : void InstructionSelector::VisitTailCall(Node* node) {
2662 : OperandGenerator g(this);
2663 119827 : auto call_descriptor = CallDescriptorOf(node->op());
2664 :
2665 : CallDescriptor* caller = linkage()->GetIncomingDescriptor();
2666 : DCHECK(caller->CanTailCall(node));
2667 119827 : const CallDescriptor* callee = CallDescriptorOf(node->op());
2668 119827 : int stack_param_delta = callee->GetStackParameterDelta(caller);
2669 119827 : CallBuffer buffer(zone(), call_descriptor, nullptr);
2670 :
2671 : // Compute InstructionOperands for inputs and outputs.
2672 : CallBufferFlags flags(kCallCodeImmediate | kCallTail);
2673 119827 : if (IsTailCallAddressImmediate()) {
2674 : flags |= kCallAddressImmediate;
2675 : }
2676 119826 : if (callee->flags() & CallDescriptor::kFixedTargetRegister) {
2677 : flags |= kCallFixedTargetRegister;
2678 : }
2679 : DCHECK_EQ(callee->flags() & CallDescriptor::kAllowCallThroughSlot, 0);
2680 119826 : InitializeCallBuffer(node, &buffer, flags, true, stack_param_delta);
2681 :
2682 : // Select the appropriate opcode based on the call type.
2683 : InstructionCode opcode;
2684 : InstructionOperandVector temps(zone());
2685 119826 : if (linkage()->GetIncomingDescriptor()->IsJSFunctionCall()) {
2686 1288 : switch (call_descriptor->kind()) {
2687 : case CallDescriptor::kCallCodeObject:
2688 : opcode = kArchTailCallCodeObjectFromJSFunction;
2689 : break;
2690 : default:
2691 0 : UNREACHABLE();
2692 : return;
2693 : }
2694 1288 : int temps_count = GetTempsCountForTailCallFromJSFunction();
2695 9016 : for (int i = 0; i < temps_count; i++) {
2696 7728 : temps.push_back(g.TempRegister());
2697 : }
2698 : } else {
2699 118538 : switch (call_descriptor->kind()) {
2700 : case CallDescriptor::kCallCodeObject:
2701 : opcode = kArchTailCallCodeObject;
2702 : break;
2703 : case CallDescriptor::kCallAddress:
2704 : opcode = kArchTailCallAddress;
2705 82768 : break;
2706 : case CallDescriptor::kCallWasmFunction:
2707 : opcode = kArchTailCallWasm;
2708 217 : break;
2709 : default:
2710 0 : UNREACHABLE();
2711 : return;
2712 : }
2713 : }
2714 119826 : opcode |= MiscField::encode(call_descriptor->flags());
2715 :
2716 : Emit(kArchPrepareTailCall, g.NoOutput());
2717 :
2718 : // Add an immediate operand that represents the first slot that is unused
2719 : // with respect to the stack pointer that has been updated for the tail call
2720 : // instruction. This is used by backends that need to pad arguments for stack
2721 : // alignment, in order to store an optional slot of padding above the
2722 : // arguments.
2723 119825 : int optional_padding_slot = callee->GetFirstUnusedStackSlot();
2724 239651 : buffer.instruction_args.push_back(g.TempImmediate(optional_padding_slot));
2725 :
2726 : int first_unused_stack_slot =
2727 : (V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? true : false) +
2728 119825 : stack_param_delta;
2729 239649 : buffer.instruction_args.push_back(g.TempImmediate(first_unused_stack_slot));
2730 :
2731 : // Emit the tailcall instruction.
2732 119824 : Emit(opcode, 0, nullptr, buffer.instruction_args.size(),
2733 : &buffer.instruction_args.front(), temps.size(),
2734 119824 : temps.empty() ? nullptr : &temps.front());
2735 119826 : }
2736 :
2737 9029252 : void InstructionSelector::VisitGoto(BasicBlock* target) {
2738 : // jump to the next block.
2739 : OperandGenerator g(this);
2740 : Emit(kArchJmp, g.NoOutput(), g.Label(target));
2741 9029357 : }
2742 :
2743 2999042 : void InstructionSelector::VisitReturn(Node* ret) {
2744 : OperandGenerator g(this);
2745 : const int input_count = linkage()->GetIncomingDescriptor()->ReturnCount() == 0
2746 : ? 1
2747 2999042 : : ret->op()->ValueInputCount();
2748 : DCHECK_GE(input_count, 1);
2749 2999042 : auto value_locations = zone()->NewArray<InstructionOperand>(input_count);
2750 : Node* pop_count = ret->InputAt(0);
2751 39896 : value_locations[0] = (pop_count->opcode() == IrOpcode::kInt32Constant ||
2752 : pop_count->opcode() == IrOpcode::kInt64Constant)
2753 : ? g.UseImmediate(pop_count)
2754 5961476 : : g.UseRegister(pop_count);
2755 8545171 : for (int i = 1; i < input_count; ++i) {
2756 : value_locations[i] =
2757 5544818 : g.UseLocation(ret->InputAt(i), linkage()->GetReturnLocation(i - 1));
2758 : }
2759 3000996 : Emit(kArchRet, 0, nullptr, input_count, value_locations);
2760 3001035 : }
2761 :
2762 5368873 : void InstructionSelector::VisitBranch(Node* branch, BasicBlock* tbranch,
2763 : BasicBlock* fbranch) {
2764 5368873 : if (NeedsPoisoning(IsSafetyCheckOf(branch->op()))) {
2765 : FlagsContinuation cont =
2766 : FlagsContinuation::ForBranchAndPoison(kNotEqual, tbranch, fbranch);
2767 0 : VisitWordCompareZero(branch, branch->InputAt(0), &cont);
2768 : } else {
2769 : FlagsContinuation cont =
2770 : FlagsContinuation::ForBranch(kNotEqual, tbranch, fbranch);
2771 5368735 : VisitWordCompareZero(branch, branch->InputAt(0), &cont);
2772 : }
2773 5368835 : }
2774 :
2775 111368 : void InstructionSelector::VisitDeoptimizeIf(Node* node) {
2776 111368 : DeoptimizeParameters p = DeoptimizeParametersOf(node->op());
2777 111368 : if (NeedsPoisoning(p.is_safety_check())) {
2778 : FlagsContinuation cont = FlagsContinuation::ForDeoptimizeAndPoison(
2779 : kNotEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2780 0 : VisitWordCompareZero(node, node->InputAt(0), &cont);
2781 : } else {
2782 : FlagsContinuation cont = FlagsContinuation::ForDeoptimize(
2783 : kNotEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2784 111369 : VisitWordCompareZero(node, node->InputAt(0), &cont);
2785 : }
2786 111370 : }
2787 :
2788 219534 : void InstructionSelector::VisitDeoptimizeUnless(Node* node) {
2789 219534 : DeoptimizeParameters p = DeoptimizeParametersOf(node->op());
2790 219534 : if (NeedsPoisoning(p.is_safety_check())) {
2791 : FlagsContinuation cont = FlagsContinuation::ForDeoptimizeAndPoison(
2792 : kEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2793 0 : VisitWordCompareZero(node, node->InputAt(0), &cont);
2794 : } else {
2795 : FlagsContinuation cont = FlagsContinuation::ForDeoptimize(
2796 : kEqual, p.kind(), p.reason(), p.feedback(), node->InputAt(1));
2797 219532 : VisitWordCompareZero(node, node->InputAt(0), &cont);
2798 : }
2799 219534 : }
2800 :
2801 6242 : void InstructionSelector::VisitTrapIf(Node* node, TrapId trap_id) {
2802 : FlagsContinuation cont =
2803 : FlagsContinuation::ForTrap(kNotEqual, trap_id, node->InputAt(1));
2804 6249 : VisitWordCompareZero(node, node->InputAt(0), &cont);
2805 6259 : }
2806 :
2807 135749 : void InstructionSelector::VisitTrapUnless(Node* node, TrapId trap_id) {
2808 : FlagsContinuation cont =
2809 : FlagsContinuation::ForTrap(kEqual, trap_id, node->InputAt(1));
2810 135762 : VisitWordCompareZero(node, node->InputAt(0), &cont);
2811 135822 : }
2812 :
2813 2420012 : void InstructionSelector::EmitIdentity(Node* node) {
2814 : OperandGenerator g(this);
2815 : MarkAsUsed(node->InputAt(0));
2816 2420012 : SetRename(node, node->InputAt(0));
2817 2420026 : }
2818 :
2819 0 : void InstructionSelector::VisitDeoptimize(DeoptimizeKind kind,
2820 : DeoptimizeReason reason,
2821 : VectorSlotPair const& feedback,
2822 : Node* value) {
2823 : EmitDeoptimize(kArchDeoptimize, 0, nullptr, 0, nullptr, kind, reason,
2824 44616 : feedback, value);
2825 0 : }
2826 :
2827 0 : void InstructionSelector::VisitThrow(Node* node) {
2828 : OperandGenerator g(this);
2829 : Emit(kArchThrowTerminator, g.NoOutput());
2830 0 : }
2831 :
2832 0 : void InstructionSelector::VisitDebugBreak(Node* node) {
2833 : OperandGenerator g(this);
2834 : Emit(kArchDebugBreak, g.NoOutput());
2835 0 : }
2836 :
2837 0 : void InstructionSelector::VisitUnreachable(Node* node) {
2838 : OperandGenerator g(this);
2839 : Emit(kArchDebugBreak, g.NoOutput());
2840 0 : }
2841 :
2842 0 : void InstructionSelector::VisitDeadValue(Node* node) {
2843 : OperandGenerator g(this);
2844 0 : MarkAsRepresentation(DeadValueRepresentationOf(node->op()), node);
2845 0 : Emit(kArchDebugBreak, g.DefineAsConstant(node));
2846 0 : }
2847 :
2848 4 : void InstructionSelector::VisitComment(Node* node) {
2849 : OperandGenerator g(this);
2850 4 : InstructionOperand operand(g.UseImmediate(node));
2851 4 : Emit(kArchComment, 0, nullptr, 1, &operand);
2852 4 : }
2853 :
2854 0 : void InstructionSelector::VisitUnsafePointerAdd(Node* node) {
2855 : #if V8_TARGET_ARCH_64_BIT
2856 6746 : VisitInt64Add(node);
2857 : #else // V8_TARGET_ARCH_64_BIT
2858 : VisitInt32Add(node);
2859 : #endif // V8_TARGET_ARCH_64_BIT
2860 0 : }
2861 :
2862 9316 : void InstructionSelector::VisitRetain(Node* node) {
2863 : OperandGenerator g(this);
2864 9316 : Emit(kArchNop, g.NoOutput(), g.UseAny(node->InputAt(0)));
2865 9316 : }
2866 :
2867 0 : bool InstructionSelector::CanProduceSignalingNaN(Node* node) {
2868 : // TODO(jarin) Improve the heuristic here.
2869 13137 : if (node->opcode() == IrOpcode::kFloat64Add ||
2870 13137 : node->opcode() == IrOpcode::kFloat64Sub ||
2871 : node->opcode() == IrOpcode::kFloat64Mul) {
2872 : return false;
2873 : }
2874 0 : return true;
2875 : }
2876 :
2877 3683978 : FrameStateDescriptor* InstructionSelector::GetFrameStateDescriptor(
2878 : Node* state) {
2879 : DCHECK_EQ(IrOpcode::kFrameState, state->opcode());
2880 : DCHECK_EQ(kFrameStateInputCount, state->InputCount());
2881 3683978 : FrameStateInfo state_info = FrameStateInfoOf(state->op());
2882 :
2883 : int parameters = static_cast<int>(
2884 3683982 : StateValuesAccess(state->InputAt(kFrameStateParametersInput)).size());
2885 : int locals = static_cast<int>(
2886 3683983 : StateValuesAccess(state->InputAt(kFrameStateLocalsInput)).size());
2887 : int stack = static_cast<int>(
2888 3683980 : StateValuesAccess(state->InputAt(kFrameStateStackInput)).size());
2889 :
2890 : DCHECK_EQ(parameters, state_info.parameter_count());
2891 : DCHECK_EQ(locals, state_info.local_count());
2892 :
2893 : FrameStateDescriptor* outer_state = nullptr;
2894 : Node* outer_node = state->InputAt(kFrameStateOuterStateInput);
2895 3683983 : if (outer_node->opcode() == IrOpcode::kFrameState) {
2896 353349 : outer_state = GetFrameStateDescriptor(outer_node);
2897 : }
2898 :
2899 : return new (instruction_zone()) FrameStateDescriptor(
2900 7367968 : instruction_zone(), state_info.type(), state_info.bailout_id(),
2901 7367968 : state_info.state_combine(), parameters, locals, stack,
2902 11051957 : state_info.shared_info(), outer_state);
2903 : }
2904 :
2905 : // static
2906 5016 : void InstructionSelector::CanonicalizeShuffle(bool inputs_equal,
2907 : uint8_t* shuffle,
2908 : bool* needs_swap,
2909 : bool* is_swizzle) {
2910 5016 : *needs_swap = false;
2911 : // Inputs equal, then it's a swizzle.
2912 5016 : if (inputs_equal) {
2913 4 : *is_swizzle = true;
2914 : } else {
2915 : // Inputs are distinct; check that both are required.
2916 : bool src0_is_used = false;
2917 : bool src1_is_used = false;
2918 165396 : for (int i = 0; i < kSimd128Size; ++i) {
2919 80192 : if (shuffle[i] < kSimd128Size) {
2920 : src0_is_used = true;
2921 : } else {
2922 : src1_is_used = true;
2923 : }
2924 : }
2925 5012 : if (src0_is_used && !src1_is_used) {
2926 1173 : *is_swizzle = true;
2927 3839 : } else if (src1_is_used && !src0_is_used) {
2928 1121 : *needs_swap = true;
2929 1121 : *is_swizzle = true;
2930 : } else {
2931 2718 : *is_swizzle = false;
2932 : // Canonicalize general 2 input shuffles so that the first input lanes are
2933 : // encountered first. This makes architectural shuffle pattern matching
2934 : // easier, since we only need to consider 1 input ordering instead of 2.
2935 2718 : if (shuffle[0] >= kSimd128Size) {
2936 : // The second operand is used first. Swap inputs and adjust the shuffle.
2937 517 : *needs_swap = true;
2938 17061 : for (int i = 0; i < kSimd128Size; ++i) {
2939 8272 : shuffle[i] ^= kSimd128Size;
2940 : }
2941 : }
2942 : }
2943 : }
2944 5016 : if (*is_swizzle) {
2945 39066 : for (int i = 0; i < kSimd128Size; ++i) shuffle[i] &= kSimd128Size - 1;
2946 : }
2947 5016 : }
2948 :
2949 5008 : void InstructionSelector::CanonicalizeShuffle(Node* node, uint8_t* shuffle,
2950 : bool* is_swizzle) {
2951 : // Get raw shuffle indices.
2952 5008 : memcpy(shuffle, OpParameter<uint8_t*>(node->op()), kSimd128Size);
2953 : bool needs_swap;
2954 5008 : bool inputs_equal = GetVirtualRegister(node->InputAt(0)) ==
2955 5008 : GetVirtualRegister(node->InputAt(1));
2956 5008 : CanonicalizeShuffle(inputs_equal, shuffle, &needs_swap, is_swizzle);
2957 5008 : if (needs_swap) {
2958 1636 : SwapShuffleInputs(node);
2959 : }
2960 : // Duplicate the first input; for some shuffles on some architectures, it's
2961 : // easiest to implement a swizzle as a shuffle so it might be used.
2962 5008 : if (*is_swizzle) {
2963 2292 : node->ReplaceInput(1, node->InputAt(0));
2964 : }
2965 5008 : }
2966 :
2967 : // static
2968 1876 : void InstructionSelector::SwapShuffleInputs(Node* node) {
2969 : Node* input0 = node->InputAt(0);
2970 : Node* input1 = node->InputAt(1);
2971 1876 : node->ReplaceInput(0, input1);
2972 1876 : node->ReplaceInput(1, input0);
2973 1876 : }
2974 :
2975 : // static
2976 523 : bool InstructionSelector::TryMatchIdentity(const uint8_t* shuffle) {
2977 9385 : for (int i = 0; i < kSimd128Size; ++i) {
2978 4713 : if (shuffle[i] != i) return false;
2979 : }
2980 : return true;
2981 : }
2982 :
2983 : // static
2984 3251 : bool InstructionSelector::TryMatch32x4Shuffle(const uint8_t* shuffle,
2985 : uint8_t* shuffle32x4) {
2986 11835 : for (int i = 0; i < 4; ++i) {
2987 6534 : if (shuffle[i * 4] % 4 != 0) return false;
2988 31957 : for (int j = 1; j < 4; ++j) {
2989 14275 : if (shuffle[i * 4 + j] - shuffle[i * 4 + j - 1] != 1) return false;
2990 : }
2991 4292 : shuffle32x4[i] = shuffle[i * 4] / 4;
2992 : }
2993 : return true;
2994 : }
2995 :
2996 : // static
2997 2243 : bool InstructionSelector::TryMatch16x8Shuffle(const uint8_t* shuffle,
2998 : uint8_t* shuffle16x8) {
2999 18263 : for (int i = 0; i < 8; ++i) {
3000 9324 : if (shuffle[i * 2] % 2 != 0) return false;
3001 24627 : for (int j = 1; j < 2; ++j) {
3002 8607 : if (shuffle[i * 2 + j] - shuffle[i * 2 + j - 1] != 1) return false;
3003 : }
3004 8010 : shuffle16x8[i] = shuffle[i * 2] / 2;
3005 : }
3006 : return true;
3007 : }
3008 :
3009 : // static
3010 5012 : bool InstructionSelector::TryMatchConcat(const uint8_t* shuffle,
3011 : uint8_t* offset) {
3012 : // Don't match the identity shuffle (e.g. [0 1 2 ... 15]).
3013 5012 : uint8_t start = shuffle[0];
3014 5012 : if (start == 0) return false;
3015 : DCHECK_GT(kSimd128Size, start); // The shuffle should be canonicalized.
3016 : // A concatenation is a series of consecutive indices, with at most one jump
3017 : // in the middle from the last lane to the first.
3018 15135 : for (int i = 1; i < kSimd128Size; ++i) {
3019 8819 : if ((shuffle[i]) != ((shuffle[i - 1] + 1))) {
3020 2866 : if (shuffle[i - 1] != 15) return false;
3021 137 : if (shuffle[i] % kSimd128Size != 0) return false;
3022 : }
3023 : }
3024 242 : *offset = start;
3025 242 : return true;
3026 : }
3027 :
3028 : // static
3029 1419 : bool InstructionSelector::TryMatchBlend(const uint8_t* shuffle) {
3030 6667 : for (int i = 0; i < 16; ++i) {
3031 3985 : if ((shuffle[i] & 0xF) != i) return false;
3032 : }
3033 : return true;
3034 : }
3035 :
3036 : // static
3037 5840 : int32_t InstructionSelector::Pack4Lanes(const uint8_t* shuffle) {
3038 : int32_t result = 0;
3039 52560 : for (int i = 3; i >= 0; --i) {
3040 23360 : result <<= 8;
3041 23360 : result |= shuffle[i];
3042 : }
3043 5840 : return result;
3044 : }
3045 :
3046 5699622 : bool InstructionSelector::NeedsPoisoning(IsSafetyCheck safety_check) const {
3047 5699622 : switch (poisoning_level_) {
3048 : case PoisoningMitigationLevel::kDontPoison:
3049 : return false;
3050 : case PoisoningMitigationLevel::kPoisonAll:
3051 24 : return safety_check != IsSafetyCheck::kNoSafetyCheck;
3052 : case PoisoningMitigationLevel::kPoisonCriticalOnly:
3053 0 : return safety_check == IsSafetyCheck::kCriticalSafetyCheck;
3054 : }
3055 0 : UNREACHABLE();
3056 : }
3057 :
3058 : } // namespace compiler
3059 : } // namespace internal
3060 121996 : } // namespace v8
|
