Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/raw-machine-assembler.h"
6 :
7 : #include "src/compiler/node-properties.h"
8 : #include "src/compiler/pipeline.h"
9 : #include "src/compiler/scheduler.h"
10 : #include "src/heap/factory-inl.h"
11 :
12 : namespace v8 {
13 : namespace internal {
14 : namespace compiler {
15 :
16 296381 : RawMachineAssembler::RawMachineAssembler(
17 853938 : Isolate* isolate, Graph* graph, CallDescriptor* call_descriptor,
18 : MachineRepresentation word, MachineOperatorBuilder::Flags flags,
19 : MachineOperatorBuilder::AlignmentRequirements alignment_requirements,
20 1446700 : PoisoningMitigationLevel poisoning_level)
21 : : isolate_(isolate),
22 : graph_(graph),
23 296381 : schedule_(new (zone()) Schedule(zone())),
24 : machine_(zone(), word, flags, alignment_requirements),
25 : common_(zone()),
26 : simplified_(zone()),
27 : call_descriptor_(call_descriptor),
28 : target_parameter_(nullptr),
29 : parameters_(parameter_count(), zone()),
30 296381 : current_block_(schedule()->start()),
31 1778286 : poisoning_level_(poisoning_level) {
32 296381 : int param_count = static_cast<int>(parameter_count());
33 : // Add an extra input for the JSFunction parameter to the start node.
34 296381 : graph->SetStart(graph->NewNode(common_.Start(param_count + 1)));
35 296381 : if (call_descriptor->IsJSFunctionCall()) {
36 : target_parameter_ = AddNode(
37 40278 : common()->Parameter(Linkage::kJSCallClosureParamIndex), graph->start());
38 : }
39 1411495 : for (size_t i = 0; i < parameter_count(); ++i) {
40 557557 : parameters_[i] =
41 1115114 : AddNode(common()->Parameter(static_cast<int>(i)), graph->start());
42 : }
43 296381 : graph->SetEnd(graph->NewNode(common_.End(0)));
44 296381 : }
45 :
46 0 : Node* RawMachineAssembler::NullConstant() {
47 0 : return HeapConstant(isolate()->factory()->null_value());
48 : }
49 :
50 5 : Node* RawMachineAssembler::UndefinedConstant() {
51 5 : return HeapConstant(isolate()->factory()->undefined_value());
52 : }
53 :
54 0 : Node* RawMachineAssembler::RelocatableIntPtrConstant(intptr_t value,
55 : RelocInfo::Mode rmode) {
56 : return kSystemPointerSize == 8
57 : ? RelocatableInt64Constant(value, rmode)
58 0 : : RelocatableInt32Constant(static_cast<int>(value), rmode);
59 : }
60 :
61 63021 : Node* RawMachineAssembler::OptimizedAllocate(Node* size,
62 : PretenureFlag pretenure) {
63 126042 : return AddNode(simplified()->AllocateRaw(Type::Any(), pretenure), size);
64 : }
65 :
66 226857 : Schedule* RawMachineAssembler::Export() {
67 : // Compute the correct codegen order.
68 : DCHECK(schedule_->rpo_order()->empty());
69 226857 : if (FLAG_trace_turbo_scheduler) {
70 0 : PrintF("--- RAW SCHEDULE -------------------------------------------\n");
71 0 : StdoutStream{} << *schedule_;
72 : }
73 226857 : schedule_->EnsureCFGWellFormedness();
74 453714 : Scheduler::ComputeSpecialRPO(zone(), schedule_);
75 226857 : schedule_->PropagateDeferredMark();
76 226857 : if (FLAG_trace_turbo_scheduler) {
77 0 : PrintF("--- EDGE SPLIT AND PROPAGATED DEFERRED SCHEDULE ------------\n");
78 0 : StdoutStream{} << *schedule_;
79 : }
80 : // Invalidate RawMachineAssembler.
81 226857 : Schedule* schedule = schedule_;
82 226857 : schedule_ = nullptr;
83 226857 : return schedule;
84 : }
85 :
86 133830 : Graph* RawMachineAssembler::ExportForOptimization() {
87 : // Compute the correct codegen order.
88 : DCHECK(schedule_->rpo_order()->empty());
89 66915 : if (FLAG_trace_turbo_scheduler) {
90 0 : PrintF("--- RAW SCHEDULE -------------------------------------------\n");
91 0 : StdoutStream{} << *schedule_;
92 : }
93 66915 : schedule_->EnsureCFGWellFormedness();
94 133830 : Scheduler::ComputeSpecialRPO(zone(), schedule_);
95 66915 : if (FLAG_trace_turbo_scheduler) {
96 0 : PrintF("--- SCHEDULE BEFORE GRAPH CREATION -------------------------\n");
97 0 : StdoutStream{} << *schedule_;
98 : }
99 66915 : MakeReschedulable();
100 : // Invalidate RawMachineAssembler.
101 66915 : schedule_ = nullptr;
102 66915 : return graph();
103 : }
104 :
105 2054014 : void RawMachineAssembler::MakeReschedulable() {
106 12446126 : std::vector<Node*> block_final_control(schedule_->all_blocks_.size());
107 133830 : std::vector<Node*> block_final_effect(schedule_->all_blocks_.size());
108 :
109 : struct LoopHeader {
110 : BasicBlock* block;
111 : Node* loop_node;
112 : Node* effect_phi;
113 : };
114 : std::vector<LoopHeader> loop_headers;
115 :
116 : // These are hoisted outside of the loop to avoid re-allocation.
117 : std::vector<Node*> merge_inputs;
118 : std::vector<Node*> effect_phi_inputs;
119 :
120 24691512 : for (BasicBlock* block : *schedule_->rpo_order()) {
121 : Node* current_control;
122 : Node* current_effect;
123 12312296 : if (block == schedule_->start()) {
124 66915 : current_control = current_effect = graph()->start();
125 6089233 : } else if (block == schedule_->end()) {
126 484402 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
127 : NodeProperties::MergeControlToEnd(
128 417492 : graph(), common(), block->PredecessorAt(i)->control_input());
129 : }
130 6022323 : } else if (block->IsLoopHeader()) {
131 : // The graph()->start() inputs are just placeholders until we computed the
132 : // real back-edges and re-structure the control flow so the loop has
133 : // exactly two predecessors.
134 : current_control = graph()->NewNode(common()->Loop(2), graph()->start(),
135 195036 : graph()->start());
136 : current_effect =
137 : graph()->NewNode(common()->EffectPhi(2), graph()->start(),
138 195036 : graph()->start(), current_control);
139 :
140 : Node* terminate = graph()->NewNode(common()->Terminate(), current_effect,
141 97518 : current_control);
142 97518 : NodeProperties::MergeControlToEnd(graph(), common(), terminate);
143 : loop_headers.push_back(
144 195036 : LoopHeader{block, current_control, current_effect});
145 5924805 : } else if (block->PredecessorCount() == 1) {
146 : BasicBlock* predecessor = block->PredecessorAt(0);
147 : DCHECK_LT(predecessor->rpo_number(), block->rpo_number());
148 10723280 : current_effect = block_final_effect[predecessor->id().ToSize()];
149 10723280 : current_control = block_final_control[predecessor->id().ToSize()];
150 : } else {
151 : // Create control merge nodes and effect phis for all predecessor blocks.
152 : merge_inputs.clear();
153 : effect_phi_inputs.clear();
154 563165 : int predecessor_count = static_cast<int>(block->PredecessorCount());
155 2218139 : for (int i = 0; i < predecessor_count; ++i) {
156 1654974 : BasicBlock* predecessor = block->PredecessorAt(i);
157 : DCHECK_LT(predecessor->rpo_number(), block->rpo_number());
158 3309948 : merge_inputs.push_back(block_final_control[predecessor->id().ToSize()]);
159 : effect_phi_inputs.push_back(
160 3309948 : block_final_effect[predecessor->id().ToSize()]);
161 : }
162 : current_control = graph()->NewNode(common()->Merge(predecessor_count),
163 563165 : static_cast<int>(merge_inputs.size()),
164 1126330 : merge_inputs.data());
165 563165 : effect_phi_inputs.push_back(current_control);
166 : current_effect = graph()->NewNode(
167 : common()->EffectPhi(predecessor_count),
168 1689495 : static_cast<int>(effect_phi_inputs.size()), effect_phi_inputs.data());
169 : }
170 :
171 74529724 : auto update_current_control_and_effect = [&](Node* node) {
172 : bool existing_effect_and_control =
173 34620010 : IrOpcode::IsIfProjectionOpcode(node->opcode()) ||
174 : IrOpcode::IsPhiOpcode(node->opcode());
175 37264862 : if (node->op()->EffectInputCount() > 0) {
176 : DCHECK_EQ(1, node->op()->EffectInputCount());
177 3386446 : if (existing_effect_and_control) {
178 12360 : NodeProperties::ReplaceEffectInput(node, current_effect);
179 : } else {
180 8054474 : node->AppendInput(graph()->zone(), current_effect);
181 : }
182 : }
183 37264862 : if (node->op()->ControlInputCount() > 0) {
184 : DCHECK_EQ(1, node->op()->ControlInputCount());
185 7937053 : if (existing_effect_and_control) {
186 3256665 : NodeProperties::ReplaceControlInput(node, current_control);
187 : } else {
188 9360776 : node->AppendInput(graph()->zone(), current_control);
189 : }
190 : }
191 37264862 : if (node->op()->EffectOutputCount() > 0) {
192 : DCHECK_EQ(1, node->op()->EffectOutputCount());
193 3177700 : current_effect = node;
194 : }
195 37264862 : if (node->op()->ControlOutputCount() > 0) {
196 4462374 : current_control = node;
197 : }
198 24788579 : };
199 :
200 29448897 : for (Node* node : *block) {
201 17136601 : update_current_control_and_effect(node);
202 : }
203 6156148 : if (block->deferred()) MarkControlDeferred(current_control);
204 :
205 6156148 : if (Node* block_terminator = block->control_input()) {
206 1495830 : update_current_control_and_effect(block_terminator);
207 : }
208 :
209 12312296 : block_final_effect[block->id().ToSize()] = current_effect;
210 12312296 : block_final_control[block->id().ToSize()] = current_control;
211 : }
212 :
213 : // Fix-up loop backedges and re-structure control flow so that loop nodes have
214 : // exactly two control predecessors.
215 231348 : for (const LoopHeader& loop_header : loop_headers) {
216 97518 : BasicBlock* block = loop_header.block;
217 : std::vector<BasicBlock*> loop_entries;
218 : std::vector<BasicBlock*> loop_backedges;
219 638328 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
220 221646 : BasicBlock* predecessor = block->PredecessorAt(i);
221 221646 : if (block->LoopContains(predecessor)) {
222 123848 : loop_backedges.push_back(predecessor);
223 : } else {
224 : DCHECK(loop_backedges.empty());
225 97798 : loop_entries.push_back(predecessor);
226 : }
227 : }
228 : DCHECK(!loop_entries.empty());
229 : DCHECK(!loop_backedges.empty());
230 :
231 195036 : int entrance_count = static_cast<int>(loop_entries.size());
232 195036 : int backedge_count = static_cast<int>(loop_backedges.size());
233 : Node* control_loop_entry = CreateNodeFromPredecessors(
234 195036 : loop_entries, block_final_control, common()->Merge(entrance_count), {});
235 : Node* control_backedge =
236 : CreateNodeFromPredecessors(loop_backedges, block_final_control,
237 195036 : common()->Merge(backedge_count), {});
238 : Node* effect_loop_entry = CreateNodeFromPredecessors(
239 : loop_entries, block_final_effect, common()->EffectPhi(entrance_count),
240 292554 : {control_loop_entry});
241 : Node* effect_backedge = CreateNodeFromPredecessors(
242 : loop_backedges, block_final_effect, common()->EffectPhi(backedge_count),
243 292554 : {control_backedge});
244 :
245 97518 : loop_header.loop_node->ReplaceInput(0, control_loop_entry);
246 97518 : loop_header.loop_node->ReplaceInput(1, control_backedge);
247 97518 : loop_header.effect_phi->ReplaceInput(0, effect_loop_entry);
248 97518 : loop_header.effect_phi->ReplaceInput(1, effect_backedge);
249 :
250 987028 : for (Node* node : *block) {
251 791992 : if (node->opcode() == IrOpcode::kPhi) {
252 159409 : MakePhiBinary(node, static_cast<int>(loop_entries.size()),
253 159409 : control_loop_entry, control_backedge);
254 : }
255 : }
256 : }
257 66915 : }
258 :
259 390072 : Node* RawMachineAssembler::CreateNodeFromPredecessors(
260 390072 : const std::vector<BasicBlock*>& predecessors,
261 443292 : const std::vector<Node*>& sidetable, const Operator* op,
262 27908 : const std::vector<Node*>& additional_inputs) {
263 390072 : if (predecessors.size() == 1) {
264 724328 : return sidetable[predecessors.front()->id().ToSize()];
265 : }
266 : std::vector<Node*> inputs;
267 136944 : for (BasicBlock* predecessor : predecessors) {
268 81128 : inputs.push_back(sidetable[predecessor->id().ToSize()]);
269 : }
270 69770 : for (Node* additional_input : additional_inputs) {
271 13954 : inputs.push_back(additional_input);
272 : }
273 83724 : return graph()->NewNode(op, static_cast<int>(inputs.size()), inputs.data());
274 : }
275 :
276 159409 : void RawMachineAssembler::MakePhiBinary(Node* phi, int split_point,
277 : Node* left_control,
278 31268 : Node* right_control) {
279 159409 : int value_count = phi->op()->ValueInputCount();
280 318818 : if (value_count == 2) return;
281 : DCHECK_LT(split_point, value_count);
282 : DCHECK_GT(split_point, 0);
283 :
284 31100 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
285 : int left_input_count = split_point;
286 31100 : int right_input_count = value_count - split_point;
287 :
288 : Node* left_input;
289 31100 : if (left_input_count == 1) {
290 30484 : left_input = NodeProperties::GetValueInput(phi, 0);
291 : } else {
292 : std::vector<Node*> inputs;
293 1848 : for (int i = 0; i < left_input_count; ++i) {
294 2464 : inputs.push_back(NodeProperties::GetValueInput(phi, i));
295 : }
296 616 : inputs.push_back(left_control);
297 : left_input =
298 : graph()->NewNode(common()->Phi(rep, static_cast<int>(left_input_count)),
299 1848 : static_cast<int>(inputs.size()), inputs.data());
300 : }
301 :
302 : Node* right_input;
303 31100 : if (right_input_count == 1) {
304 448 : right_input = NodeProperties::GetValueInput(phi, split_point);
305 : } else {
306 : std::vector<Node*> inputs;
307 127180 : for (int i = split_point; i < value_count; ++i) {
308 193056 : inputs.push_back(NodeProperties::GetValueInput(phi, i));
309 : }
310 30652 : inputs.push_back(right_control);
311 : right_input = graph()->NewNode(
312 : common()->Phi(rep, static_cast<int>(right_input_count)),
313 91956 : static_cast<int>(inputs.size()), inputs.data());
314 : }
315 :
316 31100 : Node* control = NodeProperties::GetControlInput(phi);
317 31100 : phi->TrimInputCount(3);
318 31100 : phi->ReplaceInput(0, left_input);
319 31100 : phi->ReplaceInput(1, right_input);
320 31100 : phi->ReplaceInput(2, control);
321 31100 : NodeProperties::ChangeOp(phi, common()->Phi(rep, 2));
322 : }
323 :
324 1390826 : void RawMachineAssembler::MarkControlDeferred(Node* control_node) {
325 : BranchHint new_branch_hint;
326 361553 : Node* responsible_branch = nullptr;
327 1448826 : while (responsible_branch == nullptr) {
328 618523 : switch (control_node->opcode()) {
329 : case IrOpcode::kIfException:
330 : // IfException projections are deferred by default.
331 : return;
332 : case IrOpcode::kIfSuccess:
333 896 : control_node = NodeProperties::GetControlInput(control_node);
334 896 : continue;
335 : case IrOpcode::kIfValue: {
336 952 : IfValueParameters parameters = IfValueParametersOf(control_node->op());
337 952 : if (parameters.hint() != BranchHint::kFalse) {
338 : NodeProperties::ChangeOp(
339 : control_node, common()->IfValue(parameters.value(),
340 : parameters.comparison_order(),
341 952 : BranchHint::kFalse));
342 : }
343 : return;
344 : }
345 : case IrOpcode::kIfDefault:
346 6888 : if (BranchHintOf(control_node->op()) != BranchHint::kFalse) {
347 : NodeProperties::ChangeOp(control_node,
348 6888 : common()->IfDefault(BranchHint::kFalse));
349 : }
350 : return;
351 : case IrOpcode::kIfTrue: {
352 212025 : Node* branch = NodeProperties::GetControlInput(control_node);
353 212025 : BranchHint hint = BranchOperatorInfoOf(branch->op()).hint;
354 212025 : if (hint == BranchHint::kTrue) {
355 : // The other possibility is also deferred, so the responsible branch
356 : // has to be before.
357 7504 : control_node = NodeProperties::GetControlInput(branch);
358 7504 : continue;
359 : }
360 : new_branch_hint = BranchHint::kFalse;
361 : responsible_branch = branch;
362 : break;
363 : }
364 : case IrOpcode::kIfFalse: {
365 162581 : Node* branch = NodeProperties::GetControlInput(control_node);
366 162581 : BranchHint hint = BranchOperatorInfoOf(branch->op()).hint;
367 162581 : if (hint == BranchHint::kFalse) {
368 : // The other possibility is also deferred, so the responsible branch
369 : // has to be before.
370 5549 : control_node = NodeProperties::GetControlInput(branch);
371 5549 : continue;
372 : }
373 : new_branch_hint = BranchHint::kTrue;
374 : responsible_branch = branch;
375 : break;
376 : }
377 : case IrOpcode::kMerge:
378 853219 : for (int i = 0; i < control_node->op()->ControlInputCount(); ++i) {
379 246113 : MarkControlDeferred(NodeProperties::GetControlInput(control_node, i));
380 : }
381 : return;
382 : case IrOpcode::kLoop:
383 0 : control_node = NodeProperties::GetControlInput(control_node, 0);
384 0 : continue;
385 : case IrOpcode::kBranch:
386 : case IrOpcode::kSwitch:
387 0 : UNREACHABLE();
388 : case IrOpcode::kStart:
389 : return;
390 : default:
391 : DCHECK_EQ(1, control_node->op()->ControlInputCount());
392 135824 : control_node = NodeProperties::GetControlInput(control_node);
393 135824 : continue;
394 : }
395 : }
396 :
397 361553 : BranchOperatorInfo info = BranchOperatorInfoOf(responsible_branch->op());
398 361553 : if (info.hint == new_branch_hint) return;
399 : NodeProperties::ChangeOp(
400 : responsible_branch,
401 360601 : common()->Branch(new_branch_hint, info.is_safety_check));
402 : }
403 :
404 224 : Node* RawMachineAssembler::TargetParameter() {
405 : DCHECK_NOT_NULL(target_parameter_);
406 224 : return target_parameter_;
407 : }
408 :
409 372424 : Node* RawMachineAssembler::Parameter(size_t index) {
410 : DCHECK_LT(index, parameter_count());
411 744848 : return parameters_[index];
412 : }
413 :
414 :
415 3855446 : void RawMachineAssembler::Goto(RawMachineLabel* label) {
416 : DCHECK(current_block_ != schedule()->end());
417 1927723 : schedule()->AddGoto(CurrentBlock(), Use(label));
418 1927723 : current_block_ = nullptr;
419 1927723 : }
420 :
421 :
422 1276671 : void RawMachineAssembler::Branch(Node* condition, RawMachineLabel* true_val,
423 6383355 : RawMachineLabel* false_val) {
424 : DCHECK(current_block_ != schedule()->end());
425 : Node* branch = MakeNode(
426 : common()->Branch(BranchHint::kNone, IsSafetyCheck::kNoSafetyCheck), 1,
427 2553342 : &condition);
428 1276671 : BasicBlock* true_block = schedule()->NewBasicBlock();
429 1276671 : BasicBlock* false_block = schedule()->NewBasicBlock();
430 2553342 : schedule()->AddBranch(CurrentBlock(), branch, true_block, false_block);
431 :
432 2553342 : true_block->AddNode(MakeNode(common()->IfTrue(), 1, &branch));
433 1276671 : schedule()->AddGoto(true_block, Use(true_val));
434 :
435 2553342 : false_block->AddNode(MakeNode(common()->IfFalse(), 1, &branch));
436 1276671 : schedule()->AddGoto(false_block, Use(false_val));
437 :
438 1276671 : current_block_ = nullptr;
439 1276671 : }
440 :
441 12360 : void RawMachineAssembler::Continuations(Node* call, RawMachineLabel* if_success,
442 12360 : RawMachineLabel* if_exception) {
443 : DCHECK_NOT_NULL(schedule_);
444 : DCHECK_NOT_NULL(current_block_);
445 12360 : schedule()->AddCall(CurrentBlock(), call, Use(if_success), Use(if_exception));
446 12360 : current_block_ = nullptr;
447 12360 : }
448 :
449 9962 : void RawMachineAssembler::Switch(Node* index, RawMachineLabel* default_label,
450 : const int32_t* case_values,
451 : RawMachineLabel** case_labels,
452 382694 : size_t case_count) {
453 : DCHECK_NE(schedule()->end(), current_block_);
454 9962 : size_t succ_count = case_count + 1;
455 9962 : Node* switch_node = MakeNode(common()->Switch(succ_count), 1, &index);
456 : BasicBlock** succ_blocks = zone()->NewArray<BasicBlock*>(succ_count);
457 93183 : for (size_t index = 0; index < case_count; ++index) {
458 83221 : int32_t case_value = case_values[index];
459 83221 : BasicBlock* case_block = schedule()->NewBasicBlock();
460 : Node* case_node =
461 83221 : graph()->NewNode(common()->IfValue(case_value), switch_node);
462 83221 : schedule()->AddNode(case_block, case_node);
463 166442 : schedule()->AddGoto(case_block, Use(case_labels[index]));
464 83221 : succ_blocks[index] = case_block;
465 : }
466 9962 : BasicBlock* default_block = schedule()->NewBasicBlock();
467 9962 : Node* default_node = graph()->NewNode(common()->IfDefault(), switch_node);
468 9962 : schedule()->AddNode(default_block, default_node);
469 9962 : schedule()->AddGoto(default_block, Use(default_label));
470 9962 : succ_blocks[case_count] = default_block;
471 9962 : schedule()->AddSwitch(CurrentBlock(), switch_node, succ_blocks, succ_count);
472 9962 : current_block_ = nullptr;
473 9962 : }
474 :
475 599602 : void RawMachineAssembler::Return(Node* value) {
476 299801 : Node* values[] = {Int32Constant(0), value};
477 299801 : Node* ret = MakeNode(common()->Return(1), 2, values);
478 299801 : schedule()->AddReturn(CurrentBlock(), ret);
479 299801 : current_block_ = nullptr;
480 299801 : }
481 :
482 0 : void RawMachineAssembler::Return(Node* v1, Node* v2) {
483 0 : Node* values[] = {Int32Constant(0), v1, v2};
484 0 : Node* ret = MakeNode(common()->Return(2), 3, values);
485 0 : schedule()->AddReturn(CurrentBlock(), ret);
486 0 : current_block_ = nullptr;
487 0 : }
488 :
489 0 : void RawMachineAssembler::Return(Node* v1, Node* v2, Node* v3) {
490 0 : Node* values[] = {Int32Constant(0), v1, v2, v3};
491 0 : Node* ret = MakeNode(common()->Return(3), 4, values);
492 0 : schedule()->AddReturn(CurrentBlock(), ret);
493 0 : current_block_ = nullptr;
494 0 : }
495 :
496 0 : void RawMachineAssembler::Return(Node* v1, Node* v2, Node* v3, Node* v4) {
497 0 : Node* values[] = {Int32Constant(0), v1, v2, v3, v4};
498 0 : Node* ret = MakeNode(common()->Return(4), 5, values);
499 0 : schedule()->AddReturn(CurrentBlock(), ret);
500 0 : current_block_ = nullptr;
501 0 : }
502 :
503 1882 : void RawMachineAssembler::Return(int count, Node* vs[]) {
504 : typedef Node* Node_ptr;
505 941 : Node** values = new Node_ptr[count + 1];
506 941 : values[0] = Int32Constant(0);
507 941 : for (int i = 0; i < count; ++i) values[i + 1] = vs[i];
508 941 : Node* ret = MakeNode(common()->Return(count), count + 1, values);
509 941 : schedule()->AddReturn(CurrentBlock(), ret);
510 941 : current_block_ = nullptr;
511 941 : delete[] values;
512 941 : }
513 :
514 39260 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* value) {
515 19630 : Node* values[] = {pop, value};
516 19630 : Node* ret = MakeNode(common()->Return(1), 2, values);
517 19630 : schedule()->AddReturn(CurrentBlock(), ret);
518 19630 : current_block_ = nullptr;
519 19630 : }
520 :
521 0 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* v1, Node* v2) {
522 0 : Node* values[] = {pop, v1, v2};
523 0 : Node* ret = MakeNode(common()->Return(2), 3, values);
524 0 : schedule()->AddReturn(CurrentBlock(), ret);
525 0 : current_block_ = nullptr;
526 0 : }
527 :
528 0 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* v1, Node* v2,
529 0 : Node* v3) {
530 0 : Node* values[] = {pop, v1, v2, v3};
531 0 : Node* ret = MakeNode(common()->Return(3), 4, values);
532 0 : schedule()->AddReturn(CurrentBlock(), ret);
533 0 : current_block_ = nullptr;
534 0 : }
535 :
536 0 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* v1, Node* v2, Node* v3,
537 0 : Node* v4) {
538 0 : Node* values[] = {pop, v1, v2, v3, v4};
539 0 : Node* ret = MakeNode(common()->Return(4), 5, values);
540 0 : schedule()->AddReturn(CurrentBlock(), ret);
541 0 : current_block_ = nullptr;
542 0 : }
543 :
544 525 : void RawMachineAssembler::DebugAbort(Node* message) {
545 525 : AddNode(machine()->DebugAbort(), message);
546 525 : }
547 :
548 134758 : void RawMachineAssembler::DebugBreak() { AddNode(machine()->DebugBreak()); }
549 :
550 128262 : void RawMachineAssembler::Unreachable() {
551 64131 : Node* ret = MakeNode(common()->Throw(), 0, nullptr);
552 64131 : schedule()->AddThrow(CurrentBlock(), ret);
553 64131 : current_block_ = nullptr;
554 64131 : }
555 :
556 5 : void RawMachineAssembler::Comment(std::string msg) {
557 5 : size_t length = msg.length() + 1;
558 : char* zone_buffer = zone()->NewArray<char>(length);
559 : MemCopy(zone_buffer, msg.c_str(), length);
560 5 : AddNode(machine()->Comment(zone_buffer));
561 5 : }
562 :
563 285027 : Node* RawMachineAssembler::CallN(CallDescriptor* call_descriptor,
564 : int input_count, Node* const* inputs) {
565 : DCHECK(!call_descriptor->NeedsFrameState());
566 : // +1 is for target.
567 : DCHECK_EQ(input_count, call_descriptor->ParameterCount() + 1);
568 285027 : return AddNode(common()->Call(call_descriptor), input_count, inputs);
569 : }
570 :
571 3 : Node* RawMachineAssembler::CallNWithFrameState(CallDescriptor* call_descriptor,
572 : int input_count,
573 : Node* const* inputs) {
574 : DCHECK(call_descriptor->NeedsFrameState());
575 : // +2 is for target and frame state.
576 : DCHECK_EQ(input_count, call_descriptor->ParameterCount() + 2);
577 3 : return AddNode(common()->Call(call_descriptor), input_count, inputs);
578 : }
579 :
580 62822 : Node* RawMachineAssembler::TailCallN(CallDescriptor* call_descriptor,
581 62822 : int input_count, Node* const* inputs) {
582 : // +1 is for target.
583 : DCHECK_EQ(input_count, call_descriptor->ParameterCount() + 1);
584 : Node* tail_call =
585 62822 : MakeNode(common()->TailCall(call_descriptor), input_count, inputs);
586 62822 : schedule()->AddTailCall(CurrentBlock(), tail_call);
587 62822 : current_block_ = nullptr;
588 62822 : return tail_call;
589 : }
590 :
591 5 : Node* RawMachineAssembler::CallCFunction0(MachineType return_type,
592 : Node* function) {
593 : MachineSignature::Builder builder(zone(), 1, 0);
594 : builder.AddReturn(return_type);
595 : auto call_descriptor =
596 10 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
597 :
598 10 : return AddNode(common()->Call(call_descriptor), function);
599 : }
600 :
601 :
602 634 : Node* RawMachineAssembler::CallCFunction1(MachineType return_type,
603 : MachineType arg0_type, Node* function,
604 : Node* arg0) {
605 : MachineSignature::Builder builder(zone(), 1, 1);
606 : builder.AddReturn(return_type);
607 : builder.AddParam(arg0_type);
608 : auto call_descriptor =
609 1268 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
610 :
611 1268 : return AddNode(common()->Call(call_descriptor), function, arg0);
612 : }
613 :
614 224 : Node* RawMachineAssembler::CallCFunction1WithCallerSavedRegisters(
615 : MachineType return_type, MachineType arg0_type, Node* function, Node* arg0,
616 : SaveFPRegsMode mode) {
617 : MachineSignature::Builder builder(zone(), 1, 1);
618 : builder.AddReturn(return_type);
619 : builder.AddParam(arg0_type);
620 : auto call_descriptor =
621 448 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
622 :
623 : call_descriptor->set_save_fp_mode(mode);
624 :
625 : return AddNode(common()->CallWithCallerSavedRegisters(call_descriptor),
626 448 : function, arg0);
627 : }
628 :
629 4500 : Node* RawMachineAssembler::CallCFunction2(MachineType return_type,
630 : MachineType arg0_type,
631 : MachineType arg1_type, Node* function,
632 : Node* arg0, Node* arg1) {
633 : MachineSignature::Builder builder(zone(), 1, 2);
634 : builder.AddReturn(return_type);
635 : builder.AddParam(arg0_type);
636 : builder.AddParam(arg1_type);
637 : auto call_descriptor =
638 9000 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
639 :
640 9000 : return AddNode(common()->Call(call_descriptor), function, arg0, arg1);
641 : }
642 :
643 6326 : Node* RawMachineAssembler::CallCFunction3(MachineType return_type,
644 : MachineType arg0_type,
645 : MachineType arg1_type,
646 : MachineType arg2_type, Node* function,
647 : Node* arg0, Node* arg1, Node* arg2) {
648 : MachineSignature::Builder builder(zone(), 1, 3);
649 : builder.AddReturn(return_type);
650 : builder.AddParam(arg0_type);
651 : builder.AddParam(arg1_type);
652 : builder.AddParam(arg2_type);
653 : auto call_descriptor =
654 12652 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
655 :
656 12652 : return AddNode(common()->Call(call_descriptor), function, arg0, arg1, arg2);
657 : }
658 :
659 117 : Node* RawMachineAssembler::CallCFunction3WithCallerSavedRegisters(
660 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
661 : MachineType arg2_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
662 : SaveFPRegsMode mode) {
663 : MachineSignature::Builder builder(zone(), 1, 3);
664 : builder.AddReturn(return_type);
665 : builder.AddParam(arg0_type);
666 : builder.AddParam(arg1_type);
667 : builder.AddParam(arg2_type);
668 : auto call_descriptor =
669 234 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
670 :
671 : call_descriptor->set_save_fp_mode(mode);
672 :
673 : return AddNode(common()->CallWithCallerSavedRegisters(call_descriptor),
674 234 : function, arg0, arg1, arg2);
675 : }
676 :
677 112 : Node* RawMachineAssembler::CallCFunction4(
678 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
679 : MachineType arg2_type, MachineType arg3_type, Node* function, Node* arg0,
680 : Node* arg1, Node* arg2, Node* arg3) {
681 : MachineSignature::Builder builder(zone(), 1, 4);
682 : builder.AddReturn(return_type);
683 : builder.AddParam(arg0_type);
684 : builder.AddParam(arg1_type);
685 : builder.AddParam(arg2_type);
686 : builder.AddParam(arg3_type);
687 : auto call_descriptor =
688 224 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
689 :
690 : return AddNode(common()->Call(call_descriptor), function, arg0, arg1, arg2,
691 224 : arg3);
692 : }
693 :
694 280 : Node* RawMachineAssembler::CallCFunction5(
695 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
696 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
697 : Node* function, Node* arg0, Node* arg1, Node* arg2, Node* arg3,
698 : Node* arg4) {
699 : MachineSignature::Builder builder(zone(), 1, 5);
700 : builder.AddReturn(return_type);
701 : builder.AddParam(arg0_type);
702 : builder.AddParam(arg1_type);
703 : builder.AddParam(arg2_type);
704 : builder.AddParam(arg3_type);
705 : builder.AddParam(arg4_type);
706 : auto call_descriptor =
707 560 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
708 :
709 : return AddNode(common()->Call(call_descriptor), function, arg0, arg1, arg2,
710 560 : arg3, arg4);
711 : }
712 :
713 672 : Node* RawMachineAssembler::CallCFunction6(
714 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
715 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
716 : MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
717 : Node* arg3, Node* arg4, Node* arg5) {
718 : MachineSignature::Builder builder(zone(), 1, 6);
719 : builder.AddReturn(return_type);
720 : builder.AddParam(arg0_type);
721 : builder.AddParam(arg1_type);
722 : builder.AddParam(arg2_type);
723 : builder.AddParam(arg3_type);
724 : builder.AddParam(arg4_type);
725 : builder.AddParam(arg5_type);
726 : auto call_descriptor =
727 1344 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
728 :
729 : return AddNode(common()->Call(call_descriptor), function, arg0, arg1, arg2,
730 1344 : arg3, arg4, arg5);
731 : }
732 :
733 5 : Node* RawMachineAssembler::CallCFunction8(
734 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
735 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
736 : MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
737 : Node* function, Node* arg0, Node* arg1, Node* arg2, Node* arg3, Node* arg4,
738 : Node* arg5, Node* arg6, Node* arg7) {
739 : MachineSignature::Builder builder(zone(), 1, 8);
740 : builder.AddReturn(return_type);
741 : builder.AddParam(arg0_type);
742 : builder.AddParam(arg1_type);
743 : builder.AddParam(arg2_type);
744 : builder.AddParam(arg3_type);
745 : builder.AddParam(arg4_type);
746 : builder.AddParam(arg5_type);
747 : builder.AddParam(arg6_type);
748 : builder.AddParam(arg7_type);
749 5 : Node* args[] = {function, arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7};
750 : auto call_descriptor =
751 10 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
752 5 : return AddNode(common()->Call(call_descriptor), arraysize(args), args);
753 : }
754 :
755 570 : Node* RawMachineAssembler::CallCFunction9(
756 : MachineType return_type, MachineType arg0_type, MachineType arg1_type,
757 : MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
758 : MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
759 : MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
760 : Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
761 : MachineSignature::Builder builder(zone(), 1, 9);
762 : builder.AddReturn(return_type);
763 : builder.AddParam(arg0_type);
764 : builder.AddParam(arg1_type);
765 : builder.AddParam(arg2_type);
766 : builder.AddParam(arg3_type);
767 : builder.AddParam(arg4_type);
768 : builder.AddParam(arg5_type);
769 : builder.AddParam(arg6_type);
770 : builder.AddParam(arg7_type);
771 : builder.AddParam(arg8_type);
772 : Node* args[] = {function, arg0, arg1, arg2, arg3,
773 570 : arg4, arg5, arg6, arg7, arg8};
774 : auto call_descriptor =
775 1140 : Linkage::GetSimplifiedCDescriptor(zone(), builder.Build());
776 570 : return AddNode(common()->Call(call_descriptor), arraysize(args), args);
777 : }
778 :
779 0 : BasicBlock* RawMachineAssembler::Use(RawMachineLabel* label) {
780 4598968 : label->used_ = true;
781 0 : return EnsureBlock(label);
782 : }
783 :
784 3380264 : BasicBlock* RawMachineAssembler::EnsureBlock(RawMachineLabel* label) {
785 7979232 : if (label->block_ == nullptr) {
786 3380264 : label->block_ = schedule()->NewBasicBlock();
787 : }
788 7979232 : return label->block_;
789 : }
790 :
791 3380264 : void RawMachineAssembler::Bind(RawMachineLabel* label) {
792 : DCHECK_NULL(current_block_);
793 : DCHECK(!label->bound_);
794 3380264 : label->bound_ = true;
795 3380264 : current_block_ = EnsureBlock(label);
796 3380264 : current_block_->set_deferred(label->deferred_);
797 3380264 : }
798 :
799 : #if DEBUG
800 : void RawMachineAssembler::Bind(RawMachineLabel* label,
801 : AssemblerDebugInfo info) {
802 : if (current_block_ != nullptr) {
803 : std::stringstream str;
804 : str << "Binding label without closing previous block:"
805 : << "\n# label: " << info
806 : << "\n# previous block: " << *current_block_;
807 : FATAL("%s", str.str().c_str());
808 : }
809 : Bind(label);
810 : current_block_->set_debug_info(info);
811 : }
812 :
813 : void RawMachineAssembler::PrintCurrentBlock(std::ostream& os) {
814 : os << CurrentBlock();
815 : }
816 :
817 : void RawMachineAssembler::SetInitialDebugInformation(
818 : AssemblerDebugInfo debug_info) {
819 : CurrentBlock()->set_debug_info(debug_info);
820 : }
821 : #endif // DEBUG
822 :
823 2184 : bool RawMachineAssembler::InsideBlock() { return current_block_ != nullptr; }
824 :
825 0 : BasicBlock* RawMachineAssembler::CurrentBlock() {
826 : DCHECK(current_block_);
827 24043822 : return current_block_;
828 : }
829 :
830 3988219 : Node* RawMachineAssembler::Phi(MachineRepresentation rep, int input_count,
831 3988219 : Node* const* inputs) {
832 3988219 : Node** buffer = new (zone()->New(sizeof(Node*) * (input_count + 1)))
833 3988219 : Node*[input_count + 1];
834 3988219 : std::copy(inputs, inputs + input_count, buffer);
835 7976438 : buffer[input_count] = graph()->start();
836 3988219 : return AddNode(common()->Phi(rep, input_count), input_count + 1, buffer);
837 : }
838 :
839 284557 : void RawMachineAssembler::AppendPhiInput(Node* phi, Node* new_input) {
840 : const Operator* op = phi->op();
841 284557 : const Operator* new_op = common()->ResizeMergeOrPhi(op, phi->InputCount());
842 569114 : phi->InsertInput(zone(), phi->InputCount() - 1, new_input);
843 284557 : NodeProperties::ChangeOp(phi, new_op);
844 284557 : }
845 :
846 20369781 : Node* RawMachineAssembler::AddNode(const Operator* op, int input_count,
847 20369781 : Node* const* inputs) {
848 : DCHECK_NOT_NULL(schedule_);
849 : DCHECK_NOT_NULL(current_block_);
850 : Node* node = MakeNode(op, input_count, inputs);
851 20369781 : schedule()->AddNode(CurrentBlock(), node);
852 20369781 : return node;
853 : }
854 :
855 0 : Node* RawMachineAssembler::MakeNode(const Operator* op, int input_count,
856 24657081 : Node* const* inputs) {
857 : // The raw machine assembler nodes do not have effect and control inputs,
858 : // so we disable checking input counts here.
859 24657081 : return graph()->NewNodeUnchecked(op, input_count, inputs);
860 : }
861 :
862 3485240 : RawMachineLabel::~RawMachineLabel() {
863 : #if DEBUG
864 : if (bound_ == used_) return;
865 : std::stringstream str;
866 : if (bound_) {
867 : str << "A label has been bound but it's not used."
868 : << "\n# label: " << *block_;
869 : } else {
870 : str << "A label has been used but it's not bound.";
871 : }
872 : FATAL("%s", str.str().c_str());
873 : #endif // DEBUG
874 3485240 : }
875 :
876 : } // namespace compiler
877 : } // namespace internal
878 183867 : } // namespace v8
|
