Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/raw-machine-assembler.h"
6 :
7 : #include "src/base/small-vector.h"
8 : #include "src/compiler/compiler-source-position-table.h"
9 : #include "src/compiler/node-properties.h"
10 : #include "src/compiler/pipeline.h"
11 : #include "src/compiler/scheduler.h"
12 : #include "src/heap/factory-inl.h"
13 :
14 : namespace v8 {
15 : namespace internal {
16 : namespace compiler {
17 :
18 251238 : RawMachineAssembler::RawMachineAssembler(
19 : Isolate* isolate, Graph* graph, CallDescriptor* call_descriptor,
20 : MachineRepresentation word, MachineOperatorBuilder::Flags flags,
21 : MachineOperatorBuilder::AlignmentRequirements alignment_requirements,
22 : PoisoningMitigationLevel poisoning_level)
23 : : isolate_(isolate),
24 : graph_(graph),
25 251238 : schedule_(new (zone()) Schedule(zone())),
26 251238 : source_positions_(new (zone()) SourcePositionTable(graph)),
27 : machine_(zone(), word, flags, alignment_requirements),
28 : common_(zone()),
29 : simplified_(zone()),
30 : call_descriptor_(call_descriptor),
31 : target_parameter_(nullptr),
32 : parameters_(parameter_count(), zone()),
33 : current_block_(schedule()->start()),
34 1004952 : poisoning_level_(poisoning_level) {
35 251238 : int param_count = static_cast<int>(parameter_count());
36 : // Add an extra input for the JSFunction parameter to the start node.
37 251238 : graph->SetStart(graph->NewNode(common_.Start(param_count + 1)));
38 251238 : if (call_descriptor->IsJSFunctionCall()) {
39 20136 : target_parameter_ = AddNode(
40 20136 : common()->Parameter(Linkage::kJSCallClosureParamIndex), graph->start());
41 : }
42 1263456 : for (size_t i = 0; i < parameter_count(); ++i) {
43 : parameters_[i] =
44 1012218 : AddNode(common()->Parameter(static_cast<int>(i)), graph->start());
45 : }
46 251238 : graph->SetEnd(graph->NewNode(common_.End(0)));
47 251238 : source_positions_->AddDecorator();
48 251238 : }
49 :
50 859504 : void RawMachineAssembler::SetSourcePosition(const char* file, int line) {
51 859504 : int file_id = isolate()->LookupOrAddExternallyCompiledFilename(file);
52 : SourcePosition p = SourcePosition::External(line, file_id);
53 : DCHECK(p.ExternalLine() == line);
54 : source_positions()->SetCurrentPosition(p);
55 859504 : }
56 :
57 0 : Node* RawMachineAssembler::NullConstant() {
58 0 : return HeapConstant(isolate()->factory()->null_value());
59 : }
60 :
61 5 : Node* RawMachineAssembler::UndefinedConstant() {
62 5 : return HeapConstant(isolate()->factory()->undefined_value());
63 : }
64 :
65 0 : Node* RawMachineAssembler::RelocatableIntPtrConstant(intptr_t value,
66 : RelocInfo::Mode rmode) {
67 : return kSystemPointerSize == 8
68 : ? RelocatableInt64Constant(value, rmode)
69 0 : : RelocatableInt32Constant(static_cast<int>(value), rmode);
70 : }
71 :
72 57532 : Node* RawMachineAssembler::OptimizedAllocate(Node* size,
73 : AllocationType allocation) {
74 115064 : return AddNode(simplified()->AllocateRaw(Type::Any(), allocation), size);
75 : }
76 :
77 181535 : Schedule* RawMachineAssembler::Export() {
78 : // Compute the correct codegen order.
79 : DCHECK(schedule_->rpo_order()->empty());
80 181535 : if (FLAG_trace_turbo_scheduler) {
81 0 : PrintF("--- RAW SCHEDULE -------------------------------------------\n");
82 0 : StdoutStream{} << *schedule_;
83 : }
84 181535 : schedule_->EnsureCFGWellFormedness();
85 181535 : Scheduler::ComputeSpecialRPO(zone(), schedule_);
86 181535 : schedule_->PropagateDeferredMark();
87 181535 : if (FLAG_trace_turbo_scheduler) {
88 0 : PrintF("--- EDGE SPLIT AND PROPAGATED DEFERRED SCHEDULE ------------\n");
89 0 : StdoutStream{} << *schedule_;
90 : }
91 : // Invalidate RawMachineAssembler.
92 181535 : source_positions_->RemoveDecorator();
93 181535 : Schedule* schedule = schedule_;
94 181535 : schedule_ = nullptr;
95 181535 : return schedule;
96 : }
97 :
98 67324 : Graph* RawMachineAssembler::ExportForOptimization() {
99 : // Compute the correct codegen order.
100 : DCHECK(schedule_->rpo_order()->empty());
101 67324 : if (FLAG_trace_turbo_scheduler) {
102 0 : PrintF("--- RAW SCHEDULE -------------------------------------------\n");
103 0 : StdoutStream{} << *schedule_;
104 : }
105 67324 : schedule_->EnsureCFGWellFormedness();
106 67324 : Scheduler::ComputeSpecialRPO(zone(), schedule_);
107 67324 : if (FLAG_trace_turbo_scheduler) {
108 0 : PrintF("--- SCHEDULE BEFORE GRAPH CREATION -------------------------\n");
109 0 : StdoutStream{} << *schedule_;
110 : }
111 67324 : MakeReschedulable();
112 : // Invalidate RawMachineAssembler.
113 67324 : schedule_ = nullptr;
114 67324 : return graph();
115 : }
116 :
117 67324 : void RawMachineAssembler::MakeReschedulable() {
118 134648 : std::vector<Node*> block_final_control(schedule_->all_blocks_.size());
119 134648 : std::vector<Node*> block_final_effect(schedule_->all_blocks_.size());
120 :
121 : struct LoopHeader {
122 : BasicBlock* block;
123 : Node* loop_node;
124 : Node* effect_phi;
125 : };
126 : std::vector<LoopHeader> loop_headers;
127 :
128 : // These are hoisted outside of the loop to avoid re-allocation.
129 : std::vector<Node*> merge_inputs;
130 : std::vector<Node*> effect_phi_inputs;
131 :
132 6504698 : for (BasicBlock* block : *schedule_->rpo_order()) {
133 : Node* current_control;
134 : Node* current_effect;
135 6370050 : if (block == schedule_->start()) {
136 67324 : current_control = current_effect = graph()->start();
137 6302726 : } else if (block == schedule_->end()) {
138 477424 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
139 : NodeProperties::MergeControlToEnd(
140 205052 : graph(), common(), block->PredecessorAt(i)->control_input());
141 : }
142 6235406 : } else if (block->IsLoopHeader()) {
143 : // The graph()->start() inputs are just placeholders until we computed the
144 : // real back-edges and re-structure the control flow so the loop has
145 : // exactly two predecessors.
146 90808 : current_control = graph()->NewNode(common()->Loop(2), graph()->start(),
147 90808 : graph()->start());
148 : current_effect =
149 90808 : graph()->NewNode(common()->EffectPhi(2), graph()->start(),
150 90808 : graph()->start(), current_control);
151 :
152 90808 : Node* terminate = graph()->NewNode(common()->Terminate(), current_effect,
153 : current_control);
154 90808 : NodeProperties::MergeControlToEnd(graph(), common(), terminate);
155 181616 : loop_headers.push_back(
156 : LoopHeader{block, current_control, current_effect});
157 6144598 : } else if (block->PredecessorCount() == 1) {
158 : BasicBlock* predecessor = block->PredecessorAt(0);
159 : DCHECK_LT(predecessor->rpo_number(), block->rpo_number());
160 5508671 : current_effect = block_final_effect[predecessor->id().ToSize()];
161 5508671 : current_control = block_final_control[predecessor->id().ToSize()];
162 : } else {
163 : // Create control merge nodes and effect phis for all predecessor blocks.
164 : merge_inputs.clear();
165 : effect_phi_inputs.clear();
166 635927 : int predecessor_count = static_cast<int>(block->PredecessorCount());
167 4150195 : for (int i = 0; i < predecessor_count; ++i) {
168 1757134 : BasicBlock* predecessor = block->PredecessorAt(i);
169 : DCHECK_LT(predecessor->rpo_number(), block->rpo_number());
170 1757134 : merge_inputs.push_back(block_final_control[predecessor->id().ToSize()]);
171 : effect_phi_inputs.push_back(
172 1757134 : block_final_effect[predecessor->id().ToSize()]);
173 : }
174 1271854 : current_control = graph()->NewNode(common()->Merge(predecessor_count),
175 : static_cast<int>(merge_inputs.size()),
176 635927 : merge_inputs.data());
177 635927 : effect_phi_inputs.push_back(current_control);
178 635927 : current_effect = graph()->NewNode(
179 : common()->EffectPhi(predecessor_count),
180 635927 : static_cast<int>(effect_phi_inputs.size()), effect_phi_inputs.data());
181 : }
182 :
183 18764937 : auto update_current_control_and_effect = [&](Node* node) {
184 : bool existing_effect_and_control =
185 34841268 : IrOpcode::IsIfProjectionOpcode(node->opcode()) ||
186 : IrOpcode::IsPhiOpcode(node->opcode());
187 18764937 : if (node->op()->EffectInputCount() > 0) {
188 : DCHECK_EQ(1, node->op()->EffectInputCount());
189 3560912 : if (existing_effect_and_control) {
190 3368716 : NodeProperties::ReplaceEffectInput(node, current_effect);
191 : } else {
192 8422711 : node->AppendInput(graph()->zone(), current_effect);
193 : }
194 : }
195 18764937 : if (node->op()->ControlInputCount() > 0) {
196 : DCHECK_EQ(1, node->op()->ControlInputCount());
197 8152010 : if (existing_effect_and_control) {
198 7779564 : NodeProperties::ReplaceControlInput(node, current_control);
199 : } else {
200 9749310 : node->AppendInput(graph()->zone(), current_control);
201 : }
202 : }
203 18764937 : if (node->op()->EffectOutputCount() > 0) {
204 : DCHECK_EQ(1, node->op()->EffectOutputCount());
205 3355860 : current_effect = node;
206 : }
207 18764937 : if (node->op()->ControlOutputCount() > 0) {
208 4502209 : current_control = node;
209 : }
210 25134987 : };
211 :
212 23619584 : for (Node* node : *block) {
213 17249534 : update_current_control_and_effect(node);
214 : }
215 6370050 : if (block->deferred()) MarkControlDeferred(current_control);
216 :
217 6370050 : if (Node* block_terminator = block->control_input()) {
218 1515403 : update_current_control_and_effect(block_terminator);
219 : }
220 :
221 6370050 : block_final_effect[block->id().ToSize()] = current_effect;
222 6370050 : block_final_control[block->id().ToSize()] = current_control;
223 : }
224 :
225 : // Fix-up loop backedges and re-structure control flow so that loop nodes have
226 : // exactly two control predecessors.
227 158132 : for (const LoopHeader& loop_header : loop_headers) {
228 90808 : BasicBlock* block = loop_header.block;
229 : std::vector<BasicBlock*> loop_entries;
230 : std::vector<BasicBlock*> loop_backedges;
231 511064 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
232 210128 : BasicBlock* predecessor = block->PredecessorAt(i);
233 210128 : if (block->LoopContains(predecessor)) {
234 118984 : loop_backedges.push_back(predecessor);
235 : } else {
236 : DCHECK(loop_backedges.empty());
237 91144 : loop_entries.push_back(predecessor);
238 : }
239 : }
240 : DCHECK(!loop_entries.empty());
241 : DCHECK(!loop_backedges.empty());
242 :
243 90808 : int entrance_count = static_cast<int>(loop_entries.size());
244 90808 : int backedge_count = static_cast<int>(loop_backedges.size());
245 181616 : Node* control_loop_entry = CreateNodeFromPredecessors(
246 90808 : loop_entries, block_final_control, common()->Merge(entrance_count), {});
247 : Node* control_backedge =
248 181616 : CreateNodeFromPredecessors(loop_backedges, block_final_control,
249 90808 : common()->Merge(backedge_count), {});
250 272424 : Node* effect_loop_entry = CreateNodeFromPredecessors(
251 : loop_entries, block_final_effect, common()->EffectPhi(entrance_count),
252 90808 : {control_loop_entry});
253 272424 : Node* effect_backedge = CreateNodeFromPredecessors(
254 : loop_backedges, block_final_effect, common()->EffectPhi(backedge_count),
255 90808 : {control_backedge});
256 :
257 90808 : loop_header.loop_node->ReplaceInput(0, control_loop_entry);
258 90808 : loop_header.loop_node->ReplaceInput(1, control_backedge);
259 90808 : loop_header.effect_phi->ReplaceInput(0, effect_loop_entry);
260 90808 : loop_header.effect_phi->ReplaceInput(1, effect_backedge);
261 :
262 798384 : for (Node* node : *block) {
263 707576 : if (node->opcode() == IrOpcode::kPhi) {
264 144288 : MakePhiBinary(node, static_cast<int>(loop_entries.size()),
265 144288 : control_loop_entry, control_backedge);
266 : }
267 : }
268 : }
269 67324 : }
270 :
271 363232 : Node* RawMachineAssembler::CreateNodeFromPredecessors(
272 : const std::vector<BasicBlock*>& predecessors,
273 : const std::vector<Node*>& sidetable, const Operator* op,
274 : const std::vector<Node*>& additional_inputs) {
275 363232 : if (predecessors.size() == 1) {
276 664384 : return sidetable[predecessors.front()->id().ToSize()];
277 : }
278 : std::vector<Node*> inputs;
279 119104 : for (BasicBlock* predecessor : predecessors) {
280 88064 : inputs.push_back(sidetable[predecessor->id().ToSize()]);
281 : }
282 46560 : for (Node* additional_input : additional_inputs) {
283 15520 : inputs.push_back(additional_input);
284 : }
285 31040 : return graph()->NewNode(op, static_cast<int>(inputs.size()), inputs.data());
286 : }
287 :
288 144288 : void RawMachineAssembler::MakePhiBinary(Node* phi, int split_point,
289 : Node* left_control,
290 : Node* right_control) {
291 : int value_count = phi->op()->ValueInputCount();
292 144288 : if (value_count == 2) return;
293 : DCHECK_LT(split_point, value_count);
294 : DCHECK_GT(split_point, 0);
295 :
296 33280 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
297 : int left_input_count = split_point;
298 33280 : int right_input_count = value_count - split_point;
299 :
300 : Node* left_input;
301 33280 : if (left_input_count == 1) {
302 32720 : left_input = NodeProperties::GetValueInput(phi, 0);
303 : } else {
304 : std::vector<Node*> inputs;
305 2800 : for (int i = 0; i < left_input_count; ++i) {
306 2240 : inputs.push_back(NodeProperties::GetValueInput(phi, i));
307 : }
308 560 : inputs.push_back(left_control);
309 : left_input =
310 1120 : graph()->NewNode(common()->Phi(rep, static_cast<int>(left_input_count)),
311 560 : static_cast<int>(inputs.size()), inputs.data());
312 : }
313 :
314 : Node* right_input;
315 33280 : if (right_input_count == 1) {
316 560 : right_input = NodeProperties::GetValueInput(phi, split_point);
317 : } else {
318 : std::vector<Node*> inputs;
319 236064 : for (int i = split_point; i < value_count; ++i) {
320 203344 : inputs.push_back(NodeProperties::GetValueInput(phi, i));
321 : }
322 32720 : inputs.push_back(right_control);
323 65440 : right_input = graph()->NewNode(
324 : common()->Phi(rep, static_cast<int>(right_input_count)),
325 32720 : static_cast<int>(inputs.size()), inputs.data());
326 : }
327 :
328 33280 : Node* control = NodeProperties::GetControlInput(phi);
329 33280 : phi->TrimInputCount(3);
330 33280 : phi->ReplaceInput(0, left_input);
331 33280 : phi->ReplaceInput(1, right_input);
332 33280 : phi->ReplaceInput(2, control);
333 33280 : NodeProperties::ChangeOp(phi, common()->Phi(rep, 2));
334 : }
335 :
336 553708 : void RawMachineAssembler::MarkControlDeferred(Node* control_node) {
337 : BranchHint new_branch_hint;
338 : Node* responsible_branch = nullptr;
339 1135992 : while (responsible_branch == nullptr) {
340 700656 : switch (control_node->opcode()) {
341 : case IrOpcode::kIfException:
342 : // IfException projections are deferred by default.
343 : return;
344 : case IrOpcode::kIfSuccess:
345 1008 : control_node = NodeProperties::GetControlInput(control_node);
346 1008 : continue;
347 : case IrOpcode::kIfValue: {
348 4312 : IfValueParameters parameters = IfValueParametersOf(control_node->op());
349 4312 : if (parameters.hint() != BranchHint::kFalse) {
350 4312 : NodeProperties::ChangeOp(
351 : control_node, common()->IfValue(parameters.value(),
352 : parameters.comparison_order(),
353 4312 : BranchHint::kFalse));
354 : }
355 : return;
356 : }
357 : case IrOpcode::kIfDefault:
358 7896 : if (BranchHintOf(control_node->op()) != BranchHint::kFalse) {
359 7896 : NodeProperties::ChangeOp(control_node,
360 7896 : common()->IfDefault(BranchHint::kFalse));
361 : }
362 : return;
363 : case IrOpcode::kIfTrue: {
364 209072 : Node* branch = NodeProperties::GetControlInput(control_node);
365 209072 : BranchHint hint = BranchOperatorInfoOf(branch->op()).hint;
366 209072 : if (hint == BranchHint::kTrue) {
367 : // The other possibility is also deferred, so the responsible branch
368 : // has to be before.
369 7728 : control_node = NodeProperties::GetControlInput(branch);
370 7728 : continue;
371 : }
372 : new_branch_hint = BranchHint::kFalse;
373 : responsible_branch = branch;
374 : break;
375 : }
376 : case IrOpcode::kIfFalse: {
377 242004 : Node* branch = NodeProperties::GetControlInput(control_node);
378 242004 : BranchHint hint = BranchOperatorInfoOf(branch->op()).hint;
379 242004 : if (hint == BranchHint::kFalse) {
380 : // The other possibility is also deferred, so the responsible branch
381 : // has to be before.
382 8012 : control_node = NodeProperties::GetControlInput(branch);
383 8012 : continue;
384 : }
385 : new_branch_hint = BranchHint::kTrue;
386 : responsible_branch = branch;
387 : break;
388 : }
389 : case IrOpcode::kMerge:
390 597272 : for (int i = 0; i < control_node->op()->ControlInputCount(); ++i) {
391 266476 : MarkControlDeferred(NodeProperties::GetControlInput(control_node, i));
392 : }
393 : return;
394 : case IrOpcode::kLoop:
395 0 : control_node = NodeProperties::GetControlInput(control_node, 0);
396 0 : continue;
397 : case IrOpcode::kBranch:
398 : case IrOpcode::kSwitch:
399 0 : UNREACHABLE();
400 : case IrOpcode::kStart:
401 : return;
402 : default:
403 : DCHECK_EQ(1, control_node->op()->ControlInputCount());
404 130200 : control_node = NodeProperties::GetControlInput(control_node);
405 130200 : continue;
406 : }
407 : }
408 :
409 435336 : BranchOperatorInfo info = BranchOperatorInfoOf(responsible_branch->op());
410 435336 : if (info.hint == new_branch_hint) return;
411 432200 : NodeProperties::ChangeOp(
412 : responsible_branch,
413 432200 : common()->Branch(new_branch_hint, info.is_safety_check));
414 : }
415 :
416 336 : Node* RawMachineAssembler::TargetParameter() {
417 : DCHECK_NOT_NULL(target_parameter_);
418 336 : return target_parameter_;
419 : }
420 :
421 349208 : Node* RawMachineAssembler::Parameter(size_t index) {
422 : DCHECK_LT(index, parameter_count());
423 349208 : return parameters_[index];
424 : }
425 :
426 :
427 2074157 : void RawMachineAssembler::Goto(RawMachineLabel* label) {
428 : DCHECK(current_block_ != schedule()->end());
429 2074157 : schedule()->AddGoto(CurrentBlock(), Use(label));
430 2074157 : current_block_ = nullptr;
431 2074157 : }
432 :
433 :
434 1296303 : void RawMachineAssembler::Branch(Node* condition, RawMachineLabel* true_val,
435 : RawMachineLabel* false_val) {
436 : DCHECK(current_block_ != schedule()->end());
437 1296303 : Node* branch = MakeNode(
438 : common()->Branch(BranchHint::kNone, IsSafetyCheck::kNoSafetyCheck), 1,
439 1296303 : &condition);
440 1296303 : BasicBlock* true_block = schedule()->NewBasicBlock();
441 1296303 : BasicBlock* false_block = schedule()->NewBasicBlock();
442 2592606 : schedule()->AddBranch(CurrentBlock(), branch, true_block, false_block);
443 :
444 2592606 : true_block->AddNode(MakeNode(common()->IfTrue(), 1, &branch));
445 1296303 : schedule()->AddGoto(true_block, Use(true_val));
446 :
447 2592606 : false_block->AddNode(MakeNode(common()->IfFalse(), 1, &branch));
448 1296303 : schedule()->AddGoto(false_block, Use(false_val));
449 :
450 1296303 : current_block_ = nullptr;
451 1296303 : }
452 :
453 12856 : void RawMachineAssembler::Continuations(Node* call, RawMachineLabel* if_success,
454 : RawMachineLabel* if_exception) {
455 : DCHECK_NOT_NULL(schedule_);
456 : DCHECK_NOT_NULL(current_block_);
457 12856 : schedule()->AddCall(CurrentBlock(), call, Use(if_success), Use(if_exception));
458 12856 : current_block_ = nullptr;
459 12856 : }
460 :
461 10904 : void RawMachineAssembler::Switch(Node* index, RawMachineLabel* default_label,
462 : const int32_t* case_values,
463 : RawMachineLabel** case_labels,
464 : size_t case_count) {
465 : DCHECK_NE(schedule()->end(), current_block_);
466 10904 : size_t succ_count = case_count + 1;
467 10904 : Node* switch_node = MakeNode(common()->Switch(succ_count), 1, &index);
468 : BasicBlock** succ_blocks = zone()->NewArray<BasicBlock*>(succ_count);
469 172640 : for (size_t index = 0; index < case_count; ++index) {
470 80868 : int32_t case_value = case_values[index];
471 80868 : BasicBlock* case_block = schedule()->NewBasicBlock();
472 : Node* case_node =
473 80868 : graph()->NewNode(common()->IfValue(case_value), switch_node);
474 80868 : schedule()->AddNode(case_block, case_node);
475 161736 : schedule()->AddGoto(case_block, Use(case_labels[index]));
476 80868 : succ_blocks[index] = case_block;
477 : }
478 10904 : BasicBlock* default_block = schedule()->NewBasicBlock();
479 10904 : Node* default_node = graph()->NewNode(common()->IfDefault(), switch_node);
480 10904 : schedule()->AddNode(default_block, default_node);
481 10904 : schedule()->AddGoto(default_block, Use(default_label));
482 10904 : succ_blocks[case_count] = default_block;
483 10904 : schedule()->AddSwitch(CurrentBlock(), switch_node, succ_blocks, succ_count);
484 10904 : current_block_ = nullptr;
485 10904 : }
486 :
487 252036 : void RawMachineAssembler::Return(Node* value) {
488 252036 : Node* values[] = {Int32Constant(0), value};
489 252036 : Node* ret = MakeNode(common()->Return(1), 2, values);
490 252036 : schedule()->AddReturn(CurrentBlock(), ret);
491 252036 : current_block_ = nullptr;
492 252036 : }
493 :
494 0 : void RawMachineAssembler::Return(Node* v1, Node* v2) {
495 0 : Node* values[] = {Int32Constant(0), v1, v2};
496 0 : Node* ret = MakeNode(common()->Return(2), 3, values);
497 0 : schedule()->AddReturn(CurrentBlock(), ret);
498 0 : current_block_ = nullptr;
499 0 : }
500 :
501 0 : void RawMachineAssembler::Return(Node* v1, Node* v2, Node* v3) {
502 0 : Node* values[] = {Int32Constant(0), v1, v2, v3};
503 0 : Node* ret = MakeNode(common()->Return(3), 4, values);
504 0 : schedule()->AddReturn(CurrentBlock(), ret);
505 0 : current_block_ = nullptr;
506 0 : }
507 :
508 0 : void RawMachineAssembler::Return(Node* v1, Node* v2, Node* v3, Node* v4) {
509 0 : Node* values[] = {Int32Constant(0), v1, v2, v3, v4};
510 0 : Node* ret = MakeNode(common()->Return(4), 5, values);
511 0 : schedule()->AddReturn(CurrentBlock(), ret);
512 0 : current_block_ = nullptr;
513 0 : }
514 :
515 753 : void RawMachineAssembler::Return(int count, Node* vs[]) {
516 : using Node_ptr = Node*;
517 753 : Node** values = new Node_ptr[count + 1];
518 753 : values[0] = Int32Constant(0);
519 16853 : for (int i = 0; i < count; ++i) values[i + 1] = vs[i];
520 753 : Node* ret = MakeNode(common()->Return(count), count + 1, values);
521 753 : schedule()->AddReturn(CurrentBlock(), ret);
522 753 : current_block_ = nullptr;
523 753 : delete[] values;
524 753 : }
525 :
526 13352 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* value) {
527 13352 : Node* values[] = {pop, value};
528 13352 : Node* ret = MakeNode(common()->Return(1), 2, values);
529 13352 : schedule()->AddReturn(CurrentBlock(), ret);
530 13352 : current_block_ = nullptr;
531 13352 : }
532 :
533 0 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* v1, Node* v2) {
534 0 : Node* values[] = {pop, v1, v2};
535 0 : Node* ret = MakeNode(common()->Return(2), 3, values);
536 0 : schedule()->AddReturn(CurrentBlock(), ret);
537 0 : current_block_ = nullptr;
538 0 : }
539 :
540 0 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* v1, Node* v2,
541 : Node* v3) {
542 0 : Node* values[] = {pop, v1, v2, v3};
543 0 : Node* ret = MakeNode(common()->Return(3), 4, values);
544 0 : schedule()->AddReturn(CurrentBlock(), ret);
545 0 : current_block_ = nullptr;
546 0 : }
547 :
548 0 : void RawMachineAssembler::PopAndReturn(Node* pop, Node* v1, Node* v2, Node* v3,
549 : Node* v4) {
550 0 : Node* values[] = {pop, v1, v2, v3, v4};
551 0 : Node* ret = MakeNode(common()->Return(4), 5, values);
552 0 : schedule()->AddReturn(CurrentBlock(), ret);
553 0 : current_block_ = nullptr;
554 0 : }
555 :
556 440 : void RawMachineAssembler::DebugAbort(Node* message) {
557 440 : AddNode(machine()->DebugAbort(), message);
558 440 : }
559 :
560 410220 : void RawMachineAssembler::DebugBreak() { AddNode(machine()->DebugBreak()); }
561 :
562 66720 : void RawMachineAssembler::Unreachable() {
563 66720 : Node* ret = MakeNode(common()->Throw(), 0, nullptr);
564 66720 : schedule()->AddThrow(CurrentBlock(), ret);
565 66720 : current_block_ = nullptr;
566 66720 : }
567 :
568 4 : void RawMachineAssembler::Comment(const std::string& msg) {
569 4 : size_t length = msg.length() + 1;
570 : char* zone_buffer = zone()->NewArray<char>(length);
571 : MemCopy(zone_buffer, msg.c_str(), length);
572 4 : AddNode(machine()->Comment(zone_buffer));
573 4 : }
574 :
575 264758 : Node* RawMachineAssembler::CallN(CallDescriptor* call_descriptor,
576 : int input_count, Node* const* inputs) {
577 : DCHECK(!call_descriptor->NeedsFrameState());
578 : // +1 is for target.
579 : DCHECK_EQ(input_count, call_descriptor->ParameterCount() + 1);
580 264758 : return AddNode(common()->Call(call_descriptor), input_count, inputs);
581 : }
582 :
583 3 : Node* RawMachineAssembler::CallNWithFrameState(CallDescriptor* call_descriptor,
584 : int input_count,
585 : Node* const* inputs) {
586 : DCHECK(call_descriptor->NeedsFrameState());
587 : // +2 is for target and frame state.
588 : DCHECK_EQ(input_count, call_descriptor->ParameterCount() + 2);
589 3 : return AddNode(common()->Call(call_descriptor), input_count, inputs);
590 : }
591 :
592 63104 : Node* RawMachineAssembler::TailCallN(CallDescriptor* call_descriptor,
593 : int input_count, Node* const* inputs) {
594 : // +1 is for target.
595 : DCHECK_EQ(input_count, call_descriptor->ParameterCount() + 1);
596 : Node* tail_call =
597 63104 : MakeNode(common()->TailCall(call_descriptor), input_count, inputs);
598 63104 : schedule()->AddTailCall(CurrentBlock(), tail_call);
599 63104 : current_block_ = nullptr;
600 63104 : return tail_call;
601 : }
602 :
603 : namespace {
604 :
605 13108 : Node* CallCFunctionImpl(
606 : RawMachineAssembler* rasm, Node* function, MachineType return_type,
607 : std::initializer_list<RawMachineAssembler::CFunctionArg> args,
608 : bool caller_saved_regs, SaveFPRegsMode mode) {
609 : static constexpr std::size_t kNumCArgs = 10;
610 :
611 : MachineSignature::Builder builder(rasm->zone(), 1, args.size());
612 : builder.AddReturn(return_type);
613 91644 : for (const auto& arg : args) builder.AddParam(arg.first);
614 :
615 : auto call_descriptor =
616 13108 : Linkage::GetSimplifiedCDescriptor(rasm->zone(), builder.Build());
617 :
618 13108 : if (caller_saved_regs) call_descriptor->set_save_fp_mode(mode);
619 :
620 13108 : base::SmallVector<Node*, kNumCArgs> nodes(args.size() + 1);
621 13108 : nodes[0] = function;
622 : std::transform(
623 : args.begin(), args.end(), std::next(nodes.begin()),
624 : [](const RawMachineAssembler::CFunctionArg& arg) { return arg.second; });
625 :
626 : auto common = rasm->common();
627 13108 : return rasm->AddNode(
628 : caller_saved_regs ? common->CallWithCallerSavedRegisters(call_descriptor)
629 : : common->Call(call_descriptor),
630 26216 : static_cast<int>(nodes.size()), nodes.begin());
631 : }
632 :
633 : } // namespace
634 :
635 12656 : Node* RawMachineAssembler::CallCFunction(
636 : Node* function, MachineType return_type,
637 : std::initializer_list<RawMachineAssembler::CFunctionArg> args) {
638 : return CallCFunctionImpl(this, function, return_type, args, false,
639 12656 : kDontSaveFPRegs);
640 : }
641 :
642 452 : Node* RawMachineAssembler::CallCFunctionWithCallerSavedRegisters(
643 : Node* function, MachineType return_type, SaveFPRegsMode mode,
644 : std::initializer_list<RawMachineAssembler::CFunctionArg> args) {
645 452 : return CallCFunctionImpl(this, function, return_type, args, true, mode);
646 : }
647 :
648 0 : BasicBlock* RawMachineAssembler::Use(RawMachineLabel* label) {
649 4784247 : label->used_ = true;
650 0 : return EnsureBlock(label);
651 : }
652 :
653 0 : BasicBlock* RawMachineAssembler::EnsureBlock(RawMachineLabel* label) {
654 8325569 : if (label->block_ == nullptr) {
655 3541322 : label->block_ = schedule()->NewBasicBlock();
656 : }
657 8325569 : return label->block_;
658 : }
659 :
660 3541322 : void RawMachineAssembler::Bind(RawMachineLabel* label) {
661 : DCHECK_NULL(current_block_);
662 : DCHECK(!label->bound_);
663 3541322 : label->bound_ = true;
664 3541322 : current_block_ = EnsureBlock(label);
665 3541322 : current_block_->set_deferred(label->deferred_);
666 3541322 : }
667 :
668 : #if DEBUG
669 : void RawMachineAssembler::Bind(RawMachineLabel* label,
670 : AssemblerDebugInfo info) {
671 : if (current_block_ != nullptr) {
672 : std::stringstream str;
673 : str << "Binding label without closing previous block:"
674 : << "\n# label: " << info
675 : << "\n# previous block: " << *current_block_;
676 : FATAL("%s", str.str().c_str());
677 : }
678 : Bind(label);
679 : current_block_->set_debug_info(info);
680 : }
681 :
682 : void RawMachineAssembler::PrintCurrentBlock(std::ostream& os) {
683 : os << CurrentBlock();
684 : }
685 :
686 : void RawMachineAssembler::SetInitialDebugInformation(
687 : AssemblerDebugInfo debug_info) {
688 : CurrentBlock()->set_debug_info(debug_info);
689 : }
690 : #endif // DEBUG
691 :
692 1808 : bool RawMachineAssembler::InsideBlock() { return current_block_ != nullptr; }
693 :
694 0 : BasicBlock* RawMachineAssembler::CurrentBlock() {
695 : DCHECK(current_block_);
696 24318729 : return current_block_;
697 : }
698 :
699 4540069 : Node* RawMachineAssembler::Phi(MachineRepresentation rep, int input_count,
700 : Node* const* inputs) {
701 4540069 : Node** buffer = new (zone()->New(sizeof(Node*) * (input_count + 1)))
702 4540069 : Node*[input_count + 1];
703 4540069 : std::copy(inputs, inputs + input_count, buffer);
704 4540069 : buffer[input_count] = graph()->start();
705 4540069 : return AddNode(common()->Phi(rep, input_count), input_count + 1, buffer);
706 : }
707 :
708 274120 : void RawMachineAssembler::AppendPhiInput(Node* phi, Node* new_input) {
709 : const Operator* op = phi->op();
710 274120 : const Operator* new_op = common()->ResizeMergeOrPhi(op, phi->InputCount());
711 274120 : phi->InsertInput(zone(), phi->InputCount() - 1, new_input);
712 274120 : NodeProperties::ChangeOp(phi, new_op);
713 274120 : }
714 :
715 20528544 : Node* RawMachineAssembler::AddNode(const Operator* op, int input_count,
716 : Node* const* inputs) {
717 : DCHECK_NOT_NULL(schedule_);
718 : DCHECK_NOT_NULL(current_block_);
719 : Node* node = MakeNode(op, input_count, inputs);
720 20528544 : schedule()->AddNode(CurrentBlock(), node);
721 20528544 : return node;
722 : }
723 :
724 0 : Node* RawMachineAssembler::MakeNode(const Operator* op, int input_count,
725 : Node* const* inputs) {
726 : // The raw machine assembler nodes do not have effect and control inputs,
727 : // so we disable checking input counts here.
728 24824322 : return graph()->NewNodeUnchecked(op, input_count, inputs);
729 : }
730 :
731 3640586 : RawMachineLabel::~RawMachineLabel() {
732 : #if DEBUG
733 : if (bound_ == used_) return;
734 : std::stringstream str;
735 : if (bound_) {
736 : str << "A label has been bound but it's not used."
737 : << "\n# label: " << *block_;
738 : } else {
739 : str << "A label has been used but it's not bound.";
740 : }
741 : FATAL("%s", str.str().c_str());
742 : #endif // DEBUG
743 3640586 : }
744 :
745 : } // namespace compiler
746 : } // namespace internal
747 122036 : } // namespace v8
|
