Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/pipeline.h"
6 :
7 : #include <fstream> // NOLINT(readability/streams)
8 : #include <memory>
9 : #include <sstream>
10 :
11 : #include "src/base/adapters.h"
12 : #include "src/base/platform/elapsed-timer.h"
13 : #include "src/compilation-info.h"
14 : #include "src/compiler.h"
15 : #include "src/compiler/ast-graph-builder.h"
16 : #include "src/compiler/ast-loop-assignment-analyzer.h"
17 : #include "src/compiler/basic-block-instrumentor.h"
18 : #include "src/compiler/branch-elimination.h"
19 : #include "src/compiler/bytecode-graph-builder.h"
20 : #include "src/compiler/checkpoint-elimination.h"
21 : #include "src/compiler/code-generator.h"
22 : #include "src/compiler/common-operator-reducer.h"
23 : #include "src/compiler/control-flow-optimizer.h"
24 : #include "src/compiler/dead-code-elimination.h"
25 : #include "src/compiler/effect-control-linearizer.h"
26 : #include "src/compiler/escape-analysis-reducer.h"
27 : #include "src/compiler/escape-analysis.h"
28 : #include "src/compiler/frame-elider.h"
29 : #include "src/compiler/graph-trimmer.h"
30 : #include "src/compiler/graph-visualizer.h"
31 : #include "src/compiler/instruction-selector.h"
32 : #include "src/compiler/instruction.h"
33 : #include "src/compiler/js-builtin-reducer.h"
34 : #include "src/compiler/js-call-reducer.h"
35 : #include "src/compiler/js-context-specialization.h"
36 : #include "src/compiler/js-create-lowering.h"
37 : #include "src/compiler/js-frame-specialization.h"
38 : #include "src/compiler/js-generic-lowering.h"
39 : #include "src/compiler/js-inlining-heuristic.h"
40 : #include "src/compiler/js-intrinsic-lowering.h"
41 : #include "src/compiler/js-native-context-specialization.h"
42 : #include "src/compiler/js-typed-lowering.h"
43 : #include "src/compiler/jump-threading.h"
44 : #include "src/compiler/live-range-separator.h"
45 : #include "src/compiler/load-elimination.h"
46 : #include "src/compiler/loop-analysis.h"
47 : #include "src/compiler/loop-peeling.h"
48 : #include "src/compiler/loop-variable-optimizer.h"
49 : #include "src/compiler/machine-graph-verifier.h"
50 : #include "src/compiler/machine-operator-reducer.h"
51 : #include "src/compiler/memory-optimizer.h"
52 : #include "src/compiler/move-optimizer.h"
53 : #include "src/compiler/osr.h"
54 : #include "src/compiler/pipeline-statistics.h"
55 : #include "src/compiler/redundancy-elimination.h"
56 : #include "src/compiler/register-allocator-verifier.h"
57 : #include "src/compiler/register-allocator.h"
58 : #include "src/compiler/schedule.h"
59 : #include "src/compiler/scheduler.h"
60 : #include "src/compiler/select-lowering.h"
61 : #include "src/compiler/simplified-lowering.h"
62 : #include "src/compiler/simplified-operator-reducer.h"
63 : #include "src/compiler/simplified-operator.h"
64 : #include "src/compiler/store-store-elimination.h"
65 : #include "src/compiler/tail-call-optimization.h"
66 : #include "src/compiler/typed-optimization.h"
67 : #include "src/compiler/typer.h"
68 : #include "src/compiler/value-numbering-reducer.h"
69 : #include "src/compiler/verifier.h"
70 : #include "src/compiler/zone-stats.h"
71 : #include "src/isolate-inl.h"
72 : #include "src/ostreams.h"
73 : #include "src/parsing/parse-info.h"
74 : #include "src/register-configuration.h"
75 : #include "src/trap-handler/trap-handler.h"
76 : #include "src/type-info.h"
77 : #include "src/utils.h"
78 :
79 : namespace v8 {
80 : namespace internal {
81 : namespace compiler {
82 :
83 : class PipelineData {
84 : public:
85 : // For main entry point.
86 423221 : PipelineData(ZoneStats* zone_stats, CompilationInfo* info,
87 : PipelineStatistics* pipeline_statistics)
88 : : isolate_(info->isolate()),
89 : info_(info),
90 : debug_name_(info_->GetDebugName()),
91 423221 : outer_zone_(info_->zone()),
92 : zone_stats_(zone_stats),
93 : pipeline_statistics_(pipeline_statistics),
94 : graph_zone_scope_(zone_stats_, ZONE_NAME),
95 423221 : graph_zone_(graph_zone_scope_.zone()),
96 : instruction_zone_scope_(zone_stats_, ZONE_NAME),
97 423222 : instruction_zone_(instruction_zone_scope_.zone()),
98 : register_allocation_zone_scope_(zone_stats_, ZONE_NAME),
99 2539329 : register_allocation_zone_(register_allocation_zone_scope_.zone()) {
100 : PhaseScope scope(pipeline_statistics, "init pipeline data");
101 846443 : graph_ = new (graph_zone_) Graph(graph_zone_);
102 846442 : source_positions_ = new (graph_zone_) SourcePositionTable(graph_);
103 846444 : simplified_ = new (graph_zone_) SimplifiedOperatorBuilder(graph_zone_);
104 : machine_ = new (graph_zone_) MachineOperatorBuilder(
105 : graph_zone_, MachineType::PointerRepresentation(),
106 : InstructionSelector::SupportedMachineOperatorFlags(),
107 846443 : InstructionSelector::AlignmentRequirements());
108 846443 : common_ = new (graph_zone_) CommonOperatorBuilder(graph_zone_);
109 846443 : javascript_ = new (graph_zone_) JSOperatorBuilder(graph_zone_);
110 : jsgraph_ = new (graph_zone_)
111 1269666 : JSGraph(isolate_, graph_, common_, javascript_, simplified_, machine_);
112 846444 : is_asm_ = info->shared_info()->asm_function();
113 423222 : }
114 :
115 : // For WASM compile entry point.
116 352783 : PipelineData(ZoneStats* zone_stats, CompilationInfo* info, JSGraph* jsgraph,
117 : PipelineStatistics* pipeline_statistics,
118 : SourcePositionTable* source_positions,
119 : ZoneVector<trap_handler::ProtectedInstructionData>*
120 : protected_instructions)
121 : : isolate_(info->isolate()),
122 : info_(info),
123 : debug_name_(info_->GetDebugName()),
124 : zone_stats_(zone_stats),
125 : pipeline_statistics_(pipeline_statistics),
126 : graph_zone_scope_(zone_stats_, ZONE_NAME),
127 : graph_(jsgraph->graph()),
128 : source_positions_(source_positions),
129 : machine_(jsgraph->machine()),
130 : common_(jsgraph->common()),
131 : javascript_(jsgraph->javascript()),
132 : jsgraph_(jsgraph),
133 : instruction_zone_scope_(zone_stats_, ZONE_NAME),
134 70555 : instruction_zone_(instruction_zone_scope_.zone()),
135 : register_allocation_zone_scope_(zone_stats_, ZONE_NAME),
136 70569 : register_allocation_zone_(register_allocation_zone_scope_.zone()),
137 635057 : protected_instructions_(protected_instructions) {
138 : is_asm_ =
139 141135 : info->has_shared_info() ? info->shared_info()->asm_function() : false;
140 70560 : }
141 :
142 : // For machine graph testing entry point.
143 449994 : PipelineData(ZoneStats* zone_stats, CompilationInfo* info, Graph* graph,
144 : Schedule* schedule, SourcePositionTable* source_positions)
145 : : isolate_(info->isolate()),
146 : info_(info),
147 : debug_name_(info_->GetDebugName()),
148 : zone_stats_(zone_stats),
149 : graph_zone_scope_(zone_stats_, ZONE_NAME),
150 : graph_(graph),
151 : source_positions_(source_positions),
152 : schedule_(schedule),
153 : instruction_zone_scope_(zone_stats_, ZONE_NAME),
154 449994 : instruction_zone_(instruction_zone_scope_.zone()),
155 : register_allocation_zone_scope_(zone_stats_, ZONE_NAME),
156 2249970 : register_allocation_zone_(register_allocation_zone_scope_.zone()) {
157 449994 : is_asm_ = false;
158 449994 : }
159 : // For register allocation testing entry point.
160 42 : PipelineData(ZoneStats* zone_stats, CompilationInfo* info,
161 42 : InstructionSequence* sequence)
162 : : isolate_(info->isolate()),
163 : info_(info),
164 : debug_name_(info_->GetDebugName()),
165 : zone_stats_(zone_stats),
166 : graph_zone_scope_(zone_stats_, ZONE_NAME),
167 : instruction_zone_scope_(zone_stats_, ZONE_NAME),
168 : instruction_zone_(sequence->zone()),
169 : sequence_(sequence),
170 : register_allocation_zone_scope_(zone_stats_, ZONE_NAME),
171 210 : register_allocation_zone_(register_allocation_zone_scope_.zone()) {
172 : is_asm_ =
173 84 : info->has_shared_info() ? info->shared_info()->asm_function() : false;
174 42 : }
175 :
176 943845 : ~PipelineData() {
177 943845 : DeleteRegisterAllocationZone();
178 943845 : DeleteInstructionZone();
179 943845 : DeleteGraphZone();
180 943845 : }
181 :
182 : Isolate* isolate() const { return isolate_; }
183 : CompilationInfo* info() const { return info_; }
184 : ZoneStats* zone_stats() const { return zone_stats_; }
185 : PipelineStatistics* pipeline_statistics() { return pipeline_statistics_; }
186 : bool compilation_failed() const { return compilation_failed_; }
187 0 : void set_compilation_failed() { compilation_failed_ = true; }
188 :
189 : bool is_asm() const { return is_asm_; }
190 : bool verify_graph() const { return verify_graph_; }
191 111827 : void set_verify_graph(bool value) { verify_graph_ = value; }
192 :
193 : Handle<Code> code() { return code_; }
194 : void set_code(Handle<Code> code) {
195 : DCHECK(code_.is_null());
196 912392 : code_ = code;
197 : }
198 :
199 : // RawMachineAssembler generally produces graphs which cannot be verified.
200 : bool MayHaveUnverifiableGraph() const { return outer_zone_ == nullptr; }
201 :
202 : Zone* graph_zone() const { return graph_zone_; }
203 : Graph* graph() const { return graph_; }
204 : SourcePositionTable* source_positions() const { return source_positions_; }
205 : MachineOperatorBuilder* machine() const { return machine_; }
206 : CommonOperatorBuilder* common() const { return common_; }
207 : JSOperatorBuilder* javascript() const { return javascript_; }
208 : JSGraph* jsgraph() const { return jsgraph_; }
209 1581370 : Handle<Context> native_context() const {
210 3162742 : return handle(info()->native_context(), isolate());
211 : }
212 : Handle<JSGlobalObject> global_object() const {
213 : return handle(info()->global_object(), isolate());
214 : }
215 :
216 : LoopAssignmentAnalysis* loop_assignment() const { return loop_assignment_; }
217 : void set_loop_assignment(LoopAssignmentAnalysis* loop_assignment) {
218 : DCHECK(!loop_assignment_);
219 2507 : loop_assignment_ = loop_assignment;
220 : }
221 :
222 : Schedule* schedule() const { return schedule_; }
223 : void set_schedule(Schedule* schedule) {
224 : DCHECK(!schedule_);
225 568260 : schedule_ = schedule;
226 : }
227 : void reset_schedule() { schedule_ = nullptr; }
228 :
229 : Zone* instruction_zone() const { return instruction_zone_; }
230 : InstructionSequence* sequence() const { return sequence_; }
231 : Frame* frame() const { return frame_; }
232 :
233 : Zone* register_allocation_zone() const { return register_allocation_zone_; }
234 : RegisterAllocationData* register_allocation_data() const {
235 : return register_allocation_data_;
236 : }
237 :
238 : BasicBlockProfiler::Data* profiler_data() const { return profiler_data_; }
239 : void set_profiler_data(BasicBlockProfiler::Data* profiler_data) {
240 14 : profiler_data_ = profiler_data;
241 : }
242 :
243 : std::string const& source_position_output() const {
244 : return source_position_output_;
245 : }
246 : void set_source_position_output(std::string const& source_position_output) {
247 0 : source_position_output_ = source_position_output;
248 : }
249 :
250 : ZoneVector<trap_handler::ProtectedInstructionData>* protected_instructions()
251 : const {
252 : return protected_instructions_;
253 : }
254 :
255 1859723 : void DeleteGraphZone() {
256 3719447 : if (graph_zone_ == nullptr) return;
257 423221 : graph_zone_scope_.Destroy();
258 423222 : graph_zone_ = nullptr;
259 423222 : graph_ = nullptr;
260 423222 : source_positions_ = nullptr;
261 423222 : loop_assignment_ = nullptr;
262 423222 : simplified_ = nullptr;
263 423222 : machine_ = nullptr;
264 423222 : common_ = nullptr;
265 423222 : javascript_ = nullptr;
266 423222 : jsgraph_ = nullptr;
267 423222 : schedule_ = nullptr;
268 : }
269 :
270 943845 : void DeleteInstructionZone() {
271 1887690 : if (instruction_zone_ == nullptr) return;
272 943845 : instruction_zone_scope_.Destroy();
273 943845 : instruction_zone_ = nullptr;
274 943845 : sequence_ = nullptr;
275 943845 : frame_ = nullptr;
276 : }
277 :
278 1859754 : void DeleteRegisterAllocationZone() {
279 3719522 : if (register_allocation_zone_ == nullptr) return;
280 943835 : register_allocation_zone_scope_.Destroy();
281 943849 : register_allocation_zone_ = nullptr;
282 943849 : register_allocation_data_ = nullptr;
283 : }
284 :
285 2747580 : void InitializeInstructionSequence(const CallDescriptor* descriptor) {
286 : DCHECK(sequence_ == nullptr);
287 : InstructionBlocks* instruction_blocks =
288 : InstructionSequence::InstructionBlocksFor(instruction_zone(),
289 915848 : schedule());
290 : sequence_ = new (instruction_zone()) InstructionSequence(
291 1831736 : info()->isolate(), instruction_zone(), instruction_blocks);
292 1831711 : if (descriptor && descriptor->RequiresFrameAsIncoming()) {
293 684305 : sequence_->instruction_blocks()[0]->mark_needs_frame();
294 : } else {
295 : DCHECK_EQ(0u, descriptor->CalleeSavedFPRegisters());
296 : DCHECK_EQ(0u, descriptor->CalleeSavedRegisters());
297 : }
298 915855 : }
299 :
300 1831803 : void InitializeFrameData(CallDescriptor* descriptor) {
301 : DCHECK(frame_ == nullptr);
302 : int fixed_frame_size = 0;
303 915899 : if (descriptor != nullptr) {
304 915859 : fixed_frame_size = descriptor->CalculateFixedFrameSize();
305 : }
306 915908 : frame_ = new (instruction_zone()) Frame(fixed_frame_size);
307 915882 : }
308 :
309 915904 : void InitializeRegisterAllocationData(const RegisterConfiguration* config,
310 915904 : CallDescriptor* descriptor) {
311 : DCHECK(register_allocation_data_ == nullptr);
312 : register_allocation_data_ = new (register_allocation_zone())
313 : RegisterAllocationData(config, register_allocation_zone(), frame(),
314 915929 : sequence(), debug_name());
315 915908 : }
316 :
317 3014219 : void BeginPhaseKind(const char* phase_kind_name) {
318 3014219 : if (pipeline_statistics() != nullptr) {
319 0 : pipeline_statistics()->BeginPhaseKind(phase_kind_name);
320 : }
321 : }
322 :
323 1311228 : void EndPhaseKind() {
324 1311228 : if (pipeline_statistics() != nullptr) {
325 0 : pipeline_statistics()->EndPhaseKind();
326 : }
327 : }
328 :
329 : const char* debug_name() const { return debug_name_.get(); }
330 :
331 : private:
332 : Isolate* const isolate_;
333 : CompilationInfo* const info_;
334 : std::unique_ptr<char[]> debug_name_;
335 : Zone* outer_zone_ = nullptr;
336 : ZoneStats* const zone_stats_;
337 : PipelineStatistics* pipeline_statistics_ = nullptr;
338 : bool compilation_failed_ = false;
339 : bool verify_graph_ = false;
340 : bool is_asm_ = false;
341 : Handle<Code> code_ = Handle<Code>::null();
342 :
343 : // All objects in the following group of fields are allocated in graph_zone_.
344 : // They are all set to nullptr when the graph_zone_ is destroyed.
345 : ZoneStats::Scope graph_zone_scope_;
346 : Zone* graph_zone_ = nullptr;
347 : Graph* graph_ = nullptr;
348 : SourcePositionTable* source_positions_ = nullptr;
349 : LoopAssignmentAnalysis* loop_assignment_ = nullptr;
350 : SimplifiedOperatorBuilder* simplified_ = nullptr;
351 : MachineOperatorBuilder* machine_ = nullptr;
352 : CommonOperatorBuilder* common_ = nullptr;
353 : JSOperatorBuilder* javascript_ = nullptr;
354 : JSGraph* jsgraph_ = nullptr;
355 : Schedule* schedule_ = nullptr;
356 :
357 : // All objects in the following group of fields are allocated in
358 : // instruction_zone_. They are all set to nullptr when the instruction_zone_
359 : // is
360 : // destroyed.
361 : ZoneStats::Scope instruction_zone_scope_;
362 : Zone* instruction_zone_;
363 : InstructionSequence* sequence_ = nullptr;
364 : Frame* frame_ = nullptr;
365 :
366 : // All objects in the following group of fields are allocated in
367 : // register_allocation_zone_. They are all set to nullptr when the zone is
368 : // destroyed.
369 : ZoneStats::Scope register_allocation_zone_scope_;
370 : Zone* register_allocation_zone_;
371 : RegisterAllocationData* register_allocation_data_ = nullptr;
372 :
373 : // Basic block profiling support.
374 : BasicBlockProfiler::Data* profiler_data_ = nullptr;
375 :
376 : // Source position output for --trace-turbo.
377 : std::string source_position_output_;
378 :
379 : ZoneVector<trap_handler::ProtectedInstructionData>* protected_instructions_ =
380 : nullptr;
381 :
382 : DISALLOW_COPY_AND_ASSIGN(PipelineData);
383 : };
384 :
385 : class PipelineImpl final {
386 : public:
387 943815 : explicit PipelineImpl(PipelineData* data) : data_(data) {}
388 :
389 : // Helpers for executing pipeline phases.
390 : template <typename Phase>
391 : void Run();
392 : template <typename Phase, typename Arg0>
393 : void Run(Arg0 arg_0);
394 : template <typename Phase, typename Arg0, typename Arg1>
395 : void Run(Arg0 arg_0, Arg1 arg_1);
396 :
397 : // Run the graph creation and initial optimization passes.
398 : bool CreateGraph();
399 :
400 : // Run the concurrent optimization passes.
401 : bool OptimizeGraph(Linkage* linkage);
402 :
403 : // Perform the actual code generation and return handle to a code object.
404 : Handle<Code> GenerateCode(Linkage* linkage);
405 :
406 : bool ScheduleAndSelectInstructions(Linkage* linkage, bool trim_graph);
407 : void RunPrintAndVerify(const char* phase, bool untyped = false);
408 : Handle<Code> ScheduleAndGenerateCode(CallDescriptor* call_descriptor);
409 : void AllocateRegisters(const RegisterConfiguration* config,
410 : CallDescriptor* descriptor, bool run_verifier);
411 :
412 : CompilationInfo* info() const;
413 : Isolate* isolate() const;
414 :
415 : PipelineData* const data_;
416 : };
417 :
418 : namespace {
419 :
420 0 : struct TurboCfgFile : public std::ofstream {
421 0 : explicit TurboCfgFile(Isolate* isolate)
422 : : std::ofstream(isolate->GetTurboCfgFileName().c_str(),
423 0 : std::ios_base::app) {}
424 : };
425 :
426 0 : struct TurboJsonFile : public std::ofstream {
427 0 : TurboJsonFile(CompilationInfo* info, std::ios_base::openmode mode)
428 0 : : std::ofstream(GetVisualizerLogFileName(info, nullptr, "json").get(),
429 0 : mode) {}
430 : };
431 :
432 1311173 : void TraceSchedule(CompilationInfo* info, Schedule* schedule) {
433 1311174 : if (FLAG_trace_turbo) {
434 : AllowHandleDereference allow_deref;
435 0 : TurboJsonFile json_of(info, std::ios_base::app);
436 0 : json_of << "{\"name\":\"Schedule\",\"type\":\"schedule\",\"data\":\"";
437 0 : std::stringstream schedule_stream;
438 0 : schedule_stream << *schedule;
439 : std::string schedule_string(schedule_stream.str());
440 0 : for (const auto& c : schedule_string) {
441 0 : json_of << AsEscapedUC16ForJSON(c);
442 : }
443 0 : json_of << "\"},\n";
444 : }
445 1311174 : if (FLAG_trace_turbo_graph || FLAG_trace_turbo_scheduler) {
446 : AllowHandleDereference allow_deref;
447 0 : CodeTracer::Scope tracing_scope(info->isolate()->GetCodeTracer());
448 0 : OFStream os(tracing_scope.file());
449 0 : os << "-- Schedule --------------------------------------\n" << *schedule;
450 : }
451 1311175 : }
452 :
453 :
454 : class SourcePositionWrapper final : public Reducer {
455 : public:
456 : SourcePositionWrapper(Reducer* reducer, SourcePositionTable* table)
457 1716258 : : reducer_(reducer), table_(table) {}
458 0 : ~SourcePositionWrapper() final {}
459 :
460 235524297 : Reduction Reduce(Node* node) final {
461 235524297 : SourcePosition const pos = table_->GetSourcePosition(node);
462 235524652 : SourcePositionTable::Scope position(table_, pos);
463 471047930 : return reducer_->Reduce(node);
464 : }
465 :
466 1721370 : void Finalize() final { reducer_->Finalize(); }
467 :
468 : private:
469 : Reducer* const reducer_;
470 : SourcePositionTable* const table_;
471 :
472 : DISALLOW_COPY_AND_ASSIGN(SourcePositionWrapper);
473 : };
474 :
475 :
476 : class JSGraphReducer final : public GraphReducer {
477 : public:
478 5750787 : JSGraphReducer(JSGraph* jsgraph, Zone* zone)
479 5750787 : : GraphReducer(zone, jsgraph->graph(), jsgraph->Dead()) {}
480 2875288 : ~JSGraphReducer() final {}
481 : };
482 :
483 :
484 19150426 : void AddReducer(PipelineData* data, GraphReducer* graph_reducer,
485 : Reducer* reducer) {
486 15717908 : if (data->info()->is_source_positions_enabled()) {
487 1716259 : void* const buffer = data->graph_zone()->New(sizeof(SourcePositionWrapper));
488 : SourcePositionWrapper* const wrapper =
489 1716259 : new (buffer) SourcePositionWrapper(reducer, data->source_positions());
490 1716259 : graph_reducer->AddReducer(wrapper);
491 : } else {
492 14001649 : graph_reducer->AddReducer(reducer);
493 : }
494 15717846 : }
495 :
496 :
497 50414903 : class PipelineRunScope {
498 : public:
499 75509070 : PipelineRunScope(PipelineData* data, const char* phase_name)
500 : : phase_scope_(
501 : phase_name == nullptr ? nullptr : data->pipeline_statistics(),
502 : phase_name),
503 50413902 : zone_scope_(data->zone_stats(), ZONE_NAME) {}
504 :
505 25279293 : Zone* zone() { return zone_scope_.zone(); }
506 :
507 : private:
508 : PhaseScope phase_scope_;
509 : ZoneStats::Scope zone_scope_;
510 : };
511 :
512 493789 : PipelineStatistics* CreatePipelineStatistics(CompilationInfo* info,
513 : ZoneStats* zone_stats) {
514 : PipelineStatistics* pipeline_statistics = nullptr;
515 :
516 493789 : if (FLAG_turbo_stats || FLAG_turbo_stats_nvp) {
517 0 : pipeline_statistics = new PipelineStatistics(info, zone_stats);
518 0 : pipeline_statistics->BeginPhaseKind("initializing");
519 : }
520 :
521 493790 : if (FLAG_trace_turbo) {
522 0 : TurboJsonFile json_of(info, std::ios_base::trunc);
523 0 : std::unique_ptr<char[]> function_name = info->GetDebugName();
524 0 : int pos = info->parse_info() ? info->shared_info()->start_position() : 0;
525 0 : json_of << "{\"function\":\"" << function_name.get()
526 0 : << "\", \"sourcePosition\":" << pos << ", \"source\":\"";
527 : Isolate* isolate = info->isolate();
528 : Handle<Script> script =
529 0 : info->parse_info() ? info->script() : Handle<Script>::null();
530 0 : if (!script.is_null() && !script->source()->IsUndefined(isolate)) {
531 : DisallowHeapAllocation no_allocation;
532 0 : int start = info->shared_info()->start_position();
533 0 : int len = info->shared_info()->end_position() - start;
534 : String::SubStringRange source(String::cast(script->source()), start, len);
535 0 : for (const auto& c : source) {
536 0 : json_of << AsEscapedUC16ForJSON(c);
537 : }
538 : }
539 0 : json_of << "\",\n\"phases\":[";
540 : }
541 :
542 493790 : return pipeline_statistics;
543 : }
544 :
545 : } // namespace
546 :
547 1673328 : class PipelineCompilationJob final : public CompilationJob {
548 : public:
549 418335 : PipelineCompilationJob(ParseInfo* parse_info, Handle<JSFunction> function)
550 : // Note that the CompilationInfo is not initialized at the time we pass it
551 : // to the CompilationJob constructor, but it is not dereferenced there.
552 : : CompilationJob(function->GetIsolate(), &info_, "TurboFan"),
553 : parse_info_(parse_info),
554 : zone_stats_(function->GetIsolate()->allocator()),
555 : info_(parse_info_.get()->zone(), parse_info_.get(),
556 : function->GetIsolate(), function),
557 : pipeline_statistics_(CreatePipelineStatistics(info(), &zone_stats_)),
558 : data_(&zone_stats_, info(), pipeline_statistics_.get()),
559 : pipeline_(&data_),
560 2510011 : linkage_(nullptr) {}
561 :
562 : protected:
563 : Status PrepareJobImpl() final;
564 : Status ExecuteJobImpl() final;
565 : Status FinalizeJobImpl() final;
566 :
567 : private:
568 : std::unique_ptr<ParseInfo> parse_info_;
569 : ZoneStats zone_stats_;
570 : CompilationInfo info_;
571 : std::unique_ptr<PipelineStatistics> pipeline_statistics_;
572 : PipelineData data_;
573 : PipelineImpl pipeline_;
574 : Linkage* linkage_;
575 :
576 : DISALLOW_COPY_AND_ASSIGN(PipelineCompilationJob);
577 : };
578 :
579 390457 : PipelineCompilationJob::Status PipelineCompilationJob::PrepareJobImpl() {
580 4346213 : if (info()->shared_info()->asm_function()) {
581 2191 : if (info()->osr_frame() && !info()->is_optimizing_from_bytecode()) {
582 : info()->MarkAsFrameSpecializing();
583 : }
584 : info()->MarkAsFunctionContextSpecializing();
585 : } else {
586 388346 : if (!FLAG_always_opt) {
587 : info()->MarkAsBailoutOnUninitialized();
588 : }
589 388346 : if (FLAG_turbo_loop_peeling) {
590 : info()->MarkAsLoopPeelingEnabled();
591 : }
592 : }
593 1171371 : if (info()->is_optimizing_from_bytecode() ||
594 394603 : !info()->shared_info()->asm_function()) {
595 : info()->MarkAsDeoptimizationEnabled();
596 388388 : if (FLAG_inline_accessors) {
597 : info()->MarkAsAccessorInliningEnabled();
598 : }
599 388388 : if (info()->closure()->feedback_vector_cell()->map() ==
600 388388 : isolate()->heap()->one_closure_cell_map()) {
601 : info()->MarkAsFunctionContextSpecializing();
602 : }
603 : }
604 390457 : if (!info()->is_optimizing_from_bytecode()) {
605 2073 : if (!Compiler::EnsureDeoptimizationSupport(info())) return FAILED;
606 388384 : } else if (FLAG_turbo_inlining) {
607 : info()->MarkAsInliningEnabled();
608 : }
609 :
610 390457 : linkage_ = new (info()->zone())
611 780914 : Linkage(Linkage::ComputeIncoming(info()->zone(), info()));
612 :
613 390457 : if (!pipeline_.CreateGraph()) {
614 0 : if (isolate()->has_pending_exception()) return FAILED; // Stack overflowed.
615 0 : return AbortOptimization(kGraphBuildingFailed);
616 : }
617 :
618 : return SUCCEEDED;
619 : }
620 :
621 390417 : PipelineCompilationJob::Status PipelineCompilationJob::ExecuteJobImpl() {
622 390417 : if (!pipeline_.OptimizeGraph(linkage_)) return FAILED;
623 390409 : return SUCCEEDED;
624 : }
625 :
626 386919 : PipelineCompilationJob::Status PipelineCompilationJob::FinalizeJobImpl() {
627 386919 : Handle<Code> code = pipeline_.GenerateCode(linkage_);
628 386920 : if (code.is_null()) {
629 773841 : if (info()->bailout_reason() == kNoReason) {
630 0 : return AbortOptimization(kCodeGenerationFailed);
631 : }
632 : return FAILED;
633 : }
634 386920 : info()->dependencies()->Commit(code);
635 : info()->SetCode(code);
636 386921 : if (info()->is_deoptimization_enabled()) {
637 769736 : info()->context()->native_context()->AddOptimizedCode(*code);
638 384867 : RegisterWeakObjectsInOptimizedCode(code);
639 : }
640 : return SUCCEEDED;
641 : }
642 :
643 282364 : class PipelineWasmCompilationJob final : public CompilationJob {
644 : public:
645 70586 : explicit PipelineWasmCompilationJob(
646 141164 : CompilationInfo* info, JSGraph* jsgraph, CallDescriptor* descriptor,
647 : SourcePositionTable* source_positions,
648 : ZoneVector<trap_handler::ProtectedInstructionData>* protected_insts,
649 : bool allow_signalling_nan)
650 : : CompilationJob(info->isolate(), info, "TurboFan",
651 : State::kReadyToExecute),
652 : zone_stats_(info->isolate()->allocator()),
653 : pipeline_statistics_(CreatePipelineStatistics(info, &zone_stats_)),
654 : data_(&zone_stats_, info, jsgraph, pipeline_statistics_.get(),
655 : source_positions, protected_insts),
656 : pipeline_(&data_),
657 : linkage_(descriptor),
658 282289 : allow_signalling_nan_(allow_signalling_nan) {}
659 :
660 : protected:
661 : Status PrepareJobImpl() final;
662 : Status ExecuteJobImpl() final;
663 : Status FinalizeJobImpl() final;
664 :
665 : private:
666 : ZoneStats zone_stats_;
667 : std::unique_ptr<PipelineStatistics> pipeline_statistics_;
668 : PipelineData data_;
669 : PipelineImpl pipeline_;
670 : Linkage linkage_;
671 : bool allow_signalling_nan_;
672 : };
673 :
674 : PipelineWasmCompilationJob::Status
675 0 : PipelineWasmCompilationJob::PrepareJobImpl() {
676 0 : UNREACHABLE(); // Prepare should always be skipped for WasmCompilationJob.
677 : return SUCCEEDED;
678 : }
679 :
680 : PipelineWasmCompilationJob::Status
681 70510 : PipelineWasmCompilationJob::ExecuteJobImpl() {
682 70510 : if (FLAG_trace_turbo) {
683 0 : TurboJsonFile json_of(info(), std::ios_base::trunc);
684 0 : json_of << "{\"function\":\"" << info()->GetDebugName().get()
685 0 : << "\", \"source\":\"\",\n\"phases\":[";
686 : }
687 :
688 70510 : pipeline_.RunPrintAndVerify("Machine", true);
689 70517 : if (FLAG_wasm_opt) {
690 423333 : PipelineData* data = &data_;
691 70508 : PipelineRunScope scope(data, "WASM optimization");
692 70583 : JSGraphReducer graph_reducer(data->jsgraph(), scope.zone());
693 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
694 70573 : data->common());
695 211624 : ValueNumberingReducer value_numbering(scope.zone(), data->graph()->zone());
696 : MachineOperatorReducer machine_reducer(data->jsgraph(),
697 211664 : allow_signalling_nan_);
698 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
699 70554 : data->common(), data->machine());
700 70537 : AddReducer(data, &graph_reducer, &dead_code_elimination);
701 70502 : AddReducer(data, &graph_reducer, &machine_reducer);
702 70530 : AddReducer(data, &graph_reducer, &common_reducer);
703 70535 : AddReducer(data, &graph_reducer, &value_numbering);
704 70553 : graph_reducer.ReduceGraph();
705 141035 : pipeline_.RunPrintAndVerify("Optimized Machine", true);
706 : }
707 :
708 70599 : if (!pipeline_.ScheduleAndSelectInstructions(&linkage_, true)) return FAILED;
709 70570 : return SUCCEEDED;
710 : }
711 :
712 : PipelineWasmCompilationJob::Status
713 70591 : PipelineWasmCompilationJob::FinalizeJobImpl() {
714 70591 : pipeline_.GenerateCode(&linkage_);
715 70591 : return SUCCEEDED;
716 : }
717 :
718 : template <typename Phase>
719 21886864 : void PipelineImpl::Run() {
720 21886864 : PipelineRunScope scope(this->data_, Phase::phase_name());
721 : Phase phase;
722 23725812 : phase.Run(this->data_, scope.zone());
723 21887104 : }
724 :
725 : template <typename Phase, typename Arg0>
726 3139357 : void PipelineImpl::Run(Arg0 arg_0) {
727 3139357 : PipelineRunScope scope(this->data_, Phase::phase_name());
728 : Phase phase;
729 4055323 : phase.Run(this->data_, scope.zone(), arg_0);
730 3139507 : }
731 :
732 : template <typename Phase, typename Arg0, typename Arg1>
733 111827 : void PipelineImpl::Run(Arg0 arg_0, Arg1 arg_1) {
734 111827 : PipelineRunScope scope(this->data_, Phase::phase_name());
735 : Phase phase;
736 223654 : phase.Run(this->data_, scope.zone(), arg_0, arg_1);
737 111827 : }
738 :
739 : struct LoopAssignmentAnalysisPhase {
740 : static const char* phase_name() { return "loop assignment analysis"; }
741 :
742 394630 : void Run(PipelineData* data, Zone* temp_zone) {
743 392123 : if (!data->info()->is_optimizing_from_bytecode()) {
744 2507 : AstLoopAssignmentAnalyzer analyzer(data->graph_zone(), data->info());
745 2507 : LoopAssignmentAnalysis* loop_assignment = analyzer.Analyze();
746 : data->set_loop_assignment(loop_assignment);
747 : }
748 392123 : }
749 : };
750 :
751 :
752 : struct GraphBuilderPhase {
753 : static const char* phase_name() { return "graph builder"; }
754 :
755 1183522 : void Run(PipelineData* data, Zone* temp_zone) {
756 : bool succeeded = false;
757 :
758 395343 : if (data->info()->is_optimizing_from_bytecode()) {
759 : // Bytecode graph builder assumes deoptimziation is enabled.
760 : DCHECK(data->info()->is_deoptimization_enabled());
761 : JSTypeHintLowering::Flags flags = JSTypeHintLowering::kNoFlags;
762 392836 : if (data->info()->is_bailout_on_uninitialized()) {
763 : flags |= JSTypeHintLowering::kBailoutOnUninitialized;
764 : }
765 : BytecodeGraphBuilder graph_builder(
766 : temp_zone, data->info()->shared_info(),
767 : handle(data->info()->closure()->feedback_vector()),
768 : data->info()->osr_ast_id(), data->jsgraph(), 1.0f,
769 392836 : data->source_positions(), SourcePosition::kNotInlined, flags);
770 392836 : succeeded = graph_builder.CreateGraph();
771 : } else {
772 : AstGraphBuilderWithPositions graph_builder(
773 : temp_zone, data->info(), data->jsgraph(), 1.0f,
774 2507 : data->loop_assignment(), data->source_positions());
775 2507 : succeeded = graph_builder.CreateGraph();
776 : }
777 :
778 395343 : if (!succeeded) {
779 : data->set_compilation_failed();
780 : }
781 395343 : }
782 : };
783 :
784 :
785 : struct InliningPhase {
786 : static const char* phase_name() { return "inlining"; }
787 :
788 5139455 : void Run(PipelineData* data, Zone* temp_zone) {
789 395343 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
790 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
791 395342 : data->common());
792 395343 : CheckpointElimination checkpoint_elimination(&graph_reducer);
793 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
794 395342 : data->common(), data->machine());
795 : JSCallReducer call_reducer(&graph_reducer, data->jsgraph(),
796 : data->native_context(),
797 395342 : data->info()->dependencies());
798 : JSContextSpecialization context_specialization(
799 : &graph_reducer, data->jsgraph(),
800 : data->info()->is_function_context_specializing()
801 : ? handle(data->info()->context())
802 : : MaybeHandle<Context>(),
803 : data->info()->is_function_context_specializing()
804 : ? data->info()->closure()
805 395343 : : MaybeHandle<JSFunction>());
806 : JSFrameSpecialization frame_specialization(
807 395343 : &graph_reducer, data->info()->osr_frame(), data->jsgraph());
808 : JSNativeContextSpecialization::Flags flags =
809 : JSNativeContextSpecialization::kNoFlags;
810 395343 : if (data->info()->is_accessor_inlining_enabled()) {
811 : flags |= JSNativeContextSpecialization::kAccessorInliningEnabled;
812 : }
813 395343 : if (data->info()->is_bailout_on_uninitialized()) {
814 : flags |= JSNativeContextSpecialization::kBailoutOnUninitialized;
815 : }
816 : JSNativeContextSpecialization native_context_specialization(
817 : &graph_reducer, data->jsgraph(), flags, data->native_context(),
818 790686 : data->info()->dependencies(), temp_zone);
819 : JSInliningHeuristic inlining(
820 : &graph_reducer, data->info()->is_inlining_enabled()
821 : ? JSInliningHeuristic::kGeneralInlining
822 : : JSInliningHeuristic::kRestrictedInlining,
823 790685 : temp_zone, data->info(), data->jsgraph(), data->source_positions());
824 : JSIntrinsicLowering intrinsic_lowering(
825 : &graph_reducer, data->jsgraph(),
826 : data->info()->is_deoptimization_enabled()
827 : ? JSIntrinsicLowering::kDeoptimizationEnabled
828 395342 : : JSIntrinsicLowering::kDeoptimizationDisabled);
829 395342 : AddReducer(data, &graph_reducer, &dead_code_elimination);
830 395342 : AddReducer(data, &graph_reducer, &checkpoint_elimination);
831 395343 : AddReducer(data, &graph_reducer, &common_reducer);
832 395343 : if (data->info()->is_frame_specializing()) {
833 68 : AddReducer(data, &graph_reducer, &frame_specialization);
834 : }
835 395343 : if (data->info()->is_deoptimization_enabled()) {
836 393274 : AddReducer(data, &graph_reducer, &native_context_specialization);
837 : }
838 395343 : AddReducer(data, &graph_reducer, &context_specialization);
839 395343 : AddReducer(data, &graph_reducer, &intrinsic_lowering);
840 395343 : if (data->info()->is_deoptimization_enabled()) {
841 393274 : AddReducer(data, &graph_reducer, &call_reducer);
842 : }
843 395343 : AddReducer(data, &graph_reducer, &inlining);
844 395343 : graph_reducer.ReduceGraph();
845 395343 : }
846 : };
847 :
848 :
849 : struct TyperPhase {
850 : static const char* phase_name() { return "typer"; }
851 :
852 1186029 : void Run(PipelineData* data, Zone* temp_zone, Typer* typer) {
853 : NodeVector roots(temp_zone);
854 395343 : data->jsgraph()->GetCachedNodes(&roots);
855 : LoopVariableOptimizer induction_vars(data->jsgraph()->graph(),
856 395343 : data->common(), temp_zone);
857 395343 : if (FLAG_turbo_loop_variable) induction_vars.Run();
858 395343 : typer->Run(roots, &induction_vars);
859 395343 : }
860 : };
861 :
862 : struct UntyperPhase {
863 : static const char* phase_name() { return "untyper"; }
864 :
865 : void Run(PipelineData* data, Zone* temp_zone) {
866 0 : class RemoveTypeReducer final : public Reducer {
867 : public:
868 0 : Reduction Reduce(Node* node) final {
869 0 : if (NodeProperties::IsTyped(node)) {
870 : NodeProperties::RemoveType(node);
871 : return Changed(node);
872 : }
873 : return NoChange();
874 : }
875 : };
876 :
877 : NodeVector roots(temp_zone);
878 : data->jsgraph()->GetCachedNodes(&roots);
879 : for (Node* node : roots) {
880 : NodeProperties::RemoveType(node);
881 : }
882 :
883 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
884 : RemoveTypeReducer remove_type_reducer;
885 : AddReducer(data, &graph_reducer, &remove_type_reducer);
886 : graph_reducer.ReduceGraph();
887 : }
888 : };
889 :
890 : struct OsrDeconstructionPhase {
891 : static const char* phase_name() { return "OSR deconstruction"; }
892 :
893 23252 : void Run(PipelineData* data, Zone* temp_zone) {
894 5813 : GraphTrimmer trimmer(temp_zone, data->graph());
895 : NodeVector roots(temp_zone);
896 5813 : data->jsgraph()->GetCachedNodes(&roots);
897 5813 : trimmer.TrimGraph(roots.begin(), roots.end());
898 :
899 5813 : OsrHelper osr_helper(data->info());
900 5813 : osr_helper.Deconstruct(data->jsgraph(), data->common(), temp_zone);
901 5813 : }
902 : };
903 :
904 :
905 : struct TypedLoweringPhase {
906 : static const char* phase_name() { return "typed lowering"; }
907 :
908 3953430 : void Run(PipelineData* data, Zone* temp_zone) {
909 395343 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
910 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
911 395343 : data->common());
912 : JSBuiltinReducer builtin_reducer(
913 : &graph_reducer, data->jsgraph(),
914 : data->info()->is_deoptimization_enabled()
915 : ? JSBuiltinReducer::kDeoptimizationEnabled
916 : : JSBuiltinReducer::kNoFlags,
917 1581372 : data->info()->dependencies(), data->native_context());
918 : Handle<FeedbackVector> feedback_vector(
919 : data->info()->closure()->feedback_vector());
920 : JSCreateLowering create_lowering(
921 : &graph_reducer, data->info()->dependencies(), data->jsgraph(),
922 790686 : feedback_vector, data->native_context(), temp_zone);
923 : JSTypedLowering::Flags typed_lowering_flags = JSTypedLowering::kNoFlags;
924 395343 : if (data->info()->is_deoptimization_enabled()) {
925 : typed_lowering_flags |= JSTypedLowering::kDeoptimizationEnabled;
926 : }
927 : JSTypedLowering typed_lowering(&graph_reducer, data->info()->dependencies(),
928 : typed_lowering_flags, data->jsgraph(),
929 395343 : temp_zone);
930 : TypedOptimization typed_optimization(
931 : &graph_reducer, data->info()->dependencies(),
932 : data->info()->is_deoptimization_enabled()
933 : ? TypedOptimization::kDeoptimizationEnabled
934 : : TypedOptimization::kNoFlags,
935 1186029 : data->jsgraph());
936 790686 : SimplifiedOperatorReducer simple_reducer(&graph_reducer, data->jsgraph());
937 395343 : CheckpointElimination checkpoint_elimination(&graph_reducer);
938 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
939 395343 : data->common(), data->machine());
940 395343 : AddReducer(data, &graph_reducer, &dead_code_elimination);
941 395343 : AddReducer(data, &graph_reducer, &builtin_reducer);
942 395343 : if (data->info()->is_deoptimization_enabled()) {
943 393274 : AddReducer(data, &graph_reducer, &create_lowering);
944 : }
945 395343 : AddReducer(data, &graph_reducer, &typed_optimization);
946 395343 : AddReducer(data, &graph_reducer, &typed_lowering);
947 395343 : AddReducer(data, &graph_reducer, &simple_reducer);
948 395343 : AddReducer(data, &graph_reducer, &checkpoint_elimination);
949 395343 : AddReducer(data, &graph_reducer, &common_reducer);
950 395343 : graph_reducer.ReduceGraph();
951 395343 : }
952 : };
953 :
954 :
955 : struct EscapeAnalysisPhase {
956 : static const char* phase_name() { return "escape analysis"; }
957 :
958 865906 : void Run(PipelineData* data, Zone* temp_zone) {
959 : EscapeAnalysis escape_analysis(data->graph(), data->jsgraph()->common(),
960 786386 : temp_zone);
961 746626 : if (!escape_analysis.Run()) return;
962 39760 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
963 : EscapeAnalysisReducer escape_reducer(&graph_reducer, data->jsgraph(),
964 39760 : &escape_analysis, temp_zone);
965 39760 : AddReducer(data, &graph_reducer, &escape_reducer);
966 39760 : graph_reducer.ReduceGraph();
967 39760 : if (escape_reducer.compilation_failed()) {
968 : data->set_compilation_failed();
969 : return;
970 : }
971 79520 : escape_reducer.VerifyReplacement();
972 : }
973 : };
974 :
975 : struct SimplifiedLoweringPhase {
976 : static const char* phase_name() { return "simplified lowering"; }
977 :
978 395303 : void Run(PipelineData* data, Zone* temp_zone) {
979 : SimplifiedLowering lowering(data->jsgraph(), temp_zone,
980 395303 : data->source_positions());
981 395303 : lowering.LowerAllNodes();
982 395302 : }
983 : };
984 :
985 : struct LoopPeelingPhase {
986 : static const char* phase_name() { return "loop peeling"; }
987 :
988 1553228 : void Run(PipelineData* data, Zone* temp_zone) {
989 388307 : GraphTrimmer trimmer(temp_zone, data->graph());
990 : NodeVector roots(temp_zone);
991 388307 : data->jsgraph()->GetCachedNodes(&roots);
992 388307 : trimmer.TrimGraph(roots.begin(), roots.end());
993 :
994 : LoopTree* loop_tree =
995 388307 : LoopFinder::BuildLoopTree(data->jsgraph()->graph(), temp_zone);
996 : LoopPeeler::PeelInnerLoopsOfTree(data->graph(), data->common(), loop_tree,
997 388307 : temp_zone);
998 388307 : }
999 : };
1000 :
1001 : struct LoopExitEliminationPhase {
1002 : static const char* phase_name() { return "loop exit elimination"; }
1003 :
1004 6996 : void Run(PipelineData* data, Zone* temp_zone) {
1005 6996 : LoopPeeler::EliminateLoopExits(data->graph(), temp_zone);
1006 : }
1007 : };
1008 :
1009 : struct ConcurrentOptimizationPrepPhase {
1010 : static const char* phase_name() { return "concurrency preparation"; }
1011 :
1012 1976715 : void Run(PipelineData* data, Zone* temp_zone) {
1013 : // Make sure we cache these code stubs.
1014 395343 : data->jsgraph()->CEntryStubConstant(1);
1015 395343 : data->jsgraph()->CEntryStubConstant(2);
1016 395343 : data->jsgraph()->CEntryStubConstant(3);
1017 :
1018 : // This is needed for escape analysis.
1019 395343 : NodeProperties::SetType(data->jsgraph()->FalseConstant(), Type::Boolean());
1020 395343 : NodeProperties::SetType(data->jsgraph()->TrueConstant(), Type::Boolean());
1021 395343 : }
1022 : };
1023 :
1024 : struct GenericLoweringPhase {
1025 : static const char* phase_name() { return "generic lowering"; }
1026 :
1027 790605 : void Run(PipelineData* data, Zone* temp_zone) {
1028 395302 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
1029 790604 : JSGenericLowering generic_lowering(data->jsgraph());
1030 395302 : AddReducer(data, &graph_reducer, &generic_lowering);
1031 395303 : graph_reducer.ReduceGraph();
1032 395302 : }
1033 : };
1034 :
1035 : struct EarlyOptimizationPhase {
1036 : static const char* phase_name() { return "early optimization"; }
1037 :
1038 2371817 : void Run(PipelineData* data, Zone* temp_zone) {
1039 395302 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
1040 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
1041 395303 : data->common());
1042 790603 : SimplifiedOperatorReducer simple_reducer(&graph_reducer, data->jsgraph());
1043 790602 : RedundancyElimination redundancy_elimination(&graph_reducer, temp_zone);
1044 790603 : ValueNumberingReducer value_numbering(temp_zone, data->graph()->zone());
1045 790604 : MachineOperatorReducer machine_reducer(data->jsgraph());
1046 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
1047 395303 : data->common(), data->machine());
1048 395302 : AddReducer(data, &graph_reducer, &dead_code_elimination);
1049 395303 : AddReducer(data, &graph_reducer, &simple_reducer);
1050 395303 : AddReducer(data, &graph_reducer, &redundancy_elimination);
1051 395303 : AddReducer(data, &graph_reducer, &machine_reducer);
1052 395303 : AddReducer(data, &graph_reducer, &common_reducer);
1053 395303 : AddReducer(data, &graph_reducer, &value_numbering);
1054 395303 : graph_reducer.ReduceGraph();
1055 395302 : }
1056 : };
1057 :
1058 : struct ControlFlowOptimizationPhase {
1059 : static const char* phase_name() { return "control flow optimization"; }
1060 :
1061 395303 : void Run(PipelineData* data, Zone* temp_zone) {
1062 : ControlFlowOptimizer optimizer(data->graph(), data->common(),
1063 395303 : data->machine(), temp_zone);
1064 395302 : optimizer.Optimize();
1065 395302 : }
1066 : };
1067 :
1068 : struct EffectControlLinearizationPhase {
1069 : static const char* phase_name() { return "effect linearization"; }
1070 :
1071 1976515 : void Run(PipelineData* data, Zone* temp_zone) {
1072 : // The scheduler requires the graphs to be trimmed, so trim now.
1073 : // TODO(jarin) Remove the trimming once the scheduler can handle untrimmed
1074 : // graphs.
1075 395303 : GraphTrimmer trimmer(temp_zone, data->graph());
1076 : NodeVector roots(temp_zone);
1077 395303 : data->jsgraph()->GetCachedNodes(&roots);
1078 395303 : trimmer.TrimGraph(roots.begin(), roots.end());
1079 :
1080 : // Schedule the graph without node splitting so that we can
1081 : // fix the effect and control flow for nodes with low-level side
1082 : // effects (such as changing representation to tagged or
1083 : // 'floating' allocation regions.)
1084 : Schedule* schedule = Scheduler::ComputeSchedule(temp_zone, data->graph(),
1085 395303 : Scheduler::kTempSchedule);
1086 395303 : if (FLAG_turbo_verify) ScheduleVerifier::Run(schedule);
1087 395303 : TraceSchedule(data->info(), schedule);
1088 :
1089 : // Post-pass for wiring the control/effects
1090 : // - connect allocating representation changes into the control&effect
1091 : // chains and lower them,
1092 : // - get rid of the region markers,
1093 : // - introduce effect phis and rewire effects to get SSA again.
1094 : EffectControlLinearizer linearizer(data->jsgraph(), schedule, temp_zone,
1095 395303 : data->source_positions());
1096 395301 : linearizer.Run();
1097 395296 : }
1098 : };
1099 :
1100 : // The store-store elimination greatly benefits from doing a common operator
1101 : // reducer and dead code elimination just before it, to eliminate conditional
1102 : // deopts with a constant condition.
1103 :
1104 : struct DeadCodeEliminationPhase {
1105 : static const char* phase_name() { return "dead code elimination"; }
1106 :
1107 1185905 : void Run(PipelineData* data, Zone* temp_zone) {
1108 395301 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
1109 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
1110 395302 : data->common());
1111 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
1112 395302 : data->common(), data->machine());
1113 395301 : AddReducer(data, &graph_reducer, &dead_code_elimination);
1114 395301 : AddReducer(data, &graph_reducer, &common_reducer);
1115 395301 : graph_reducer.ReduceGraph();
1116 395303 : }
1117 : };
1118 :
1119 : struct StoreStoreEliminationPhase {
1120 : static const char* phase_name() { return "store-store elimination"; }
1121 :
1122 1185909 : void Run(PipelineData* data, Zone* temp_zone) {
1123 395303 : GraphTrimmer trimmer(temp_zone, data->graph());
1124 : NodeVector roots(temp_zone);
1125 395303 : data->jsgraph()->GetCachedNodes(&roots);
1126 395303 : trimmer.TrimGraph(roots.begin(), roots.end());
1127 :
1128 395303 : StoreStoreElimination::Run(data->jsgraph(), temp_zone);
1129 395300 : }
1130 : };
1131 :
1132 : struct LoadEliminationPhase {
1133 : static const char* phase_name() { return "load elimination"; }
1134 :
1135 2359050 : void Run(PipelineData* data, Zone* temp_zone) {
1136 393175 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
1137 : BranchElimination branch_condition_elimination(&graph_reducer,
1138 786350 : data->jsgraph(), temp_zone);
1139 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
1140 393175 : data->common());
1141 786350 : RedundancyElimination redundancy_elimination(&graph_reducer, temp_zone);
1142 : LoadElimination load_elimination(&graph_reducer, data->jsgraph(),
1143 : temp_zone);
1144 393175 : CheckpointElimination checkpoint_elimination(&graph_reducer);
1145 786350 : ValueNumberingReducer value_numbering(temp_zone, data->graph()->zone());
1146 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
1147 393175 : data->common(), data->machine());
1148 393175 : AddReducer(data, &graph_reducer, &branch_condition_elimination);
1149 393175 : AddReducer(data, &graph_reducer, &dead_code_elimination);
1150 393175 : AddReducer(data, &graph_reducer, &redundancy_elimination);
1151 393175 : AddReducer(data, &graph_reducer, &load_elimination);
1152 393175 : AddReducer(data, &graph_reducer, &checkpoint_elimination);
1153 393175 : AddReducer(data, &graph_reducer, &common_reducer);
1154 393175 : AddReducer(data, &graph_reducer, &value_numbering);
1155 393175 : graph_reducer.ReduceGraph();
1156 393175 : }
1157 : };
1158 :
1159 : struct MemoryOptimizationPhase {
1160 : static const char* phase_name() { return "memory optimization"; }
1161 :
1162 1185909 : void Run(PipelineData* data, Zone* temp_zone) {
1163 : // The memory optimizer requires the graphs to be trimmed, so trim now.
1164 395303 : GraphTrimmer trimmer(temp_zone, data->graph());
1165 : NodeVector roots(temp_zone);
1166 395303 : data->jsgraph()->GetCachedNodes(&roots);
1167 395303 : trimmer.TrimGraph(roots.begin(), roots.end());
1168 :
1169 : // Optimize allocations and load/store operations.
1170 395303 : MemoryOptimizer optimizer(data->jsgraph(), temp_zone);
1171 790603 : optimizer.Optimize();
1172 395300 : }
1173 : };
1174 :
1175 : struct LateOptimizationPhase {
1176 : static const char* phase_name() { return "late optimization"; }
1177 :
1178 3162419 : void Run(PipelineData* data, Zone* temp_zone) {
1179 395303 : JSGraphReducer graph_reducer(data->jsgraph(), temp_zone);
1180 : BranchElimination branch_condition_elimination(&graph_reducer,
1181 790601 : data->jsgraph(), temp_zone);
1182 : DeadCodeElimination dead_code_elimination(&graph_reducer, data->graph(),
1183 395303 : data->common());
1184 790601 : ValueNumberingReducer value_numbering(temp_zone, data->graph()->zone());
1185 790597 : MachineOperatorReducer machine_reducer(data->jsgraph());
1186 : CommonOperatorReducer common_reducer(&graph_reducer, data->graph(),
1187 395302 : data->common(), data->machine());
1188 : SelectLowering select_lowering(data->jsgraph()->graph(),
1189 790604 : data->jsgraph()->common());
1190 : TailCallOptimization tco(data->common(), data->graph());
1191 395302 : AddReducer(data, &graph_reducer, &branch_condition_elimination);
1192 395302 : AddReducer(data, &graph_reducer, &dead_code_elimination);
1193 395303 : AddReducer(data, &graph_reducer, &machine_reducer);
1194 395303 : AddReducer(data, &graph_reducer, &common_reducer);
1195 395303 : AddReducer(data, &graph_reducer, &select_lowering);
1196 395303 : AddReducer(data, &graph_reducer, &tco);
1197 395303 : AddReducer(data, &graph_reducer, &value_numbering);
1198 395303 : graph_reducer.ReduceGraph();
1199 395301 : }
1200 : };
1201 :
1202 : struct EarlyGraphTrimmingPhase {
1203 : static const char* phase_name() { return "early graph trimming"; }
1204 790686 : void Run(PipelineData* data, Zone* temp_zone) {
1205 395343 : GraphTrimmer trimmer(temp_zone, data->graph());
1206 : NodeVector roots(temp_zone);
1207 395343 : data->jsgraph()->GetCachedNodes(&roots);
1208 395343 : trimmer.TrimGraph(roots.begin(), roots.end());
1209 395343 : }
1210 : };
1211 :
1212 :
1213 : struct LateGraphTrimmingPhase {
1214 : static const char* phase_name() { return "late graph trimming"; }
1215 931748 : void Run(PipelineData* data, Zone* temp_zone) {
1216 465861 : GraphTrimmer trimmer(temp_zone, data->graph());
1217 : NodeVector roots(temp_zone);
1218 465887 : if (data->jsgraph()) {
1219 465888 : data->jsgraph()->GetCachedNodes(&roots);
1220 : }
1221 465880 : trimmer.TrimGraph(roots.begin(), roots.end());
1222 465869 : }
1223 : };
1224 :
1225 :
1226 : struct ComputeSchedulePhase {
1227 : static const char* phase_name() { return "scheduling"; }
1228 :
1229 1136518 : void Run(PipelineData* data, Zone* temp_zone) {
1230 : Schedule* schedule = Scheduler::ComputeSchedule(
1231 : temp_zone, data->graph(), data->info()->is_splitting_enabled()
1232 : ? Scheduler::kSplitNodes
1233 1136518 : : Scheduler::kNoFlags);
1234 568260 : if (FLAG_turbo_verify) ScheduleVerifier::Run(schedule);
1235 : data->set_schedule(schedule);
1236 568260 : }
1237 : };
1238 :
1239 :
1240 : struct InstructionSelectionPhase {
1241 : static const char* phase_name() { return "select instructions"; }
1242 :
1243 2747526 : void Run(PipelineData* data, Zone* temp_zone, Linkage* linkage) {
1244 : InstructionSelector selector(
1245 : temp_zone, data->graph()->NodeCount(), linkage, data->sequence(),
1246 : data->schedule(), data->source_positions(), data->frame(),
1247 : data->info()->is_source_positions_enabled()
1248 : ? InstructionSelector::kAllSourcePositions
1249 : : InstructionSelector::kCallSourcePositions,
1250 : InstructionSelector::SupportedFeatures(),
1251 : FLAG_turbo_instruction_scheduling
1252 : ? InstructionSelector::kEnableScheduling
1253 : : InstructionSelector::kDisableScheduling,
1254 : data->info()->will_serialize()
1255 : ? InstructionSelector::kEnableSerialization
1256 3663374 : : InstructionSelector::kDisableSerialization);
1257 915877 : if (!selector.SelectInstructions()) {
1258 : data->set_compilation_failed();
1259 : }
1260 915847 : }
1261 : };
1262 :
1263 :
1264 : struct MeetRegisterConstraintsPhase {
1265 : static const char* phase_name() { return "meet register constraints"; }
1266 :
1267 915913 : void Run(PipelineData* data, Zone* temp_zone) {
1268 915913 : ConstraintBuilder builder(data->register_allocation_data());
1269 915909 : builder.MeetRegisterConstraints();
1270 915922 : }
1271 : };
1272 :
1273 :
1274 : struct ResolvePhisPhase {
1275 : static const char* phase_name() { return "resolve phis"; }
1276 :
1277 915921 : void Run(PipelineData* data, Zone* temp_zone) {
1278 915921 : ConstraintBuilder builder(data->register_allocation_data());
1279 915920 : builder.ResolvePhis();
1280 915907 : }
1281 : };
1282 :
1283 :
1284 : struct BuildLiveRangesPhase {
1285 : static const char* phase_name() { return "build live ranges"; }
1286 :
1287 915890 : void Run(PipelineData* data, Zone* temp_zone) {
1288 915890 : LiveRangeBuilder builder(data->register_allocation_data(), temp_zone);
1289 915896 : builder.BuildLiveRanges();
1290 915919 : }
1291 : };
1292 :
1293 :
1294 : struct SplinterLiveRangesPhase {
1295 : static const char* phase_name() { return "splinter live ranges"; }
1296 :
1297 915920 : void Run(PipelineData* data, Zone* temp_zone) {
1298 : LiveRangeSeparator live_range_splinterer(data->register_allocation_data(),
1299 : temp_zone);
1300 915920 : live_range_splinterer.Splinter();
1301 : }
1302 : };
1303 :
1304 :
1305 : template <typename RegAllocator>
1306 : struct AllocateGeneralRegistersPhase {
1307 : static const char* phase_name() { return "allocate general registers"; }
1308 :
1309 915911 : void Run(PipelineData* data, Zone* temp_zone) {
1310 : RegAllocator allocator(data->register_allocation_data(), GENERAL_REGISTERS,
1311 915911 : temp_zone);
1312 915922 : allocator.AllocateRegisters();
1313 915890 : }
1314 : };
1315 :
1316 : template <typename RegAllocator>
1317 : struct AllocateFPRegistersPhase {
1318 : static const char* phase_name() { return "allocate f.p. registers"; }
1319 :
1320 915923 : void Run(PipelineData* data, Zone* temp_zone) {
1321 : RegAllocator allocator(data->register_allocation_data(), FP_REGISTERS,
1322 915923 : temp_zone);
1323 915928 : allocator.AllocateRegisters();
1324 915926 : }
1325 : };
1326 :
1327 :
1328 : struct MergeSplintersPhase {
1329 : static const char* phase_name() { return "merge splintered ranges"; }
1330 915926 : void Run(PipelineData* pipeline_data, Zone* temp_zone) {
1331 : RegisterAllocationData* data = pipeline_data->register_allocation_data();
1332 : LiveRangeMerger live_range_merger(data, temp_zone);
1333 915926 : live_range_merger.Merge();
1334 : }
1335 : };
1336 :
1337 :
1338 : struct LocateSpillSlotsPhase {
1339 : static const char* phase_name() { return "locate spill slots"; }
1340 :
1341 915927 : void Run(PipelineData* data, Zone* temp_zone) {
1342 915927 : SpillSlotLocator locator(data->register_allocation_data());
1343 915924 : locator.LocateSpillSlots();
1344 915927 : }
1345 : };
1346 :
1347 :
1348 : struct AssignSpillSlotsPhase {
1349 : static const char* phase_name() { return "assign spill slots"; }
1350 :
1351 915924 : void Run(PipelineData* data, Zone* temp_zone) {
1352 915924 : OperandAssigner assigner(data->register_allocation_data());
1353 915918 : assigner.AssignSpillSlots();
1354 915883 : }
1355 : };
1356 :
1357 :
1358 : struct CommitAssignmentPhase {
1359 : static const char* phase_name() { return "commit assignment"; }
1360 :
1361 915923 : void Run(PipelineData* data, Zone* temp_zone) {
1362 915923 : OperandAssigner assigner(data->register_allocation_data());
1363 915923 : assigner.CommitAssignment();
1364 915919 : }
1365 : };
1366 :
1367 :
1368 : struct PopulateReferenceMapsPhase {
1369 : static const char* phase_name() { return "populate pointer maps"; }
1370 :
1371 915922 : void Run(PipelineData* data, Zone* temp_zone) {
1372 915922 : ReferenceMapPopulator populator(data->register_allocation_data());
1373 915922 : populator.PopulateReferenceMaps();
1374 915890 : }
1375 : };
1376 :
1377 :
1378 : struct ConnectRangesPhase {
1379 : static const char* phase_name() { return "connect ranges"; }
1380 :
1381 915919 : void Run(PipelineData* data, Zone* temp_zone) {
1382 915919 : LiveRangeConnector connector(data->register_allocation_data());
1383 915913 : connector.ConnectRanges(temp_zone);
1384 915899 : }
1385 : };
1386 :
1387 :
1388 : struct ResolveControlFlowPhase {
1389 : static const char* phase_name() { return "resolve control flow"; }
1390 :
1391 915923 : void Run(PipelineData* data, Zone* temp_zone) {
1392 915923 : LiveRangeConnector connector(data->register_allocation_data());
1393 915923 : connector.ResolveControlFlow(temp_zone);
1394 915901 : }
1395 : };
1396 :
1397 :
1398 : struct OptimizeMovesPhase {
1399 : static const char* phase_name() { return "optimize moves"; }
1400 :
1401 915928 : void Run(PipelineData* data, Zone* temp_zone) {
1402 915928 : MoveOptimizer move_optimizer(temp_zone, data->sequence());
1403 915914 : move_optimizer.Run();
1404 915899 : }
1405 : };
1406 :
1407 :
1408 : struct FrameElisionPhase {
1409 : static const char* phase_name() { return "frame elision"; }
1410 :
1411 915884 : void Run(PipelineData* data, Zone* temp_zone) {
1412 915884 : FrameElider(data->sequence()).Run();
1413 915838 : }
1414 : };
1415 :
1416 :
1417 : struct JumpThreadingPhase {
1418 : static const char* phase_name() { return "jump threading"; }
1419 :
1420 2389016 : void Run(PipelineData* data, Zone* temp_zone, bool frame_at_start) {
1421 : ZoneVector<RpoNumber> result(temp_zone);
1422 915866 : if (JumpThreading::ComputeForwarding(temp_zone, result, data->sequence(),
1423 915866 : frame_at_start)) {
1424 557284 : JumpThreading::ApplyForwarding(result, data->sequence());
1425 : }
1426 915882 : }
1427 : };
1428 :
1429 :
1430 : struct GenerateCodePhase {
1431 : static const char* phase_name() { return "generate code"; }
1432 :
1433 912392 : void Run(PipelineData* data, Zone* temp_zone, Linkage* linkage) {
1434 : CodeGenerator generator(data->frame(), linkage, data->sequence(),
1435 912392 : data->info());
1436 1824784 : data->set_code(generator.GenerateCode());
1437 912392 : }
1438 : };
1439 :
1440 :
1441 : struct PrintGraphPhase {
1442 : static const char* phase_name() { return nullptr; }
1443 :
1444 0 : void Run(PipelineData* data, Zone* temp_zone, const char* phase) {
1445 0 : CompilationInfo* info = data->info();
1446 : Graph* graph = data->graph();
1447 :
1448 : { // Print JSON.
1449 : AllowHandleDereference allow_deref;
1450 0 : TurboJsonFile json_of(info, std::ios_base::app);
1451 0 : json_of << "{\"name\":\"" << phase << "\",\"type\":\"graph\",\"data\":"
1452 0 : << AsJSON(*graph, data->source_positions()) << "},\n";
1453 : }
1454 :
1455 0 : if (FLAG_trace_turbo_graph) { // Simple textual RPO.
1456 : AllowHandleDereference allow_deref;
1457 0 : CodeTracer::Scope tracing_scope(info->isolate()->GetCodeTracer());
1458 0 : OFStream os(tracing_scope.file());
1459 0 : os << "-- Graph after " << phase << " -- " << std::endl;
1460 0 : os << AsRPO(*graph);
1461 : }
1462 0 : }
1463 : };
1464 :
1465 :
1466 : struct VerifyGraphPhase {
1467 : static const char* phase_name() { return nullptr; }
1468 :
1469 111827 : void Run(PipelineData* data, Zone* temp_zone, const bool untyped,
1470 : bool values_only = false) {
1471 : Verifier::Run(data->graph(), !untyped ? Verifier::TYPED : Verifier::UNTYPED,
1472 223654 : values_only ? Verifier::kValuesOnly : Verifier::kAll);
1473 : }
1474 : };
1475 :
1476 7666982 : void PipelineImpl::RunPrintAndVerify(const char* phase, bool untyped) {
1477 7666982 : if (FLAG_trace_turbo) {
1478 0 : Run<PrintGraphPhase>(phase);
1479 : }
1480 7667033 : if (FLAG_turbo_verify) {
1481 0 : Run<VerifyGraphPhase>(untyped);
1482 : }
1483 7667033 : }
1484 :
1485 395343 : bool PipelineImpl::CreateGraph() {
1486 1581372 : PipelineData* data = this->data_;
1487 :
1488 : data->BeginPhaseKind("graph creation");
1489 :
1490 395343 : if (FLAG_trace_turbo) {
1491 0 : CodeTracer::Scope tracing_scope(isolate()->GetCodeTracer());
1492 0 : OFStream os(tracing_scope.file());
1493 0 : os << "---------------------------------------------------\n"
1494 0 : << "Begin compiling method " << info()->GetDebugName().get()
1495 0 : << " using Turbofan" << std::endl;
1496 0 : TurboCfgFile tcf(isolate());
1497 0 : tcf << AsC1VCompilation(info());
1498 : }
1499 :
1500 395343 : data->source_positions()->AddDecorator();
1501 :
1502 395343 : if (FLAG_loop_assignment_analysis) {
1503 392123 : Run<LoopAssignmentAnalysisPhase>();
1504 : }
1505 :
1506 395343 : Run<GraphBuilderPhase>();
1507 395343 : if (data->compilation_failed()) {
1508 : data->EndPhaseKind();
1509 : return false;
1510 : }
1511 395343 : RunPrintAndVerify("Initial untyped", true);
1512 :
1513 : // Perform OSR deconstruction.
1514 395343 : if (info()->is_osr()) {
1515 5813 : Run<OsrDeconstructionPhase>();
1516 5813 : RunPrintAndVerify("OSR deconstruction", true);
1517 : }
1518 :
1519 : // Perform function context specialization and inlining (if enabled).
1520 395343 : Run<InliningPhase>();
1521 395343 : RunPrintAndVerify("Inlined", true);
1522 :
1523 : // Remove dead->live edges from the graph.
1524 395343 : Run<EarlyGraphTrimmingPhase>();
1525 395343 : RunPrintAndVerify("Early trimmed", true);
1526 :
1527 : // Run the type-sensitive lowerings and optimizations on the graph.
1528 : {
1529 : // Determine the Typer operation flags.
1530 : Typer::Flags flags = Typer::kNoFlags;
1531 1495924 : if (is_sloppy(info()->shared_info()->language_mode()) &&
1532 1015133 : info()->shared_info()->IsUserJavaScript()) {
1533 : // Sloppy mode functions always have an Object for this.
1534 : flags |= Typer::kThisIsReceiver;
1535 : }
1536 790686 : if (IsClassConstructor(info()->shared_info()->kind())) {
1537 : // Class constructors cannot be [[Call]]ed.
1538 : flags |= Typer::kNewTargetIsReceiver;
1539 : }
1540 :
1541 : // Type the graph and keep the Typer running on newly created nodes within
1542 : // this scope; the Typer is automatically unlinked from the Graph once we
1543 : // leave this scope below.
1544 395343 : Typer typer(isolate(), flags, data->graph());
1545 395343 : Run<TyperPhase>(&typer);
1546 395343 : RunPrintAndVerify("Typed");
1547 :
1548 : // Lower JSOperators where we can determine types.
1549 395343 : Run<TypedLoweringPhase>();
1550 395343 : RunPrintAndVerify("Lowered typed");
1551 : }
1552 :
1553 : // Do some hacky things to prepare for the optimization phase.
1554 : // (caching handles, etc.).
1555 395343 : Run<ConcurrentOptimizationPrepPhase>();
1556 :
1557 : data->EndPhaseKind();
1558 :
1559 : return true;
1560 : }
1561 :
1562 395303 : bool PipelineImpl::OptimizeGraph(Linkage* linkage) {
1563 1974405 : PipelineData* data = this->data_;
1564 :
1565 : data->BeginPhaseKind("lowering");
1566 :
1567 395303 : if (data->info()->is_loop_peeling_enabled()) {
1568 388307 : Run<LoopPeelingPhase>();
1569 388307 : RunPrintAndVerify("Loops peeled", true);
1570 : } else {
1571 6996 : Run<LoopExitEliminationPhase>();
1572 6996 : RunPrintAndVerify("Loop exits eliminated", true);
1573 : }
1574 :
1575 395303 : if (!data->is_asm()) {
1576 393193 : if (FLAG_turbo_load_elimination) {
1577 393175 : Run<LoadEliminationPhase>();
1578 393175 : RunPrintAndVerify("Load eliminated");
1579 : }
1580 :
1581 393193 : if (FLAG_turbo_escape) {
1582 393193 : Run<EscapeAnalysisPhase>();
1583 393193 : if (data->compilation_failed()) {
1584 : info()->AbortOptimization(kCyclicObjectStateDetectedInEscapeAnalysis);
1585 : data->EndPhaseKind();
1586 : return false;
1587 : }
1588 393193 : RunPrintAndVerify("Escape Analysed");
1589 : }
1590 : }
1591 :
1592 : // Perform simplified lowering. This has to run w/o the Typer decorator,
1593 : // because we cannot compute meaningful types anyways, and the computed types
1594 : // might even conflict with the representation/truncation logic.
1595 395303 : Run<SimplifiedLoweringPhase>();
1596 395303 : RunPrintAndVerify("Simplified lowering", true);
1597 :
1598 : #ifdef DEBUG
1599 : // From now on it is invalid to look at types on the nodes, because:
1600 : //
1601 : // (a) The remaining passes (might) run concurrent to the main thread and
1602 : // therefore must not access the Heap or the Isolate in an uncontrolled
1603 : // way (as done by the type system), and
1604 : // (b) the types on the nodes might not make sense after representation
1605 : // selection due to the way we handle truncations; if we'd want to look
1606 : // at types afterwards we'd essentially need to re-type (large portions
1607 : // of) the graph.
1608 : //
1609 : // In order to catch bugs related to type access after this point we remove
1610 : // the types from the nodes at this point (currently only in Debug builds).
1611 : Run<UntyperPhase>();
1612 : RunPrintAndVerify("Untyped", true);
1613 : #endif
1614 :
1615 : // Run generic lowering pass.
1616 395303 : Run<GenericLoweringPhase>();
1617 395303 : RunPrintAndVerify("Generic lowering", true);
1618 :
1619 : data->BeginPhaseKind("block building");
1620 :
1621 : // Run early optimization pass.
1622 395303 : Run<EarlyOptimizationPhase>();
1623 395303 : RunPrintAndVerify("Early optimized", true);
1624 :
1625 395303 : Run<EffectControlLinearizationPhase>();
1626 395302 : RunPrintAndVerify("Effect and control linearized", true);
1627 :
1628 395302 : Run<DeadCodeEliminationPhase>();
1629 395303 : RunPrintAndVerify("Dead code elimination", true);
1630 :
1631 395303 : if (FLAG_turbo_store_elimination) {
1632 395303 : Run<StoreStoreEliminationPhase>();
1633 395303 : RunPrintAndVerify("Store-store elimination", true);
1634 : }
1635 :
1636 : // Optimize control flow.
1637 395303 : if (FLAG_turbo_cf_optimization) {
1638 395303 : Run<ControlFlowOptimizationPhase>();
1639 395303 : RunPrintAndVerify("Control flow optimized", true);
1640 : }
1641 :
1642 : // Optimize memory access and allocation operations.
1643 395302 : Run<MemoryOptimizationPhase>();
1644 : // TODO(jarin, rossberg): Remove UNTYPED once machine typing works.
1645 395303 : RunPrintAndVerify("Memory optimized", true);
1646 :
1647 : // Lower changes that have been inserted before.
1648 395303 : Run<LateOptimizationPhase>();
1649 : // TODO(jarin, rossberg): Remove UNTYPED once machine typing works.
1650 395303 : RunPrintAndVerify("Late optimized", true);
1651 :
1652 395303 : data->source_positions()->RemoveDecorator();
1653 :
1654 395302 : return ScheduleAndSelectInstructions(linkage, true);
1655 : }
1656 :
1657 335479 : Handle<Code> Pipeline::GenerateCodeForCodeStub(Isolate* isolate,
1658 : CallDescriptor* call_descriptor,
1659 111827 : Graph* graph, Schedule* schedule,
1660 : Code::Flags flags,
1661 : const char* debug_name) {
1662 111827 : CompilationInfo info(CStrVector(debug_name), isolate, graph->zone(), flags);
1663 111826 : if (isolate->serializer_enabled()) info.PrepareForSerializing();
1664 :
1665 : // Construct a pipeline for scheduling and code generation.
1666 223653 : ZoneStats zone_stats(isolate->allocator());
1667 111827 : SourcePositionTable source_positions(graph);
1668 223654 : PipelineData data(&zone_stats, &info, graph, schedule, &source_positions);
1669 : data.set_verify_graph(FLAG_verify_csa);
1670 : std::unique_ptr<PipelineStatistics> pipeline_statistics;
1671 111827 : if (FLAG_turbo_stats || FLAG_turbo_stats_nvp) {
1672 0 : pipeline_statistics.reset(new PipelineStatistics(&info, &zone_stats));
1673 0 : pipeline_statistics->BeginPhaseKind("stub codegen");
1674 : }
1675 :
1676 : PipelineImpl pipeline(&data);
1677 : DCHECK_NOT_NULL(data.schedule());
1678 :
1679 111826 : if (FLAG_trace_turbo) {
1680 : {
1681 0 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
1682 0 : OFStream os(tracing_scope.file());
1683 0 : os << "---------------------------------------------------\n"
1684 0 : << "Begin compiling " << debug_name << " using Turbofan" << std::endl;
1685 : }
1686 : {
1687 0 : TurboJsonFile json_of(&info, std::ios_base::trunc);
1688 0 : json_of << "{\"function\":\"" << info.GetDebugName().get()
1689 0 : << "\", \"source\":\"\",\n\"phases\":[";
1690 : }
1691 0 : pipeline.Run<PrintGraphPhase>("Machine");
1692 : }
1693 :
1694 111826 : pipeline.Run<VerifyGraphPhase>(false, true);
1695 223654 : return pipeline.ScheduleAndGenerateCode(call_descriptor);
1696 : }
1697 :
1698 : // static
1699 4886 : Handle<Code> Pipeline::GenerateCodeForTesting(CompilationInfo* info) {
1700 4886 : ZoneStats zone_stats(info->isolate()->allocator());
1701 : std::unique_ptr<PipelineStatistics> pipeline_statistics(
1702 4886 : CreatePipelineStatistics(info, &zone_stats));
1703 9772 : PipelineData data(&zone_stats, info, pipeline_statistics.get());
1704 : PipelineImpl pipeline(&data);
1705 :
1706 4886 : Linkage linkage(Linkage::ComputeIncoming(data.instruction_zone(), info));
1707 :
1708 4886 : if (!pipeline.CreateGraph()) return Handle<Code>::null();
1709 4886 : if (!pipeline.OptimizeGraph(&linkage)) return Handle<Code>::null();
1710 9772 : return pipeline.GenerateCode(&linkage);
1711 : }
1712 :
1713 : // static
1714 7 : Handle<Code> Pipeline::GenerateCodeForTesting(CompilationInfo* info,
1715 : Graph* graph,
1716 : Schedule* schedule) {
1717 : CallDescriptor* call_descriptor =
1718 7 : Linkage::ComputeIncoming(info->zone(), info);
1719 7 : return GenerateCodeForTesting(info, call_descriptor, graph, schedule);
1720 : }
1721 :
1722 : // static
1723 338167 : Handle<Code> Pipeline::GenerateCodeForTesting(
1724 674065 : CompilationInfo* info, CallDescriptor* call_descriptor, Graph* graph,
1725 : Schedule* schedule, SourcePositionTable* source_positions) {
1726 : // Construct a pipeline for scheduling and code generation.
1727 338167 : ZoneStats zone_stats(info->isolate()->allocator());
1728 : // TODO(wasm): Refactor code generation to check for non-existing source
1729 : // table, then remove this conditional allocation.
1730 338167 : if (!source_positions)
1731 335898 : source_positions = new (info->zone()) SourcePositionTable(graph);
1732 676334 : PipelineData data(&zone_stats, info, graph, schedule, source_positions);
1733 : std::unique_ptr<PipelineStatistics> pipeline_statistics;
1734 338167 : if (FLAG_turbo_stats || FLAG_turbo_stats_nvp) {
1735 0 : pipeline_statistics.reset(new PipelineStatistics(info, &zone_stats));
1736 0 : pipeline_statistics->BeginPhaseKind("test codegen");
1737 : }
1738 :
1739 : PipelineImpl pipeline(&data);
1740 :
1741 338167 : if (FLAG_trace_turbo) {
1742 0 : TurboJsonFile json_of(info, std::ios_base::trunc);
1743 0 : json_of << "{\"function\":\"" << info->GetDebugName().get()
1744 0 : << "\", \"source\":\"\",\n\"phases\":[";
1745 : }
1746 : // TODO(rossberg): Should this really be untyped?
1747 338167 : pipeline.RunPrintAndVerify("Machine", true);
1748 :
1749 676334 : return pipeline.ScheduleAndGenerateCode(call_descriptor);
1750 : }
1751 :
1752 : // static
1753 418335 : CompilationJob* Pipeline::NewCompilationJob(Handle<JSFunction> function,
1754 : bool has_script) {
1755 : Handle<SharedFunctionInfo> shared = handle(function->shared());
1756 : ParseInfo* parse_info;
1757 418335 : if (!has_script) {
1758 0 : parse_info = ParseInfo::AllocateWithoutScript(shared);
1759 : } else {
1760 418335 : parse_info = new ParseInfo(shared);
1761 : }
1762 418336 : return new PipelineCompilationJob(parse_info, function);
1763 : }
1764 :
1765 : // static
1766 70580 : CompilationJob* Pipeline::NewWasmCompilationJob(
1767 : CompilationInfo* info, JSGraph* jsgraph, CallDescriptor* descriptor,
1768 : SourcePositionTable* source_positions,
1769 : ZoneVector<trap_handler::ProtectedInstructionData>* protected_instructions,
1770 : bool allow_signalling_nan) {
1771 : return new PipelineWasmCompilationJob(
1772 : info, jsgraph, descriptor, source_positions, protected_instructions,
1773 70580 : allow_signalling_nan);
1774 : }
1775 :
1776 42 : bool Pipeline::AllocateRegistersForTesting(const RegisterConfiguration* config,
1777 84 : InstructionSequence* sequence,
1778 : bool run_verifier) {
1779 : CompilationInfo info(ArrayVector("testing"), sequence->isolate(),
1780 42 : sequence->zone(), Code::ComputeFlags(Code::STUB));
1781 84 : ZoneStats zone_stats(sequence->isolate()->allocator());
1782 84 : PipelineData data(&zone_stats, &info, sequence);
1783 : PipelineImpl pipeline(&data);
1784 42 : pipeline.data_->InitializeFrameData(nullptr);
1785 42 : pipeline.AllocateRegisters(config, nullptr, run_verifier);
1786 84 : return !data.compilation_failed();
1787 : }
1788 :
1789 915874 : bool PipelineImpl::ScheduleAndSelectInstructions(Linkage* linkage,
1790 : bool trim_graph) {
1791 : CallDescriptor* call_descriptor = linkage->GetIncomingDescriptor();
1792 5495284 : PipelineData* data = this->data_;
1793 :
1794 : DCHECK_NOT_NULL(data->graph());
1795 :
1796 915874 : if (trim_graph) {
1797 465878 : Run<LateGraphTrimmingPhase>();
1798 465891 : RunPrintAndVerify("Late trimmed", true);
1799 : }
1800 915878 : if (data->schedule() == nullptr) Run<ComputeSchedulePhase>();
1801 915886 : TraceSchedule(data->info(), data->schedule());
1802 :
1803 915871 : if (FLAG_turbo_profiling) {
1804 : data->set_profiler_data(BasicBlockInstrumentor::Instrument(
1805 14 : info(), data->graph(), data->schedule()));
1806 : }
1807 :
1808 : bool verify_stub_graph = data->verify_graph();
1809 2747605 : if (verify_stub_graph ||
1810 915863 : (FLAG_turbo_verify_machine_graph != nullptr &&
1811 0 : (!strcmp(FLAG_turbo_verify_machine_graph, "*") ||
1812 0 : !strcmp(FLAG_turbo_verify_machine_graph, data->debug_name())))) {
1813 0 : if (FLAG_trace_verify_csa) {
1814 : AllowHandleDereference allow_deref;
1815 0 : CompilationInfo* info = data->info();
1816 0 : CodeTracer::Scope tracing_scope(info->isolate()->GetCodeTracer());
1817 0 : OFStream os(tracing_scope.file());
1818 0 : os << "--------------------------------------------------\n"
1819 0 : << "--- Verifying " << data->debug_name() << " generated by TurboFan\n"
1820 0 : << "--------------------------------------------------\n"
1821 0 : << *data->schedule()
1822 0 : << "--------------------------------------------------\n"
1823 0 : << "--- End of " << data->debug_name() << " generated by TurboFan\n"
1824 0 : << "--------------------------------------------------\n";
1825 : }
1826 0 : Zone temp_zone(data->isolate()->allocator(), ZONE_NAME);
1827 : MachineGraphVerifier::Run(data->graph(), data->schedule(), linkage,
1828 0 : data->info()->IsStub(), data->debug_name(),
1829 0 : &temp_zone);
1830 : }
1831 :
1832 915871 : data->InitializeInstructionSequence(call_descriptor);
1833 :
1834 915863 : data->InitializeFrameData(call_descriptor);
1835 : // Select and schedule instructions covering the scheduled graph.
1836 915833 : Run<InstructionSelectionPhase>(linkage);
1837 915888 : if (data->compilation_failed()) {
1838 : info()->AbortOptimization(kCodeGenerationFailed);
1839 : data->EndPhaseKind();
1840 : return false;
1841 : }
1842 :
1843 915888 : if (FLAG_trace_turbo && !data->MayHaveUnverifiableGraph()) {
1844 : AllowHandleDereference allow_deref;
1845 0 : TurboCfgFile tcf(isolate());
1846 : tcf << AsC1V("CodeGen", data->schedule(), data->source_positions(),
1847 0 : data->sequence());
1848 : }
1849 :
1850 915888 : if (FLAG_trace_turbo) {
1851 0 : std::ostringstream source_position_output;
1852 : // Output source position information before the graph is deleted.
1853 915873 : data_->source_positions()->Print(source_position_output);
1854 0 : data_->set_source_position_output(source_position_output.str());
1855 : }
1856 :
1857 915888 : data->DeleteGraphZone();
1858 :
1859 : data->BeginPhaseKind("register allocation");
1860 :
1861 915880 : bool run_verifier = FLAG_turbo_verify_allocation;
1862 :
1863 : // Allocate registers.
1864 : AllocateRegisters(RegisterConfiguration::Turbofan(), call_descriptor,
1865 915880 : run_verifier);
1866 915878 : Run<FrameElisionPhase>();
1867 915873 : if (data->compilation_failed()) {
1868 : info()->AbortOptimization(kNotEnoughVirtualRegistersRegalloc);
1869 : data->EndPhaseKind();
1870 : return false;
1871 : }
1872 :
1873 : // TODO(mtrofin): move this off to the register allocator.
1874 : bool generate_frame_at_start =
1875 2747619 : data_->sequence()->instruction_blocks().front()->must_construct_frame();
1876 : // Optimimize jumps.
1877 915873 : if (FLAG_turbo_jt) {
1878 915832 : Run<JumpThreadingPhase>(generate_frame_at_start);
1879 : }
1880 :
1881 : data->EndPhaseKind();
1882 :
1883 : return true;
1884 : }
1885 :
1886 912390 : Handle<Code> PipelineImpl::GenerateCode(Linkage* linkage) {
1887 912390 : PipelineData* data = this->data_;
1888 :
1889 : data->BeginPhaseKind("code generation");
1890 :
1891 : // Generate final machine code.
1892 912390 : Run<GenerateCodePhase>(linkage);
1893 :
1894 : Handle<Code> code = data->code();
1895 : if (data->profiler_data()) {
1896 : #if ENABLE_DISASSEMBLER
1897 : std::ostringstream os;
1898 : code->Disassemble(nullptr, os);
1899 : data->profiler_data()->SetCode(&os);
1900 : #endif
1901 : }
1902 :
1903 : info()->SetCode(code);
1904 912392 : v8::internal::CodeGenerator::PrintCode(code, info());
1905 :
1906 912391 : if (FLAG_trace_turbo) {
1907 0 : TurboJsonFile json_of(info(), std::ios_base::app);
1908 0 : json_of << "{\"name\":\"disassembly\",\"type\":\"disassembly\",\"data\":\"";
1909 : #if ENABLE_DISASSEMBLER
1910 : std::stringstream disassembly_stream;
1911 : code->Disassemble(nullptr, disassembly_stream);
1912 : std::string disassembly_string(disassembly_stream.str());
1913 : for (const auto& c : disassembly_string) {
1914 : json_of << AsEscapedUC16ForJSON(c);
1915 : }
1916 : #endif // ENABLE_DISASSEMBLER
1917 0 : json_of << "\"}\n],\n";
1918 0 : json_of << "\"nodePositions\":";
1919 : json_of << data->source_position_output();
1920 0 : json_of << "}";
1921 :
1922 0 : CodeTracer::Scope tracing_scope(isolate()->GetCodeTracer());
1923 0 : OFStream os(tracing_scope.file());
1924 0 : os << "---------------------------------------------------\n"
1925 0 : << "Finished compiling method " << info()->GetDebugName().get()
1926 0 : << " using Turbofan" << std::endl;
1927 : }
1928 :
1929 912391 : return code;
1930 : }
1931 :
1932 449994 : Handle<Code> PipelineImpl::ScheduleAndGenerateCode(
1933 : CallDescriptor* call_descriptor) {
1934 : Linkage linkage(call_descriptor);
1935 :
1936 : // Schedule the graph, perform instruction selection and register allocation.
1937 449994 : if (!ScheduleAndSelectInstructions(&linkage, false)) return Handle<Code>();
1938 :
1939 : // Generate the final machine code.
1940 449994 : return GenerateCode(&linkage);
1941 : }
1942 :
1943 915904 : void PipelineImpl::AllocateRegisters(const RegisterConfiguration* config,
1944 : CallDescriptor* descriptor,
1945 : bool run_verifier) {
1946 921843 : PipelineData* data = this->data_;
1947 : // Don't track usage for this zone in compiler stats.
1948 : std::unique_ptr<Zone> verifier_zone;
1949 : RegisterAllocatorVerifier* verifier = nullptr;
1950 915904 : if (run_verifier) {
1951 42 : verifier_zone.reset(new Zone(isolate()->allocator(), ZONE_NAME));
1952 : verifier = new (verifier_zone.get()) RegisterAllocatorVerifier(
1953 42 : verifier_zone.get(), config, data->sequence());
1954 : }
1955 :
1956 : #ifdef DEBUG
1957 : data_->sequence()->ValidateEdgeSplitForm();
1958 : data_->sequence()->ValidateDeferredBlockEntryPaths();
1959 : data_->sequence()->ValidateDeferredBlockExitPaths();
1960 : #endif
1961 :
1962 915904 : data->InitializeRegisterAllocationData(config, descriptor);
1963 915905 : if (info()->is_osr()) {
1964 : AllowHandleDereference allow_deref;
1965 5813 : OsrHelper osr_helper(info());
1966 5813 : osr_helper.SetupFrame(data->frame());
1967 : }
1968 :
1969 915905 : Run<MeetRegisterConstraintsPhase>();
1970 915927 : Run<ResolvePhisPhase>();
1971 915924 : Run<BuildLiveRangesPhase>();
1972 915921 : if (FLAG_trace_turbo_graph) {
1973 : AllowHandleDereference allow_deref;
1974 0 : CodeTracer::Scope tracing_scope(isolate()->GetCodeTracer());
1975 0 : OFStream os(tracing_scope.file());
1976 0 : os << "----- Instruction sequence before register allocation -----\n"
1977 0 : << PrintableInstructionSequence({config, data->sequence()});
1978 : }
1979 915921 : if (verifier != nullptr) {
1980 42 : CHECK(!data->register_allocation_data()->ExistsUseWithoutDefinition());
1981 42 : CHECK(data->register_allocation_data()
1982 : ->RangesDefinedInDeferredStayInDeferred());
1983 : }
1984 :
1985 915921 : if (FLAG_turbo_preprocess_ranges) {
1986 915908 : Run<SplinterLiveRangesPhase>();
1987 : }
1988 :
1989 915939 : Run<AllocateGeneralRegistersPhase<LinearScanAllocator>>();
1990 915927 : Run<AllocateFPRegistersPhase<LinearScanAllocator>>();
1991 :
1992 915930 : if (FLAG_turbo_preprocess_ranges) {
1993 915929 : Run<MergeSplintersPhase>();
1994 : }
1995 :
1996 915927 : Run<AssignSpillSlotsPhase>();
1997 :
1998 915929 : Run<CommitAssignmentPhase>();
1999 915928 : Run<PopulateReferenceMapsPhase>();
2000 915926 : Run<ConnectRangesPhase>();
2001 915928 : Run<ResolveControlFlowPhase>();
2002 915927 : if (FLAG_turbo_move_optimization) {
2003 915927 : Run<OptimizeMovesPhase>();
2004 : }
2005 :
2006 915929 : Run<LocateSpillSlotsPhase>();
2007 :
2008 915930 : if (FLAG_trace_turbo_graph) {
2009 : AllowHandleDereference allow_deref;
2010 0 : CodeTracer::Scope tracing_scope(isolate()->GetCodeTracer());
2011 0 : OFStream os(tracing_scope.file());
2012 0 : os << "----- Instruction sequence after register allocation -----\n"
2013 0 : << PrintableInstructionSequence({config, data->sequence()});
2014 : }
2015 :
2016 915930 : if (verifier != nullptr) {
2017 42 : verifier->VerifyAssignment();
2018 42 : verifier->VerifyGapMoves();
2019 : }
2020 :
2021 915912 : if (FLAG_trace_turbo && !data->MayHaveUnverifiableGraph()) {
2022 0 : TurboCfgFile tcf(data->isolate());
2023 : tcf << AsC1VRegisterAllocationData("CodeGen",
2024 0 : data->register_allocation_data());
2025 : }
2026 :
2027 915912 : data->DeleteRegisterAllocationZone();
2028 915929 : }
2029 :
2030 4632012 : CompilationInfo* PipelineImpl::info() const { return data_->info(); }
2031 :
2032 395385 : Isolate* PipelineImpl::isolate() const { return info()->isolate(); }
2033 :
2034 : } // namespace compiler
2035 : } // namespace internal
2036 : } // namespace v8
|
