Line data Source code
1 : // Copyright 2013 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/backend/code-generator.h"
6 :
7 : #include "src/address-map.h"
8 : #include "src/assembler-inl.h"
9 : #include "src/base/adapters.h"
10 : #include "src/compiler/backend/code-generator-impl.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/pipeline.h"
13 : #include "src/compiler/wasm-compiler.h"
14 : #include "src/counters.h"
15 : #include "src/eh-frame.h"
16 : #include "src/frames.h"
17 : #include "src/log.h"
18 : #include "src/macro-assembler-inl.h"
19 : #include "src/objects/smi.h"
20 : #include "src/optimized-compilation-info.h"
21 : #include "src/string-constants.h"
22 :
23 : namespace v8 {
24 : namespace internal {
25 : namespace compiler {
26 :
27 : class CodeGenerator::JumpTable final : public ZoneObject {
28 : public:
29 : JumpTable(JumpTable* next, Label** targets, size_t target_count)
30 320 : : next_(next), targets_(targets), target_count_(target_count) {}
31 :
32 641 : Label* label() { return &label_; }
33 : JumpTable* next() const { return next_; }
34 : Label** targets() const { return targets_; }
35 : size_t target_count() const { return target_count_; }
36 :
37 : private:
38 : Label label_;
39 : JumpTable* const next_;
40 : Label** const targets_;
41 : size_t const target_count_;
42 : };
43 :
44 2641052 : CodeGenerator::CodeGenerator(
45 : Zone* codegen_zone, Frame* frame, Linkage* linkage,
46 : InstructionSequence* instructions, OptimizedCompilationInfo* info,
47 : Isolate* isolate, base::Optional<OsrHelper> osr_helper,
48 : int start_source_position, JumpOptimizationInfo* jump_opt,
49 : PoisoningMitigationLevel poisoning_level, const AssemblerOptions& options,
50 : int32_t builtin_index, std::unique_ptr<AssemblerBuffer> buffer)
51 : : zone_(codegen_zone),
52 : isolate_(isolate),
53 : frame_access_state_(nullptr),
54 : linkage_(linkage),
55 : instructions_(instructions),
56 : unwinding_info_writer_(zone()),
57 : info_(info),
58 2640269 : labels_(zone()->NewArray<Label>(instructions->InstructionBlockCount())),
59 : current_block_(RpoNumber::Invalid()),
60 : start_source_position_(start_source_position),
61 : current_source_position_(SourcePosition::Unknown()),
62 : tasm_(isolate, options, CodeObjectRequired::kNo, std::move(buffer)),
63 : resolver_(this),
64 : safepoints_(zone()),
65 : handlers_(zone()),
66 : deoptimization_exits_(zone()),
67 : deoptimization_states_(zone()),
68 : deoptimization_literals_(zone()),
69 : translations_(zone()),
70 : caller_registers_saved_(false),
71 : jump_tables_(nullptr),
72 : ools_(nullptr),
73 : osr_helper_(std::move(osr_helper)),
74 : osr_pc_offset_(-1),
75 : optimized_out_literal_id_(-1),
76 : source_position_table_builder_(
77 : SourcePositionTableBuilder::RECORD_SOURCE_POSITIONS),
78 : protected_instructions_(zone()),
79 : result_(kSuccess),
80 : poisoning_level_(poisoning_level),
81 : block_starts_(zone()),
82 23770507 : instr_starts_(zone()) {
83 43504707 : for (int i = 0; i < instructions->InstructionBlockCount(); ++i) {
84 20431953 : new (&labels_[i]) Label;
85 : }
86 2640801 : CreateFrameAccessState(frame);
87 2640426 : CHECK_EQ(info->is_osr(), osr_helper_.has_value());
88 : tasm_.set_jump_optimization_info(jump_opt);
89 : Code::Kind code_kind = info->code_kind();
90 5280852 : if (code_kind == Code::WASM_FUNCTION ||
91 2640426 : code_kind == Code::WASM_TO_JS_FUNCTION ||
92 6927172 : code_kind == Code::WASM_INTERPRETER_ENTRY ||
93 123088 : (Builtins::IsBuiltinId(builtin_index) &&
94 123088 : Builtins::IsWasmRuntimeStub(builtin_index))) {
95 : tasm_.set_abort_hard(true);
96 : }
97 : tasm_.set_builtin_index(builtin_index);
98 2640426 : }
99 :
100 384044 : bool CodeGenerator::wasm_runtime_exception_support() const {
101 : DCHECK_NOT_NULL(info_);
102 768088 : return info_->wasm_runtime_exception_support();
103 : }
104 :
105 241775 : void CodeGenerator::AddProtectedInstructionLanding(uint32_t instr_offset,
106 : uint32_t landing_offset) {
107 483858 : protected_instructions_.push_back({instr_offset, landing_offset});
108 242083 : }
109 :
110 2640767 : void CodeGenerator::CreateFrameAccessState(Frame* frame) {
111 2640767 : FinishFrame(frame);
112 2640418 : frame_access_state_ = new (zone()) FrameAccessState(frame);
113 2640418 : }
114 :
115 3330686 : CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
116 : int deoptimization_id, SourcePosition pos) {
117 3330686 : if (deoptimization_id > Deoptimizer::kMaxNumberOfEntries) {
118 : return kTooManyDeoptimizationBailouts;
119 : }
120 :
121 : DeoptimizeKind deopt_kind = GetDeoptimizationKind(deoptimization_id);
122 : DeoptimizeReason deoptimization_reason =
123 : GetDeoptimizationReason(deoptimization_id);
124 : Address deopt_entry =
125 3330689 : Deoptimizer::GetDeoptimizationEntry(tasm()->isolate(), deopt_kind);
126 3330693 : if (info()->is_source_positions_enabled()) {
127 101858 : tasm()->RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
128 : }
129 3330693 : tasm()->CallForDeoptimization(deopt_entry, deoptimization_id);
130 3330695 : return kSuccess;
131 : }
132 :
133 36 : void CodeGenerator::MaybeEmitOutOfLineConstantPool() {
134 : tasm()->MaybeEmitOutOfLineConstantPool();
135 36 : }
136 :
137 2641622 : void CodeGenerator::AssembleCode() {
138 : OptimizedCompilationInfo* info = this->info();
139 :
140 : // Open a frame scope to indicate that there is a frame on the stack. The
141 : // MANUAL indicates that the scope shouldn't actually generate code to set up
142 : // the frame (that is done in AssemblePrologue).
143 2640395 : FrameScope frame_scope(tasm(), StackFrame::MANUAL);
144 :
145 2641622 : if (info->is_source_positions_enabled()) {
146 11356 : AssembleSourcePosition(start_source_position());
147 : }
148 :
149 : // Check that {kJavaScriptCallCodeStartRegister} has been set correctly.
150 2641455 : if (FLAG_debug_code && (info->code_kind() == Code::OPTIMIZED_FUNCTION ||
151 : info->code_kind() == Code::BYTECODE_HANDLER)) {
152 114 : tasm()->RecordComment("-- Prologue: check code start register --");
153 114 : AssembleCodeStartRegisterCheck();
154 : }
155 :
156 : // We want to bailout only from JS functions, which are the only ones
157 : // that are optimized.
158 2641455 : if (info->IsOptimizing()) {
159 : DCHECK(linkage()->GetIncomingDescriptor()->IsJSFunctionCall());
160 464393 : tasm()->RecordComment("-- Prologue: check for deoptimization --");
161 464394 : BailoutIfDeoptimized();
162 : }
163 :
164 2641459 : InitializeSpeculationPoison();
165 :
166 : // Define deoptimization literals for all inlined functions.
167 : DCHECK_EQ(0u, deoptimization_literals_.size());
168 2705948 : for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
169 : info->inlined_functions()) {
170 65529 : if (!inlined.shared_info.equals(info->shared_info())) {
171 65118 : int index = DefineDeoptimizationLiteral(
172 65118 : DeoptimizationLiteral(inlined.shared_info));
173 : inlined.RegisterInlinedFunctionId(index);
174 : }
175 : }
176 2640419 : inlined_function_count_ = deoptimization_literals_.size();
177 :
178 : // Define deoptimization literals for all BytecodeArrays to which we might
179 : // deopt to ensure they are strongly held by the optimized code.
180 2640419 : if (info->has_bytecode_array()) {
181 464397 : DefineDeoptimizationLiteral(DeoptimizationLiteral(info->bytecode_array()));
182 : }
183 2705939 : for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
184 : info->inlined_functions()) {
185 65529 : DefineDeoptimizationLiteral(DeoptimizationLiteral(inlined.bytecode_array));
186 : }
187 :
188 : unwinding_info_writer_.SetNumberOfInstructionBlocks(
189 : instructions()->InstructionBlockCount());
190 :
191 2639986 : if (info->trace_turbo_json_enabled()) {
192 27 : block_starts_.assign(instructions()->instruction_blocks().size(), -1);
193 27 : instr_starts_.assign(instructions()->instructions().size(), -1);
194 : }
195 :
196 : // Assemble instructions in assembly order.
197 23075164 : for (const InstructionBlock* block : instructions()->ao_blocks()) {
198 : // Align loop headers on vendor recommended boundaries.
199 20432115 : if (block->ShouldAlign() && !tasm()->jump_optimization_info()) {
200 56518 : tasm()->CodeTargetAlign();
201 : }
202 20432135 : if (info->trace_turbo_json_enabled()) {
203 42 : block_starts_[block->rpo_number().ToInt()] = tasm()->pc_offset();
204 : }
205 : // Bind a label for a block.
206 20432135 : current_block_ = block->rpo_number();
207 20432135 : unwinding_info_writer_.BeginInstructionBlock(tasm()->pc_offset(), block);
208 20432007 : if (FLAG_code_comments) {
209 4006 : std::ostringstream buffer;
210 2003 : buffer << "-- B" << block->rpo_number().ToInt() << " start";
211 2003 : if (block->IsDeferred()) buffer << " (deferred)";
212 2003 : if (!block->needs_frame()) buffer << " (no frame)";
213 2003 : if (block->must_construct_frame()) buffer << " (construct frame)";
214 2003 : if (block->must_deconstruct_frame()) buffer << " (deconstruct frame)";
215 :
216 2003 : if (block->IsLoopHeader()) {
217 17 : buffer << " (loop up to " << block->loop_end().ToInt() << ")";
218 : }
219 2003 : if (block->loop_header().IsValid()) {
220 793 : buffer << " (in loop " << block->loop_header().ToInt() << ")";
221 : }
222 2003 : buffer << " --";
223 4006 : tasm()->RecordComment(buffer.str().c_str());
224 : }
225 :
226 20432007 : frame_access_state()->MarkHasFrame(block->needs_frame());
227 :
228 20432499 : tasm()->bind(GetLabel(current_block_));
229 :
230 20432081 : TryInsertBranchPoisoning(block);
231 :
232 20433553 : if (block->must_construct_frame()) {
233 2662641 : AssembleConstructFrame();
234 : // We need to setup the root register after we assemble the prologue, to
235 : // avoid clobbering callee saved registers in case of C linkage and
236 : // using the roots.
237 : // TODO(mtrofin): investigate how we can avoid doing this repeatedly.
238 2662904 : if (linkage()->GetIncomingDescriptor()->InitializeRootRegister()) {
239 888556 : tasm()->InitializeRootRegister();
240 : }
241 : }
242 :
243 : if (FLAG_enable_embedded_constant_pool && !block->needs_frame()) {
244 : ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
245 : result_ = AssembleBlock(block);
246 : } else {
247 20433816 : result_ = AssembleBlock(block);
248 : }
249 20435370 : if (result_ != kSuccess) return;
250 20435546 : unwinding_info_writer_.EndInstructionBlock(block);
251 : }
252 :
253 : // Assemble all out-of-line code.
254 2643049 : if (ools_) {
255 248661 : tasm()->RecordComment("-- Out of line code --");
256 1008282 : for (OutOfLineCode* ool = ools_; ool; ool = ool->next()) {
257 759639 : tasm()->bind(ool->entry());
258 759663 : ool->Generate();
259 1135632 : if (ool->exit()->is_bound()) tasm()->jmp(ool->exit());
260 : }
261 : }
262 :
263 : // This nop operation is needed to ensure that the trampoline is not
264 : // confused with the pc of the call before deoptimization.
265 : // The test regress/regress-259 is an example of where we need it.
266 2643031 : tasm()->nop();
267 :
268 : // Assemble deoptimization exits.
269 : int last_updated = 0;
270 5926665 : for (DeoptimizationExit* exit : deoptimization_exits_) {
271 3286152 : tasm()->bind(exit->label());
272 : int trampoline_pc = tasm()->pc_offset();
273 : int deoptimization_id = exit->deoptimization_id();
274 3286153 : DeoptimizationState* ds = deoptimization_states_[deoptimization_id];
275 :
276 3286153 : if (ds->kind() == DeoptimizeKind::kLazy) {
277 : last_updated = safepoints()->UpdateDeoptimizationInfo(
278 2958451 : ds->pc_offset(), trampoline_pc, last_updated);
279 : }
280 3286154 : result_ = AssembleDeoptimizerCall(deoptimization_id, exit->pos());
281 3286161 : if (result_ != kSuccess) return;
282 : }
283 :
284 : // TODO(jgruber): Move all inlined metadata generation into a new,
285 : // architecture-independent version of FinishCode. Currently, this includes
286 : // the safepoint table, handler table, constant pool, and code comments, in
287 : // that order.
288 2640513 : FinishCode();
289 :
290 : // Emit the jump tables.
291 2640150 : if (jump_tables_) {
292 319 : tasm()->Align(kSystemPointerSize);
293 640 : for (JumpTable* table = jump_tables_; table; table = table->next()) {
294 321 : tasm()->bind(table->label());
295 321 : AssembleJumpTable(table->targets(), table->target_count());
296 : }
297 : }
298 :
299 : // The PerfJitLogger logs code up until here, excluding the safepoint
300 : // table. Resolve the unwinding info now so it is aware of the same code
301 : // size as reported by perf.
302 : unwinding_info_writer_.Finish(tasm()->pc_offset());
303 :
304 2640150 : safepoints()->Emit(tasm(), frame()->GetTotalFrameSlotCount());
305 :
306 : // Emit the exception handler table.
307 2640656 : if (!handlers_.empty()) {
308 20374 : handler_table_offset_ = HandlerTable::EmitReturnTableStart(tasm());
309 474380 : for (size_t i = 0; i < handlers_.size(); ++i) {
310 227003 : HandlerTable::EmitReturnEntry(tasm(), handlers_[i].pc_offset,
311 454006 : handlers_[i].handler->pos());
312 : }
313 : }
314 :
315 : tasm()->MaybeEmitOutOfLineConstantPool();
316 2640656 : tasm()->FinalizeJumpOptimizationInfo();
317 :
318 2640395 : result_ = kSuccess;
319 : }
320 :
321 20433085 : void CodeGenerator::TryInsertBranchPoisoning(const InstructionBlock* block) {
322 : // See if our predecessor was a basic block terminated by a branch_and_poison
323 : // instruction. If yes, then perform the masking based on the flags.
324 26917891 : if (block->PredecessorCount() != 1) return;
325 13947889 : RpoNumber pred_rpo = (block->predecessors())[0];
326 13947889 : const InstructionBlock* pred = instructions()->InstructionBlockAt(pred_rpo);
327 13948044 : if (pred->code_start() == pred->code_end()) return;
328 13948434 : Instruction* instr = instructions()->InstructionAt(pred->code_end() - 1);
329 13948634 : FlagsMode mode = FlagsModeField::decode(instr->opcode());
330 13948634 : switch (mode) {
331 : case kFlags_branch_and_poison: {
332 : BranchInfo branch;
333 0 : RpoNumber target = ComputeBranchInfo(&branch, instr);
334 0 : if (!target.IsValid()) {
335 : // Non-trivial branch, add the masking code.
336 0 : FlagsCondition condition = branch.condition;
337 0 : if (branch.false_label == GetLabel(block->rpo_number())) {
338 : condition = NegateFlagsCondition(condition);
339 : }
340 0 : AssembleBranchPoisoning(condition, instr);
341 : }
342 : break;
343 : }
344 : case kFlags_deoptimize_and_poison: {
345 0 : UNREACHABLE();
346 : break;
347 : }
348 : default:
349 : break;
350 : }
351 : }
352 :
353 118446 : void CodeGenerator::AssembleArchBinarySearchSwitchRange(
354 : Register input, RpoNumber def_block, std::pair<int32_t, Label*>* begin,
355 : std::pair<int32_t, Label*>* end) {
356 118446 : if (end - begin < kBinarySearchSwitchMinimalCases) {
357 465369 : while (begin != end) {
358 388652 : tasm()->JumpIfEqual(input, begin->first, begin->second);
359 194326 : ++begin;
360 : }
361 76717 : AssembleArchJump(def_block);
362 76717 : return;
363 : }
364 41729 : auto middle = begin + (end - begin) / 2;
365 41729 : Label less_label;
366 83458 : tasm()->JumpIfLessThan(input, middle->first, &less_label);
367 41729 : AssembleArchBinarySearchSwitchRange(input, def_block, middle, end);
368 41729 : tasm()->bind(&less_label);
369 41729 : AssembleArchBinarySearchSwitchRange(input, def_block, begin, middle);
370 : }
371 :
372 993367 : OwnedVector<byte> CodeGenerator::GetSourcePositionTable() {
373 993367 : return source_position_table_builder_.ToSourcePositionTableVector();
374 : }
375 :
376 : OwnedVector<trap_handler::ProtectedInstructionData>
377 993767 : CodeGenerator::GetProtectedInstructions() {
378 : return OwnedVector<trap_handler::ProtectedInstructionData>::Of(
379 993767 : protected_instructions_);
380 : }
381 :
382 1589584 : MaybeHandle<Code> CodeGenerator::FinalizeCode() {
383 1589584 : if (result_ != kSuccess) {
384 : tasm()->AbortedCodeGeneration();
385 0 : return MaybeHandle<Code>();
386 : }
387 :
388 : // Allocate the source position table.
389 : Handle<ByteArray> source_positions =
390 1589584 : source_position_table_builder_.ToSourcePositionTable(isolate());
391 :
392 : // Allocate deoptimization data.
393 1589582 : Handle<DeoptimizationData> deopt_data = GenerateDeoptimizationData();
394 :
395 : // Allocate and install the code.
396 1589581 : CodeDesc desc;
397 3179162 : tasm()->GetCode(isolate(), &desc, safepoints(), handler_table_offset_);
398 :
399 : #if defined(V8_OS_WIN_X64)
400 : if (Builtins::IsBuiltinId(info_->builtin_index())) {
401 : isolate_->SetBuiltinUnwindData(info_->builtin_index(),
402 : tasm()->GetUnwindInfo());
403 : }
404 : #endif
405 :
406 1589583 : if (unwinding_info_writer_.eh_frame_writer()) {
407 22 : unwinding_info_writer_.eh_frame_writer()->GetEhFrame(&desc);
408 : }
409 :
410 : MaybeHandle<Code> maybe_code = isolate()->factory()->TryNewCode(
411 : desc, info()->code_kind(), Handle<Object>(), info()->builtin_index(),
412 : source_positions, deopt_data, kMovable, true,
413 1589583 : frame()->GetTotalFrameSlotCount());
414 :
415 : Handle<Code> code;
416 1589583 : if (!maybe_code.ToHandle(&code)) {
417 : tasm()->AbortedCodeGeneration();
418 0 : return MaybeHandle<Code>();
419 : }
420 :
421 : isolate()->counters()->total_compiled_code_size()->Increment(
422 1589583 : code->raw_instruction_size());
423 :
424 1589657 : LOG_CODE_EVENT(isolate(),
425 : CodeLinePosInfoRecordEvent(code->raw_instruction_start(),
426 : *source_positions));
427 :
428 1589586 : return code;
429 : }
430 :
431 15741374 : bool CodeGenerator::IsNextInAssemblyOrder(RpoNumber block) const {
432 : return instructions()
433 : ->InstructionBlockAt(current_block_)
434 15741474 : ->ao_number()
435 31482883 : .IsNext(instructions()->InstructionBlockAt(block)->ao_number());
436 : }
437 :
438 6268715 : void CodeGenerator::RecordSafepoint(ReferenceMap* references,
439 : Safepoint::Kind kind,
440 : Safepoint::DeoptMode deopt_mode) {
441 6268715 : Safepoint safepoint = safepoints()->DefineSafepoint(tasm(), kind, deopt_mode);
442 : int stackSlotToSpillSlotDelta =
443 6269619 : frame()->GetTotalFrameSlotCount() - frame()->GetSpillSlotCount();
444 87046068 : for (const InstructionOperand& operand : references->reference_operands()) {
445 80776354 : if (operand.IsStackSlot()) {
446 : int index = LocationOperand::cast(operand).index();
447 : DCHECK_LE(0, index);
448 : // We might index values in the fixed part of the frame (i.e. the
449 : // closure pointer or the context pointer); these are not spill slots
450 : // and therefore don't work with the SafepointTable currently, but
451 : // we also don't need to worry about them, since the GC has special
452 : // knowledge about those fields anyway.
453 15833445 : if (index < stackSlotToSpillSlotDelta) continue;
454 : safepoint.DefinePointerSlot(index);
455 64942909 : } else if (operand.IsRegister() && (kind & Safepoint::kWithRegisters)) {
456 0 : Register reg = LocationOperand::cast(operand).GetRegister();
457 0 : safepoint.DefinePointerRegister(reg);
458 : }
459 : }
460 6269714 : }
461 :
462 7866331 : bool CodeGenerator::IsMaterializableFromRoot(Handle<HeapObject> object,
463 : RootIndex* index_return) {
464 : const CallDescriptor* incoming_descriptor =
465 : linkage()->GetIncomingDescriptor();
466 7866331 : if (incoming_descriptor->flags() & CallDescriptor::kCanUseRoots) {
467 8927491 : return isolate()->roots_table().IsRootHandle(object, index_return) &&
468 1904004 : RootsTable::IsImmortalImmovable(*index_return);
469 : }
470 : return false;
471 : }
472 :
473 20433806 : CodeGenerator::CodeGenResult CodeGenerator::AssembleBlock(
474 : const InstructionBlock* block) {
475 158069716 : for (int i = block->code_start(); i < block->code_end(); ++i) {
476 68818504 : if (info()->trace_turbo_json_enabled()) {
477 120 : instr_starts_[i] = tasm()->pc_offset();
478 : }
479 : Instruction* instr = instructions()->InstructionAt(i);
480 68818239 : CodeGenResult result = AssembleInstruction(instr, block);
481 68817117 : if (result != kSuccess) return result;
482 : }
483 : return kSuccess;
484 : }
485 :
486 39404 : bool CodeGenerator::IsValidPush(InstructionOperand source,
487 : CodeGenerator::PushTypeFlags push_type) {
488 39404 : if (source.IsImmediate() &&
489 : ((push_type & CodeGenerator::kImmediatePush) != 0)) {
490 : return true;
491 : }
492 64740 : if (source.IsRegister() &&
493 : ((push_type & CodeGenerator::kRegisterPush) != 0)) {
494 : return true;
495 : }
496 27148 : if (source.IsStackSlot() &&
497 : ((push_type & CodeGenerator::kStackSlotPush) != 0)) {
498 : return true;
499 : }
500 988 : return false;
501 : }
502 :
503 119863 : void CodeGenerator::GetPushCompatibleMoves(Instruction* instr,
504 : PushTypeFlags push_type,
505 : ZoneVector<MoveOperands*>* pushes) {
506 : pushes->clear();
507 205388 : for (int i = Instruction::FIRST_GAP_POSITION;
508 325251 : i <= Instruction::LAST_GAP_POSITION; ++i) {
509 : Instruction::GapPosition inner_pos =
510 : static_cast<Instruction::GapPosition>(i);
511 : ParallelMove* parallel_move = instr->GetParallelMove(inner_pos);
512 222557 : if (parallel_move != nullptr) {
513 554559 : for (auto move : *parallel_move) {
514 435110 : InstructionOperand source = move->source();
515 435110 : InstructionOperand destination = move->destination();
516 : int first_push_compatible_index =
517 : V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0;
518 : // If there are any moves from slots that will be overridden by pushes,
519 : // then the full gap resolver must be used since optimization with
520 : // pushes don't participate in the parallel move and might clobber
521 : // values needed for the gap resolve.
522 465871 : if (source.IsStackSlot() && LocationOperand::cast(source).index() >=
523 : first_push_compatible_index) {
524 : pushes->clear();
525 : return;
526 : }
527 : // TODO(danno): Right now, only consider moves from the FIRST gap for
528 : // pushes. Theoretically, we could extract pushes for both gaps (there
529 : // are cases where this happens), but the logic for that would also have
530 : // to check to make sure that non-memory inputs to the pushes from the
531 : // LAST gap don't get clobbered in the FIRST gap.
532 417941 : if (i == Instruction::FIRST_GAP_POSITION) {
533 476812 : if (destination.IsStackSlot() &&
534 : LocationOperand::cast(destination).index() >=
535 : first_push_compatible_index) {
536 : int index = LocationOperand::cast(destination).index();
537 39404 : if (IsValidPush(source, push_type)) {
538 38416 : if (index >= static_cast<int>(pushes->size())) {
539 38416 : pushes->resize(index + 1);
540 : }
541 76832 : (*pushes)[index] = move;
542 : }
543 : }
544 : }
545 : }
546 : }
547 : }
548 :
549 : // For now, only support a set of continuous pushes at the end of the list.
550 : size_t push_count_upper_bound = pushes->size();
551 : size_t push_begin = push_count_upper_bound;
552 134390 : for (auto move : base::Reversed(*pushes)) {
553 44764 : if (move == nullptr) break;
554 31696 : push_begin--;
555 : }
556 102694 : size_t push_count = pushes->size() - push_begin;
557 : std::copy(pushes->begin() + push_begin,
558 : pushes->begin() + push_begin + push_count, pushes->begin());
559 102694 : pushes->resize(push_count);
560 : }
561 :
562 37806035 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferMove(
563 : InstructionOperand* source, InstructionOperand* destination) {
564 37806035 : if (source->IsConstant()) {
565 18997356 : if (destination->IsAnyRegister()) {
566 : return MoveType::kConstantToRegister;
567 : } else {
568 : DCHECK(destination->IsAnyStackSlot());
569 45671 : return MoveType::kConstantToStack;
570 : }
571 : }
572 : DCHECK(LocationOperand::cast(source)->IsCompatible(
573 : LocationOperand::cast(destination)));
574 18808679 : if (source->IsAnyRegister()) {
575 9864143 : if (destination->IsAnyRegister()) {
576 : return MoveType::kRegisterToRegister;
577 : } else {
578 : DCHECK(destination->IsAnyStackSlot());
579 5678526 : return MoveType::kRegisterToStack;
580 : }
581 : } else {
582 : DCHECK(source->IsAnyStackSlot());
583 8944536 : if (destination->IsAnyRegister()) {
584 : return MoveType::kStackToRegister;
585 : } else {
586 : DCHECK(destination->IsAnyStackSlot());
587 50391 : return MoveType::kStackToStack;
588 : }
589 : }
590 : }
591 :
592 77356 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferSwap(
593 : InstructionOperand* source, InstructionOperand* destination) {
594 : DCHECK(LocationOperand::cast(source)->IsCompatible(
595 : LocationOperand::cast(destination)));
596 77356 : if (source->IsAnyRegister()) {
597 73660 : if (destination->IsAnyRegister()) {
598 : return MoveType::kRegisterToRegister;
599 : } else {
600 : DCHECK(destination->IsAnyStackSlot());
601 7020 : return MoveType::kRegisterToStack;
602 : }
603 : } else {
604 : DCHECK(source->IsAnyStackSlot());
605 : DCHECK(destination->IsAnyStackSlot());
606 : return MoveType::kStackToStack;
607 : }
608 : }
609 :
610 5352962 : RpoNumber CodeGenerator::ComputeBranchInfo(BranchInfo* branch,
611 : Instruction* instr) {
612 : // Assemble a branch after this instruction.
613 : InstructionOperandConverter i(this, instr);
614 5352962 : RpoNumber true_rpo = i.InputRpo(instr->InputCount() - 2);
615 5353002 : RpoNumber false_rpo = i.InputRpo(instr->InputCount() - 1);
616 :
617 5353052 : if (true_rpo == false_rpo) {
618 2287 : return true_rpo;
619 : }
620 5350765 : FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
621 5350765 : if (IsNextInAssemblyOrder(true_rpo)) {
622 : // true block is next, can fall through if condition negated.
623 : std::swap(true_rpo, false_rpo);
624 : condition = NegateFlagsCondition(condition);
625 : }
626 5350764 : branch->condition = condition;
627 5350764 : branch->true_label = GetLabel(true_rpo);
628 5350764 : branch->false_label = GetLabel(false_rpo);
629 5350764 : branch->fallthru = IsNextInAssemblyOrder(false_rpo);
630 : return RpoNumber::Invalid();
631 : }
632 :
633 68818157 : CodeGenerator::CodeGenResult CodeGenerator::AssembleInstruction(
634 : Instruction* instr, const InstructionBlock* block) {
635 : int first_unused_stack_slot;
636 68818157 : FlagsMode mode = FlagsModeField::decode(instr->opcode());
637 68818157 : if (mode != kFlags_trap) {
638 68677503 : AssembleSourcePosition(instr);
639 : }
640 : bool adjust_stack =
641 68822943 : GetSlotAboveSPBeforeTailCall(instr, &first_unused_stack_slot);
642 68824023 : if (adjust_stack) AssembleTailCallBeforeGap(instr, first_unused_stack_slot);
643 : AssembleGaps(instr);
644 68818971 : if (adjust_stack) AssembleTailCallAfterGap(instr, first_unused_stack_slot);
645 : DCHECK_IMPLIES(
646 : block->must_deconstruct_frame(),
647 : instr != instructions()->InstructionAt(block->last_instruction_index()) ||
648 : instr->IsRet() || instr->IsJump());
649 68819143 : if (instr->IsJump() && block->must_deconstruct_frame()) {
650 62062 : AssembleDeconstructFrame();
651 : }
652 : // Assemble architecture-specific code for the instruction.
653 68819143 : CodeGenResult result = AssembleArchInstruction(instr);
654 68816850 : if (result != kSuccess) return result;
655 :
656 68817128 : FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
657 68817128 : switch (mode) {
658 : case kFlags_branch:
659 : case kFlags_branch_and_poison: {
660 : BranchInfo branch;
661 5353020 : RpoNumber target = ComputeBranchInfo(&branch, instr);
662 5353100 : if (target.IsValid()) {
663 : // redundant branch.
664 2287 : if (!IsNextInAssemblyOrder(target)) {
665 365 : AssembleArchJump(target);
666 : }
667 2287 : return kSuccess;
668 : }
669 : // Assemble architecture-specific branch.
670 5350813 : AssembleArchBranch(instr, &branch);
671 5350662 : break;
672 : }
673 : case kFlags_deoptimize:
674 : case kFlags_deoptimize_and_poison: {
675 : // Assemble a conditional eager deoptimization after this instruction.
676 : InstructionOperandConverter i(this, instr);
677 327700 : size_t frame_state_offset = MiscField::decode(instr->opcode());
678 : DeoptimizationExit* const exit =
679 327700 : AddDeoptimizationExit(instr, frame_state_offset);
680 327698 : Label continue_label;
681 : BranchInfo branch;
682 327698 : branch.condition = condition;
683 327698 : branch.true_label = exit->label();
684 327698 : branch.false_label = &continue_label;
685 327698 : branch.fallthru = true;
686 : // Assemble architecture-specific branch.
687 327698 : AssembleArchDeoptBranch(instr, &branch);
688 327703 : tasm()->bind(&continue_label);
689 327704 : if (mode == kFlags_deoptimize_and_poison) {
690 0 : AssembleBranchPoisoning(NegateFlagsCondition(branch.condition), instr);
691 : }
692 : break;
693 : }
694 : case kFlags_set: {
695 : // Assemble a boolean materialization after this instruction.
696 376413 : AssembleArchBoolean(instr, condition);
697 376409 : break;
698 : }
699 : case kFlags_trap: {
700 142047 : AssembleArchTrap(instr, condition);
701 142068 : break;
702 : }
703 : case kFlags_none: {
704 : break;
705 : }
706 : }
707 :
708 : // TODO(jarin) We should thread the flag through rather than set it.
709 68814791 : if (instr->IsCall()) {
710 : ResetSpeculationPoison();
711 : }
712 :
713 : return kSuccess;
714 : }
715 :
716 68910623 : void CodeGenerator::AssembleSourcePosition(Instruction* instr) {
717 68910623 : SourcePosition source_position = SourcePosition::Unknown();
718 133063061 : if (instr->IsNop() && instr->AreMovesRedundant()) return;
719 43174891 : if (!instructions()->GetSourcePosition(instr, &source_position)) return;
720 4761058 : AssembleSourcePosition(source_position);
721 : }
722 :
723 4772266 : void CodeGenerator::AssembleSourcePosition(SourcePosition source_position) {
724 4772266 : if (source_position == current_source_position_) return;
725 3641091 : current_source_position_ = source_position;
726 3641091 : if (!source_position.IsKnown()) return;
727 3641197 : source_position_table_builder_.AddPosition(tasm()->pc_offset(),
728 3641197 : source_position, false);
729 3641703 : if (FLAG_code_comments) {
730 : OptimizedCompilationInfo* info = this->info();
731 1163 : if (info->IsNotOptimizedFunctionOrWasmFunction()) return;
732 2326 : std::ostringstream buffer;
733 1163 : buffer << "-- ";
734 : // Turbolizer only needs the source position, as it can reconstruct
735 : // the inlining stack from other information.
736 1163 : if (info->trace_turbo_json_enabled() || !tasm()->isolate() ||
737 : tasm()->isolate()->concurrent_recompilation_enabled()) {
738 169 : buffer << source_position;
739 : } else {
740 : AllowHeapAllocation allocation;
741 : AllowHandleAllocation handles;
742 : AllowHandleDereference deref;
743 1988 : buffer << source_position.InliningStack(info);
744 : }
745 1163 : buffer << " --";
746 2326 : tasm()->RecordComment(buffer.str().c_str());
747 : }
748 : }
749 :
750 68822765 : bool CodeGenerator::GetSlotAboveSPBeforeTailCall(Instruction* instr,
751 : int* slot) {
752 68822765 : if (instr->IsTailCall()) {
753 : InstructionOperandConverter g(this, instr);
754 239700 : *slot = g.InputInt32(instr->InputCount() - 1);
755 : return true;
756 : } else {
757 : return false;
758 : }
759 : }
760 :
761 567709 : StubCallMode CodeGenerator::DetermineStubCallMode() const {
762 : Code::Kind code_kind = info()->code_kind();
763 567709 : return (code_kind == Code::WASM_FUNCTION ||
764 : code_kind == Code::WASM_TO_JS_FUNCTION)
765 : ? StubCallMode::kCallWasmRuntimeStub
766 567709 : : StubCallMode::kCallCodeObject;
767 : }
768 :
769 12 : void CodeGenerator::AssembleGaps(Instruction* instr) {
770 137638193 : for (int i = Instruction::FIRST_GAP_POSITION;
771 206452106 : i <= Instruction::LAST_GAP_POSITION; i++) {
772 : Instruction::GapPosition inner_pos =
773 : static_cast<Instruction::GapPosition>(i);
774 : ParallelMove* move = instr->GetParallelMove(inner_pos);
775 186456340 : if (move != nullptr) resolver()->Resolve(move);
776 : }
777 12 : }
778 :
779 : namespace {
780 :
781 464251 : Handle<PodArray<InliningPosition>> CreateInliningPositions(
782 : OptimizedCompilationInfo* info, Isolate* isolate) {
783 : const OptimizedCompilationInfo::InlinedFunctionList& inlined_functions =
784 : info->inlined_functions();
785 464251 : if (inlined_functions.size() == 0) {
786 : return Handle<PodArray<InliningPosition>>::cast(
787 : isolate->factory()->empty_byte_array());
788 : }
789 : Handle<PodArray<InliningPosition>> inl_positions =
790 : PodArray<InliningPosition>::New(
791 : isolate, static_cast<int>(inlined_functions.size()),
792 13700 : AllocationType::kOld);
793 144228 : for (size_t i = 0; i < inlined_functions.size(); ++i) {
794 65264 : inl_positions->set(static_cast<int>(i), inlined_functions[i].position);
795 : }
796 13700 : return inl_positions;
797 : }
798 :
799 : } // namespace
800 :
801 1589575 : Handle<DeoptimizationData> CodeGenerator::GenerateDeoptimizationData() {
802 : OptimizedCompilationInfo* info = this->info();
803 1589575 : int deopt_count = static_cast<int>(deoptimization_states_.size());
804 2714905 : if (deopt_count == 0 && !info->is_osr()) {
805 1125329 : return DeoptimizationData::Empty(isolate());
806 : }
807 : Handle<DeoptimizationData> data =
808 464246 : DeoptimizationData::New(isolate(), deopt_count, AllocationType::kOld);
809 :
810 : Handle<ByteArray> translation_array =
811 464250 : translations_.CreateByteArray(isolate()->factory());
812 :
813 928500 : data->SetTranslationByteArray(*translation_array);
814 464251 : data->SetInlinedFunctionCount(
815 464251 : Smi::FromInt(static_cast<int>(inlined_function_count_)));
816 : data->SetOptimizationId(Smi::FromInt(info->optimization_id()));
817 :
818 464251 : if (info->has_shared_info()) {
819 928502 : data->SetSharedFunctionInfo(*info->shared_info());
820 : } else {
821 0 : data->SetSharedFunctionInfo(Smi::kZero);
822 : }
823 :
824 : Handle<FixedArray> literals = isolate()->factory()->NewFixedArray(
825 464251 : static_cast<int>(deoptimization_literals_.size()), AllocationType::kOld);
826 6345611 : for (unsigned i = 0; i < deoptimization_literals_.size(); i++) {
827 1805703 : Handle<Object> object = deoptimization_literals_[i].Reify(isolate());
828 3611404 : literals->set(i, *object);
829 : }
830 928502 : data->SetLiteralArray(*literals);
831 :
832 : Handle<PodArray<InliningPosition>> inl_pos =
833 464251 : CreateInliningPositions(info, isolate());
834 928502 : data->SetInliningPositions(*inl_pos);
835 :
836 464251 : if (info->is_osr()) {
837 : DCHECK_LE(0, osr_pc_offset_);
838 4632 : data->SetOsrBytecodeOffset(Smi::FromInt(info_->osr_offset().ToInt()));
839 4632 : data->SetOsrPcOffset(Smi::FromInt(osr_pc_offset_));
840 : } else {
841 : BailoutId osr_offset = BailoutId::None();
842 : data->SetOsrBytecodeOffset(Smi::FromInt(osr_offset.ToInt()));
843 : data->SetOsrPcOffset(Smi::FromInt(-1));
844 : }
845 :
846 : // Populate deoptimization entries.
847 7120561 : for (int i = 0; i < deopt_count; i++) {
848 3328155 : DeoptimizationState* deoptimization_state = deoptimization_states_[i];
849 : data->SetBytecodeOffset(i, deoptimization_state->bailout_id());
850 3328155 : CHECK(deoptimization_state);
851 : data->SetTranslationIndex(
852 : i, Smi::FromInt(deoptimization_state->translation_id()));
853 : data->SetPc(i, Smi::FromInt(deoptimization_state->pc_offset()));
854 : }
855 :
856 464251 : return data;
857 : }
858 :
859 320 : Label* CodeGenerator::AddJumpTable(Label** targets, size_t target_count) {
860 640 : jump_tables_ = new (zone()) JumpTable(jump_tables_, targets, target_count);
861 320 : return jump_tables_->label();
862 : }
863 :
864 6038601 : void CodeGenerator::RecordCallPosition(Instruction* instr) {
865 6038601 : CallDescriptor::Flags flags(MiscField::decode(instr->opcode()));
866 :
867 : bool needs_frame_state = (flags & CallDescriptor::kNeedsFrameState);
868 :
869 6038601 : RecordSafepoint(
870 : instr->reference_map(), Safepoint::kSimple,
871 6038601 : needs_frame_state ? Safepoint::kLazyDeopt : Safepoint::kNoLazyDeopt);
872 :
873 6038950 : if (flags & CallDescriptor::kHasExceptionHandler) {
874 : InstructionOperandConverter i(this, instr);
875 227002 : RpoNumber handler_rpo = i.InputRpo(instr->InputCount() - 1);
876 681007 : handlers_.push_back({GetLabel(handler_rpo), tasm()->pc_offset()});
877 : }
878 :
879 6038951 : if (needs_frame_state) {
880 : MarkLazyDeoptSite();
881 : // If the frame state is present, it starts at argument 2 - after
882 : // the code address and the poison-alias index.
883 : size_t frame_state_offset = 2;
884 : FrameStateDescriptor* descriptor =
885 2958448 : GetDeoptimizationEntry(instr, frame_state_offset).descriptor();
886 : int pc_offset = tasm()->pc_offset();
887 : int deopt_state_id = BuildTranslation(instr, pc_offset, frame_state_offset,
888 2958454 : descriptor->state_combine());
889 :
890 : DeoptimizationExit* const exit = new (zone())
891 5916919 : DeoptimizationExit(deopt_state_id, current_source_position_);
892 2958460 : deoptimization_exits_.push_back(exit);
893 2958460 : safepoints()->RecordLazyDeoptimizationIndex(deopt_state_id);
894 : }
895 6038961 : }
896 :
897 7720336 : int CodeGenerator::DefineDeoptimizationLiteral(DeoptimizationLiteral literal) {
898 7720336 : int result = static_cast<int>(deoptimization_literals_.size());
899 146860556 : for (unsigned i = 0; i < deoptimization_literals_.size(); ++i) {
900 49722511 : if (deoptimization_literals_[i] == literal) return i;
901 : }
902 1806892 : deoptimization_literals_.push_back(literal);
903 1806884 : return result;
904 : }
905 :
906 6289125 : DeoptimizationEntry const& CodeGenerator::GetDeoptimizationEntry(
907 : Instruction* instr, size_t frame_state_offset) {
908 : InstructionOperandConverter i(this, instr);
909 : int const state_id = i.InputInt32(frame_state_offset);
910 6289127 : return instructions()->GetDeoptimizationEntry(state_id);
911 : }
912 :
913 0 : DeoptimizeKind CodeGenerator::GetDeoptimizationKind(
914 : int deoptimization_id) const {
915 : size_t const index = static_cast<size_t>(deoptimization_id);
916 : DCHECK_LT(index, deoptimization_states_.size());
917 3330694 : return deoptimization_states_[index]->kind();
918 : }
919 :
920 0 : DeoptimizeReason CodeGenerator::GetDeoptimizationReason(
921 : int deoptimization_id) const {
922 : size_t const index = static_cast<size_t>(deoptimization_id);
923 : DCHECK_LT(index, deoptimization_states_.size());
924 3330689 : return deoptimization_states_[index]->reason();
925 : }
926 :
927 49676515 : void CodeGenerator::TranslateStateValueDescriptor(
928 : StateValueDescriptor* desc, StateValueList* nested,
929 : Translation* translation, InstructionOperandIterator* iter) {
930 : // Note:
931 : // If translation is null, we just skip the relevant instruction operands.
932 49676515 : if (desc->IsNested()) {
933 87662 : if (translation != nullptr) {
934 87662 : translation->BeginCapturedObject(static_cast<int>(nested->size()));
935 : }
936 610035 : for (auto field : *nested) {
937 : TranslateStateValueDescriptor(field.desc, field.nested, translation,
938 522373 : iter);
939 : }
940 49588853 : } else if (desc->IsArgumentsElements()) {
941 5983 : if (translation != nullptr) {
942 5983 : translation->ArgumentsElements(desc->arguments_type());
943 : }
944 49582870 : } else if (desc->IsArgumentsLength()) {
945 6291 : if (translation != nullptr) {
946 6291 : translation->ArgumentsLength(desc->arguments_type());
947 : }
948 49576579 : } else if (desc->IsDuplicate()) {
949 18690 : if (translation != nullptr) {
950 18690 : translation->DuplicateObject(static_cast<int>(desc->id()));
951 : }
952 49557889 : } else if (desc->IsPlain()) {
953 : InstructionOperand* op = iter->Advance();
954 21907063 : if (translation != nullptr) {
955 : AddTranslationForOperand(translation, iter->instruction(), op,
956 21907488 : desc->type());
957 : }
958 : } else {
959 : DCHECK(desc->IsOptimizedOut());
960 27650826 : if (translation != nullptr) {
961 27650864 : if (optimized_out_literal_id_ == -1) {
962 928404 : optimized_out_literal_id_ = DefineDeoptimizationLiteral(
963 464199 : DeoptimizationLiteral(isolate()->factory()->optimized_out()));
964 : }
965 27650858 : translation->StoreLiteral(optimized_out_literal_id_);
966 : }
967 : }
968 49675518 : }
969 :
970 3681003 : void CodeGenerator::TranslateFrameStateDescriptorOperands(
971 : FrameStateDescriptor* desc, InstructionOperandIterator* iter,
972 : Translation* translation) {
973 : size_t index = 0;
974 : StateValueList* values = desc->GetStateValueDescriptors();
975 101975459 : for (StateValueList::iterator it = values->begin(); it != values->end();
976 : ++it, ++index) {
977 49147267 : TranslateStateValueDescriptor((*it).desc, (*it).nested, translation, iter);
978 : }
979 : DCHECK_EQ(desc->GetSize(), index);
980 3680964 : }
981 :
982 3681148 : void CodeGenerator::BuildTranslationForFrameStateDescriptor(
983 : FrameStateDescriptor* descriptor, InstructionOperandIterator* iter,
984 : Translation* translation, OutputFrameStateCombine state_combine) {
985 : // Outer-most state must be added to translation first.
986 3681148 : if (descriptor->outer_state() != nullptr) {
987 : BuildTranslationForFrameStateDescriptor(descriptor->outer_state(), iter,
988 350474 : translation, state_combine);
989 : }
990 :
991 : Handle<SharedFunctionInfo> shared_info;
992 3680947 : if (!descriptor->shared_info().ToHandle(&shared_info)) {
993 2088 : if (!info()->has_shared_info()) {
994 : return; // Stub with no SharedFunctionInfo.
995 : }
996 : shared_info = info()->shared_info();
997 : }
998 : int shared_info_id =
999 3680947 : DefineDeoptimizationLiteral(DeoptimizationLiteral(shared_info));
1000 :
1001 3681058 : switch (descriptor->type()) {
1002 : case FrameStateType::kInterpretedFunction: {
1003 : int return_offset = 0;
1004 : int return_count = 0;
1005 3555535 : if (!state_combine.IsOutputIgnored()) {
1006 2119388 : return_offset = static_cast<int>(state_combine.GetOffsetToPokeAt());
1007 2119388 : return_count = static_cast<int>(iter->instruction()->OutputCount());
1008 : }
1009 3555535 : translation->BeginInterpretedFrame(
1010 : descriptor->bailout_id(), shared_info_id,
1011 : static_cast<unsigned int>(descriptor->locals_count() + 1),
1012 3555535 : return_offset, return_count);
1013 3555393 : break;
1014 : }
1015 : case FrameStateType::kArgumentsAdaptor:
1016 84106 : translation->BeginArgumentsAdaptorFrame(
1017 : shared_info_id,
1018 84106 : static_cast<unsigned int>(descriptor->parameters_count()));
1019 84106 : break;
1020 : case FrameStateType::kConstructStub:
1021 : DCHECK(descriptor->bailout_id().IsValidForConstructStub());
1022 26707 : translation->BeginConstructStubFrame(
1023 : descriptor->bailout_id(), shared_info_id,
1024 26707 : static_cast<unsigned int>(descriptor->parameters_count() + 1));
1025 26707 : break;
1026 : case FrameStateType::kBuiltinContinuation: {
1027 2088 : BailoutId bailout_id = descriptor->bailout_id();
1028 : int parameter_count =
1029 : static_cast<unsigned int>(descriptor->parameters_count());
1030 2088 : translation->BeginBuiltinContinuationFrame(bailout_id, shared_info_id,
1031 2088 : parameter_count);
1032 : break;
1033 : }
1034 : case FrameStateType::kJavaScriptBuiltinContinuation: {
1035 12352 : BailoutId bailout_id = descriptor->bailout_id();
1036 : int parameter_count =
1037 : static_cast<unsigned int>(descriptor->parameters_count());
1038 12352 : translation->BeginJavaScriptBuiltinContinuationFrame(
1039 12352 : bailout_id, shared_info_id, parameter_count);
1040 : break;
1041 : }
1042 : case FrameStateType::kJavaScriptBuiltinContinuationWithCatch: {
1043 281 : BailoutId bailout_id = descriptor->bailout_id();
1044 : int parameter_count =
1045 : static_cast<unsigned int>(descriptor->parameters_count());
1046 281 : translation->BeginJavaScriptBuiltinContinuationWithCatchFrame(
1047 281 : bailout_id, shared_info_id, parameter_count);
1048 : break;
1049 : }
1050 : }
1051 :
1052 3680916 : TranslateFrameStateDescriptorOperands(descriptor, iter, translation);
1053 : }
1054 :
1055 3330678 : int CodeGenerator::BuildTranslation(Instruction* instr, int pc_offset,
1056 : size_t frame_state_offset,
1057 : OutputFrameStateCombine state_combine) {
1058 : DeoptimizationEntry const& entry =
1059 3330678 : GetDeoptimizationEntry(instr, frame_state_offset);
1060 : FrameStateDescriptor* const descriptor = entry.descriptor();
1061 3330686 : frame_state_offset++;
1062 :
1063 3330686 : int update_feedback_count = entry.feedback().IsValid() ? 1 : 0;
1064 : Translation translation(&translations_,
1065 3330683 : static_cast<int>(descriptor->GetFrameCount()),
1066 3330686 : static_cast<int>(descriptor->GetJSFrameCount()),
1067 6661369 : update_feedback_count, zone());
1068 3330690 : if (entry.feedback().IsValid()) {
1069 : DeoptimizationLiteral literal =
1070 : DeoptimizationLiteral(entry.feedback().vector());
1071 40463 : int literal_id = DefineDeoptimizationLiteral(literal);
1072 40463 : translation.AddUpdateFeedback(literal_id, entry.feedback().slot().ToInt());
1073 : }
1074 : InstructionOperandIterator iter(instr, frame_state_offset);
1075 : BuildTranslationForFrameStateDescriptor(descriptor, &iter, &translation,
1076 3330690 : state_combine);
1077 :
1078 3330698 : int deoptimization_id = static_cast<int>(deoptimization_states_.size());
1079 :
1080 6661381 : deoptimization_states_.push_back(new (zone()) DeoptimizationState(
1081 : descriptor->bailout_id(), translation.index(), pc_offset, entry.kind(),
1082 : entry.reason()));
1083 :
1084 3330689 : return deoptimization_id;
1085 : }
1086 :
1087 21911232 : void CodeGenerator::AddTranslationForOperand(Translation* translation,
1088 : Instruction* instr,
1089 : InstructionOperand* op,
1090 : MachineType type) {
1091 21911232 : if (op->IsStackSlot()) {
1092 14784371 : if (type.representation() == MachineRepresentation::kBit) {
1093 21533 : translation->StoreBoolStackSlot(LocationOperand::cast(op)->index());
1094 44286329 : } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
1095 : type == MachineType::Int32()) {
1096 209905 : translation->StoreInt32StackSlot(LocationOperand::cast(op)->index());
1097 43658839 : } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
1098 : type == MachineType::Uint32()) {
1099 6899 : translation->StoreUint32StackSlot(LocationOperand::cast(op)->index());
1100 14546034 : } else if (type == MachineType::Int64()) {
1101 114 : translation->StoreInt64StackSlot(LocationOperand::cast(op)->index());
1102 : } else {
1103 : #if defined(V8_COMPRESS_POINTERS)
1104 : CHECK(MachineRepresentation::kTagged == type.representation() ||
1105 : MachineRepresentation::kCompressed == type.representation());
1106 : #else
1107 14545920 : CHECK(MachineRepresentation::kTagged == type.representation());
1108 : #endif
1109 14545920 : translation->StoreStackSlot(LocationOperand::cast(op)->index());
1110 : }
1111 7126861 : } else if (op->IsFPStackSlot()) {
1112 258946 : if (type.representation() == MachineRepresentation::kFloat64) {
1113 258035 : translation->StoreDoubleStackSlot(LocationOperand::cast(op)->index());
1114 : } else {
1115 911 : CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
1116 911 : translation->StoreFloatStackSlot(LocationOperand::cast(op)->index());
1117 : }
1118 6867915 : } else if (op->IsRegister()) {
1119 : InstructionOperandConverter converter(this, instr);
1120 496547 : if (type.representation() == MachineRepresentation::kBit) {
1121 3241 : translation->StoreBoolRegister(converter.ToRegister(op));
1122 1479327 : } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
1123 : type == MachineType::Int32()) {
1124 34180 : translation->StoreInt32Register(converter.ToRegister(op));
1125 1377378 : } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
1126 : type == MachineType::Uint32()) {
1127 1149 : translation->StoreUint32Register(converter.ToRegister(op));
1128 457977 : } else if (type == MachineType::Int64()) {
1129 14 : translation->StoreInt64Register(converter.ToRegister(op));
1130 : } else {
1131 : #if defined(V8_COMPRESS_POINTERS)
1132 : CHECK(MachineRepresentation::kTagged == type.representation() ||
1133 : MachineRepresentation::kCompressed == type.representation());
1134 : #else
1135 457963 : CHECK(MachineRepresentation::kTagged == type.representation());
1136 : #endif
1137 457963 : translation->StoreRegister(converter.ToRegister(op));
1138 : }
1139 6371368 : } else if (op->IsFPRegister()) {
1140 : InstructionOperandConverter converter(this, instr);
1141 64064 : if (type.representation() == MachineRepresentation::kFloat64) {
1142 63939 : translation->StoreDoubleRegister(converter.ToDoubleRegister(op));
1143 : } else {
1144 125 : CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
1145 125 : translation->StoreFloatRegister(converter.ToFloatRegister(op));
1146 : }
1147 : } else {
1148 6307304 : CHECK(op->IsImmediate());
1149 : InstructionOperandConverter converter(this, instr);
1150 6307304 : Constant constant = converter.ToConstant(op);
1151 : DeoptimizationLiteral literal;
1152 6308363 : switch (constant.type()) {
1153 : case Constant::kInt32:
1154 13879 : if (type.representation() == MachineRepresentation::kTagged) {
1155 : // When pointers are 4 bytes, we can use int32 constants to represent
1156 : // Smis.
1157 : DCHECK_EQ(4, kSystemPointerSize);
1158 1060 : Smi smi(static_cast<Address>(constant.ToInt32()));
1159 : DCHECK(smi->IsSmi());
1160 1060 : literal = DeoptimizationLiteral(smi->value());
1161 12819 : } else if (type.representation() == MachineRepresentation::kBit) {
1162 783 : if (constant.ToInt32() == 0) {
1163 : literal =
1164 340 : DeoptimizationLiteral(isolate()->factory()->false_value());
1165 : } else {
1166 : DCHECK_EQ(1, constant.ToInt32());
1167 443 : literal = DeoptimizationLiteral(isolate()->factory()->true_value());
1168 : }
1169 : } else {
1170 : DCHECK(type == MachineType::Int32() ||
1171 : type == MachineType::Uint32() ||
1172 : type.representation() == MachineRepresentation::kWord32 ||
1173 : type.representation() == MachineRepresentation::kNone);
1174 : DCHECK(type.representation() != MachineRepresentation::kNone ||
1175 : constant.ToInt32() == FrameStateDescriptor::kImpossibleValue);
1176 12036 : if (type == MachineType::Uint32()) {
1177 : literal = DeoptimizationLiteral(
1178 8 : static_cast<uint32_t>(constant.ToInt32()));
1179 : } else {
1180 12028 : literal = DeoptimizationLiteral(constant.ToInt32());
1181 : }
1182 : }
1183 : break;
1184 : case Constant::kInt64:
1185 : DCHECK_EQ(8, kSystemPointerSize);
1186 562389 : if (type.representation() == MachineRepresentation::kWord64) {
1187 : literal =
1188 0 : DeoptimizationLiteral(static_cast<double>(constant.ToInt64()));
1189 : } else {
1190 : // When pointers are 8 bytes, we can use int64 constants to represent
1191 : // Smis.
1192 : DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
1193 : Smi smi(static_cast<Address>(constant.ToInt64()));
1194 : DCHECK(smi->IsSmi());
1195 562389 : literal = DeoptimizationLiteral(smi->value());
1196 : }
1197 : break;
1198 : case Constant::kFloat32:
1199 : DCHECK(type.representation() == MachineRepresentation::kFloat32 ||
1200 : type.representation() == MachineRepresentation::kTagged);
1201 14 : literal = DeoptimizationLiteral(constant.ToFloat32());
1202 14 : break;
1203 : case Constant::kFloat64:
1204 : DCHECK(type.representation() == MachineRepresentation::kFloat64 ||
1205 : type.representation() == MachineRepresentation::kTagged);
1206 89792 : literal = DeoptimizationLiteral(constant.ToFloat64().value());
1207 89792 : break;
1208 : case Constant::kHeapObject:
1209 : DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
1210 5639822 : literal = DeoptimizationLiteral(constant.ToHeapObject());
1211 5639745 : break;
1212 : case Constant::kDelayedStringConstant:
1213 : DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
1214 2467 : literal = DeoptimizationLiteral(constant.ToDelayedStringConstant());
1215 2467 : break;
1216 : default:
1217 0 : UNREACHABLE();
1218 : }
1219 6308286 : if (literal.object().equals(info()->closure())) {
1220 3365126 : translation->StoreJSFrameFunction();
1221 : } else {
1222 2943160 : int literal_id = DefineDeoptimizationLiteral(literal);
1223 2942273 : translation->StoreLiteral(literal_id);
1224 : }
1225 : }
1226 21910431 : }
1227 :
1228 0 : void CodeGenerator::MarkLazyDeoptSite() {
1229 2958448 : last_lazy_deopt_pc_ = tasm()->pc_offset();
1230 0 : }
1231 :
1232 327701 : DeoptimizationExit* CodeGenerator::AddDeoptimizationExit(
1233 : Instruction* instr, size_t frame_state_offset) {
1234 : int const deoptimization_id = BuildTranslation(
1235 327701 : instr, -1, frame_state_offset, OutputFrameStateCombine::Ignore());
1236 :
1237 : DeoptimizationExit* const exit = new (zone())
1238 655398 : DeoptimizationExit(deoptimization_id, current_source_position_);
1239 327698 : deoptimization_exits_.push_back(exit);
1240 327698 : return exit;
1241 : }
1242 :
1243 2640432 : void CodeGenerator::InitializeSpeculationPoison() {
1244 2640432 : if (poisoning_level_ == PoisoningMitigationLevel::kDontPoison) return;
1245 :
1246 : // Initialize {kSpeculationPoisonRegister} either by comparing the expected
1247 : // with the actual call target, or by unconditionally using {-1} initially.
1248 : // Masking register arguments with it only makes sense in the first case.
1249 0 : if (info()->called_with_code_start_register()) {
1250 0 : tasm()->RecordComment("-- Prologue: generate speculation poison --");
1251 0 : GenerateSpeculationPoisonFromCodeStartRegister();
1252 0 : if (info()->is_poisoning_register_arguments()) {
1253 0 : AssembleRegisterArgumentPoisoning();
1254 : }
1255 : } else {
1256 : ResetSpeculationPoison();
1257 : }
1258 : }
1259 :
1260 4632 : void CodeGenerator::ResetSpeculationPoison() {
1261 6072625 : if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison) {
1262 0 : tasm()->ResetSpeculationPoisonRegister();
1263 : }
1264 4632 : }
1265 :
1266 759736 : OutOfLineCode::OutOfLineCode(CodeGenerator* gen)
1267 2279208 : : frame_(gen->frame()), tasm_(gen->tasm()), next_(gen->ools_) {
1268 759736 : gen->ools_ = this;
1269 759736 : }
1270 :
1271 : OutOfLineCode::~OutOfLineCode() = default;
1272 :
1273 1805703 : Handle<Object> DeoptimizationLiteral::Reify(Isolate* isolate) const {
1274 1805703 : switch (kind_) {
1275 : case DeoptimizationLiteralKind::kObject: {
1276 1725679 : return object_;
1277 : }
1278 : case DeoptimizationLiteralKind::kNumber: {
1279 79393 : return isolate->factory()->NewNumber(number_);
1280 : }
1281 : case DeoptimizationLiteralKind::kString: {
1282 631 : return string_->AllocateStringConstant(isolate);
1283 : }
1284 : }
1285 0 : UNREACHABLE();
1286 : }
1287 :
1288 : } // namespace compiler
1289 : } // namespace internal
1290 122036 : } // namespace v8
|
