LCOV - code coverage report
Current view: top level - src/compiler/backend - code-generator.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 499 524 95.2 %
Date: 2019-03-21 Functions: 44 47 93.6 %

          Line data    Source code
       1             : // Copyright 2013 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #include "src/compiler/backend/code-generator.h"
       6             : 
       7             : #include "src/address-map.h"
       8             : #include "src/assembler-inl.h"
       9             : #include "src/base/adapters.h"
      10             : #include "src/compiler/backend/code-generator-impl.h"
      11             : #include "src/compiler/linkage.h"
      12             : #include "src/compiler/pipeline.h"
      13             : #include "src/compiler/wasm-compiler.h"
      14             : #include "src/counters.h"
      15             : #include "src/eh-frame.h"
      16             : #include "src/frames.h"
      17             : #include "src/log.h"
      18             : #include "src/macro-assembler-inl.h"
      19             : #include "src/objects/smi.h"
      20             : #include "src/optimized-compilation-info.h"
      21             : #include "src/string-constants.h"
      22             : 
      23             : namespace v8 {
      24             : namespace internal {
      25             : namespace compiler {
      26             : 
      27             : class CodeGenerator::JumpTable final : public ZoneObject {
      28             :  public:
      29             :   JumpTable(JumpTable* next, Label** targets, size_t target_count)
      30         307 :       : next_(next), targets_(targets), target_count_(target_count) {}
      31             : 
      32         614 :   Label* label() { return &label_; }
      33             :   JumpTable* next() const { return next_; }
      34             :   Label** targets() const { return targets_; }
      35             :   size_t target_count() const { return target_count_; }
      36             : 
      37             :  private:
      38             :   Label label_;
      39             :   JumpTable* const next_;
      40             :   Label** const targets_;
      41             :   size_t const target_count_;
      42             : };
      43             : 
      44     2515838 : CodeGenerator::CodeGenerator(
      45             :     Zone* codegen_zone, Frame* frame, Linkage* linkage,
      46             :     InstructionSequence* instructions, OptimizedCompilationInfo* info,
      47             :     Isolate* isolate, base::Optional<OsrHelper> osr_helper,
      48             :     int start_source_position, JumpOptimizationInfo* jump_opt,
      49             :     PoisoningMitigationLevel poisoning_level, const AssemblerOptions& options,
      50             :     int32_t builtin_index, std::unique_ptr<AssemblerBuffer> buffer)
      51             :     : zone_(codegen_zone),
      52             :       isolate_(isolate),
      53             :       frame_access_state_(nullptr),
      54             :       linkage_(linkage),
      55             :       instructions_(instructions),
      56             :       unwinding_info_writer_(zone()),
      57             :       info_(info),
      58     2515099 :       labels_(zone()->NewArray<Label>(instructions->InstructionBlockCount())),
      59             :       current_block_(RpoNumber::Invalid()),
      60             :       start_source_position_(start_source_position),
      61             :       current_source_position_(SourcePosition::Unknown()),
      62             :       tasm_(isolate, options, CodeObjectRequired::kNo, std::move(buffer)),
      63             :       resolver_(this),
      64             :       safepoints_(zone()),
      65             :       handlers_(zone()),
      66             :       deoptimization_exits_(zone()),
      67             :       deoptimization_states_(zone()),
      68             :       deoptimization_literals_(zone()),
      69             :       translations_(zone()),
      70             :       caller_registers_saved_(false),
      71             :       jump_tables_(nullptr),
      72             :       ools_(nullptr),
      73             :       osr_helper_(std::move(osr_helper)),
      74             :       osr_pc_offset_(-1),
      75             :       optimized_out_literal_id_(-1),
      76             :       source_position_table_builder_(
      77             :           SourcePositionTableBuilder::RECORD_SOURCE_POSITIONS),
      78             :       protected_instructions_(zone()),
      79             :       result_(kSuccess),
      80             :       poisoning_level_(poisoning_level),
      81             :       block_starts_(zone()),
      82    22640420 :       instr_starts_(zone()) {
      83    43127584 :   for (int i = 0; i < instructions->InstructionBlockCount(); ++i) {
      84    20306072 :     new (&labels_[i]) Label;
      85             :   }
      86     2515440 :   CreateFrameAccessState(frame);
      87     2515427 :   CHECK_EQ(info->is_osr(), osr_helper_.has_value());
      88             :   tasm_.set_jump_optimization_info(jump_opt);
      89             :   Code::Kind code_kind = info->code_kind();
      90     5030854 :   if (code_kind == Code::WASM_FUNCTION ||
      91     2515427 :       code_kind == Code::WASM_TO_JS_FUNCTION ||
      92     6675468 :       code_kind == Code::WASM_INTERPRETER_ENTRY ||
      93      122472 :       (Builtins::IsBuiltinId(builtin_index) &&
      94      122472 :        Builtins::IsWasmRuntimeStub(builtin_index))) {
      95             :     tasm_.set_abort_hard(true);
      96             :   }
      97             :   tasm_.set_builtin_index(builtin_index);
      98     2515427 : }
      99             : 
     100      272021 : bool CodeGenerator::wasm_runtime_exception_support() const {
     101             :   DCHECK_NOT_NULL(info_);
     102      544042 :   return info_->wasm_runtime_exception_support();
     103             : }
     104             : 
     105      237421 : void CodeGenerator::AddProtectedInstructionLanding(uint32_t instr_offset,
     106             :                                                    uint32_t landing_offset) {
     107      474989 :   protected_instructions_.push_back({instr_offset, landing_offset});
     108      237568 : }
     109             : 
     110     2515451 : void CodeGenerator::CreateFrameAccessState(Frame* frame) {
     111     2515451 :   FinishFrame(frame);
     112     2515418 :   frame_access_state_ = new (zone()) FrameAccessState(frame);
     113     2515418 : }
     114             : 
     115     3354647 : CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
     116             :     int deoptimization_id, SourcePosition pos) {
     117     3354647 :   if (deoptimization_id > Deoptimizer::kMaxNumberOfEntries) {
     118             :     return kTooManyDeoptimizationBailouts;
     119             :   }
     120             : 
     121             :   DeoptimizeKind deopt_kind = GetDeoptimizationKind(deoptimization_id);
     122             :   DeoptimizeReason deoptimization_reason =
     123             :       GetDeoptimizationReason(deoptimization_id);
     124             :   Address deopt_entry =
     125     3354659 :       Deoptimizer::GetDeoptimizationEntry(tasm()->isolate(), deopt_kind);
     126     3354664 :   if (info()->is_source_positions_enabled()) {
     127      101547 :     tasm()->RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
     128             :   }
     129     3354664 :   tasm()->CallForDeoptimization(deopt_entry, deoptimization_id);
     130     3354658 :   return kSuccess;
     131             : }
     132             : 
     133          36 : void CodeGenerator::MaybeEmitOutOfLineConstantPool() {
     134             :   tasm()->MaybeEmitOutOfLineConstantPool();
     135          36 : }
     136             : 
     137     2516176 : void CodeGenerator::AssembleCode() {
     138             :   OptimizedCompilationInfo* info = this->info();
     139             : 
     140             :   // Open a frame scope to indicate that there is a frame on the stack.  The
     141             :   // MANUAL indicates that the scope shouldn't actually generate code to set up
     142             :   // the frame (that is done in AssemblePrologue).
     143     2515381 :   FrameScope frame_scope(tasm(), StackFrame::MANUAL);
     144             : 
     145     2516176 :   if (info->is_source_positions_enabled()) {
     146       11334 :     AssembleSourcePosition(start_source_position());
     147             :   }
     148             : 
     149             :   // Check that {kJavaScriptCallCodeStartRegister} has been set correctly.
     150     2515860 :   if (FLAG_debug_code && (info->code_kind() == Code::OPTIMIZED_FUNCTION ||
     151             :                           info->code_kind() == Code::BYTECODE_HANDLER)) {
     152         114 :     tasm()->RecordComment("-- Prologue: check code start register --");
     153         114 :     AssembleCodeStartRegisterCheck();
     154             :   }
     155             : 
     156             :   // We want to bailout only from JS functions, which are the only ones
     157             :   // that are optimized.
     158     2515860 :   if (info->IsOptimizing()) {
     159             :     DCHECK(linkage()->GetIncomingDescriptor()->IsJSFunctionCall());
     160      463826 :     tasm()->RecordComment("-- Prologue: check for deoptimization --");
     161      463827 :     BailoutIfDeoptimized();
     162             :   }
     163             : 
     164     2515862 :   InitializeSpeculationPoison();
     165             : 
     166             :   // Define deoptimization literals for all inlined functions.
     167             :   DCHECK_EQ(0u, deoptimization_literals_.size());
     168     2580490 :   for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
     169             :        info->inlined_functions()) {
     170       65105 :     if (!inlined.shared_info.equals(info->shared_info())) {
     171       64675 :       int index = DefineDeoptimizationLiteral(
     172       64675 :           DeoptimizationLiteral(inlined.shared_info));
     173             :       inlined.RegisterInlinedFunctionId(index);
     174             :     }
     175             :   }
     176     2515385 :   inlined_function_count_ = deoptimization_literals_.size();
     177             : 
     178             :   // Define deoptimization literals for all BytecodeArrays to which we might
     179             :   // deopt to ensure they are strongly held by the optimized code.
     180     2515385 :   if (info->has_bytecode_array()) {
     181      463825 :     DefineDeoptimizationLiteral(DeoptimizationLiteral(info->bytecode_array()));
     182             :   }
     183     2580485 :   for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
     184             :        info->inlined_functions()) {
     185       65105 :     DefineDeoptimizationLiteral(DeoptimizationLiteral(inlined.bytecode_array));
     186             :   }
     187             : 
     188             :   unwinding_info_writer_.SetNumberOfInstructionBlocks(
     189             :       instructions()->InstructionBlockCount());
     190             : 
     191     2514940 :   if (info->trace_turbo_json_enabled()) {
     192           3 :     block_starts_.assign(instructions()->instruction_blocks().size(), -1);
     193           3 :     instr_starts_.assign(instructions()->instructions().size(), -1);
     194             :   }
     195             : 
     196             :   // Assemble instructions in assembly order.
     197    22823405 :   for (const InstructionBlock* block : instructions()->ao_blocks()) {
     198             :     // Align loop headers on vendor recommended boundaries.
     199    20306243 :     if (block->ShouldAlign() && !tasm()->jump_optimization_info()) {
     200       56542 :       tasm()->CodeTargetAlign();
     201             :     }
     202    20306245 :     if (info->trace_turbo_json_enabled()) {
     203          20 :       block_starts_[block->rpo_number().ToInt()] = tasm()->pc_offset();
     204             :     }
     205             :     // Bind a label for a block.
     206    20306245 :     current_block_ = block->rpo_number();
     207    20306245 :     unwinding_info_writer_.BeginInstructionBlock(tasm()->pc_offset(), block);
     208    20306373 :     if (FLAG_code_comments) {
     209        4156 :       std::ostringstream buffer;
     210        2078 :       buffer << "-- B" << block->rpo_number().ToInt() << " start";
     211        2078 :       if (block->IsDeferred()) buffer << " (deferred)";
     212        2078 :       if (!block->needs_frame()) buffer << " (no frame)";
     213        2078 :       if (block->must_construct_frame()) buffer << " (construct frame)";
     214        2078 :       if (block->must_deconstruct_frame()) buffer << " (deconstruct frame)";
     215             : 
     216        2078 :       if (block->IsLoopHeader()) {
     217          17 :         buffer << " (loop up to " << block->loop_end().ToInt() << ")";
     218             :       }
     219        2078 :       if (block->loop_header().IsValid()) {
     220         802 :         buffer << " (in loop " << block->loop_header().ToInt() << ")";
     221             :       }
     222        2078 :       buffer << " --";
     223        4156 :       tasm()->RecordComment(buffer.str().c_str());
     224             :     }
     225             : 
     226    20306373 :     frame_access_state()->MarkHasFrame(block->needs_frame());
     227             : 
     228    20306433 :     tasm()->bind(GetLabel(current_block_));
     229             : 
     230    20306469 :     TryInsertBranchPoisoning(block);
     231             : 
     232    20306038 :     if (block->must_construct_frame()) {
     233     2538026 :       AssembleConstructFrame();
     234             :       // We need to setup the root register after we assemble the prologue, to
     235             :       // avoid clobbering callee saved registers in case of C linkage and
     236             :       // using the roots.
     237             :       // TODO(mtrofin): investigate how we can avoid doing this repeatedly.
     238     2538138 :       if (linkage()->GetIncomingDescriptor()->InitializeRootRegister()) {
     239      888572 :         tasm()->InitializeRootRegister();
     240             :       }
     241             :     }
     242             : 
     243             :     if (FLAG_enable_embedded_constant_pool && !block->needs_frame()) {
     244             :       ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
     245             :       result_ = AssembleBlock(block);
     246             :     } else {
     247    20306150 :       result_ = AssembleBlock(block);
     248             :     }
     249    20308570 :     if (result_ != kSuccess) return;
     250    20308658 :     unwinding_info_writer_.EndInstructionBlock(block);
     251             :   }
     252             : 
     253             :   // Assemble all out-of-line code.
     254     2517162 :   if (ools_) {
     255      143311 :     tasm()->RecordComment("-- Out of line code --");
     256      800389 :     for (OutOfLineCode* ool = ools_; ool; ool = ool->next()) {
     257      657125 :       tasm()->bind(ool->entry());
     258      657132 :       ool->Generate();
     259     1042183 :       if (ool->exit()->is_bound()) tasm()->jmp(ool->exit());
     260             :     }
     261             :   }
     262             : 
     263             :   // This nop operation is needed to ensure that the trampoline is not
     264             :   // confused with the pc of the call before deoptimization.
     265             :   // The test regress/regress-259 is an example of where we need it.
     266     2517115 :   tasm()->nop();
     267             : 
     268             :   // Assemble deoptimization exits.
     269             :   int last_updated = 0;
     270     5825823 :   for (DeoptimizationExit* exit : deoptimization_exits_) {
     271     3310963 :     tasm()->bind(exit->label());
     272             :     int trampoline_pc = tasm()->pc_offset();
     273             :     int deoptimization_id = exit->deoptimization_id();
     274     3310968 :     DeoptimizationState* ds = deoptimization_states_[deoptimization_id];
     275             : 
     276     3310968 :     if (ds->kind() == DeoptimizeKind::kLazy) {
     277             :       last_updated = safepoints()->UpdateDeoptimizationInfo(
     278     2979004 :           ds->pc_offset(), trampoline_pc, last_updated);
     279             :     }
     280     3310967 :     result_ = AssembleDeoptimizerCall(deoptimization_id, exit->pos());
     281     3310982 :     if (result_ != kSuccess) return;
     282             :   }
     283             : 
     284             :   // TODO(jgruber): Move all inlined metadata generation into a new,
     285             :   // architecture-independent version of FinishCode. Currently, this includes
     286             :   // the safepoint table, handler table, constant pool, and code comments, in
     287             :   // that order.
     288     2514860 :   FinishCode();
     289             : 
     290             :   // Emit the jump tables.
     291     2514837 :   if (jump_tables_) {
     292         305 :     tasm()->Align(kSystemPointerSize);
     293         612 :     for (JumpTable* table = jump_tables_; table; table = table->next()) {
     294         307 :       tasm()->bind(table->label());
     295         307 :       AssembleJumpTable(table->targets(), table->target_count());
     296             :     }
     297             :   }
     298             : 
     299             :   // The PerfJitLogger logs code up until here, excluding the safepoint
     300             :   // table. Resolve the unwinding info now so it is aware of the same code
     301             :   // size as reported by perf.
     302             :   unwinding_info_writer_.Finish(tasm()->pc_offset());
     303             : 
     304     2514837 :   safepoints()->Emit(tasm(), frame()->GetTotalFrameSlotCount());
     305             : 
     306             :   // Emit the exception handler table.
     307     2515676 :   if (!handlers_.empty()) {
     308       19814 :     handler_table_offset_ = HandlerTable::EmitReturnTableStart(
     309       19813 :         tasm(), static_cast<int>(handlers_.size()));
     310      459057 :     for (size_t i = 0; i < handlers_.size(); ++i) {
     311      219622 :       HandlerTable::EmitReturnEntry(tasm(), handlers_[i].pc_offset,
     312      439244 :                                     handlers_[i].handler->pos());
     313             :     }
     314             :   }
     315             : 
     316             :   tasm()->MaybeEmitOutOfLineConstantPool();
     317     2515675 :   tasm()->FinalizeJumpOptimizationInfo();
     318             : 
     319     2515381 :   result_ = kSuccess;
     320             : }
     321             : 
     322    20306698 : void CodeGenerator::TryInsertBranchPoisoning(const InstructionBlock* block) {
     323             :   // See if our predecessor was a basic block terminated by a branch_and_poison
     324             :   // instruction. If yes, then perform the masking based on the flags.
     325    26603282 :   if (block->PredecessorCount() != 1) return;
     326    14009997 :   RpoNumber pred_rpo = (block->predecessors())[0];
     327    14009997 :   const InstructionBlock* pred = instructions()->InstructionBlockAt(pred_rpo);
     328    14009337 :   if (pred->code_start() == pred->code_end()) return;
     329    14009454 :   Instruction* instr = instructions()->InstructionAt(pred->code_end() - 1);
     330    14009612 :   FlagsMode mode = FlagsModeField::decode(instr->opcode());
     331    14009612 :   switch (mode) {
     332             :     case kFlags_branch_and_poison: {
     333             :       BranchInfo branch;
     334           0 :       RpoNumber target = ComputeBranchInfo(&branch, instr);
     335           0 :       if (!target.IsValid()) {
     336             :         // Non-trivial branch, add the masking code.
     337           0 :         FlagsCondition condition = branch.condition;
     338           0 :         if (branch.false_label == GetLabel(block->rpo_number())) {
     339             :           condition = NegateFlagsCondition(condition);
     340             :         }
     341           0 :         AssembleBranchPoisoning(condition, instr);
     342             :       }
     343             :       break;
     344             :     }
     345             :     case kFlags_deoptimize_and_poison: {
     346           0 :       UNREACHABLE();
     347             :       break;
     348             :     }
     349             :     default:
     350             :       break;
     351             :   }
     352             : }
     353             : 
     354      115199 : void CodeGenerator::AssembleArchBinarySearchSwitchRange(
     355             :     Register input, RpoNumber def_block, std::pair<int32_t, Label*>* begin,
     356             :     std::pair<int32_t, Label*>* end) {
     357      115199 :   if (end - begin < kBinarySearchSwitchMinimalCases) {
     358      451181 :     while (begin != end) {
     359      376644 :       tasm()->JumpIfEqual(input, begin->first, begin->second);
     360      188323 :       ++begin;
     361             :     }
     362       74536 :     AssembleArchJump(def_block);
     363       74537 :     return;
     364             :   }
     365       40664 :   auto middle = begin + (end - begin) / 2;
     366       40664 :   Label less_label;
     367       81328 :   tasm()->JumpIfLessThan(input, middle->first, &less_label);
     368       40664 :   AssembleArchBinarySearchSwitchRange(input, def_block, middle, end);
     369       40665 :   tasm()->bind(&less_label);
     370       40665 :   AssembleArchBinarySearchSwitchRange(input, def_block, begin, middle);
     371             : }
     372             : 
     373      869818 : OwnedVector<byte> CodeGenerator::GetSourcePositionTable() {
     374      869818 :   return source_position_table_builder_.ToSourcePositionTableVector();
     375             : }
     376             : 
     377             : OwnedVector<trap_handler::ProtectedInstructionData>
     378      870145 : CodeGenerator::GetProtectedInstructions() {
     379             :   return OwnedVector<trap_handler::ProtectedInstructionData>::Of(
     380      870145 :       protected_instructions_);
     381             : }
     382             : 
     383     1588151 : MaybeHandle<Code> CodeGenerator::FinalizeCode() {
     384     1588151 :   if (result_ != kSuccess) {
     385             :     tasm()->AbortedCodeGeneration();
     386           8 :     return MaybeHandle<Code>();
     387             :   }
     388             : 
     389             :   // Allocate the source position table.
     390             :   Handle<ByteArray> source_positions =
     391     1588143 :       source_position_table_builder_.ToSourcePositionTable(isolate());
     392             : 
     393             :   // Allocate deoptimization data.
     394     1588160 :   Handle<DeoptimizationData> deopt_data = GenerateDeoptimizationData();
     395             : 
     396             :   // Allocate and install the code.
     397     1588161 :   CodeDesc desc;
     398     3176322 :   tasm()->GetCode(isolate(), &desc, safepoints(), handler_table_offset_);
     399     1588163 :   if (unwinding_info_writer_.eh_frame_writer()) {
     400          22 :     unwinding_info_writer_.eh_frame_writer()->GetEhFrame(&desc);
     401             :   }
     402             : 
     403             :   MaybeHandle<Code> maybe_code = isolate()->factory()->TryNewCode(
     404             :       desc, info()->code_kind(), Handle<Object>(), info()->builtin_index(),
     405             :       source_positions, deopt_data, kMovable, true,
     406     1588163 :       frame()->GetTotalFrameSlotCount());
     407             : 
     408             :   Handle<Code> code;
     409     1588167 :   if (!maybe_code.ToHandle(&code)) {
     410             :     tasm()->AbortedCodeGeneration();
     411           0 :     return MaybeHandle<Code>();
     412             :   }
     413             : 
     414             :   isolate()->counters()->total_compiled_code_size()->Increment(
     415     1588167 :       code->raw_instruction_size());
     416             : 
     417     1588239 :   LOG_CODE_EVENT(isolate(),
     418             :                  CodeLinePosInfoRecordEvent(code->raw_instruction_start(),
     419             :                                             *source_positions));
     420             : 
     421     1588168 :   return code;
     422             : }
     423             : 
     424    15938063 : bool CodeGenerator::IsNextInAssemblyOrder(RpoNumber block) const {
     425             :   return instructions()
     426             :       ->InstructionBlockAt(current_block_)
     427    15938130 :       ->ao_number()
     428    31876223 :       .IsNext(instructions()->InstructionBlockAt(block)->ao_number());
     429             : }
     430             : 
     431     6253753 : void CodeGenerator::RecordSafepoint(ReferenceMap* references,
     432             :                                     Safepoint::Kind kind,
     433             :                                     Safepoint::DeoptMode deopt_mode) {
     434     6253753 :   Safepoint safepoint = safepoints()->DefineSafepoint(tasm(), kind, deopt_mode);
     435             :   int stackSlotToSpillSlotDelta =
     436     6254215 :       frame()->GetTotalFrameSlotCount() - frame()->GetSpillSlotCount();
     437    87924052 :   for (const InstructionOperand& operand : references->reference_operands()) {
     438    81669771 :     if (operand.IsStackSlot()) {
     439             :       int index = LocationOperand::cast(operand).index();
     440             :       DCHECK_LE(0, index);
     441             :       // We might index values in the fixed part of the frame (i.e. the
     442             :       // closure pointer or the context pointer); these are not spill slots
     443             :       // and therefore don't work with the SafepointTable currently, but
     444             :       // we also don't need to worry about them, since the GC has special
     445             :       // knowledge about those fields anyway.
     446    16228756 :       if (index < stackSlotToSpillSlotDelta) continue;
     447             :       safepoint.DefinePointerSlot(index);
     448    65441015 :     } else if (operand.IsRegister() && (kind & Safepoint::kWithRegisters)) {
     449           0 :       Register reg = LocationOperand::cast(operand).GetRegister();
     450           0 :       safepoint.DefinePointerRegister(reg);
     451             :     }
     452             :   }
     453     6254281 : }
     454             : 
     455     8439226 : bool CodeGenerator::IsMaterializableFromRoot(Handle<HeapObject> object,
     456             :                                              RootIndex* index_return) {
     457             :   const CallDescriptor* incoming_descriptor =
     458             :       linkage()->GetIncomingDescriptor();
     459     8439226 :   if (incoming_descriptor->flags() & CallDescriptor::kCanUseRoots) {
     460     9517344 :     return isolate()->roots_table().IsRootHandle(object, index_return) &&
     461     1920673 :            RootsTable::IsImmortalImmovable(*index_return);
     462             :   }
     463             :   return false;
     464             : }
     465             : 
     466    20306040 : CodeGenerator::CodeGenResult CodeGenerator::AssembleBlock(
     467             :     const InstructionBlock* block) {
     468   159580246 :   for (int i = block->code_start(); i < block->code_end(); ++i) {
     469    69635114 :     if (info()->trace_turbo_json_enabled()) {
     470          82 :       instr_starts_[i] = tasm()->pc_offset();
     471             :     }
     472             :     Instruction* instr = instructions()->InstructionAt(i);
     473    69636049 :     CodeGenResult result = AssembleInstruction(instr, block);
     474    69636552 :     if (result != kSuccess) return result;
     475             :   }
     476             :   return kSuccess;
     477             : }
     478             : 
     479       38892 : bool CodeGenerator::IsValidPush(InstructionOperand source,
     480             :                                 CodeGenerator::PushTypeFlags push_type) {
     481       38892 :   if (source.IsImmediate() &&
     482             :       ((push_type & CodeGenerator::kImmediatePush) != 0)) {
     483             :     return true;
     484             :   }
     485       64228 :   if (source.IsRegister() &&
     486             :       ((push_type & CodeGenerator::kRegisterPush) != 0)) {
     487             :     return true;
     488             :   }
     489       26496 :   if (source.IsStackSlot() &&
     490             :       ((push_type & CodeGenerator::kStackSlotPush) != 0)) {
     491             :     return true;
     492             :   }
     493         616 :   return false;
     494             : }
     495             : 
     496      119724 : void CodeGenerator::GetPushCompatibleMoves(Instruction* instr,
     497             :                                            PushTypeFlags push_type,
     498             :                                            ZoneVector<MoveOperands*>* pushes) {
     499             :   pushes->clear();
     500      204968 :   for (int i = Instruction::FIRST_GAP_POSITION;
     501      324692 :        i <= Instruction::LAST_GAP_POSITION; ++i) {
     502             :     Instruction::GapPosition inner_pos =
     503             :         static_cast<Instruction::GapPosition>(i);
     504             :     ParallelMove* parallel_move = instr->GetParallelMove(inner_pos);
     505      222208 :     if (parallel_move != nullptr) {
     506      551634 :       for (auto move : *parallel_move) {
     507      432502 :         InstructionOperand source = move->source();
     508      432502 :         InstructionOperand destination = move->destination();
     509             :         int first_push_compatible_index =
     510             :             V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0;
     511             :         // If there are any moves from slots that will be overridden by pushes,
     512             :         // then the full gap resolver must be used since optimization with
     513             :         // pushes don't participate in the parallel move and might clobber
     514             :         // values needed for the gap resolve.
     515      463214 :         if (source.IsStackSlot() && LocationOperand::cast(source).index() >=
     516             :                                         first_push_compatible_index) {
     517             :           pushes->clear();
     518             :           return;
     519             :         }
     520             :         // TODO(danno): Right now, only consider moves from the FIRST gap for
     521             :         // pushes. Theoretically, we could extract pushes for both gaps (there
     522             :         // are cases where this happens), but the logic for that would also have
     523             :         // to check to make sure that non-memory inputs to the pushes from the
     524             :         // LAST gap don't get clobbered in the FIRST gap.
     525      415262 :         if (i == Instruction::FIRST_GAP_POSITION) {
     526      473354 :           if (destination.IsStackSlot() &&
     527             :               LocationOperand::cast(destination).index() >=
     528             :                   first_push_compatible_index) {
     529             :             int index = LocationOperand::cast(destination).index();
     530       38892 :             if (IsValidPush(source, push_type)) {
     531       38276 :               if (index >= static_cast<int>(pushes->size())) {
     532       38276 :                 pushes->resize(index + 1);
     533             :               }
     534       76552 :               (*pushes)[index] = move;
     535             :             }
     536             :           }
     537             :         }
     538             :       }
     539             :     }
     540             :   }
     541             : 
     542             :   // For now, only support a set of continuous pushes at the end of the list.
     543             :   size_t push_count_upper_bound = pushes->size();
     544             :   size_t push_begin = push_count_upper_bound;
     545      133704 :   for (auto move : base::Reversed(*pushes)) {
     546       44148 :     if (move == nullptr) break;
     547       31220 :     push_begin--;
     548             :   }
     549      102484 :   size_t push_count = pushes->size() - push_begin;
     550             :   std::copy(pushes->begin() + push_begin,
     551             :             pushes->begin() + push_begin + push_count, pushes->begin());
     552      102484 :   pushes->resize(push_count);
     553             : }
     554             : 
     555    38515122 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferMove(
     556             :     InstructionOperand* source, InstructionOperand* destination) {
     557    38515122 :   if (source->IsConstant()) {
     558    19496188 :     if (destination->IsAnyRegister()) {
     559             :       return MoveType::kConstantToRegister;
     560             :     } else {
     561             :       DCHECK(destination->IsAnyStackSlot());
     562       44616 :       return MoveType::kConstantToStack;
     563             :     }
     564             :   }
     565             :   DCHECK(LocationOperand::cast(source)->IsCompatible(
     566             :       LocationOperand::cast(destination)));
     567    19018934 :   if (source->IsAnyRegister()) {
     568     9943752 :     if (destination->IsAnyRegister()) {
     569             :       return MoveType::kRegisterToRegister;
     570             :     } else {
     571             :       DCHECK(destination->IsAnyStackSlot());
     572     5794303 :       return MoveType::kRegisterToStack;
     573             :     }
     574             :   } else {
     575             :     DCHECK(source->IsAnyStackSlot());
     576     9075182 :     if (destination->IsAnyRegister()) {
     577             :       return MoveType::kStackToRegister;
     578             :     } else {
     579             :       DCHECK(destination->IsAnyStackSlot());
     580       44958 :       return MoveType::kStackToStack;
     581             :     }
     582             :   }
     583             : }
     584             : 
     585       66091 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferSwap(
     586             :     InstructionOperand* source, InstructionOperand* destination) {
     587             :   DCHECK(LocationOperand::cast(source)->IsCompatible(
     588             :       LocationOperand::cast(destination)));
     589       66091 :   if (source->IsAnyRegister()) {
     590       62435 :     if (destination->IsAnyRegister()) {
     591             :       return MoveType::kRegisterToRegister;
     592             :     } else {
     593             :       DCHECK(destination->IsAnyStackSlot());
     594        6767 :       return MoveType::kRegisterToStack;
     595             :     }
     596             :   } else {
     597             :     DCHECK(source->IsAnyStackSlot());
     598             :     DCHECK(destination->IsAnyStackSlot());
     599             :     return MoveType::kStackToStack;
     600             :   }
     601             : }
     602             : 
     603     5409138 : RpoNumber CodeGenerator::ComputeBranchInfo(BranchInfo* branch,
     604             :                                            Instruction* instr) {
     605             :   // Assemble a branch after this instruction.
     606             :   InstructionOperandConverter i(this, instr);
     607     5409138 :   RpoNumber true_rpo = i.InputRpo(instr->InputCount() - 2);
     608     5409155 :   RpoNumber false_rpo = i.InputRpo(instr->InputCount() - 1);
     609             : 
     610     5409200 :   if (true_rpo == false_rpo) {
     611        2200 :     return true_rpo;
     612             :   }
     613     5407000 :   FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
     614     5407000 :   if (IsNextInAssemblyOrder(true_rpo)) {
     615             :     // true block is next, can fall through if condition negated.
     616             :     std::swap(true_rpo, false_rpo);
     617             :     condition = NegateFlagsCondition(condition);
     618             :   }
     619     5407008 :   branch->condition = condition;
     620     5407008 :   branch->true_label = GetLabel(true_rpo);
     621     5407008 :   branch->false_label = GetLabel(false_rpo);
     622     5407008 :   branch->fallthru = IsNextInAssemblyOrder(false_rpo);
     623             :   return RpoNumber::Invalid();
     624             : }
     625             : 
     626    69635972 : CodeGenerator::CodeGenResult CodeGenerator::AssembleInstruction(
     627             :     Instruction* instr, const InstructionBlock* block) {
     628             :   int first_unused_stack_slot;
     629    69635972 :   FlagsMode mode = FlagsModeField::decode(instr->opcode());
     630    69635972 :   if (mode != kFlags_trap) {
     631    69602115 :     AssembleSourcePosition(instr);
     632             :   }
     633             :   bool adjust_stack =
     634    69638861 :       GetSlotAboveSPBeforeTailCall(instr, &first_unused_stack_slot);
     635    69639053 :   if (adjust_stack) AssembleTailCallBeforeGap(instr, first_unused_stack_slot);
     636             :   AssembleGaps(instr);
     637    69637892 :   if (adjust_stack) AssembleTailCallAfterGap(instr, first_unused_stack_slot);
     638             :   DCHECK_IMPLIES(
     639             :       block->must_deconstruct_frame(),
     640             :       instr != instructions()->InstructionAt(block->last_instruction_index()) ||
     641             :           instr->IsRet() || instr->IsJump());
     642    69638116 :   if (instr->IsJump() && block->must_deconstruct_frame()) {
     643       60142 :     AssembleDeconstructFrame();
     644             :   }
     645             :   // Assemble architecture-specific code for the instruction.
     646    69638116 :   CodeGenResult result = AssembleArchInstruction(instr);
     647    69636304 :   if (result != kSuccess) return result;
     648             : 
     649    69636474 :   FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
     650    69636474 :   switch (mode) {
     651             :     case kFlags_branch:
     652             :     case kFlags_branch_and_poison: {
     653             :       BranchInfo branch;
     654     5409171 :       RpoNumber target = ComputeBranchInfo(&branch, instr);
     655     5409223 :       if (target.IsValid()) {
     656             :         // redundant branch.
     657        2200 :         if (!IsNextInAssemblyOrder(target)) {
     658         365 :           AssembleArchJump(target);
     659             :         }
     660        2200 :         return kSuccess;
     661             :       }
     662             :       // Assemble architecture-specific branch.
     663     5407023 :       AssembleArchBranch(instr, &branch);
     664     5406921 :       break;
     665             :     }
     666             :     case kFlags_deoptimize:
     667             :     case kFlags_deoptimize_and_poison: {
     668             :       // Assemble a conditional eager deoptimization after this instruction.
     669             :       InstructionOperandConverter i(this, instr);
     670      332000 :       size_t frame_state_offset = MiscField::decode(instr->opcode());
     671             :       DeoptimizationExit* const exit =
     672      332000 :           AddDeoptimizationExit(instr, frame_state_offset);
     673      331998 :       Label continue_label;
     674             :       BranchInfo branch;
     675      331998 :       branch.condition = condition;
     676      331998 :       branch.true_label = exit->label();
     677      331998 :       branch.false_label = &continue_label;
     678      331998 :       branch.fallthru = true;
     679             :       // Assemble architecture-specific branch.
     680      331998 :       AssembleArchDeoptBranch(instr, &branch);
     681      332006 :       tasm()->bind(&continue_label);
     682      332004 :       if (mode == kFlags_deoptimize_and_poison) {
     683           0 :         AssembleBranchPoisoning(NegateFlagsCondition(branch.condition), instr);
     684             :       }
     685             :       break;
     686             :     }
     687             :     case kFlags_set: {
     688             :       // Assemble a boolean materialization after this instruction.
     689      375716 :       AssembleArchBoolean(instr, condition);
     690      375715 :       break;
     691             :     }
     692             :     case kFlags_trap: {
     693       34514 :       AssembleArchTrap(instr, condition);
     694       34473 :       break;
     695             :     }
     696             :     case kFlags_none: {
     697             :       break;
     698             :     }
     699             :   }
     700             : 
     701             :   // TODO(jarin) We should thread the flag through rather than set it.
     702    69634187 :   if (instr->IsCall()) {
     703             :     ResetSpeculationPoison();
     704             :   }
     705             : 
     706             :   return kSuccess;
     707             : }
     708             : 
     709    69721570 : void CodeGenerator::AssembleSourcePosition(Instruction* instr) {
     710    69721570 :   SourcePosition source_position = SourcePosition::Unknown();
     711   134754721 :   if (instr->IsNop() && instr->AreMovesRedundant()) return;
     712    44302459 :   if (!instructions()->GetSourcePosition(instr, &source_position)) return;
     713     4690259 :   AssembleSourcePosition(source_position);
     714             : }
     715             : 
     716     4701449 : void CodeGenerator::AssembleSourcePosition(SourcePosition source_position) {
     717     4701449 :   if (source_position == current_source_position_) return;
     718     3558998 :   current_source_position_ = source_position;
     719     3558998 :   if (!source_position.IsKnown()) return;
     720     3559101 :   source_position_table_builder_.AddPosition(tasm()->pc_offset(),
     721     3559101 :                                              source_position, false);
     722     3559380 :   if (FLAG_code_comments) {
     723             :     OptimizedCompilationInfo* info = this->info();
     724        1147 :     if (info->IsNotOptimizedFunctionOrWasmFunction()) return;
     725        2294 :     std::ostringstream buffer;
     726        1147 :     buffer << "-- ";
     727             :     // Turbolizer only needs the source position, as it can reconstruct
     728             :     // the inlining stack from other information.
     729        1147 :     if (info->trace_turbo_json_enabled() || !tasm()->isolate() ||
     730             :         tasm()->isolate()->concurrent_recompilation_enabled()) {
     731         153 :       buffer << source_position;
     732             :     } else {
     733             :       AllowHeapAllocation allocation;
     734             :       AllowHandleAllocation handles;
     735             :       AllowHandleDereference deref;
     736        1988 :       buffer << source_position.InliningStack(info);
     737             :     }
     738        1147 :     buffer << " --";
     739        2294 :     tasm()->RecordComment(buffer.str().c_str());
     740             :   }
     741             : }
     742             : 
     743    69637604 : bool CodeGenerator::GetSlotAboveSPBeforeTailCall(Instruction* instr,
     744             :                                                  int* slot) {
     745    69637604 :   if (instr->IsTailCall()) {
     746             :     InstructionOperandConverter g(this, instr);
     747      239424 :     *slot = g.InputInt32(instr->InputCount() - 1);
     748             :     return true;
     749             :   } else {
     750             :     return false;
     751             :   }
     752             : }
     753             : 
     754      477450 : StubCallMode CodeGenerator::DetermineStubCallMode() const {
     755             :   Code::Kind code_kind = info()->code_kind();
     756      477450 :   return (code_kind == Code::WASM_FUNCTION ||
     757             :           code_kind == Code::WASM_TO_JS_FUNCTION)
     758             :              ? StubCallMode::kCallWasmRuntimeStub
     759      477450 :              : StubCallMode::kCallCodeObject;
     760             : }
     761             : 
     762          12 : void CodeGenerator::AssembleGaps(Instruction* instr) {
     763   139274323 :   for (int i = Instruction::FIRST_GAP_POSITION;
     764   208912190 :        i <= Instruction::LAST_GAP_POSITION; i++) {
     765             :     Instruction::GapPosition inner_pos =
     766             :         static_cast<Instruction::GapPosition>(i);
     767             :     ParallelMove* move = instr->GetParallelMove(inner_pos);
     768   188893622 :     if (move != nullptr) resolver()->Resolve(move);
     769             :   }
     770          12 : }
     771             : 
     772             : namespace {
     773             : 
     774      463632 : Handle<PodArray<InliningPosition>> CreateInliningPositions(
     775             :     OptimizedCompilationInfo* info, Isolate* isolate) {
     776             :   const OptimizedCompilationInfo::InlinedFunctionList& inlined_functions =
     777             :       info->inlined_functions();
     778      463632 :   if (inlined_functions.size() == 0) {
     779             :     return Handle<PodArray<InliningPosition>>::cast(
     780             :         isolate->factory()->empty_byte_array());
     781             :   }
     782             :   Handle<PodArray<InliningPosition>> inl_positions =
     783             :       PodArray<InliningPosition>::New(
     784             :           isolate, static_cast<int>(inlined_functions.size()),
     785       13458 :           AllocationType::kOld);
     786      143032 :   for (size_t i = 0; i < inlined_functions.size(); ++i) {
     787       64787 :     inl_positions->set(static_cast<int>(i), inlined_functions[i].position);
     788             :   }
     789       13458 :   return inl_positions;
     790             : }
     791             : 
     792             : }  // namespace
     793             : 
     794     1588155 : Handle<DeoptimizationData> CodeGenerator::GenerateDeoptimizationData() {
     795             :   OptimizedCompilationInfo* info = this->info();
     796     1588155 :   int deopt_count = static_cast<int>(deoptimization_states_.size());
     797     2712682 :   if (deopt_count == 0 && !info->is_osr()) {
     798     1124526 :     return DeoptimizationData::Empty(isolate());
     799             :   }
     800             :   Handle<DeoptimizationData> data =
     801      463629 :       DeoptimizationData::New(isolate(), deopt_count, AllocationType::kOld);
     802             : 
     803             :   Handle<ByteArray> translation_array =
     804      463631 :       translations_.CreateByteArray(isolate()->factory());
     805             : 
     806      927267 :   data->SetTranslationByteArray(*translation_array);
     807      463634 :   data->SetInlinedFunctionCount(
     808      463634 :       Smi::FromInt(static_cast<int>(inlined_function_count_)));
     809             :   data->SetOptimizationId(Smi::FromInt(info->optimization_id()));
     810             : 
     811      463634 :   if (info->has_shared_info()) {
     812      927268 :     data->SetSharedFunctionInfo(*info->shared_info());
     813             :   } else {
     814           0 :     data->SetSharedFunctionInfo(Smi::kZero);
     815             :   }
     816             : 
     817             :   Handle<FixedArray> literals = isolate()->factory()->NewFixedArray(
     818      463634 :       static_cast<int>(deoptimization_literals_.size()), AllocationType::kOld);
     819     6342270 :   for (unsigned i = 0; i < deoptimization_literals_.size(); i++) {
     820     1805001 :     Handle<Object> object = deoptimization_literals_[i].Reify(isolate());
     821     3609996 :     literals->set(i, *object);
     822             :   }
     823      927266 :   data->SetLiteralArray(*literals);
     824             : 
     825             :   Handle<PodArray<InliningPosition>> inl_pos =
     826      463633 :       CreateInliningPositions(info, isolate());
     827      927265 :   data->SetInliningPositions(*inl_pos);
     828             : 
     829      463633 :   if (info->is_osr()) {
     830             :     DCHECK_LE(0, osr_pc_offset_);
     831        4700 :     data->SetOsrBytecodeOffset(Smi::FromInt(info_->osr_offset().ToInt()));
     832        4700 :     data->SetOsrPcOffset(Smi::FromInt(osr_pc_offset_));
     833             :   } else {
     834             :     BailoutId osr_offset = BailoutId::None();
     835             :     data->SetOsrBytecodeOffset(Smi::FromInt(osr_offset.ToInt()));
     836             :     data->SetOsrPcOffset(Smi::FromInt(-1));
     837             :   }
     838             : 
     839             :   // Populate deoptimization entries.
     840     7133591 :   for (int i = 0; i < deopt_count; i++) {
     841     3334979 :     DeoptimizationState* deoptimization_state = deoptimization_states_[i];
     842             :     data->SetBytecodeOffset(i, deoptimization_state->bailout_id());
     843     3334979 :     CHECK(deoptimization_state);
     844             :     data->SetTranslationIndex(
     845             :         i, Smi::FromInt(deoptimization_state->translation_id()));
     846             :     data->SetPc(i, Smi::FromInt(deoptimization_state->pc_offset()));
     847             :   }
     848             : 
     849      463632 :   return data;
     850             : }
     851             : 
     852         307 : Label* CodeGenerator::AddJumpTable(Label** targets, size_t target_count) {
     853         614 :   jump_tables_ = new (zone()) JumpTable(jump_tables_, targets, target_count);
     854         307 :   return jump_tables_->label();
     855             : }
     856             : 
     857     6135391 : void CodeGenerator::RecordCallPosition(Instruction* instr) {
     858     6135391 :   CallDescriptor::Flags flags(MiscField::decode(instr->opcode()));
     859             : 
     860             :   bool needs_frame_state = (flags & CallDescriptor::kNeedsFrameState);
     861             : 
     862     6135391 :   RecordSafepoint(
     863             :       instr->reference_map(), Safepoint::kSimple,
     864     6135391 :       needs_frame_state ? Safepoint::kLazyDeopt : Safepoint::kNoLazyDeopt);
     865             : 
     866     6135545 :   if (flags & CallDescriptor::kHasExceptionHandler) {
     867             :     InstructionOperandConverter i(this, instr);
     868      219623 :     RpoNumber handler_rpo = i.InputRpo(instr->InputCount() - 1);
     869      658863 :     handlers_.push_back({GetLabel(handler_rpo), tasm()->pc_offset()});
     870             :   }
     871             : 
     872     6135543 :   if (needs_frame_state) {
     873             :     MarkLazyDeoptSite();
     874             :     // If the frame state is present, it starts at argument 2 - after
     875             :     // the code address and the poison-alias index.
     876             :     size_t frame_state_offset = 2;
     877             :     FrameStateDescriptor* descriptor =
     878     3224770 :         GetDeoptimizationEntry(instr, frame_state_offset).descriptor();
     879             :     int pc_offset = tasm()->pc_offset();
     880             :     int deopt_state_id = BuildTranslation(instr, pc_offset, frame_state_offset,
     881     3224779 :                                           descriptor->state_combine());
     882             : 
     883             :     DeoptimizationExit* const exit = new (zone())
     884     6449532 :         DeoptimizationExit(deopt_state_id, current_source_position_);
     885     3224763 :     deoptimization_exits_.push_back(exit);
     886     3224764 :     safepoints()->RecordLazyDeoptimizationIndex(deopt_state_id);
     887             :   }
     888     6135555 : }
     889             : 
     890     8055462 : int CodeGenerator::DefineDeoptimizationLiteral(DeoptimizationLiteral literal) {
     891     8055462 :   int result = static_cast<int>(deoptimization_literals_.size());
     892   215999640 :   for (unsigned i = 0; i < deoptimization_literals_.size(); ++i) {
     893    72878780 :     if (deoptimization_literals_[i] == literal) return i;
     894             :   }
     895     1806295 :   deoptimization_literals_.push_back(literal);
     896     1806287 :   return result;
     897             : }
     898             : 
     899     6825223 : DeoptimizationEntry const& CodeGenerator::GetDeoptimizationEntry(
     900             :     Instruction* instr, size_t frame_state_offset) {
     901             :   InstructionOperandConverter i(this, instr);
     902             :   int const state_id = i.InputInt32(frame_state_offset);
     903     6825234 :   return instructions()->GetDeoptimizationEntry(state_id);
     904             : }
     905             : 
     906           0 : DeoptimizeKind CodeGenerator::GetDeoptimizationKind(
     907             :     int deoptimization_id) const {
     908             :   size_t const index = static_cast<size_t>(deoptimization_id);
     909             :   DCHECK_LT(index, deoptimization_states_.size());
     910     3354655 :   return deoptimization_states_[index]->kind();
     911             : }
     912             : 
     913           0 : DeoptimizeReason CodeGenerator::GetDeoptimizationReason(
     914             :     int deoptimization_id) const {
     915             :   size_t const index = static_cast<size_t>(deoptimization_id);
     916             :   DCHECK_LT(index, deoptimization_states_.size());
     917     3354659 :   return deoptimization_states_[index]->reason();
     918             : }
     919             : 
     920    51986510 : void CodeGenerator::TranslateStateValueDescriptor(
     921             :     StateValueDescriptor* desc, StateValueList* nested,
     922             :     Translation* translation, InstructionOperandIterator* iter) {
     923             :   // Note:
     924             :   // If translation is null, we just skip the relevant instruction operands.
     925    51986510 :   if (desc->IsNested()) {
     926       90086 :     if (translation != nullptr) {
     927       90086 :       translation->BeginCapturedObject(static_cast<int>(nested->size()));
     928             :     }
     929      624431 :     for (auto field : *nested) {
     930             :       TranslateStateValueDescriptor(field.desc, field.nested, translation,
     931      534345 :                                     iter);
     932             :     }
     933    51896424 :   } else if (desc->IsArgumentsElements()) {
     934        5954 :     if (translation != nullptr) {
     935        5954 :       translation->ArgumentsElements(desc->arguments_type());
     936             :     }
     937    51890470 :   } else if (desc->IsArgumentsLength()) {
     938        6262 :     if (translation != nullptr) {
     939        6262 :       translation->ArgumentsLength(desc->arguments_type());
     940             :     }
     941    51884208 :   } else if (desc->IsDuplicate()) {
     942       19797 :     if (translation != nullptr) {
     943       19797 :       translation->DuplicateObject(static_cast<int>(desc->id()));
     944             :     }
     945    51864411 :   } else if (desc->IsPlain()) {
     946             :     InstructionOperand* op = iter->Advance();
     947    23298472 :     if (translation != nullptr) {
     948             :       AddTranslationForOperand(translation, iter->instruction(), op,
     949    23298540 :                                desc->type());
     950             :     }
     951             :   } else {
     952             :     DCHECK(desc->IsOptimizedOut());
     953    28565939 :     if (translation != nullptr) {
     954    28565938 :       if (optimized_out_literal_id_ == -1) {
     955      927280 :         optimized_out_literal_id_ = DefineDeoptimizationLiteral(
     956      463640 :             DeoptimizationLiteral(isolate()->factory()->optimized_out()));
     957             :       }
     958    28565938 :       translation->StoreLiteral(optimized_out_literal_id_);
     959             :     }
     960             :   }
     961    51986465 : }
     962             : 
     963     3955820 : void CodeGenerator::TranslateFrameStateDescriptorOperands(
     964             :     FrameStateDescriptor* desc, InstructionOperandIterator* iter,
     965             :     Translation* translation) {
     966             :   size_t index = 0;
     967             :   StateValueList* values = desc->GetStateValueDescriptors();
     968   106858268 :   for (StateValueList::iterator it = values->begin(); it != values->end();
     969             :        ++it, ++index) {
     970    51451167 :     TranslateStateValueDescriptor((*it).desc, (*it).nested, translation, iter);
     971             :   }
     972             :   DCHECK_EQ(desc->GetSize(), index);
     973     3955877 : }
     974             : 
     975     3955861 : void CodeGenerator::BuildTranslationForFrameStateDescriptor(
     976             :     FrameStateDescriptor* descriptor, InstructionOperandIterator* iter,
     977             :     Translation* translation, OutputFrameStateCombine state_combine) {
     978             :   // Outer-most state must be added to translation first.
     979     3955861 :   if (descriptor->outer_state() != nullptr) {
     980             :     BuildTranslationForFrameStateDescriptor(descriptor->outer_state(), iter,
     981      355407 :                                             translation, state_combine);
     982             :   }
     983             : 
     984             :   Handle<SharedFunctionInfo> shared_info;
     985     3955857 :   if (!descriptor->shared_info().ToHandle(&shared_info)) {
     986        2086 :     if (!info()->has_shared_info()) {
     987             :       return;  // Stub with no SharedFunctionInfo.
     988             :     }
     989             :     shared_info = info()->shared_info();
     990             :   }
     991             :   int shared_info_id =
     992     3955857 :       DefineDeoptimizationLiteral(DeoptimizationLiteral(shared_info));
     993             : 
     994     3955858 :   switch (descriptor->type()) {
     995             :     case FrameStateType::kInterpretedFunction: {
     996             :       int return_offset = 0;
     997             :       int return_count = 0;
     998     3830516 :       if (!state_combine.IsOutputIgnored()) {
     999     2387531 :         return_offset = static_cast<int>(state_combine.GetOffsetToPokeAt());
    1000     2387531 :         return_count = static_cast<int>(iter->instruction()->OutputCount());
    1001             :       }
    1002     3830516 :       translation->BeginInterpretedFrame(
    1003             :           descriptor->bailout_id(), shared_info_id,
    1004             :           static_cast<unsigned int>(descriptor->locals_count() + 1),
    1005     3830516 :           return_offset, return_count);
    1006     3830523 :       break;
    1007             :     }
    1008             :     case FrameStateType::kArgumentsAdaptor:
    1009       84306 :       translation->BeginArgumentsAdaptorFrame(
    1010             :           shared_info_id,
    1011       84306 :           static_cast<unsigned int>(descriptor->parameters_count()));
    1012       84306 :       break;
    1013             :     case FrameStateType::kConstructStub:
    1014             :       DCHECK(descriptor->bailout_id().IsValidForConstructStub());
    1015       26390 :       translation->BeginConstructStubFrame(
    1016             :           descriptor->bailout_id(), shared_info_id,
    1017       26390 :           static_cast<unsigned int>(descriptor->parameters_count() + 1));
    1018       26390 :       break;
    1019             :     case FrameStateType::kBuiltinContinuation: {
    1020        2086 :       BailoutId bailout_id = descriptor->bailout_id();
    1021             :       int parameter_count =
    1022             :           static_cast<unsigned int>(descriptor->parameters_count());
    1023        2086 :       translation->BeginBuiltinContinuationFrame(bailout_id, shared_info_id,
    1024        2086 :                                                  parameter_count);
    1025             :       break;
    1026             :     }
    1027             :     case FrameStateType::kJavaScriptBuiltinContinuation: {
    1028       12285 :       BailoutId bailout_id = descriptor->bailout_id();
    1029             :       int parameter_count =
    1030             :           static_cast<unsigned int>(descriptor->parameters_count());
    1031       12285 :       translation->BeginJavaScriptBuiltinContinuationFrame(
    1032       12285 :           bailout_id, shared_info_id, parameter_count);
    1033             :       break;
    1034             :     }
    1035             :     case FrameStateType::kJavaScriptBuiltinContinuationWithCatch: {
    1036         274 :       BailoutId bailout_id = descriptor->bailout_id();
    1037             :       int parameter_count =
    1038             :           static_cast<unsigned int>(descriptor->parameters_count());
    1039         274 :       translation->BeginJavaScriptBuiltinContinuationWithCatchFrame(
    1040         274 :           bailout_id, shared_info_id, parameter_count);
    1041             :       break;
    1042             :     }
    1043             :   }
    1044             : 
    1045     3955865 :   TranslateFrameStateDescriptorOperands(descriptor, iter, translation);
    1046             : }
    1047             : 
    1048     3600453 : int CodeGenerator::BuildTranslation(Instruction* instr, int pc_offset,
    1049             :                                     size_t frame_state_offset,
    1050             :                                     OutputFrameStateCombine state_combine) {
    1051             :   DeoptimizationEntry const& entry =
    1052     3600453 :       GetDeoptimizationEntry(instr, frame_state_offset);
    1053             :   FrameStateDescriptor* const descriptor = entry.descriptor();
    1054     3600461 :   frame_state_offset++;
    1055             : 
    1056     3600461 :   int update_feedback_count = entry.feedback().IsValid() ? 1 : 0;
    1057             :   Translation translation(&translations_,
    1058     3600462 :                           static_cast<int>(descriptor->GetFrameCount()),
    1059     3600461 :                           static_cast<int>(descriptor->GetJSFrameCount()),
    1060     7200930 :                           update_feedback_count, zone());
    1061     3600475 :   if (entry.feedback().IsValid()) {
    1062             :     DeoptimizationLiteral literal =
    1063             :         DeoptimizationLiteral(entry.feedback().vector());
    1064       40808 :     int literal_id = DefineDeoptimizationLiteral(literal);
    1065       40807 :     translation.AddUpdateFeedback(literal_id, entry.feedback().slot().ToInt());
    1066             :   }
    1067             :   InstructionOperandIterator iter(instr, frame_state_offset);
    1068             :   BuildTranslationForFrameStateDescriptor(descriptor, &iter, &translation,
    1069     3600475 :                                           state_combine);
    1070             : 
    1071     3600478 :   int deoptimization_id = static_cast<int>(deoptimization_states_.size());
    1072             : 
    1073     7200921 :   deoptimization_states_.push_back(new (zone()) DeoptimizationState(
    1074             :       descriptor->bailout_id(), translation.index(), pc_offset, entry.kind(),
    1075             :       entry.reason()));
    1076             : 
    1077     3600452 :   return deoptimization_id;
    1078             : }
    1079             : 
    1080    23298958 : void CodeGenerator::AddTranslationForOperand(Translation* translation,
    1081             :                                              Instruction* instr,
    1082             :                                              InstructionOperand* op,
    1083             :                                              MachineType type) {
    1084    23298958 :   if (op->IsStackSlot()) {
    1085    15841771 :     if (type.representation() == MachineRepresentation::kBit) {
    1086       21489 :       translation->StoreBoolStackSlot(LocationOperand::cast(op)->index());
    1087    47459729 :     } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
    1088             :                type == MachineType::Int32()) {
    1089      481408 :       translation->StoreInt32StackSlot(LocationOperand::cast(op)->index());
    1090    46016615 :     } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
    1091             :                type == MachineType::Uint32()) {
    1092        7150 :       translation->StoreUint32StackSlot(LocationOperand::cast(op)->index());
    1093    15331724 :     } else if (type == MachineType::Int64()) {
    1094         237 :       translation->StoreInt64StackSlot(LocationOperand::cast(op)->index());
    1095             :     } else {
    1096    15331487 :       CHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1097    15331487 :       translation->StoreStackSlot(LocationOperand::cast(op)->index());
    1098             :     }
    1099     7457187 :   } else if (op->IsFPStackSlot()) {
    1100      259632 :     if (type.representation() == MachineRepresentation::kFloat64) {
    1101      258735 :       translation->StoreDoubleStackSlot(LocationOperand::cast(op)->index());
    1102             :     } else {
    1103         897 :       CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
    1104         897 :       translation->StoreFloatStackSlot(LocationOperand::cast(op)->index());
    1105             :     }
    1106     7197555 :   } else if (op->IsRegister()) {
    1107             :     InstructionOperandConverter converter(this, instr);
    1108      501228 :     if (type.representation() == MachineRepresentation::kBit) {
    1109        3205 :       translation->StoreBoolRegister(converter.ToRegister(op));
    1110     1493485 :     } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
    1111             :                type == MachineType::Int32()) {
    1112       35070 :       translation->StoreInt32Register(converter.ToRegister(op));
    1113     1388863 :     } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
    1114             :                type == MachineType::Uint32()) {
    1115        1104 :       translation->StoreUint32Register(converter.ToRegister(op));
    1116      461849 :     } else if (type == MachineType::Int64()) {
    1117          14 :       translation->StoreInt64Register(converter.ToRegister(op));
    1118             :     } else {
    1119      461835 :       CHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1120      461835 :       translation->StoreRegister(converter.ToRegister(op));
    1121             :     }
    1122     6696327 :   } else if (op->IsFPRegister()) {
    1123             :     InstructionOperandConverter converter(this, instr);
    1124       64190 :     if (type.representation() == MachineRepresentation::kFloat64) {
    1125       64072 :       translation->StoreDoubleRegister(converter.ToDoubleRegister(op));
    1126             :     } else {
    1127         118 :       CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
    1128         118 :       translation->StoreFloatRegister(converter.ToFloatRegister(op));
    1129             :     }
    1130             :   } else {
    1131     6632137 :     CHECK(op->IsImmediate());
    1132             :     InstructionOperandConverter converter(this, instr);
    1133     6632137 :     Constant constant = converter.ToConstant(op);
    1134             :     DeoptimizationLiteral literal;
    1135     6632199 :     switch (constant.type()) {
    1136             :       case Constant::kInt32:
    1137       17695 :         if (type.representation() == MachineRepresentation::kTagged) {
    1138             :           // When pointers are 4 bytes, we can use int32 constants to represent
    1139             :           // Smis.
    1140             :           DCHECK_EQ(4, kSystemPointerSize);
    1141             :           Smi smi(static_cast<Address>(constant.ToInt32()));
    1142             :           DCHECK(smi->IsSmi());
    1143          52 :           literal = DeoptimizationLiteral(smi->value());
    1144       17643 :         } else if (type.representation() == MachineRepresentation::kBit) {
    1145         783 :           if (constant.ToInt32() == 0) {
    1146             :             literal =
    1147         340 :                 DeoptimizationLiteral(isolate()->factory()->false_value());
    1148             :           } else {
    1149             :             DCHECK_EQ(1, constant.ToInt32());
    1150         443 :             literal = DeoptimizationLiteral(isolate()->factory()->true_value());
    1151             :           }
    1152             :         } else {
    1153             :           DCHECK(type == MachineType::Int32() ||
    1154             :                  type == MachineType::Uint32() ||
    1155             :                  type.representation() == MachineRepresentation::kWord32 ||
    1156             :                  type.representation() == MachineRepresentation::kNone);
    1157             :           DCHECK(type.representation() != MachineRepresentation::kNone ||
    1158             :                  constant.ToInt32() == FrameStateDescriptor::kImpossibleValue);
    1159       16860 :           if (type == MachineType::Uint32()) {
    1160             :             literal = DeoptimizationLiteral(
    1161         828 :                 static_cast<uint32_t>(constant.ToInt32()));
    1162             :           } else {
    1163       16032 :             literal = DeoptimizationLiteral(constant.ToInt32());
    1164             :           }
    1165             :         }
    1166             :         break;
    1167             :       case Constant::kInt64:
    1168             :         DCHECK_EQ(8, kSystemPointerSize);
    1169      556302 :         if (type.representation() == MachineRepresentation::kWord64) {
    1170             :           literal =
    1171         556 :               DeoptimizationLiteral(static_cast<double>(constant.ToInt64()));
    1172             :         } else {
    1173             :           // When pointers are 8 bytes, we can use int64 constants to represent
    1174             :           // Smis.
    1175             :           DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1176      555746 :           Smi smi(static_cast<Address>(constant.ToInt64()));
    1177             :           DCHECK(smi->IsSmi());
    1178      555746 :           literal = DeoptimizationLiteral(smi->value());
    1179             :         }
    1180             :         break;
    1181             :       case Constant::kFloat32:
    1182             :         DCHECK(type.representation() == MachineRepresentation::kFloat32 ||
    1183             :                type.representation() == MachineRepresentation::kTagged);
    1184           0 :         literal = DeoptimizationLiteral(constant.ToFloat32());
    1185           0 :         break;
    1186             :       case Constant::kFloat64:
    1187             :         DCHECK(type.representation() == MachineRepresentation::kFloat64 ||
    1188             :                type.representation() == MachineRepresentation::kTagged);
    1189       97743 :         literal = DeoptimizationLiteral(constant.ToFloat64().value());
    1190       97743 :         break;
    1191             :       case Constant::kHeapObject:
    1192             :         DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1193     5957992 :         literal = DeoptimizationLiteral(constant.ToHeapObject());
    1194     5957991 :         break;
    1195             :       case Constant::kDelayedStringConstant:
    1196             :         DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1197        2467 :         literal = DeoptimizationLiteral(constant.ToDelayedStringConstant());
    1198        2467 :         break;
    1199             :       default:
    1200           0 :         UNREACHABLE();
    1201             :     }
    1202     6632198 :     if (literal.object().equals(info()->closure())) {
    1203     3630544 :       translation->StoreJSFrameFunction();
    1204             :     } else {
    1205     3001654 :       int literal_id = DefineDeoptimizationLiteral(literal);
    1206     3001610 :       translation->StoreLiteral(literal_id);
    1207             :     }
    1208             :   }
    1209    23298930 : }
    1210             : 
    1211           0 : void CodeGenerator::MarkLazyDeoptSite() {
    1212     3224770 :   last_lazy_deopt_pc_ = tasm()->pc_offset();
    1213           0 : }
    1214             : 
    1215      332000 : DeoptimizationExit* CodeGenerator::AddDeoptimizationExit(
    1216             :     Instruction* instr, size_t frame_state_offset) {
    1217             :   int const deoptimization_id = BuildTranslation(
    1218      332000 :       instr, -1, frame_state_offset, OutputFrameStateCombine::Ignore());
    1219             : 
    1220             :   DeoptimizationExit* const exit = new (zone())
    1221      664001 :       DeoptimizationExit(deoptimization_id, current_source_position_);
    1222      332000 :   deoptimization_exits_.push_back(exit);
    1223      331998 :   return exit;
    1224             : }
    1225             : 
    1226     2515339 : void CodeGenerator::InitializeSpeculationPoison() {
    1227     2515339 :   if (poisoning_level_ == PoisoningMitigationLevel::kDontPoison) return;
    1228             : 
    1229             :   // Initialize {kSpeculationPoisonRegister} either by comparing the expected
    1230             :   // with the actual call target, or by unconditionally using {-1} initially.
    1231             :   // Masking register arguments with it only makes sense in the first case.
    1232           0 :   if (info()->called_with_code_start_register()) {
    1233           0 :     tasm()->RecordComment("-- Prologue: generate speculation poison --");
    1234           0 :     GenerateSpeculationPoisonFromCodeStartRegister();
    1235           0 :     if (info()->is_poisoning_register_arguments()) {
    1236           0 :       AssembleRegisterArgumentPoisoning();
    1237             :     }
    1238             :   } else {
    1239             :     ResetSpeculationPoison();
    1240             :   }
    1241             : }
    1242             : 
    1243        4698 : void CodeGenerator::ResetSpeculationPoison() {
    1244     6169108 :   if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison) {
    1245           0 :     tasm()->ResetSpeculationPoisonRegister();
    1246             :   }
    1247        4698 : }
    1248             : 
    1249      656969 : OutOfLineCode::OutOfLineCode(CodeGenerator* gen)
    1250     1970907 :     : frame_(gen->frame()), tasm_(gen->tasm()), next_(gen->ools_) {
    1251      656969 :   gen->ools_ = this;
    1252      656969 : }
    1253             : 
    1254             : OutOfLineCode::~OutOfLineCode() = default;
    1255             : 
    1256     1804998 : Handle<Object> DeoptimizationLiteral::Reify(Isolate* isolate) const {
    1257     1804998 :   switch (kind_) {
    1258             :     case DeoptimizationLiteralKind::kObject: {
    1259     1724196 :       return object_;
    1260             :     }
    1261             :     case DeoptimizationLiteralKind::kNumber: {
    1262       80183 :       return isolate->factory()->NewNumber(number_);
    1263             :     }
    1264             :     case DeoptimizationLiteralKind::kString: {
    1265         619 :       return string_->AllocateStringConstant(isolate);
    1266             :     }
    1267             :   }
    1268           0 :   UNREACHABLE();
    1269             : }
    1270             : 
    1271             : }  // namespace compiler
    1272             : }  // namespace internal
    1273      120216 : }  // namespace v8

Generated by: LCOV version 1.10