LCOV - code coverage report
Current view: top level - src/compiler/backend - code-generator.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 498 523 95.2 %
Date: 2019-04-18 Functions: 44 47 93.6 %

          Line data    Source code
       1             : // Copyright 2013 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #include "src/compiler/backend/code-generator.h"
       6             : 
       7             : #include "src/address-map.h"
       8             : #include "src/assembler-inl.h"
       9             : #include "src/base/adapters.h"
      10             : #include "src/compiler/backend/code-generator-impl.h"
      11             : #include "src/compiler/linkage.h"
      12             : #include "src/compiler/pipeline.h"
      13             : #include "src/compiler/wasm-compiler.h"
      14             : #include "src/counters.h"
      15             : #include "src/eh-frame.h"
      16             : #include "src/frames.h"
      17             : #include "src/log.h"
      18             : #include "src/macro-assembler-inl.h"
      19             : #include "src/objects/smi.h"
      20             : #include "src/optimized-compilation-info.h"
      21             : #include "src/string-constants.h"
      22             : 
      23             : namespace v8 {
      24             : namespace internal {
      25             : namespace compiler {
      26             : 
      27             : class CodeGenerator::JumpTable final : public ZoneObject {
      28             :  public:
      29             :   JumpTable(JumpTable* next, Label** targets, size_t target_count)
      30         315 :       : next_(next), targets_(targets), target_count_(target_count) {}
      31             : 
      32         630 :   Label* label() { return &label_; }
      33             :   JumpTable* next() const { return next_; }
      34             :   Label** targets() const { return targets_; }
      35             :   size_t target_count() const { return target_count_; }
      36             : 
      37             :  private:
      38             :   Label label_;
      39             :   JumpTable* const next_;
      40             :   Label** const targets_;
      41             :   size_t const target_count_;
      42             : };
      43             : 
      44     2630650 : CodeGenerator::CodeGenerator(
      45             :     Zone* codegen_zone, Frame* frame, Linkage* linkage,
      46             :     InstructionSequence* instructions, OptimizedCompilationInfo* info,
      47             :     Isolate* isolate, base::Optional<OsrHelper> osr_helper,
      48             :     int start_source_position, JumpOptimizationInfo* jump_opt,
      49             :     PoisoningMitigationLevel poisoning_level, const AssemblerOptions& options,
      50             :     int32_t builtin_index, std::unique_ptr<AssemblerBuffer> buffer)
      51             :     : zone_(codegen_zone),
      52             :       isolate_(isolate),
      53             :       frame_access_state_(nullptr),
      54             :       linkage_(linkage),
      55             :       instructions_(instructions),
      56             :       unwinding_info_writer_(zone()),
      57             :       info_(info),
      58     2629659 :       labels_(zone()->NewArray<Label>(instructions->InstructionBlockCount())),
      59             :       current_block_(RpoNumber::Invalid()),
      60             :       start_source_position_(start_source_position),
      61             :       current_source_position_(SourcePosition::Unknown()),
      62             :       tasm_(isolate, options, CodeObjectRequired::kNo, std::move(buffer)),
      63             :       resolver_(this),
      64             :       safepoints_(zone()),
      65             :       handlers_(zone()),
      66             :       deoptimization_exits_(zone()),
      67             :       deoptimization_states_(zone()),
      68             :       deoptimization_literals_(zone()),
      69             :       translations_(zone()),
      70             :       caller_registers_saved_(false),
      71             :       jump_tables_(nullptr),
      72             :       ools_(nullptr),
      73             :       osr_helper_(std::move(osr_helper)),
      74             :       osr_pc_offset_(-1),
      75             :       optimized_out_literal_id_(-1),
      76             :       source_position_table_builder_(
      77             :           SourcePositionTableBuilder::RECORD_SOURCE_POSITIONS),
      78             :       protected_instructions_(zone()),
      79             :       result_(kSuccess),
      80             :       poisoning_level_(poisoning_level),
      81             :       block_starts_(zone()),
      82    23677252 :       instr_starts_(zone()) {
      83    43438127 :   for (int i = 0; i < instructions->InstructionBlockCount(); ++i) {
      84    20403807 :     new (&labels_[i]) Label;
      85             :   }
      86     2630513 :   CreateFrameAccessState(frame);
      87     2629966 :   CHECK_EQ(info->is_osr(), osr_helper_.has_value());
      88             :   tasm_.set_jump_optimization_info(jump_opt);
      89             :   Code::Kind code_kind = info->code_kind();
      90     5259932 :   if (code_kind == Code::WASM_FUNCTION ||
      91     2629966 :       code_kind == Code::WASM_TO_JS_FUNCTION ||
      92     6905696 :       code_kind == Code::WASM_INTERPRETER_ENTRY ||
      93      123088 :       (Builtins::IsBuiltinId(builtin_index) &&
      94      123088 :        Builtins::IsWasmRuntimeStub(builtin_index))) {
      95             :     tasm_.set_abort_hard(true);
      96             :   }
      97             :   tasm_.set_builtin_index(builtin_index);
      98     2629966 : }
      99             : 
     100      365508 : bool CodeGenerator::wasm_runtime_exception_support() const {
     101             :   DCHECK_NOT_NULL(info_);
     102      731016 :   return info_->wasm_runtime_exception_support();
     103             : }
     104             : 
     105      223339 : void CodeGenerator::AddProtectedInstructionLanding(uint32_t instr_offset,
     106             :                                                    uint32_t landing_offset) {
     107      446881 :   protected_instructions_.push_back({instr_offset, landing_offset});
     108      223542 : }
     109             : 
     110     2630455 : void CodeGenerator::CreateFrameAccessState(Frame* frame) {
     111     2630455 :   FinishFrame(frame);
     112     2629957 :   frame_access_state_ = new (zone()) FrameAccessState(frame);
     113     2629957 : }
     114             : 
     115     3329972 : CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
     116             :     int deoptimization_id, SourcePosition pos) {
     117     3329972 :   if (deoptimization_id > Deoptimizer::kMaxNumberOfEntries) {
     118             :     return kTooManyDeoptimizationBailouts;
     119             :   }
     120             : 
     121             :   DeoptimizeKind deopt_kind = GetDeoptimizationKind(deoptimization_id);
     122             :   DeoptimizeReason deoptimization_reason =
     123             :       GetDeoptimizationReason(deoptimization_id);
     124             :   Address deopt_entry =
     125     3329977 :       Deoptimizer::GetDeoptimizationEntry(tasm()->isolate(), deopt_kind);
     126     3329977 :   if (info()->is_source_positions_enabled()) {
     127      101964 :     tasm()->RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
     128             :   }
     129     3329977 :   tasm()->CallForDeoptimization(deopt_entry, deoptimization_id);
     130     3329979 :   return kSuccess;
     131             : }
     132             : 
     133          36 : void CodeGenerator::MaybeEmitOutOfLineConstantPool() {
     134             :   tasm()->MaybeEmitOutOfLineConstantPool();
     135          36 : }
     136             : 
     137     2631593 : void CodeGenerator::AssembleCode() {
     138             :   OptimizedCompilationInfo* info = this->info();
     139             : 
     140             :   // Open a frame scope to indicate that there is a frame on the stack.  The
     141             :   // MANUAL indicates that the scope shouldn't actually generate code to set up
     142             :   // the frame (that is done in AssemblePrologue).
     143     2629910 :   FrameScope frame_scope(tasm(), StackFrame::MANUAL);
     144             : 
     145     2631593 :   if (info->is_source_positions_enabled()) {
     146       11360 :     AssembleSourcePosition(start_source_position());
     147             :   }
     148             : 
     149             :   // Check that {kJavaScriptCallCodeStartRegister} has been set correctly.
     150     2631401 :   if (FLAG_debug_code && (info->code_kind() == Code::OPTIMIZED_FUNCTION ||
     151             :                           info->code_kind() == Code::BYTECODE_HANDLER)) {
     152         114 :     tasm()->RecordComment("-- Prologue: check code start register --");
     153         114 :     AssembleCodeStartRegisterCheck();
     154             :   }
     155             : 
     156             :   // We want to bailout only from JS functions, which are the only ones
     157             :   // that are optimized.
     158     2631401 :   if (info->IsOptimizing()) {
     159             :     DCHECK(linkage()->GetIncomingDescriptor()->IsJSFunctionCall());
     160      463882 :     tasm()->RecordComment("-- Prologue: check for deoptimization --");
     161      463883 :     BailoutIfDeoptimized();
     162             :   }
     163             : 
     164     2631406 :   InitializeSpeculationPoison();
     165             : 
     166             :   // Define deoptimization literals for all inlined functions.
     167             :   DCHECK_EQ(0u, deoptimization_literals_.size());
     168     2695857 :   for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
     169             :        info->inlined_functions()) {
     170       65817 :     if (!inlined.shared_info.equals(info->shared_info())) {
     171       65390 :       int index = DefineDeoptimizationLiteral(
     172       65390 :           DeoptimizationLiteral(inlined.shared_info));
     173             :       inlined.RegisterInlinedFunctionId(index);
     174             :     }
     175             :   }
     176     2630040 :   inlined_function_count_ = deoptimization_literals_.size();
     177             : 
     178             :   // Define deoptimization literals for all BytecodeArrays to which we might
     179             :   // deopt to ensure they are strongly held by the optimized code.
     180     2630040 :   if (info->has_bytecode_array()) {
     181      463887 :     DefineDeoptimizationLiteral(DeoptimizationLiteral(info->bytecode_array()));
     182             :   }
     183     2695849 :   for (OptimizedCompilationInfo::InlinedFunctionHolder& inlined :
     184             :        info->inlined_functions()) {
     185       65816 :     DefineDeoptimizationLiteral(DeoptimizationLiteral(inlined.bytecode_array));
     186             :   }
     187             : 
     188             :   unwinding_info_writer_.SetNumberOfInstructionBlocks(
     189             :       instructions()->InstructionBlockCount());
     190             : 
     191     2629485 :   if (info->trace_turbo_json_enabled()) {
     192          27 :     block_starts_.assign(instructions()->instruction_blocks().size(), -1);
     193          27 :     instr_starts_.assign(instructions()->instructions().size(), -1);
     194             :   }
     195             : 
     196             :   // Assemble instructions in assembly order.
     197    23036986 :   for (const InstructionBlock* block : instructions()->ao_blocks()) {
     198             :     // Align loop headers on vendor recommended boundaries.
     199    20403854 :     if (block->ShouldAlign() && !tasm()->jump_optimization_info()) {
     200       56561 :       tasm()->CodeTargetAlign();
     201             :     }
     202    20403881 :     if (info->trace_turbo_json_enabled()) {
     203          42 :       block_starts_[block->rpo_number().ToInt()] = tasm()->pc_offset();
     204             :     }
     205             :     // Bind a label for a block.
     206    20403881 :     current_block_ = block->rpo_number();
     207    20403881 :     unwinding_info_writer_.BeginInstructionBlock(tasm()->pc_offset(), block);
     208    20403711 :     if (FLAG_code_comments) {
     209        4006 :       std::ostringstream buffer;
     210        2003 :       buffer << "-- B" << block->rpo_number().ToInt() << " start";
     211        2003 :       if (block->IsDeferred()) buffer << " (deferred)";
     212        2003 :       if (!block->needs_frame()) buffer << " (no frame)";
     213        2003 :       if (block->must_construct_frame()) buffer << " (construct frame)";
     214        2003 :       if (block->must_deconstruct_frame()) buffer << " (deconstruct frame)";
     215             : 
     216        2003 :       if (block->IsLoopHeader()) {
     217          17 :         buffer << " (loop up to " << block->loop_end().ToInt() << ")";
     218             :       }
     219        2003 :       if (block->loop_header().IsValid()) {
     220         793 :         buffer << " (in loop " << block->loop_header().ToInt() << ")";
     221             :       }
     222        2003 :       buffer << " --";
     223        4006 :       tasm()->RecordComment(buffer.str().c_str());
     224             :     }
     225             : 
     226    20403711 :     frame_access_state()->MarkHasFrame(block->needs_frame());
     227             : 
     228    20404146 :     tasm()->bind(GetLabel(current_block_));
     229             : 
     230    20403697 :     TryInsertBranchPoisoning(block);
     231             : 
     232    20405751 :     if (block->must_construct_frame()) {
     233     2652128 :       AssembleConstructFrame();
     234             :       // We need to setup the root register after we assemble the prologue, to
     235             :       // avoid clobbering callee saved registers in case of C linkage and
     236             :       // using the roots.
     237             :       // TODO(mtrofin): investigate how we can avoid doing this repeatedly.
     238     2651992 :       if (linkage()->GetIncomingDescriptor()->InitializeRootRegister()) {
     239      888556 :         tasm()->InitializeRootRegister();
     240             :       }
     241             :     }
     242             : 
     243             :     if (FLAG_enable_embedded_constant_pool && !block->needs_frame()) {
     244             :       ConstantPoolUnavailableScope constant_pool_unavailable(tasm());
     245             :       result_ = AssembleBlock(block);
     246             :     } else {
     247    20405615 :       result_ = AssembleBlock(block);
     248             :     }
     249    20407661 :     if (result_ != kSuccess) return;
     250    20407906 :     unwinding_info_writer_.EndInstructionBlock(block);
     251             :   }
     252             : 
     253             :   // Assemble all out-of-line code.
     254     2633132 :   if (ools_) {
     255      239027 :     tasm()->RecordComment("-- Out of line code --");
     256      978363 :     for (OutOfLineCode* ool = ools_; ool; ool = ool->next()) {
     257      739345 :       tasm()->bind(ool->entry());
     258      739359 :       ool->Generate();
     259     1113353 :       if (ool->exit()->is_bound()) tasm()->jmp(ool->exit());
     260             :     }
     261             :   }
     262             : 
     263             :   // This nop operation is needed to ensure that the trampoline is not
     264             :   // confused with the pc of the call before deoptimization.
     265             :   // The test regress/regress-259 is an example of where we need it.
     266     2633123 :   tasm()->nop();
     267             : 
     268             :   // Assemble deoptimization exits.
     269             :   int last_updated = 0;
     270     5915206 :   for (DeoptimizationExit* exit : deoptimization_exits_) {
     271     3285433 :     tasm()->bind(exit->label());
     272             :     int trampoline_pc = tasm()->pc_offset();
     273             :     int deoptimization_id = exit->deoptimization_id();
     274     3285438 :     DeoptimizationState* ds = deoptimization_states_[deoptimization_id];
     275             : 
     276     3285438 :     if (ds->kind() == DeoptimizeKind::kLazy) {
     277             :       last_updated = safepoints()->UpdateDeoptimizationInfo(
     278     2957378 :           ds->pc_offset(), trampoline_pc, last_updated);
     279             :     }
     280     3285442 :     result_ = AssembleDeoptimizerCall(deoptimization_id, exit->pos());
     281     3285444 :     if (result_ != kSuccess) return;
     282             :   }
     283             : 
     284             :   // TODO(jgruber): Move all inlined metadata generation into a new,
     285             :   // architecture-independent version of FinishCode. Currently, this includes
     286             :   // the safepoint table, handler table, constant pool, and code comments, in
     287             :   // that order.
     288     2629773 :   FinishCode();
     289             : 
     290             :   // Emit the jump tables.
     291     2629454 :   if (jump_tables_) {
     292         313 :     tasm()->Align(kSystemPointerSize);
     293         628 :     for (JumpTable* table = jump_tables_; table; table = table->next()) {
     294         315 :       tasm()->bind(table->label());
     295         315 :       AssembleJumpTable(table->targets(), table->target_count());
     296             :     }
     297             :   }
     298             : 
     299             :   // The PerfJitLogger logs code up until here, excluding the safepoint
     300             :   // table. Resolve the unwinding info now so it is aware of the same code
     301             :   // size as reported by perf.
     302             :   unwinding_info_writer_.Finish(tasm()->pc_offset());
     303             : 
     304     2629454 :   safepoints()->Emit(tasm(), frame()->GetTotalFrameSlotCount());
     305             : 
     306             :   // Emit the exception handler table.
     307     2630148 :   if (!handlers_.empty()) {
     308       20348 :     handler_table_offset_ = HandlerTable::EmitReturnTableStart(tasm());
     309      474654 :     for (size_t i = 0; i < handlers_.size(); ++i) {
     310      227153 :       HandlerTable::EmitReturnEntry(tasm(), handlers_[i].pc_offset,
     311      454306 :                                     handlers_[i].handler->pos());
     312             :     }
     313             :   }
     314             : 
     315             :   tasm()->MaybeEmitOutOfLineConstantPool();
     316     2630148 :   tasm()->FinalizeJumpOptimizationInfo();
     317             : 
     318     2629910 :   result_ = kSuccess;
     319             : }
     320             : 
     321    20404760 : void CodeGenerator::TryInsertBranchPoisoning(const InstructionBlock* block) {
     322             :   // See if our predecessor was a basic block terminated by a branch_and_poison
     323             :   // instruction. If yes, then perform the masking based on the flags.
     324    26877594 :   if (block->PredecessorCount() != 1) return;
     325    13931485 :   RpoNumber pred_rpo = (block->predecessors())[0];
     326    13931485 :   const InstructionBlock* pred = instructions()->InstructionBlockAt(pred_rpo);
     327    13931762 :   if (pred->code_start() == pred->code_end()) return;
     328    13932203 :   Instruction* instr = instructions()->InstructionAt(pred->code_end() - 1);
     329    13932473 :   FlagsMode mode = FlagsModeField::decode(instr->opcode());
     330    13932473 :   switch (mode) {
     331             :     case kFlags_branch_and_poison: {
     332             :       BranchInfo branch;
     333           0 :       RpoNumber target = ComputeBranchInfo(&branch, instr);
     334           0 :       if (!target.IsValid()) {
     335             :         // Non-trivial branch, add the masking code.
     336           0 :         FlagsCondition condition = branch.condition;
     337           0 :         if (branch.false_label == GetLabel(block->rpo_number())) {
     338             :           condition = NegateFlagsCondition(condition);
     339             :         }
     340           0 :         AssembleBranchPoisoning(condition, instr);
     341             :       }
     342             :       break;
     343             :     }
     344             :     case kFlags_deoptimize_and_poison: {
     345           0 :       UNREACHABLE();
     346             :       break;
     347             :     }
     348             :     default:
     349             :       break;
     350             :   }
     351             : }
     352             : 
     353      117516 : void CodeGenerator::AssembleArchBinarySearchSwitchRange(
     354             :     Register input, RpoNumber def_block, std::pair<int32_t, Label*>* begin,
     355             :     std::pair<int32_t, Label*>* end) {
     356      117516 :   if (end - begin < kBinarySearchSwitchMinimalCases) {
     357      460794 :     while (begin != end) {
     358      384994 :       tasm()->JumpIfEqual(input, begin->first, begin->second);
     359      192495 :       ++begin;
     360             :     }
     361       75802 :     AssembleArchJump(def_block);
     362       75803 :     return;
     363             :   }
     364       41712 :   auto middle = begin + (end - begin) / 2;
     365       41712 :   Label less_label;
     366       83424 :   tasm()->JumpIfLessThan(input, middle->first, &less_label);
     367       41712 :   AssembleArchBinarySearchSwitchRange(input, def_block, middle, end);
     368       41712 :   tasm()->bind(&less_label);
     369       41712 :   AssembleArchBinarySearchSwitchRange(input, def_block, begin, middle);
     370             : }
     371             : 
     372      983426 : OwnedVector<byte> CodeGenerator::GetSourcePositionTable() {
     373      983426 :   return source_position_table_builder_.ToSourcePositionTableVector();
     374             : }
     375             : 
     376             : OwnedVector<trap_handler::ProtectedInstructionData>
     377      983703 : CodeGenerator::GetProtectedInstructions() {
     378             :   return OwnedVector<trap_handler::ProtectedInstructionData>::Of(
     379      983703 :       protected_instructions_);
     380             : }
     381             : 
     382     1589030 : MaybeHandle<Code> CodeGenerator::FinalizeCode() {
     383     1589030 :   if (result_ != kSuccess) {
     384             :     tasm()->AbortedCodeGeneration();
     385           0 :     return MaybeHandle<Code>();
     386             :   }
     387             : 
     388             :   // Allocate the source position table.
     389             :   Handle<ByteArray> source_positions =
     390     1589030 :       source_position_table_builder_.ToSourcePositionTable(isolate());
     391             : 
     392             :   // Allocate deoptimization data.
     393     1589032 :   Handle<DeoptimizationData> deopt_data = GenerateDeoptimizationData();
     394             : 
     395             :   // Allocate and install the code.
     396     1589033 :   CodeDesc desc;
     397     3178066 :   tasm()->GetCode(isolate(), &desc, safepoints(), handler_table_offset_);
     398             : 
     399             : #if defined(V8_OS_WIN_X64)
     400             :   if (Builtins::IsBuiltinId(info_->builtin_index())) {
     401             :     isolate_->SetBuiltinUnwindData(info_->builtin_index(),
     402             :                                    tasm()->GetUnwindInfo());
     403             :   }
     404             : #endif
     405             : 
     406     1589030 :   if (unwinding_info_writer_.eh_frame_writer()) {
     407          22 :     unwinding_info_writer_.eh_frame_writer()->GetEhFrame(&desc);
     408             :   }
     409             : 
     410             :   MaybeHandle<Code> maybe_code = isolate()->factory()->TryNewCode(
     411             :       desc, info()->code_kind(), Handle<Object>(), info()->builtin_index(),
     412             :       source_positions, deopt_data, kMovable, true,
     413     1589030 :       frame()->GetTotalFrameSlotCount());
     414             : 
     415             :   Handle<Code> code;
     416     1589034 :   if (!maybe_code.ToHandle(&code)) {
     417             :     tasm()->AbortedCodeGeneration();
     418           0 :     return MaybeHandle<Code>();
     419             :   }
     420             : 
     421             :   isolate()->counters()->total_compiled_code_size()->Increment(
     422     1589034 :       code->raw_instruction_size());
     423             : 
     424     1589106 :   LOG_CODE_EVENT(isolate(),
     425             :                  CodeLinePosInfoRecordEvent(code->raw_instruction_start(),
     426             :                                             *source_positions));
     427             : 
     428     1589035 :   return code;
     429             : }
     430             : 
     431    15736100 : bool CodeGenerator::IsNextInAssemblyOrder(RpoNumber block) const {
     432             :   return instructions()
     433             :       ->InstructionBlockAt(current_block_)
     434    15736255 :       ->ao_number()
     435    31472417 :       .IsNext(instructions()->InstructionBlockAt(block)->ao_number());
     436             : }
     437             : 
     438     6248564 : void CodeGenerator::RecordSafepoint(ReferenceMap* references,
     439             :                                     Safepoint::Kind kind,
     440             :                                     Safepoint::DeoptMode deopt_mode) {
     441     6248564 :   Safepoint safepoint = safepoints()->DefineSafepoint(tasm(), kind, deopt_mode);
     442             :   int stackSlotToSpillSlotDelta =
     443     6249384 :       frame()->GetTotalFrameSlotCount() - frame()->GetSpillSlotCount();
     444    86862790 :   for (const InstructionOperand& operand : references->reference_operands()) {
     445    80613269 :     if (operand.IsStackSlot()) {
     446             :       int index = LocationOperand::cast(operand).index();
     447             :       DCHECK_LE(0, index);
     448             :       // We might index values in the fixed part of the frame (i.e. the
     449             :       // closure pointer or the context pointer); these are not spill slots
     450             :       // and therefore don't work with the SafepointTable currently, but
     451             :       // we also don't need to worry about them, since the GC has special
     452             :       // knowledge about those fields anyway.
     453    15833727 :       if (index < stackSlotToSpillSlotDelta) continue;
     454             :       safepoint.DefinePointerSlot(index);
     455    64779542 :     } else if (operand.IsRegister() && (kind & Safepoint::kWithRegisters)) {
     456           0 :       Register reg = LocationOperand::cast(operand).GetRegister();
     457           0 :       safepoint.DefinePointerRegister(reg);
     458             :     }
     459             :   }
     460     6249521 : }
     461             : 
     462     7863242 : bool CodeGenerator::IsMaterializableFromRoot(Handle<HeapObject> object,
     463             :                                              RootIndex* index_return) {
     464             :   const CallDescriptor* incoming_descriptor =
     465             :       linkage()->GetIncomingDescriptor();
     466     7863242 :   if (incoming_descriptor->flags() & CallDescriptor::kCanUseRoots) {
     467     8922193 :     return isolate()->roots_table().IsRootHandle(object, index_return) &&
     468     1901749 :            RootsTable::IsImmortalImmovable(*index_return);
     469             :   }
     470             :   return false;
     471             : }
     472             : 
     473    20405824 : CodeGenerator::CodeGenResult CodeGenerator::AssembleBlock(
     474             :     const InstructionBlock* block) {
     475   157704746 :   for (int i = block->code_start(); i < block->code_end(); ++i) {
     476    68649462 :     if (info()->trace_turbo_json_enabled()) {
     477         120 :       instr_starts_[i] = tasm()->pc_offset();
     478             :     }
     479             :     Instruction* instr = instructions()->InstructionAt(i);
     480    68649617 :     CodeGenResult result = AssembleInstruction(instr, block);
     481    68648638 :     if (result != kSuccess) return result;
     482             :   }
     483             :   return kSuccess;
     484             : }
     485             : 
     486       38980 : bool CodeGenerator::IsValidPush(InstructionOperand source,
     487             :                                 CodeGenerator::PushTypeFlags push_type) {
     488       38980 :   if (source.IsImmediate() &&
     489             :       ((push_type & CodeGenerator::kImmediatePush) != 0)) {
     490             :     return true;
     491             :   }
     492       64316 :   if (source.IsRegister() &&
     493             :       ((push_type & CodeGenerator::kRegisterPush) != 0)) {
     494             :     return true;
     495             :   }
     496       26696 :   if (source.IsStackSlot() &&
     497             :       ((push_type & CodeGenerator::kStackSlotPush) != 0)) {
     498             :     return true;
     499             :   }
     500         592 :   return false;
     501             : }
     502             : 
     503      119863 : void CodeGenerator::GetPushCompatibleMoves(Instruction* instr,
     504             :                                            PushTypeFlags push_type,
     505             :                                            ZoneVector<MoveOperands*>* pushes) {
     506             :   pushes->clear();
     507      205388 :   for (int i = Instruction::FIRST_GAP_POSITION;
     508      325251 :        i <= Instruction::LAST_GAP_POSITION; ++i) {
     509             :     Instruction::GapPosition inner_pos =
     510             :         static_cast<Instruction::GapPosition>(i);
     511             :     ParallelMove* parallel_move = instr->GetParallelMove(inner_pos);
     512      222557 :     if (parallel_move != nullptr) {
     513      553715 :       for (auto move : *parallel_move) {
     514      434268 :         InstructionOperand source = move->source();
     515      434268 :         InstructionOperand destination = move->destination();
     516             :         int first_push_compatible_index =
     517             :             V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0;
     518             :         // If there are any moves from slots that will be overridden by pushes,
     519             :         // then the full gap resolver must be used since optimization with
     520             :         // pushes don't participate in the parallel move and might clobber
     521             :         // values needed for the gap resolve.
     522      465025 :         if (source.IsStackSlot() && LocationOperand::cast(source).index() >=
     523             :                                         first_push_compatible_index) {
     524             :           pushes->clear();
     525             :           return;
     526             :         }
     527             :         // TODO(danno): Right now, only consider moves from the FIRST gap for
     528             :         // pushes. Theoretically, we could extract pushes for both gaps (there
     529             :         // are cases where this happens), but the logic for that would also have
     530             :         // to check to make sure that non-memory inputs to the pushes from the
     531             :         // LAST gap don't get clobbered in the FIRST gap.
     532      417099 :         if (i == Instruction::FIRST_GAP_POSITION) {
     533      475399 :           if (destination.IsStackSlot() &&
     534             :               LocationOperand::cast(destination).index() >=
     535             :                   first_push_compatible_index) {
     536             :             int index = LocationOperand::cast(destination).index();
     537       38980 :             if (IsValidPush(source, push_type)) {
     538       38388 :               if (index >= static_cast<int>(pushes->size())) {
     539       38388 :                 pushes->resize(index + 1);
     540             :               }
     541       76776 :               (*pushes)[index] = move;
     542             :             }
     543             :           }
     544             :         }
     545             :       }
     546             :     }
     547             :   }
     548             : 
     549             :   // For now, only support a set of continuous pushes at the end of the list.
     550             :   size_t push_count_upper_bound = pushes->size();
     551             :   size_t push_begin = push_count_upper_bound;
     552      134362 :   for (auto move : base::Reversed(*pushes)) {
     553       44708 :     if (move == nullptr) break;
     554       31668 :     push_begin--;
     555             :   }
     556      102694 :   size_t push_count = pushes->size() - push_begin;
     557             :   std::copy(pushes->begin() + push_begin,
     558             :             pushes->begin() + push_begin + push_count, pushes->begin());
     559      102694 :   pushes->resize(push_count);
     560             : }
     561             : 
     562    37781501 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferMove(
     563             :     InstructionOperand* source, InstructionOperand* destination) {
     564    37781501 :   if (source->IsConstant()) {
     565    18987845 :     if (destination->IsAnyRegister()) {
     566             :       return MoveType::kConstantToRegister;
     567             :     } else {
     568             :       DCHECK(destination->IsAnyStackSlot());
     569       45200 :       return MoveType::kConstantToStack;
     570             :     }
     571             :   }
     572             :   DCHECK(LocationOperand::cast(source)->IsCompatible(
     573             :       LocationOperand::cast(destination)));
     574    18793656 :   if (source->IsAnyRegister()) {
     575     9864423 :     if (destination->IsAnyRegister()) {
     576             :       return MoveType::kRegisterToRegister;
     577             :     } else {
     578             :       DCHECK(destination->IsAnyStackSlot());
     579     5686696 :       return MoveType::kRegisterToStack;
     580             :     }
     581             :   } else {
     582             :     DCHECK(source->IsAnyStackSlot());
     583     8929233 :     if (destination->IsAnyRegister()) {
     584             :       return MoveType::kStackToRegister;
     585             :     } else {
     586             :       DCHECK(destination->IsAnyStackSlot());
     587       50173 :       return MoveType::kStackToStack;
     588             :     }
     589             :   }
     590             : }
     591             : 
     592       77532 : CodeGenerator::MoveType::Type CodeGenerator::MoveType::InferSwap(
     593             :     InstructionOperand* source, InstructionOperand* destination) {
     594             :   DCHECK(LocationOperand::cast(source)->IsCompatible(
     595             :       LocationOperand::cast(destination)));
     596       77532 :   if (source->IsAnyRegister()) {
     597       73535 :     if (destination->IsAnyRegister()) {
     598             :       return MoveType::kRegisterToRegister;
     599             :     } else {
     600             :       DCHECK(destination->IsAnyStackSlot());
     601        6603 :       return MoveType::kRegisterToStack;
     602             :     }
     603             :   } else {
     604             :     DCHECK(source->IsAnyStackSlot());
     605             :     DCHECK(destination->IsAnyStackSlot());
     606             :     return MoveType::kStackToStack;
     607             :   }
     608             : }
     609             : 
     610     5351525 : RpoNumber CodeGenerator::ComputeBranchInfo(BranchInfo* branch,
     611             :                                            Instruction* instr) {
     612             :   // Assemble a branch after this instruction.
     613             :   InstructionOperandConverter i(this, instr);
     614     5351525 :   RpoNumber true_rpo = i.InputRpo(instr->InputCount() - 2);
     615     5351557 :   RpoNumber false_rpo = i.InputRpo(instr->InputCount() - 1);
     616             : 
     617     5351675 :   if (true_rpo == false_rpo) {
     618        2290 :     return true_rpo;
     619             :   }
     620     5349385 :   FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
     621     5349385 :   if (IsNextInAssemblyOrder(true_rpo)) {
     622             :     // true block is next, can fall through if condition negated.
     623             :     std::swap(true_rpo, false_rpo);
     624             :     condition = NegateFlagsCondition(condition);
     625             :   }
     626     5349398 :   branch->condition = condition;
     627     5349398 :   branch->true_label = GetLabel(true_rpo);
     628     5349398 :   branch->false_label = GetLabel(false_rpo);
     629     5349398 :   branch->fallthru = IsNextInAssemblyOrder(false_rpo);
     630             :   return RpoNumber::Invalid();
     631             : }
     632             : 
     633    68649486 : CodeGenerator::CodeGenResult CodeGenerator::AssembleInstruction(
     634             :     Instruction* instr, const InstructionBlock* block) {
     635             :   int first_unused_stack_slot;
     636    68649486 :   FlagsMode mode = FlagsModeField::decode(instr->opcode());
     637    68649486 :   if (mode != kFlags_trap) {
     638    68508644 :     AssembleSourcePosition(instr);
     639             :   }
     640             :   bool adjust_stack =
     641    68654542 :       GetSlotAboveSPBeforeTailCall(instr, &first_unused_stack_slot);
     642    68655554 :   if (adjust_stack) AssembleTailCallBeforeGap(instr, first_unused_stack_slot);
     643             :   AssembleGaps(instr);
     644    68651340 :   if (adjust_stack) AssembleTailCallAfterGap(instr, first_unused_stack_slot);
     645             :   DCHECK_IMPLIES(
     646             :       block->must_deconstruct_frame(),
     647             :       instr != instructions()->InstructionAt(block->last_instruction_index()) ||
     648             :           instr->IsRet() || instr->IsJump());
     649    68651478 :   if (instr->IsJump() && block->must_deconstruct_frame()) {
     650       62062 :     AssembleDeconstructFrame();
     651             :   }
     652             :   // Assemble architecture-specific code for the instruction.
     653    68651478 :   CodeGenResult result = AssembleArchInstruction(instr);
     654    68648414 :   if (result != kSuccess) return result;
     655             : 
     656    68648698 :   FlagsCondition condition = FlagsConditionField::decode(instr->opcode());
     657    68648698 :   switch (mode) {
     658             :     case kFlags_branch:
     659             :     case kFlags_branch_and_poison: {
     660             :       BranchInfo branch;
     661     5351584 :       RpoNumber target = ComputeBranchInfo(&branch, instr);
     662     5351745 :       if (target.IsValid()) {
     663             :         // redundant branch.
     664        2290 :         if (!IsNextInAssemblyOrder(target)) {
     665         365 :           AssembleArchJump(target);
     666             :         }
     667        2290 :         return kSuccess;
     668             :       }
     669             :       // Assemble architecture-specific branch.
     670     5349455 :       AssembleArchBranch(instr, &branch);
     671     5349211 :       break;
     672             :     }
     673             :     case kFlags_deoptimize:
     674             :     case kFlags_deoptimize_and_poison: {
     675             :       // Assemble a conditional eager deoptimization after this instruction.
     676             :       InstructionOperandConverter i(this, instr);
     677      328054 :       size_t frame_state_offset = MiscField::decode(instr->opcode());
     678             :       DeoptimizationExit* const exit =
     679      328054 :           AddDeoptimizationExit(instr, frame_state_offset);
     680      328055 :       Label continue_label;
     681             :       BranchInfo branch;
     682      328055 :       branch.condition = condition;
     683      328055 :       branch.true_label = exit->label();
     684      328055 :       branch.false_label = &continue_label;
     685      328055 :       branch.fallthru = true;
     686             :       // Assemble architecture-specific branch.
     687      328055 :       AssembleArchDeoptBranch(instr, &branch);
     688      328058 :       tasm()->bind(&continue_label);
     689      328059 :       if (mode == kFlags_deoptimize_and_poison) {
     690           0 :         AssembleBranchPoisoning(NegateFlagsCondition(branch.condition), instr);
     691             :       }
     692             :       break;
     693             :     }
     694             :     case kFlags_set: {
     695             :       // Assemble a boolean materialization after this instruction.
     696      376436 :       AssembleArchBoolean(instr, condition);
     697      376433 :       break;
     698             :     }
     699             :     case kFlags_trap: {
     700      142125 :       AssembleArchTrap(instr, condition);
     701      142157 :       break;
     702             :     }
     703             :     case kFlags_none: {
     704             :       break;
     705             :     }
     706             :   }
     707             : 
     708             :   // TODO(jarin) We should thread the flag through rather than set it.
     709    68646359 :   if (instr->IsCall()) {
     710             :     ResetSpeculationPoison();
     711             :   }
     712             : 
     713             :   return kSuccess;
     714             : }
     715             : 
     716    68723030 : void CodeGenerator::AssembleSourcePosition(Instruction* instr) {
     717    68723030 :   SourcePosition source_position = SourcePosition::Unknown();
     718   132726477 :   if (instr->IsNop() && instr->AreMovesRedundant()) return;
     719    43057021 :   if (!instructions()->GetSourcePosition(instr, &source_position)) return;
     720     4722985 :   AssembleSourcePosition(source_position);
     721             : }
     722             : 
     723     4734240 : void CodeGenerator::AssembleSourcePosition(SourcePosition source_position) {
     724     4734240 :   if (source_position == current_source_position_) return;
     725     3612178 :   current_source_position_ = source_position;
     726     3612178 :   if (!source_position.IsKnown()) return;
     727     3612252 :   source_position_table_builder_.AddPosition(tasm()->pc_offset(),
     728     3612252 :                                              source_position, false);
     729     3612688 :   if (FLAG_code_comments) {
     730             :     OptimizedCompilationInfo* info = this->info();
     731        1163 :     if (info->IsNotOptimizedFunctionOrWasmFunction()) return;
     732        2326 :     std::ostringstream buffer;
     733        1163 :     buffer << "-- ";
     734             :     // Turbolizer only needs the source position, as it can reconstruct
     735             :     // the inlining stack from other information.
     736        1163 :     if (info->trace_turbo_json_enabled() || !tasm()->isolate() ||
     737             :         tasm()->isolate()->concurrent_recompilation_enabled()) {
     738         169 :       buffer << source_position;
     739             :     } else {
     740             :       AllowHeapAllocation allocation;
     741             :       AllowHandleAllocation handles;
     742             :       AllowHandleDereference deref;
     743        1988 :       buffer << source_position.InliningStack(info);
     744             :     }
     745        1163 :     buffer << " --";
     746        2326 :     tasm()->RecordComment(buffer.str().c_str());
     747             :   }
     748             : }
     749             : 
     750    68654203 : bool CodeGenerator::GetSlotAboveSPBeforeTailCall(Instruction* instr,
     751             :                                                  int* slot) {
     752    68654203 :   if (instr->IsTailCall()) {
     753             :     InstructionOperandConverter g(this, instr);
     754      239702 :     *slot = g.InputInt32(instr->InputCount() - 1);
     755             :     return true;
     756             :   } else {
     757             :     return false;
     758             :   }
     759             : }
     760             : 
     761      565709 : StubCallMode CodeGenerator::DetermineStubCallMode() const {
     762             :   Code::Kind code_kind = info()->code_kind();
     763      565709 :   return (code_kind == Code::WASM_FUNCTION ||
     764             :           code_kind == Code::WASM_TO_JS_FUNCTION)
     765             :              ? StubCallMode::kCallWasmRuntimeStub
     766      565709 :              : StubCallMode::kCallCodeObject;
     767             : }
     768             : 
     769          12 : void CodeGenerator::AssembleGaps(Instruction* instr) {
     770   137302854 :   for (int i = Instruction::FIRST_GAP_POSITION;
     771   205949097 :        i <= Instruction::LAST_GAP_POSITION; i++) {
     772             :     Instruction::GapPosition inner_pos =
     773             :         static_cast<Instruction::GapPosition>(i);
     774             :     ParallelMove* move = instr->GetParallelMove(inner_pos);
     775   186024185 :     if (move != nullptr) resolver()->Resolve(move);
     776             :   }
     777          12 : }
     778             : 
     779             : namespace {
     780             : 
     781      463748 : Handle<PodArray<InliningPosition>> CreateInliningPositions(
     782             :     OptimizedCompilationInfo* info, Isolate* isolate) {
     783             :   const OptimizedCompilationInfo::InlinedFunctionList& inlined_functions =
     784             :       info->inlined_functions();
     785      463748 :   if (inlined_functions.size() == 0) {
     786             :     return Handle<PodArray<InliningPosition>>::cast(
     787             :         isolate->factory()->empty_byte_array());
     788             :   }
     789             :   Handle<PodArray<InliningPosition>> inl_positions =
     790             :       PodArray<InliningPosition>::New(
     791             :           isolate, static_cast<int>(inlined_functions.size()),
     792       13716 :           AllocationType::kOld);
     793      144868 :   for (size_t i = 0; i < inlined_functions.size(); ++i) {
     794       65576 :     inl_positions->set(static_cast<int>(i), inlined_functions[i].position);
     795             :   }
     796       13716 :   return inl_positions;
     797             : }
     798             : 
     799             : }  // namespace
     800             : 
     801     1589031 : Handle<DeoptimizationData> CodeGenerator::GenerateDeoptimizationData() {
     802             :   OptimizedCompilationInfo* info = this->info();
     803     1589031 :   int deopt_count = static_cast<int>(deoptimization_states_.size());
     804     2714316 :   if (deopt_count == 0 && !info->is_osr()) {
     805     1125285 :     return DeoptimizationData::Empty(isolate());
     806             :   }
     807             :   Handle<DeoptimizationData> data =
     808      463746 :       DeoptimizationData::New(isolate(), deopt_count, AllocationType::kOld);
     809             : 
     810             :   Handle<ByteArray> translation_array =
     811      463745 :       translations_.CreateByteArray(isolate()->factory());
     812             : 
     813      927495 :   data->SetTranslationByteArray(*translation_array);
     814      463748 :   data->SetInlinedFunctionCount(
     815      463748 :       Smi::FromInt(static_cast<int>(inlined_function_count_)));
     816             :   data->SetOptimizationId(Smi::FromInt(info->optimization_id()));
     817             : 
     818      463748 :   if (info->has_shared_info()) {
     819      927496 :     data->SetSharedFunctionInfo(*info->shared_info());
     820             :   } else {
     821           0 :     data->SetSharedFunctionInfo(Smi::kZero);
     822             :   }
     823             : 
     824             :   Handle<FixedArray> literals = isolate()->factory()->NewFixedArray(
     825      463748 :       static_cast<int>(deoptimization_literals_.size()), AllocationType::kOld);
     826     6341504 :   for (unsigned i = 0; i < deoptimization_literals_.size(); i++) {
     827     1804668 :     Handle<Object> object = deoptimization_literals_[i].Reify(isolate());
     828     3609344 :     literals->set(i, *object);
     829             :   }
     830      927496 :   data->SetLiteralArray(*literals);
     831             : 
     832             :   Handle<PodArray<InliningPosition>> inl_pos =
     833      463748 :       CreateInliningPositions(info, isolate());
     834      927495 :   data->SetInliningPositions(*inl_pos);
     835             : 
     836      463748 :   if (info->is_osr()) {
     837             :     DCHECK_LE(0, osr_pc_offset_);
     838        4629 :     data->SetOsrBytecodeOffset(Smi::FromInt(info_->osr_offset().ToInt()));
     839        4629 :     data->SetOsrPcOffset(Smi::FromInt(osr_pc_offset_));
     840             :   } else {
     841             :     BailoutId osr_offset = BailoutId::None();
     842             :     data->SetOsrBytecodeOffset(Smi::FromInt(osr_offset.ToInt()));
     843             :     data->SetOsrPcOffset(Smi::FromInt(-1));
     844             :   }
     845             : 
     846             :   // Populate deoptimization entries.
     847     7119202 :   for (int i = 0; i < deopt_count; i++) {
     848     3327727 :     DeoptimizationState* deoptimization_state = deoptimization_states_[i];
     849             :     data->SetBytecodeOffset(i, deoptimization_state->bailout_id());
     850     3327727 :     CHECK(deoptimization_state);
     851             :     data->SetTranslationIndex(
     852             :         i, Smi::FromInt(deoptimization_state->translation_id()));
     853             :     data->SetPc(i, Smi::FromInt(deoptimization_state->pc_offset()));
     854             :   }
     855             : 
     856      463748 :   return data;
     857             : }
     858             : 
     859         315 : Label* CodeGenerator::AddJumpTable(Label** targets, size_t target_count) {
     860         630 :   jump_tables_ = new (zone()) JumpTable(jump_tables_, targets, target_count);
     861         315 :   return jump_tables_->label();
     862             : }
     863             : 
     864     6036915 : void CodeGenerator::RecordCallPosition(Instruction* instr) {
     865     6036915 :   CallDescriptor::Flags flags(MiscField::decode(instr->opcode()));
     866             : 
     867             :   bool needs_frame_state = (flags & CallDescriptor::kNeedsFrameState);
     868             : 
     869     6036915 :   RecordSafepoint(
     870             :       instr->reference_map(), Safepoint::kSimple,
     871     6036915 :       needs_frame_state ? Safepoint::kLazyDeopt : Safepoint::kNoLazyDeopt);
     872             : 
     873     6037421 :   if (flags & CallDescriptor::kHasExceptionHandler) {
     874             :     InstructionOperandConverter i(this, instr);
     875      227151 :     RpoNumber handler_rpo = i.InputRpo(instr->InputCount() - 1);
     876      681457 :     handlers_.push_back({GetLabel(handler_rpo), tasm()->pc_offset()});
     877             :   }
     878             : 
     879     6037423 :   if (needs_frame_state) {
     880             :     MarkLazyDeoptSite();
     881             :     // If the frame state is present, it starts at argument 2 - after
     882             :     // the code address and the poison-alias index.
     883             :     size_t frame_state_offset = 2;
     884             :     FrameStateDescriptor* descriptor =
     885     2957389 :         GetDeoptimizationEntry(instr, frame_state_offset).descriptor();
     886             :     int pc_offset = tasm()->pc_offset();
     887             :     int deopt_state_id = BuildTranslation(instr, pc_offset, frame_state_offset,
     888     2957388 :                                           descriptor->state_combine());
     889             : 
     890             :     DeoptimizationExit* const exit = new (zone())
     891     5914774 :         DeoptimizationExit(deopt_state_id, current_source_position_);
     892     2957387 :     deoptimization_exits_.push_back(exit);
     893     2957388 :     safepoints()->RecordLazyDeoptimizationIndex(deopt_state_id);
     894             :   }
     895     6037421 : }
     896             : 
     897     7739849 : int CodeGenerator::DefineDeoptimizationLiteral(DeoptimizationLiteral literal) {
     898     7739849 :   int result = static_cast<int>(deoptimization_literals_.size());
     899   147880327 :   for (unsigned i = 0; i < deoptimization_literals_.size(); ++i) {
     900    50067692 :     if (deoptimization_literals_[i] == literal) return i;
     901             :   }
     902     1805710 :   deoptimization_literals_.push_back(literal);
     903     1805696 :   return result;
     904             : }
     905             : 
     906     6287349 : DeoptimizationEntry const& CodeGenerator::GetDeoptimizationEntry(
     907             :     Instruction* instr, size_t frame_state_offset) {
     908             :   InstructionOperandConverter i(this, instr);
     909             :   int const state_id = i.InputInt32(frame_state_offset);
     910     6287351 :   return instructions()->GetDeoptimizationEntry(state_id);
     911             : }
     912             : 
     913           0 : DeoptimizeKind CodeGenerator::GetDeoptimizationKind(
     914             :     int deoptimization_id) const {
     915             :   size_t const index = static_cast<size_t>(deoptimization_id);
     916             :   DCHECK_LT(index, deoptimization_states_.size());
     917     3329979 :   return deoptimization_states_[index]->kind();
     918             : }
     919             : 
     920           0 : DeoptimizeReason CodeGenerator::GetDeoptimizationReason(
     921             :     int deoptimization_id) const {
     922             :   size_t const index = static_cast<size_t>(deoptimization_id);
     923             :   DCHECK_LT(index, deoptimization_states_.size());
     924     3329977 :   return deoptimization_states_[index]->reason();
     925             : }
     926             : 
     927    49728985 : void CodeGenerator::TranslateStateValueDescriptor(
     928             :     StateValueDescriptor* desc, StateValueList* nested,
     929             :     Translation* translation, InstructionOperandIterator* iter) {
     930             :   // Note:
     931             :   // If translation is null, we just skip the relevant instruction operands.
     932    49728985 :   if (desc->IsNested()) {
     933       88866 :     if (translation != nullptr) {
     934       88866 :       translation->BeginCapturedObject(static_cast<int>(nested->size()));
     935             :     }
     936      618426 :     for (auto field : *nested) {
     937             :       TranslateStateValueDescriptor(field.desc, field.nested, translation,
     938      529560 :                                     iter);
     939             :     }
     940    49640119 :   } else if (desc->IsArgumentsElements()) {
     941        5981 :     if (translation != nullptr) {
     942        5981 :       translation->ArgumentsElements(desc->arguments_type());
     943             :     }
     944    49634138 :   } else if (desc->IsArgumentsLength()) {
     945        6289 :     if (translation != nullptr) {
     946        6289 :       translation->ArgumentsLength(desc->arguments_type());
     947             :     }
     948    49627849 :   } else if (desc->IsDuplicate()) {
     949       19067 :     if (translation != nullptr) {
     950       19067 :       translation->DuplicateObject(static_cast<int>(desc->id()));
     951             :     }
     952    49608782 :   } else if (desc->IsPlain()) {
     953             :     InstructionOperand* op = iter->Advance();
     954    21934400 :     if (translation != nullptr) {
     955             :       AddTranslationForOperand(translation, iter->instruction(), op,
     956    21934423 :                                desc->type());
     957             :     }
     958             :   } else {
     959             :     DCHECK(desc->IsOptimizedOut());
     960    27674382 :     if (translation != nullptr) {
     961    27674377 :       if (optimized_out_literal_id_ == -1) {
     962      927385 :         optimized_out_literal_id_ = DefineDeoptimizationLiteral(
     963      463688 :             DeoptimizationLiteral(isolate()->factory()->optimized_out()));
     964             :       }
     965    27674368 :       translation->StoreLiteral(optimized_out_literal_id_);
     966             :     }
     967             :   }
     968    49728935 : }
     969             : 
     970     3682448 : void CodeGenerator::TranslateFrameStateDescriptorOperands(
     971             :     FrameStateDescriptor* desc, InstructionOperandIterator* iter,
     972             :     Translation* translation) {
     973             :   size_t index = 0;
     974             :   StateValueList* values = desc->GetStateValueDescriptors();
     975   102080012 :   for (StateValueList::iterator it = values->begin(); it != values->end();
     976             :        ++it, ++index) {
     977    49198796 :     TranslateStateValueDescriptor((*it).desc, (*it).nested, translation, iter);
     978             :   }
     979             :   DCHECK_EQ(desc->GetSize(), index);
     980     3682434 : }
     981             : 
     982     3682418 : void CodeGenerator::BuildTranslationForFrameStateDescriptor(
     983             :     FrameStateDescriptor* descriptor, InstructionOperandIterator* iter,
     984             :     Translation* translation, OutputFrameStateCombine state_combine) {
     985             :   // Outer-most state must be added to translation first.
     986     3682418 :   if (descriptor->outer_state() != nullptr) {
     987             :     BuildTranslationForFrameStateDescriptor(descriptor->outer_state(), iter,
     988      352448 :                                             translation, state_combine);
     989             :   }
     990             : 
     991             :   Handle<SharedFunctionInfo> shared_info;
     992     3682424 :   if (!descriptor->shared_info().ToHandle(&shared_info)) {
     993        2083 :     if (!info()->has_shared_info()) {
     994             :       return;  // Stub with no SharedFunctionInfo.
     995             :     }
     996             :     shared_info = info()->shared_info();
     997             :   }
     998             :   int shared_info_id =
     999     3682424 :       DefineDeoptimizationLiteral(DeoptimizationLiteral(shared_info));
    1000             : 
    1001     3682427 :   switch (descriptor->type()) {
    1002             :     case FrameStateType::kInterpretedFunction: {
    1003             :       int return_offset = 0;
    1004             :       int return_count = 0;
    1005     3556135 :       if (!state_combine.IsOutputIgnored()) {
    1006     2119695 :         return_offset = static_cast<int>(state_combine.GetOffsetToPokeAt());
    1007     2119695 :         return_count = static_cast<int>(iter->instruction()->OutputCount());
    1008             :       }
    1009     3556135 :       translation->BeginInterpretedFrame(
    1010             :           descriptor->bailout_id(), shared_info_id,
    1011             :           static_cast<unsigned int>(descriptor->locals_count() + 1),
    1012     3556135 :           return_offset, return_count);
    1013     3556134 :       break;
    1014             :     }
    1015             :     case FrameStateType::kArgumentsAdaptor:
    1016       84832 :       translation->BeginArgumentsAdaptorFrame(
    1017             :           shared_info_id,
    1018       84832 :           static_cast<unsigned int>(descriptor->parameters_count()));
    1019       84832 :       break;
    1020             :     case FrameStateType::kConstructStub:
    1021             :       DCHECK(descriptor->bailout_id().IsValidForConstructStub());
    1022       26719 :       translation->BeginConstructStubFrame(
    1023             :           descriptor->bailout_id(), shared_info_id,
    1024       26719 :           static_cast<unsigned int>(descriptor->parameters_count() + 1));
    1025       26719 :       break;
    1026             :     case FrameStateType::kBuiltinContinuation: {
    1027        2083 :       BailoutId bailout_id = descriptor->bailout_id();
    1028             :       int parameter_count =
    1029             :           static_cast<unsigned int>(descriptor->parameters_count());
    1030        2083 :       translation->BeginBuiltinContinuationFrame(bailout_id, shared_info_id,
    1031        2083 :                                                  parameter_count);
    1032             :       break;
    1033             :     }
    1034             :     case FrameStateType::kJavaScriptBuiltinContinuation: {
    1035       12381 :       BailoutId bailout_id = descriptor->bailout_id();
    1036             :       int parameter_count =
    1037             :           static_cast<unsigned int>(descriptor->parameters_count());
    1038       12381 :       translation->BeginJavaScriptBuiltinContinuationFrame(
    1039       12381 :           bailout_id, shared_info_id, parameter_count);
    1040             :       break;
    1041             :     }
    1042             :     case FrameStateType::kJavaScriptBuiltinContinuationWithCatch: {
    1043         274 :       BailoutId bailout_id = descriptor->bailout_id();
    1044             :       int parameter_count =
    1045             :           static_cast<unsigned int>(descriptor->parameters_count());
    1046         274 :       translation->BeginJavaScriptBuiltinContinuationWithCatchFrame(
    1047         274 :           bailout_id, shared_info_id, parameter_count);
    1048             :       break;
    1049             :     }
    1050             :   }
    1051             : 
    1052     3682426 :   TranslateFrameStateDescriptorOperands(descriptor, iter, translation);
    1053             : }
    1054             : 
    1055     3329965 : int CodeGenerator::BuildTranslation(Instruction* instr, int pc_offset,
    1056             :                                     size_t frame_state_offset,
    1057             :                                     OutputFrameStateCombine state_combine) {
    1058             :   DeoptimizationEntry const& entry =
    1059     3329965 :       GetDeoptimizationEntry(instr, frame_state_offset);
    1060             :   FrameStateDescriptor* const descriptor = entry.descriptor();
    1061     3329967 :   frame_state_offset++;
    1062             : 
    1063     3329967 :   int update_feedback_count = entry.feedback().IsValid() ? 1 : 0;
    1064             :   Translation translation(&translations_,
    1065     3329968 :                           static_cast<int>(descriptor->GetFrameCount()),
    1066     3329967 :                           static_cast<int>(descriptor->GetJSFrameCount()),
    1067     6659940 :                           update_feedback_count, zone());
    1068     3329979 :   if (entry.feedback().IsValid()) {
    1069             :     DeoptimizationLiteral literal =
    1070             :         DeoptimizationLiteral(entry.feedback().vector());
    1071       40532 :     int literal_id = DefineDeoptimizationLiteral(literal);
    1072       40531 :     translation.AddUpdateFeedback(literal_id, entry.feedback().slot().ToInt());
    1073             :   }
    1074             :   InstructionOperandIterator iter(instr, frame_state_offset);
    1075             :   BuildTranslationForFrameStateDescriptor(descriptor, &iter, &translation,
    1076     3329978 :                                           state_combine);
    1077             : 
    1078     3329984 :   int deoptimization_id = static_cast<int>(deoptimization_states_.size());
    1079             : 
    1080     6659958 :   deoptimization_states_.push_back(new (zone()) DeoptimizationState(
    1081             :       descriptor->bailout_id(), translation.index(), pc_offset, entry.kind(),
    1082             :       entry.reason()));
    1083             : 
    1084     3329978 :   return deoptimization_id;
    1085             : }
    1086             : 
    1087    21934646 : void CodeGenerator::AddTranslationForOperand(Translation* translation,
    1088             :                                              Instruction* instr,
    1089             :                                              InstructionOperand* op,
    1090             :                                              MachineType type) {
    1091    21934646 :   if (op->IsStackSlot()) {
    1092    14792399 :     if (type.representation() == MachineRepresentation::kBit) {
    1093       21524 :       translation->StoreBoolStackSlot(LocationOperand::cast(op)->index());
    1094    44311573 :     } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
    1095             :                type == MachineType::Int32()) {
    1096      210243 :       translation->StoreInt32StackSlot(LocationOperand::cast(op)->index());
    1097    43681898 :     } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
    1098             :                type == MachineType::Uint32()) {
    1099        7043 :       translation->StoreUint32StackSlot(LocationOperand::cast(op)->index());
    1100    14553589 :     } else if (type == MachineType::Int64()) {
    1101         114 :       translation->StoreInt64StackSlot(LocationOperand::cast(op)->index());
    1102             :     } else {
    1103             : #if defined(V8_COMPRESS_POINTERS)
    1104             :       CHECK(MachineRepresentation::kTagged == type.representation() ||
    1105             :             MachineRepresentation::kCompressed == type.representation());
    1106             : #else
    1107    14553475 :       CHECK(MachineRepresentation::kTagged == type.representation());
    1108             : #endif
    1109    14553475 :       translation->StoreStackSlot(LocationOperand::cast(op)->index());
    1110             :     }
    1111     7142247 :   } else if (op->IsFPStackSlot()) {
    1112      259005 :     if (type.representation() == MachineRepresentation::kFloat64) {
    1113      257977 :       translation->StoreDoubleStackSlot(LocationOperand::cast(op)->index());
    1114             :     } else {
    1115        1028 :       CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
    1116        1028 :       translation->StoreFloatStackSlot(LocationOperand::cast(op)->index());
    1117             :     }
    1118     6883242 :   } else if (op->IsRegister()) {
    1119             :     InstructionOperandConverter converter(this, instr);
    1120      496733 :     if (type.representation() == MachineRepresentation::kBit) {
    1121        3241 :       translation->StoreBoolRegister(converter.ToRegister(op));
    1122     1479887 :     } else if (type == MachineType::Int8() || type == MachineType::Int16() ||
    1123             :                type == MachineType::Int32()) {
    1124       34306 :       translation->StoreInt32Register(converter.ToRegister(op));
    1125     1377555 :     } else if (type == MachineType::Uint8() || type == MachineType::Uint16() ||
    1126             :                type == MachineType::Uint32()) {
    1127        1159 :       translation->StoreUint32Register(converter.ToRegister(op));
    1128      458027 :     } else if (type == MachineType::Int64()) {
    1129          14 :       translation->StoreInt64Register(converter.ToRegister(op));
    1130             :     } else {
    1131             : #if defined(V8_COMPRESS_POINTERS)
    1132             :       CHECK(MachineRepresentation::kTagged == type.representation() ||
    1133             :             MachineRepresentation::kCompressed == type.representation());
    1134             : #else
    1135      458013 :       CHECK(MachineRepresentation::kTagged == type.representation());
    1136             : #endif
    1137      458013 :       translation->StoreRegister(converter.ToRegister(op));
    1138             :     }
    1139     6386509 :   } else if (op->IsFPRegister()) {
    1140             :     InstructionOperandConverter converter(this, instr);
    1141       63854 :     if (type.representation() == MachineRepresentation::kFloat64) {
    1142       63732 :       translation->StoreDoubleRegister(converter.ToDoubleRegister(op));
    1143             :     } else {
    1144         122 :       CHECK_EQ(MachineRepresentation::kFloat32, type.representation());
    1145         122 :       translation->StoreFloatRegister(converter.ToFloatRegister(op));
    1146             :     }
    1147             :   } else {
    1148     6322655 :     CHECK(op->IsImmediate());
    1149             :     InstructionOperandConverter converter(this, instr);
    1150     6322655 :     Constant constant = converter.ToConstant(op);
    1151             :     DeoptimizationLiteral literal;
    1152     6322688 :     switch (constant.type()) {
    1153             :       case Constant::kInt32:
    1154       12804 :         if (type.representation() == MachineRepresentation::kTagged) {
    1155             :           // When pointers are 4 bytes, we can use int32 constants to represent
    1156             :           // Smis.
    1157             :           DCHECK_EQ(4, kSystemPointerSize);
    1158          18 :           Smi smi(static_cast<Address>(constant.ToInt32()));
    1159             :           DCHECK(smi->IsSmi());
    1160          18 :           literal = DeoptimizationLiteral(smi->value());
    1161       12786 :         } else if (type.representation() == MachineRepresentation::kBit) {
    1162         783 :           if (constant.ToInt32() == 0) {
    1163             :             literal =
    1164         340 :                 DeoptimizationLiteral(isolate()->factory()->false_value());
    1165             :           } else {
    1166             :             DCHECK_EQ(1, constant.ToInt32());
    1167         443 :             literal = DeoptimizationLiteral(isolate()->factory()->true_value());
    1168             :           }
    1169             :         } else {
    1170             :           DCHECK(type == MachineType::Int32() ||
    1171             :                  type == MachineType::Uint32() ||
    1172             :                  type.representation() == MachineRepresentation::kWord32 ||
    1173             :                  type.representation() == MachineRepresentation::kNone);
    1174             :           DCHECK(type.representation() != MachineRepresentation::kNone ||
    1175             :                  constant.ToInt32() == FrameStateDescriptor::kImpossibleValue);
    1176       12003 :           if (type == MachineType::Uint32()) {
    1177             :             literal = DeoptimizationLiteral(
    1178           8 :                 static_cast<uint32_t>(constant.ToInt32()));
    1179             :           } else {
    1180       11995 :             literal = DeoptimizationLiteral(constant.ToInt32());
    1181             :           }
    1182             :         }
    1183             :         break;
    1184             :       case Constant::kInt64:
    1185             :         DCHECK_EQ(8, kSystemPointerSize);
    1186      563441 :         if (type.representation() == MachineRepresentation::kWord64) {
    1187             :           literal =
    1188           0 :               DeoptimizationLiteral(static_cast<double>(constant.ToInt64()));
    1189             :         } else {
    1190             :           // When pointers are 8 bytes, we can use int64 constants to represent
    1191             :           // Smis.
    1192             :           DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1193             :           Smi smi(static_cast<Address>(constant.ToInt64()));
    1194             :           DCHECK(smi->IsSmi());
    1195      563441 :           literal = DeoptimizationLiteral(smi->value());
    1196             :         }
    1197             :         break;
    1198             :       case Constant::kFloat32:
    1199             :         DCHECK(type.representation() == MachineRepresentation::kFloat32 ||
    1200             :                type.representation() == MachineRepresentation::kTagged);
    1201          14 :         literal = DeoptimizationLiteral(constant.ToFloat32());
    1202          14 :         break;
    1203             :       case Constant::kFloat64:
    1204             :         DCHECK(type.representation() == MachineRepresentation::kFloat64 ||
    1205             :                type.representation() == MachineRepresentation::kTagged);
    1206       89776 :         literal = DeoptimizationLiteral(constant.ToFloat64().value());
    1207       89776 :         break;
    1208             :       case Constant::kHeapObject:
    1209             :         DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1210     5654115 :         literal = DeoptimizationLiteral(constant.ToHeapObject());
    1211     5654114 :         break;
    1212             :       case Constant::kDelayedStringConstant:
    1213             :         DCHECK_EQ(MachineRepresentation::kTagged, type.representation());
    1214        2538 :         literal = DeoptimizationLiteral(constant.ToDelayedStringConstant());
    1215        2538 :         break;
    1216             :       default:
    1217           0 :         UNREACHABLE();
    1218             :     }
    1219     6322687 :     if (literal.object().equals(info()->closure())) {
    1220     3364551 :       translation->StoreJSFrameFunction();
    1221             :     } else {
    1222     2958136 :       int literal_id = DefineDeoptimizationLiteral(literal);
    1223     2958124 :       translation->StoreLiteral(literal_id);
    1224             :     }
    1225             :   }
    1226    21934654 : }
    1227             : 
    1228           0 : void CodeGenerator::MarkLazyDeoptSite() {
    1229     2957389 :   last_lazy_deopt_pc_ = tasm()->pc_offset();
    1230           0 : }
    1231             : 
    1232      328055 : DeoptimizationExit* CodeGenerator::AddDeoptimizationExit(
    1233             :     Instruction* instr, size_t frame_state_offset) {
    1234             :   int const deoptimization_id = BuildTranslation(
    1235      328055 :       instr, -1, frame_state_offset, OutputFrameStateCombine::Ignore());
    1236             : 
    1237             :   DeoptimizationExit* const exit = new (zone())
    1238      656113 :       DeoptimizationExit(deoptimization_id, current_source_position_);
    1239      328056 :   deoptimization_exits_.push_back(exit);
    1240      328055 :   return exit;
    1241             : }
    1242             : 
    1243     2630017 : void CodeGenerator::InitializeSpeculationPoison() {
    1244     2630017 :   if (poisoning_level_ == PoisoningMitigationLevel::kDontPoison) return;
    1245             : 
    1246             :   // Initialize {kSpeculationPoisonRegister} either by comparing the expected
    1247             :   // with the actual call target, or by unconditionally using {-1} initially.
    1248             :   // Masking register arguments with it only makes sense in the first case.
    1249           0 :   if (info()->called_with_code_start_register()) {
    1250           0 :     tasm()->RecordComment("-- Prologue: generate speculation poison --");
    1251           0 :     GenerateSpeculationPoisonFromCodeStartRegister();
    1252           0 :     if (info()->is_poisoning_register_arguments()) {
    1253           0 :       AssembleRegisterArgumentPoisoning();
    1254             :     }
    1255             :   } else {
    1256             :     ResetSpeculationPoison();
    1257             :   }
    1258             : }
    1259             : 
    1260        4629 : void CodeGenerator::ResetSpeculationPoison() {
    1261     6071095 :   if (poisoning_level_ != PoisoningMitigationLevel::kDontPoison) {
    1262           0 :     tasm()->ResetSpeculationPoisonRegister();
    1263             :   }
    1264        4629 : }
    1265             : 
    1266      739303 : OutOfLineCode::OutOfLineCode(CodeGenerator* gen)
    1267     2217909 :     : frame_(gen->frame()), tasm_(gen->tasm()), next_(gen->ools_) {
    1268      739303 :   gen->ools_ = this;
    1269      739303 : }
    1270             : 
    1271             : OutOfLineCode::~OutOfLineCode() = default;
    1272             : 
    1273     1804669 : Handle<Object> DeoptimizationLiteral::Reify(Isolate* isolate) const {
    1274     1804669 :   switch (kind_) {
    1275             :     case DeoptimizationLiteralKind::kObject: {
    1276     1724616 :       return object_;
    1277             :     }
    1278             :     case DeoptimizationLiteralKind::kNumber: {
    1279       79409 :       return isolate->factory()->NewNumber(number_);
    1280             :     }
    1281             :     case DeoptimizationLiteralKind::kString: {
    1282         644 :       return string_->AllocateStringConstant(isolate);
    1283             :     }
    1284             :   }
    1285           0 :   UNREACHABLE();
    1286             : }
    1287             : 
    1288             : }  // namespace compiler
    1289             : }  // namespace internal
    1290      122036 : }  // namespace v8

Generated by: LCOV version 1.10