LCOV - code coverage report
Current view: top level - src/wasm/baseline - liftoff-compiler.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 650 703 92.5 %
Date: 2019-04-17 Functions: 177 179 98.9 %

          Line data    Source code
       1             : // Copyright 2017 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #include "src/wasm/baseline/liftoff-compiler.h"
       6             : 
       7             : #include "src/assembler-inl.h"
       8             : #include "src/base/optional.h"
       9             : // TODO(clemensh): Remove dependences on compiler stuff.
      10             : #include "src/compiler/linkage.h"
      11             : #include "src/compiler/wasm-compiler.h"
      12             : #include "src/counters.h"
      13             : #include "src/interface-descriptors.h"
      14             : #include "src/log.h"
      15             : #include "src/macro-assembler-inl.h"
      16             : #include "src/objects/smi.h"
      17             : #include "src/ostreams.h"
      18             : #include "src/tracing/trace-event.h"
      19             : #include "src/utils.h"
      20             : #include "src/wasm/baseline/liftoff-assembler.h"
      21             : #include "src/wasm/function-body-decoder-impl.h"
      22             : #include "src/wasm/function-compiler.h"
      23             : #include "src/wasm/memory-tracing.h"
      24             : #include "src/wasm/object-access.h"
      25             : #include "src/wasm/wasm-engine.h"
      26             : #include "src/wasm/wasm-linkage.h"
      27             : #include "src/wasm/wasm-objects.h"
      28             : #include "src/wasm/wasm-opcodes.h"
      29             : 
      30             : namespace v8 {
      31             : namespace internal {
      32             : namespace wasm {
      33             : 
      34             : constexpr auto kRegister = LiftoffAssembler::VarState::kRegister;
      35             : constexpr auto kIntConst = LiftoffAssembler::VarState::kIntConst;
      36             : constexpr auto kStack = LiftoffAssembler::VarState::kStack;
      37             : 
      38             : namespace {
      39             : 
      40             : #define __ asm_.
      41             : 
      42             : #define TRACE(...)                                            \
      43             :   do {                                                        \
      44             :     if (FLAG_trace_liftoff) PrintF("[liftoff] " __VA_ARGS__); \
      45             :   } while (false)
      46             : 
      47             : #define WASM_INSTANCE_OBJECT_FIELD_OFFSET(name) \
      48             :   ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
      49             : 
      50             : template <int expected_size, int actual_size>
      51             : struct assert_field_size {
      52             :   static_assert(expected_size == actual_size,
      53             :                 "field in WasmInstance does not have the expected size");
      54             :   static constexpr int size = actual_size;
      55             : };
      56             : 
      57             : #define WASM_INSTANCE_OBJECT_FIELD_SIZE(name) \
      58             :   FIELD_SIZE(WasmInstanceObject::k##name##Offset)
      59             : 
      60             : #define LOAD_INSTANCE_FIELD(dst, name, load_size)                              \
      61             :   __ LoadFromInstance(dst, WASM_INSTANCE_OBJECT_FIELD_OFFSET(name),            \
      62             :                       assert_field_size<WASM_INSTANCE_OBJECT_FIELD_SIZE(name), \
      63             :                                         load_size>::size);
      64             : 
      65             : #define LOAD_TAGGED_PTR_INSTANCE_FIELD(dst, name)                         \
      66             :   static_assert(WASM_INSTANCE_OBJECT_FIELD_SIZE(name) == kTaggedSize,     \
      67             :                 "field in WasmInstance does not have the expected size"); \
      68             :   __ LoadTaggedPointerFromInstance(dst,                                   \
      69             :                                    WASM_INSTANCE_OBJECT_FIELD_OFFSET(name));
      70             : 
      71             : #ifdef DEBUG
      72             : #define DEBUG_CODE_COMMENT(str) \
      73             :   do {                          \
      74             :     __ RecordComment(str);      \
      75             :   } while (false)
      76             : #else
      77             : #define DEBUG_CODE_COMMENT(str) ((void)0)
      78             : #endif
      79             : 
      80             : constexpr LoadType::LoadTypeValue kPointerLoadType =
      81             :     kSystemPointerSize == 8 ? LoadType::kI64Load : LoadType::kI32Load;
      82             : 
      83             : #if V8_TARGET_ARCH_ARM64
      84             : // On ARM64, the Assembler keeps track of pointers to Labels to resolve
      85             : // branches to distant targets. Moving labels would confuse the Assembler,
      86             : // thus store the label on the heap and keep a unique_ptr.
      87             : class MovableLabel {
      88             :  public:
      89             :   MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(MovableLabel);
      90             :   MovableLabel() : label_(new Label()) {}
      91             : 
      92             :   Label* get() { return label_.get(); }
      93             : 
      94             :  private:
      95             :   std::unique_ptr<Label> label_;
      96             : };
      97             : #else
      98             : // On all other platforms, just store the Label directly.
      99             : class MovableLabel {
     100             :  public:
     101             :   MOVE_ONLY_WITH_DEFAULT_CONSTRUCTORS(MovableLabel);
     102             : 
     103      950304 :   Label* get() { return &label_; }
     104             : 
     105             :  private:
     106             :   Label label_;
     107             : };
     108             : #endif
     109             : 
     110             : compiler::CallDescriptor* GetLoweredCallDescriptor(
     111             :     Zone* zone, compiler::CallDescriptor* call_desc) {
     112             :   return kSystemPointerSize == 4
     113             :              ? compiler::GetI32WasmCallDescriptor(zone, call_desc)
     114             :              : call_desc;
     115             : }
     116             : 
     117             : constexpr ValueType kSupportedTypesArr[] = {kWasmI32, kWasmI64, kWasmF32,
     118             :                                             kWasmF64};
     119             : constexpr Vector<const ValueType> kSupportedTypes =
     120             :     ArrayVector(kSupportedTypesArr);
     121             : 
     122      921963 : class LiftoffCompiler {
     123             :  public:
     124             :   // TODO(clemensh): Make this a template parameter.
     125             :   static constexpr Decoder::ValidateFlag validate = Decoder::kValidate;
     126             : 
     127             :   using Value = ValueBase;
     128             : 
     129        1370 :   struct ElseState {
     130             :     MovableLabel label;
     131             :     LiftoffAssembler::CacheState state;
     132             :   };
     133             : 
     134     1205760 :   struct Control : public ControlBase<Value> {
     135             :     std::unique_ptr<ElseState> else_state;
     136             :     LiftoffAssembler::CacheState label_state;
     137             :     MovableLabel label;
     138             : 
     139      151392 :     MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(Control);
     140             : 
     141             :     template <typename... Args>
     142             :     explicit Control(Args&&... args) V8_NOEXCEPT
     143             :         : ControlBase(std::forward<Args>(args)...) {}
     144             :   };
     145             : 
     146             :   using FullDecoder = WasmFullDecoder<validate, LiftoffCompiler>;
     147             : 
     148             :   struct OutOfLineCode {
     149             :     MovableLabel label;
     150             :     MovableLabel continuation;
     151             :     WasmCode::RuntimeStubId stub;
     152             :     WasmCodePosition position;
     153             :     LiftoffRegList regs_to_save;
     154             :     uint32_t pc;  // for trap handler.
     155             : 
     156             :     // Named constructors:
     157             :     static OutOfLineCode Trap(WasmCode::RuntimeStubId s, WasmCodePosition pos,
     158             :                               uint32_t pc) {
     159             :       DCHECK_LT(0, pos);
     160      297607 :       return {{}, {}, s, pos, {}, pc};
     161             :     }
     162             :     static OutOfLineCode StackCheck(WasmCodePosition pos, LiftoffRegList regs) {
     163       99751 :       return {{}, {}, WasmCode::kWasmStackGuard, pos, regs, 0};
     164             :     }
     165             :   };
     166             : 
     167      461026 :   LiftoffCompiler(compiler::CallDescriptor* call_descriptor,
     168             :                   CompilationEnv* env, Zone* compilation_zone,
     169             :                   std::unique_ptr<AssemblerBuffer> buffer)
     170             :       : asm_(std::move(buffer)),
     171             :         descriptor_(
     172             :             GetLoweredCallDescriptor(compilation_zone, call_descriptor)),
     173             :         env_(env),
     174             :         compilation_zone_(compilation_zone),
     175     2304904 :         safepoint_table_builder_(compilation_zone_) {}
     176             : 
     177             :   bool ok() const { return ok_; }
     178             : 
     179             :   void GetCode(CodeDesc* desc) {
     180             :     asm_.GetCode(nullptr, desc, &safepoint_table_builder_,
     181      441429 :                  Assembler::kNoHandlerTable);
     182             :   }
     183             : 
     184             :   OwnedVector<uint8_t> GetSourcePositionTable() {
     185      441169 :     return source_position_table_builder_.ToSourcePositionTableVector();
     186             :   }
     187             : 
     188             :   OwnedVector<trap_handler::ProtectedInstructionData> GetProtectedInstructions()
     189             :       const {
     190             :     return OwnedVector<trap_handler::ProtectedInstructionData>::Of(
     191      441582 :         protected_instructions_);
     192             :   }
     193             : 
     194             :   uint32_t GetTotalFrameSlotCount() const {
     195             :     return __ GetTotalFrameSlotCount();
     196             :   }
     197             : 
     198             :   void unsupported(FullDecoder* decoder, const char* reason) {
     199       16379 :     ok_ = false;
     200             :     TRACE("unsupported: %s\n", reason);
     201        1640 :     decoder->errorf(decoder->pc_offset(), "unsupported liftoff operation: %s",
     202       16379 :                     reason);
     203             :     UnuseLabels(decoder);
     204             :   }
     205             : 
     206     1342217 :   bool DidAssemblerBailout(FullDecoder* decoder) {
     207     1342217 :     if (decoder->failed() || !__ did_bailout()) return false;
     208             :     unsupported(decoder, __ bailout_reason());
     209           0 :     return true;
     210             :   }
     211             : 
     212      411727 :   bool CheckSupportedType(FullDecoder* decoder,
     213             :                           Vector<const ValueType> supported_types,
     214             :                           ValueType type, const char* context) {
     215             :     char buffer[128];
     216             :     // Check supported types.
     217      947007 :     for (ValueType supported : supported_types) {
     218      679097 :       if (type == supported) return true;
     219             :     }
     220         270 :     SNPrintF(ArrayVector(buffer), "%s %s", ValueTypes::TypeName(type), context);
     221             :     unsupported(decoder, buffer);
     222         270 :     return false;
     223             :   }
     224             : 
     225             :   int GetSafepointTableOffset() const {
     226             :     return safepoint_table_builder_.GetCodeOffset();
     227             :   }
     228             : 
     229             :   void UnuseLabels(FullDecoder* decoder) {
     230             : #ifdef DEBUG
     231             :     auto Unuse = [](Label* label) {
     232             :       label->Unuse();
     233             :       label->UnuseNear();
     234             :     };
     235             :     // Unuse all labels now, otherwise their destructor will fire a DCHECK error
     236             :     // if they where referenced before.
     237             :     uint32_t control_depth = decoder ? decoder->control_depth() : 0;
     238             :     for (uint32_t i = 0; i < control_depth; ++i) {
     239             :       Control* c = decoder->control_at(i);
     240             :       Unuse(c->label.get());
     241             :       if (c->else_state) Unuse(c->else_state->label.get());
     242             :     }
     243             :     for (auto& ool : out_of_line_code_) Unuse(ool.label.get());
     244             : #endif
     245             :   }
     246             : 
     247      460699 :   void StartFunction(FullDecoder* decoder) {
     248      460699 :     int num_locals = decoder->num_locals();
     249      460699 :     __ set_num_locals(num_locals);
     250      834196 :     for (int i = 0; i < num_locals; ++i) {
     251      186735 :       __ set_local_type(i, decoder->GetLocalType(i));
     252             :     }
     253      460726 :   }
     254             : 
     255             :   // Returns the number of inputs processed (1 or 2).
     256      116573 :   uint32_t ProcessParameter(ValueType type, uint32_t input_idx) {
     257             :     const int num_lowered_params = 1 + needs_reg_pair(type);
     258             :     ValueType lowered_type = needs_reg_pair(type) ? kWasmI32 : type;
     259             :     RegClass rc = reg_class_for(lowered_type);
     260             :     // Initialize to anything, will be set in the loop and used afterwards.
     261             :     LiftoffRegister reg = kGpCacheRegList.GetFirstRegSet();
     262             :     LiftoffRegList pinned;
     263      349441 :     for (int pair_idx = 0; pair_idx < num_lowered_params; ++pair_idx) {
     264             :       compiler::LinkageLocation param_loc =
     265      116557 :           descriptor_->GetInputLocation(input_idx + pair_idx);
     266             :       // Initialize to anything, will be set in both arms of the if.
     267             :       LiftoffRegister in_reg = kGpCacheRegList.GetFirstRegSet();
     268      116557 :       if (param_loc.IsRegister()) {
     269             :         DCHECK(!param_loc.IsAnyRegister());
     270             :         int reg_code = param_loc.AsRegister();
     271             : #if V8_TARGET_ARCH_ARM
     272             :         // Liftoff assumes a one-to-one mapping between float registers and
     273             :         // double registers, and so does not distinguish between f32 and f64
     274             :         // registers. The f32 register code must therefore be halved in order to
     275             :         // pass the f64 code to Liftoff.
     276             :         DCHECK_IMPLIES(type == kWasmF32, (reg_code % 2) == 0);
     277             :         if (type == kWasmF32) {
     278             :           reg_code /= 2;
     279             :         }
     280             : #endif
     281             :         RegList cache_regs = rc == kGpReg ? kLiftoffAssemblerGpCacheRegs
     282       97256 :                                           : kLiftoffAssemblerFpCacheRegs;
     283       97256 :         if (cache_regs & (1ULL << reg_code)) {
     284             :           // This is a cache register, just use it.
     285       97256 :           in_reg = LiftoffRegister::from_code(rc, reg_code);
     286             :         } else {
     287             :           // Move to a cache register (spill one if necessary).
     288             :           // Note that we cannot create a {LiftoffRegister} for reg_code, since
     289             :           // {LiftoffRegister} can only store cache regs.
     290           0 :           in_reg = __ GetUnusedRegister(rc, pinned);
     291           0 :           if (rc == kGpReg) {
     292           0 :             __ Move(in_reg.gp(), Register::from_code(reg_code), lowered_type);
     293             :           } else {
     294           0 :             __ Move(in_reg.fp(), DoubleRegister::from_code(reg_code),
     295           0 :                     lowered_type);
     296             :           }
     297             :         }
     298       19301 :       } else if (param_loc.IsCallerFrameSlot()) {
     299       19301 :         in_reg = __ GetUnusedRegister(rc, pinned);
     300       38602 :         __ LoadCallerFrameSlot(in_reg, -param_loc.AsCallerFrameSlot(),
     301       19301 :                                lowered_type);
     302             :       }
     303             :       reg = pair_idx == 0 ? in_reg
     304      116434 :                           : LiftoffRegister::ForPair(reg.gp(), in_reg.gp());
     305             :       pinned.set(reg);
     306             :     }
     307             :     __ PushRegister(type, reg);
     308      116528 :     return num_lowered_params;
     309             :   }
     310             : 
     311      461551 :   void StackCheck(WasmCodePosition position) {
     312      823351 :     if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) return;
     313       99751 :     out_of_line_code_.push_back(
     314      100518 :         OutOfLineCode::StackCheck(position, __ cache_state()->used_registers));
     315             :     OutOfLineCode& ool = out_of_line_code_.back();
     316      200395 :     Register limit_address = __ GetUnusedRegister(kGpReg).gp();
     317       99877 :     LOAD_INSTANCE_FIELD(limit_address, StackLimitAddress, kSystemPointerSize);
     318      100048 :     __ StackCheck(ool.label.get(), limit_address);
     319       99509 :     __ bind(ool.continuation.get());
     320             :   }
     321             : 
     322      460561 :   void StartFunctionBody(FullDecoder* decoder, Control* block) {
     323      832635 :     for (uint32_t i = 0; i < __ num_locals(); ++i) {
     324      186156 :       if (!CheckSupportedType(decoder, kSupportedTypes, __ local_type(i),
     325             :                               "param"))
     326         177 :         return;
     327             :     }
     328             : 
     329             :     // Input 0 is the call target, the instance is at 1.
     330             :     constexpr int kInstanceParameterIndex = 1;
     331             :     // Store the instance parameter to a special stack slot.
     332             :     compiler::LinkageLocation instance_loc =
     333      460442 :         descriptor_->GetInputLocation(kInstanceParameterIndex);
     334             :     DCHECK(instance_loc.IsRegister());
     335             :     DCHECK(!instance_loc.IsAnyRegister());
     336      460442 :     Register instance_reg = Register::from_code(instance_loc.AsRegister());
     337             :     DCHECK_EQ(kWasmInstanceRegister, instance_reg);
     338             : 
     339             :     // Parameter 0 is the instance parameter.
     340             :     uint32_t num_params =
     341      460442 :         static_cast<uint32_t>(decoder->sig_->parameter_count());
     342             : 
     343      460442 :     __ EnterFrame(StackFrame::WASM_COMPILED);
     344             :     __ set_has_frame(true);
     345      460263 :     pc_offset_stack_frame_construction_ = __ PrepareStackFrame();
     346             :     // {PrepareStackFrame} is the first platform-specific assembler method.
     347             :     // If this failed, we can bail out immediately, avoiding runtime overhead
     348             :     // and potential failures because of other unimplemented methods.
     349             :     // A platform implementing {PrepareStackFrame} must ensure that we can
     350             :     // finish compilation without errors even if we hit unimplemented
     351             :     // LiftoffAssembler methods.
     352      460263 :     if (DidAssemblerBailout(decoder)) return;
     353             : 
     354      460334 :     __ SpillInstance(instance_reg);
     355             :     // Input 0 is the code target, 1 is the instance. First parameter at 2.
     356             :     uint32_t input_idx = kInstanceParameterIndex + 1;
     357      693553 :     for (uint32_t param_idx = 0; param_idx < num_params; ++param_idx) {
     358      116507 :       input_idx += ProcessParameter(__ local_type(param_idx), input_idx);
     359             :     }
     360             :     DCHECK_EQ(input_idx, descriptor_->InputCount());
     361             :     // Set to a gp register, to mark this uninitialized.
     362             :     LiftoffRegister zero_double_reg = kGpCacheRegList.GetFirstRegSet();
     363             :     DCHECK(zero_double_reg.is_gp());
     364      599557 :     for (uint32_t param_idx = num_params; param_idx < __ num_locals();
     365             :          ++param_idx) {
     366             :       ValueType type = decoder->GetLocalType(param_idx);
     367       69523 :       switch (type) {
     368             :         case kWasmI32:
     369       33880 :           __ cache_state()->stack_state.emplace_back(kWasmI32, uint32_t{0});
     370             :           break;
     371             :         case kWasmI64:
     372       35026 :           __ cache_state()->stack_state.emplace_back(kWasmI64, uint32_t{0});
     373             :           break;
     374             :         case kWasmF32:
     375             :         case kWasmF64:
     376         617 :           if (zero_double_reg.is_gp()) {
     377             :             // Note: This might spill one of the registers used to hold
     378             :             // parameters.
     379             :             zero_double_reg = __ GetUnusedRegister(kFpReg);
     380             :             // Zero is represented by the bit pattern 0 for both f32 and f64.
     381         400 :             __ LoadConstant(zero_double_reg, WasmValue(0.));
     382             :           }
     383             :           __ PushRegister(type, zero_double_reg);
     384             :           break;
     385             :         default:
     386           0 :           UNIMPLEMENTED();
     387             :       }
     388             :     }
     389             : 
     390             :     // The function-prologue stack check is associated with position 0, which
     391             :     // is never a position of any instruction in the function.
     392      460513 :     StackCheck(0);
     393             : 
     394             :     DCHECK_EQ(__ num_locals(), __ cache_state()->stack_height());
     395             :   }
     396             : 
     397      352052 :   void GenerateOutOfLineCode(OutOfLineCode& ool) {
     398      352052 :     __ bind(ool.label.get());
     399      352023 :     const bool is_stack_check = ool.stub == WasmCode::kWasmStackGuard;
     400             :     const bool is_mem_out_of_bounds =
     401             :         ool.stub == WasmCode::kThrowWasmTrapMemOutOfBounds;
     402             : 
     403      352023 :     if (is_mem_out_of_bounds && env_->use_trap_handler) {
     404      153626 :       uint32_t pc = static_cast<uint32_t>(__ pc_offset());
     405             :       DCHECK_EQ(pc, __ pc_offset());
     406      307279 :       protected_instructions_.emplace_back(
     407      307252 :           trap_handler::ProtectedInstructionData{ool.pc, pc});
     408             :     }
     409             : 
     410      352050 :     if (!env_->runtime_exception_support) {
     411             :       // We cannot test calls to the runtime in cctest/test-run-wasm.
     412             :       // Therefore we emit a call to C here instead of a call to the runtime.
     413             :       // In this mode, we never generate stack checks.
     414             :       DCHECK(!is_stack_check);
     415      217656 :       __ CallTrapCallbackForTesting();
     416      217656 :       __ LeaveFrame(StackFrame::WASM_COMPILED);
     417      217656 :       __ DropStackSlotsAndRet(
     418      217656 :           static_cast<uint32_t>(descriptor_->StackParameterCount()));
     419             :       return;
     420             :     }
     421             : 
     422      134394 :     if (!ool.regs_to_save.is_empty()) __ PushRegisters(ool.regs_to_save);
     423             : 
     424      404036 :     source_position_table_builder_.AddPosition(
     425      134429 :         __ pc_offset(), SourcePosition(ool.position), false);
     426      135178 :     __ CallRuntimeStub(ool.stub);
     427             :     safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
     428      134417 :                                              Safepoint::kNoLazyDeopt);
     429             :     DCHECK_EQ(ool.continuation.get()->is_bound(), is_stack_check);
     430      135047 :     if (!ool.regs_to_save.is_empty()) __ PopRegisters(ool.regs_to_save);
     431      135092 :     if (is_stack_check) {
     432             :       __ emit_jump(ool.continuation.get());
     433             :     } else {
     434             :       __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
     435             :     }
     436             :   }
     437             : 
     438      441289 :   void FinishFunction(FullDecoder* decoder) {
     439      441289 :     if (DidAssemblerBailout(decoder)) return;
     440      793510 :     for (OutOfLineCode& ool : out_of_line_code_) {
     441      352071 :       GenerateOutOfLineCode(ool);
     442             :     }
     443      441439 :     __ PatchPrepareStackFrame(pc_offset_stack_frame_construction_,
     444      441439 :                               __ GetTotalFrameSlotCount());
     445             :     __ FinishCode();
     446      442031 :     safepoint_table_builder_.Emit(&asm_, __ GetTotalFrameSlotCount());
     447             :     __ MaybeEmitOutOfLineConstantPool();
     448             :     // The previous calls may have also generated a bailout.
     449      441648 :     DidAssemblerBailout(decoder);
     450             :   }
     451             : 
     452             :   void OnFirstError(FullDecoder* decoder) {
     453       38256 :     ok_ = false;
     454             :     UnuseLabels(decoder);
     455             :     asm_.AbortCompilation();
     456             :   }
     457             : 
     458             :   void NextInstruction(FullDecoder* decoder, WasmOpcode opcode) {
     459             :     TraceCacheState(decoder);
     460             :     SLOW_DCHECK(__ ValidateCacheState());
     461             :     DEBUG_CODE_COMMENT(WasmOpcodes::OpcodeName(opcode));
     462             :   }
     463             : 
     464             :   void Block(FullDecoder* decoder, Control* block) {}
     465             : 
     466         983 :   void Loop(FullDecoder* decoder, Control* loop) {
     467             :     // Before entering a loop, spill all locals to the stack, in order to free
     468             :     // the cache registers, and to avoid unnecessarily reloading stack values
     469             :     // into registers at branches.
     470             :     // TODO(clemensh): Come up with a better strategy here, involving
     471             :     // pre-analysis of the function.
     472         983 :     __ SpillLocals();
     473             : 
     474             :     // Loop labels bind at the beginning of the block.
     475         984 :     __ bind(loop->label.get());
     476             : 
     477             :     // Save the current cache state for the merge when jumping to this loop.
     478         984 :     loop->label_state.Split(*__ cache_state());
     479             : 
     480             :     // Execute a stack check in the loop header.
     481         984 :     StackCheck(decoder->position());
     482         982 :   }
     483             : 
     484             :   void Try(FullDecoder* decoder, Control* block) {
     485             :     unsupported(decoder, "try");
     486             :   }
     487             : 
     488             :   void Catch(FullDecoder* decoder, Control* block, Value* exception) {
     489             :     unsupported(decoder, "catch");
     490             :   }
     491             : 
     492        1382 :   void If(FullDecoder* decoder, const Value& cond, Control* if_block) {
     493             :     DCHECK_EQ(if_block, decoder->control_at(0));
     494             :     DCHECK(if_block->is_if());
     495             : 
     496        1382 :     if (if_block->start_merge.arity > 0 || if_block->end_merge.arity > 1)
     497          13 :       return unsupported(decoder, "multi-value if");
     498             : 
     499             :     // Allocate the else state.
     500        2738 :     if_block->else_state = base::make_unique<ElseState>();
     501             : 
     502             :     // Test the condition, jump to else if zero.
     503        2737 :     Register value = __ PopToRegister().gp();
     504             :     __ emit_cond_jump(kEqual, if_block->else_state->label.get(), kWasmI32,
     505        1368 :                       value);
     506             : 
     507             :     // Store the state (after popping the value) for executing the else branch.
     508        1369 :     if_block->else_state->state.Split(*__ cache_state());
     509             :   }
     510             : 
     511           0 :   void FallThruTo(FullDecoder* decoder, Control* c) {
     512           0 :     if (c->end_merge.reached) {
     513           0 :       __ MergeFullStackWith(c->label_state, *__ cache_state());
     514             :     } else {
     515           0 :       c->label_state.Split(*__ cache_state());
     516             :     }
     517             :     TraceCacheState(decoder);
     518           0 :   }
     519             : 
     520         505 :   void FinishOneArmedIf(FullDecoder* decoder, Control* c) {
     521             :     DCHECK(c->is_onearmed_if());
     522         505 :     if (c->end_merge.reached) {
     523             :       // Someone already merged to the end of the if. Merge both arms into that.
     524          27 :       if (c->reachable()) {
     525             :         // Merge the if state into the end state.
     526           0 :         __ MergeFullStackWith(c->label_state, *__ cache_state());
     527             :         __ emit_jump(c->label.get());
     528             :       }
     529             :       // Merge the else state into the end state.
     530          27 :       __ bind(c->else_state->label.get());
     531          27 :       __ MergeFullStackWith(c->label_state, c->else_state->state);
     532          27 :       __ cache_state()->Steal(c->label_state);
     533         478 :     } else if (c->reachable()) {
     534             :       // No merge yet at the end of the if, but we need to create a merge for
     535             :       // the both arms of this if. Thus init the merge point from the else
     536             :       // state, then merge the if state into that.
     537             :       DCHECK_EQ(0, c->end_merge.arity);
     538         482 :       c->label_state.InitMerge(c->else_state->state, __ num_locals(), 0,
     539         241 :                                c->stack_depth);
     540         241 :       __ MergeFullStackWith(c->label_state, *__ cache_state());
     541             :       __ emit_jump(c->label.get());
     542             :       // Merge the else state into the end state.
     543         241 :       __ bind(c->else_state->label.get());
     544         241 :       __ MergeFullStackWith(c->label_state, c->else_state->state);
     545         241 :       __ cache_state()->Steal(c->label_state);
     546             :     } else {
     547             :       // No merge needed, just continue with the else state.
     548         237 :       __ bind(c->else_state->label.get());
     549         474 :       __ cache_state()->Steal(c->else_state->state);
     550             :     }
     551         505 :   }
     552             : 
     553       65356 :   void PopControl(FullDecoder* decoder, Control* c) {
     554       65356 :     if (c->is_loop()) return;  // A loop just falls through.
     555       64530 :     if (c->is_onearmed_if()) {
     556             :       // Special handling for one-armed ifs.
     557         505 :       FinishOneArmedIf(decoder, c);
     558       64025 :     } else if (c->end_merge.reached) {
     559             :       // There is a merge already. Merge our state into that, then continue with
     560             :       // that state.
     561       52774 :       if (c->reachable()) {
     562         943 :         __ MergeFullStackWith(c->label_state, *__ cache_state());
     563             :       }
     564      105548 :       __ cache_state()->Steal(c->label_state);
     565             :     } else {
     566             :       // No merge, just continue with our current state.
     567             :     }
     568             : 
     569      129054 :     if (!c->label.get()->is_bound()) __ bind(c->label.get());
     570             :   }
     571             : 
     572             :   void EndControl(FullDecoder* decoder, Control* c) {}
     573             : 
     574             :   enum CCallReturn : bool { kHasReturn = true, kNoReturn = false };
     575             : 
     576       26932 :   void GenerateCCall(const LiftoffRegister* result_regs, FunctionSig* sig,
     577             :                      ValueType out_argument_type,
     578             :                      const LiftoffRegister* arg_regs,
     579             :                      ExternalReference ext_ref) {
     580             :     // Before making a call, spill all cache registers.
     581       26932 :     __ SpillAllRegisters();
     582             : 
     583             :     // Store arguments on our stack, then align the stack for calling to C.
     584       26932 :     int param_bytes = 0;
     585      134660 :     for (ValueType param_type : sig->parameters()) {
     586       53864 :       param_bytes += ValueTypes::MemSize(param_type);
     587             :     }
     588             :     int out_arg_bytes = out_argument_type == kWasmStmt
     589             :                             ? 0
     590       53864 :                             : ValueTypes::MemSize(out_argument_type);
     591       26932 :     int stack_bytes = std::max(param_bytes, out_arg_bytes);
     592       26932 :     __ CallC(sig, arg_regs, result_regs, out_argument_type, stack_bytes,
     593       26932 :              ext_ref);
     594       26932 :   }
     595             : 
     596             :   template <ValueType src_type, ValueType result_type, class EmitFn>
     597       40832 :   void EmitUnOp(EmitFn fn) {
     598             :     static RegClass src_rc = reg_class_for(src_type);
     599             :     static RegClass result_rc = reg_class_for(result_type);
     600       40832 :     LiftoffRegister src = __ PopToRegister();
     601             :     LiftoffRegister dst = src_rc == result_rc
     602       81664 :                               ? __ GetUnusedRegister(result_rc, {src})
     603       81664 :                               : __ GetUnusedRegister(result_rc);
     604         136 :     fn(dst, src);
     605             :     __ PushRegister(result_type, dst);
     606       40831 :   }
     607             : 
     608             :   void EmitI32UnOpWithCFallback(bool (LiftoffAssembler::*emit_fn)(Register,
     609             :                                                                   Register),
     610             :                                 ExternalReference (*fallback_fn)()) {
     611          56 :     auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) {
     612         168 :       if (emit_fn && (asm_.*emit_fn)(dst.gp(), src.gp())) return;
     613           0 :       ExternalReference ext_ref = fallback_fn();
     614           0 :       ValueType sig_i_i_reps[] = {kWasmI32, kWasmI32};
     615             :       FunctionSig sig_i_i(1, 1, sig_i_i_reps);
     616           0 :       GenerateCCall(&dst, &sig_i_i, kWasmStmt, &src, ext_ref);
     617          56 :     };
     618          56 :     EmitUnOp<kWasmI32, kWasmI32>(emit_with_c_fallback);
     619             :   }
     620             : 
     621             :   template <ValueType type>
     622             :   void EmitFloatUnOpWithCFallback(
     623             :       bool (LiftoffAssembler::*emit_fn)(DoubleRegister, DoubleRegister),
     624             :       ExternalReference (*fallback_fn)()) {
     625          80 :     auto emit_with_c_fallback = [=](LiftoffRegister dst, LiftoffRegister src) {
     626         240 :       if ((asm_.*emit_fn)(dst.fp(), src.fp())) return;
     627           0 :       ExternalReference ext_ref = fallback_fn();
     628           0 :       ValueType sig_reps[] = {type};
     629             :       FunctionSig sig(0, 1, sig_reps);
     630           0 :       GenerateCCall(&dst, &sig, type, &src, ext_ref);
     631          80 :     };
     632          80 :     EmitUnOp<type, type>(emit_with_c_fallback);
     633             :   }
     634             : 
     635             :   enum TypeConversionTrapping : bool { kCanTrap = true, kNoTrap = false };
     636             :   template <ValueType dst_type, ValueType src_type,
     637             :             TypeConversionTrapping can_trap>
     638       52398 :   void EmitTypeConversion(WasmOpcode opcode, ExternalReference (*fallback_fn)(),
     639             :                           WasmCodePosition trap_position) {
     640             :     static constexpr RegClass src_rc = reg_class_for(src_type);
     641             :     static constexpr RegClass dst_rc = reg_class_for(dst_type);
     642       52398 :     LiftoffRegister src = __ PopToRegister();
     643       26883 :     LiftoffRegister dst = src_rc == dst_rc ? __ GetUnusedRegister(dst_rc, {src})
     644       51840 :                                            : __ GetUnusedRegister(dst_rc);
     645             :     DCHECK_EQ(!!can_trap, trap_position > 0);
     646             :     Label* trap = can_trap ? AddOutOfLineTrap(
     647             :                                  trap_position,
     648             :                                  WasmCode::kThrowWasmTrapFloatUnrepresentable)
     649             :                            : nullptr;
     650       52416 :     if (!__ emit_type_conversion(opcode, dst, src, trap)) {
     651             :       DCHECK_NOT_NULL(fallback_fn);
     652           0 :       ExternalReference ext_ref = fallback_fn();
     653             :       if (can_trap) {
     654             :         // External references for potentially trapping conversions return int.
     655           0 :         ValueType sig_reps[] = {kWasmI32, src_type};
     656             :         FunctionSig sig(1, 1, sig_reps);
     657             :         LiftoffRegister ret_reg =
     658             :             __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst));
     659           0 :         LiftoffRegister dst_regs[] = {ret_reg, dst};
     660           0 :         GenerateCCall(dst_regs, &sig, dst_type, &src, ext_ref);
     661           0 :         __ emit_cond_jump(kEqual, trap, kWasmI32, ret_reg.gp());
     662             :       } else {
     663           0 :         ValueType sig_reps[] = {src_type};
     664             :         FunctionSig sig(0, 1, sig_reps);
     665           0 :         GenerateCCall(&dst, &sig, dst_type, &src, ext_ref);
     666             :       }
     667             :     }
     668             :     __ PushRegister(dst_type, dst);
     669       52440 :   }
     670             : 
     671       93334 :   void UnOp(FullDecoder* decoder, WasmOpcode opcode, const Value& value,
     672             :             Value* result) {
     673             : #define CASE_I32_UNOP(opcode, fn)                       \
     674             :   case WasmOpcode::kExpr##opcode:                       \
     675             :     EmitUnOp<kWasmI32, kWasmI32>(                       \
     676             :         [=](LiftoffRegister dst, LiftoffRegister src) { \
     677             :           __ emit_##fn(dst.gp(), src.gp());             \
     678             :         });                                             \
     679             :     break;
     680             : #define CASE_I32_SIGN_EXTENSION(opcode, fn)             \
     681             :   case WasmOpcode::kExpr##opcode:                       \
     682             :     EmitUnOp<kWasmI32, kWasmI32>(                       \
     683             :         [=](LiftoffRegister dst, LiftoffRegister src) { \
     684             :           __ emit_##fn(dst.gp(), src.gp());             \
     685             :         });                                             \
     686             :     break;
     687             : #define CASE_I64_SIGN_EXTENSION(opcode, fn)             \
     688             :   case WasmOpcode::kExpr##opcode:                       \
     689             :     EmitUnOp<kWasmI64, kWasmI64>(                       \
     690             :         [=](LiftoffRegister dst, LiftoffRegister src) { \
     691             :           __ emit_##fn(dst, src);                       \
     692             :         });                                             \
     693             :     break;
     694             : #define CASE_FLOAT_UNOP(opcode, type, fn)               \
     695             :   case WasmOpcode::kExpr##opcode:                       \
     696             :     EmitUnOp<kWasm##type, kWasm##type>(                 \
     697             :         [=](LiftoffRegister dst, LiftoffRegister src) { \
     698             :           __ emit_##fn(dst.fp(), src.fp());             \
     699             :         });                                             \
     700             :     break;
     701             : #define CASE_FLOAT_UNOP_WITH_CFALLBACK(opcode, type, fn)                    \
     702             :   case WasmOpcode::kExpr##opcode:                                           \
     703             :     EmitFloatUnOpWithCFallback<kWasm##type>(&LiftoffAssembler::emit_##fn,   \
     704             :                                             &ExternalReference::wasm_##fn); \
     705             :     break;
     706             : #define CASE_TYPE_CONVERSION(opcode, dst_type, src_type, ext_ref, can_trap) \
     707             :   case WasmOpcode::kExpr##opcode:                                           \
     708             :     EmitTypeConversion<kWasm##dst_type, kWasm##src_type, can_trap>(         \
     709             :         kExpr##opcode, ext_ref, can_trap ? decoder->position() : 0);        \
     710             :     break;
     711       93334 :     switch (opcode) {
     712       78742 :       CASE_I32_UNOP(I32Eqz, i32_eqz)
     713        1188 :       CASE_I32_UNOP(I32Clz, i32_clz)
     714         582 :       CASE_I32_UNOP(I32Ctz, i32_ctz)
     715          44 :       CASE_FLOAT_UNOP(F32Abs, F32, f32_abs)
     716         186 :       CASE_FLOAT_UNOP(F32Neg, F32, f32_neg)
     717             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Ceil, F32, f32_ceil)
     718             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Floor, F32, f32_floor)
     719             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32Trunc, F32, f32_trunc)
     720             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F32NearestInt, F32, f32_nearest_int)
     721         124 :       CASE_FLOAT_UNOP(F32Sqrt, F32, f32_sqrt)
     722          44 :       CASE_FLOAT_UNOP(F64Abs, F64, f64_abs)
     723         186 :       CASE_FLOAT_UNOP(F64Neg, F64, f64_neg)
     724             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Ceil, F64, f64_ceil)
     725             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Floor, F64, f64_floor)
     726             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64Trunc, F64, f64_trunc)
     727             :       CASE_FLOAT_UNOP_WITH_CFALLBACK(F64NearestInt, F64, f64_nearest_int)
     728         106 :       CASE_FLOAT_UNOP(F64Sqrt, F64, f64_sqrt)
     729         587 :       CASE_TYPE_CONVERSION(I32ConvertI64, I32, I64, nullptr, kNoTrap)
     730          87 :       CASE_TYPE_CONVERSION(I32SConvertF32, I32, F32, nullptr, kCanTrap)
     731          19 :       CASE_TYPE_CONVERSION(I32UConvertF32, I32, F32, nullptr, kCanTrap)
     732          87 :       CASE_TYPE_CONVERSION(I32SConvertF64, I32, F64, nullptr, kCanTrap)
     733          16 :       CASE_TYPE_CONVERSION(I32UConvertF64, I32, F64, nullptr, kCanTrap)
     734       19098 :       CASE_TYPE_CONVERSION(I32ReinterpretF32, I32, F32, nullptr, kNoTrap)
     735          32 :       CASE_TYPE_CONVERSION(I64SConvertI32, I64, I32, nullptr, kNoTrap)
     736       12729 :       CASE_TYPE_CONVERSION(I64UConvertI32, I64, I32, nullptr, kNoTrap)
     737          16 :       CASE_TYPE_CONVERSION(I64SConvertF32, I64, F32,
     738             :                            &ExternalReference::wasm_float32_to_int64, kCanTrap)
     739          16 :       CASE_TYPE_CONVERSION(I64UConvertF32, I64, F32,
     740             :                            &ExternalReference::wasm_float32_to_uint64, kCanTrap)
     741         301 :       CASE_TYPE_CONVERSION(I64SConvertF64, I64, F64,
     742             :                            &ExternalReference::wasm_float64_to_int64, kCanTrap)
     743          19 :       CASE_TYPE_CONVERSION(I64UConvertF64, I64, F64,
     744             :                            &ExternalReference::wasm_float64_to_uint64, kCanTrap)
     745       18983 :       CASE_TYPE_CONVERSION(I64ReinterpretF64, I64, F64, nullptr, kNoTrap)
     746          32 :       CASE_TYPE_CONVERSION(F32SConvertI32, F32, I32, nullptr, kNoTrap)
     747          12 :       CASE_TYPE_CONVERSION(F32UConvertI32, F32, I32, nullptr, kNoTrap)
     748          19 :       CASE_TYPE_CONVERSION(F32SConvertI64, F32, I64,
     749             :                            &ExternalReference::wasm_int64_to_float32, kNoTrap)
     750          13 :       CASE_TYPE_CONVERSION(F32UConvertI64, F32, I64,
     751             :                            &ExternalReference::wasm_uint64_to_float32, kNoTrap)
     752          30 :       CASE_TYPE_CONVERSION(F32ConvertF64, F32, F64, nullptr, kNoTrap)
     753          52 :       CASE_TYPE_CONVERSION(F32ReinterpretI32, F32, I32, nullptr, kNoTrap)
     754          25 :       CASE_TYPE_CONVERSION(F64SConvertI32, F64, I32, nullptr, kNoTrap)
     755          27 :       CASE_TYPE_CONVERSION(F64UConvertI32, F64, I32, nullptr, kNoTrap)
     756          26 :       CASE_TYPE_CONVERSION(F64SConvertI64, F64, I64,
     757             :                            &ExternalReference::wasm_int64_to_float64, kNoTrap)
     758          52 :       CASE_TYPE_CONVERSION(F64UConvertI64, F64, I64,
     759             :                            &ExternalReference::wasm_uint64_to_float64, kNoTrap)
     760          54 :       CASE_TYPE_CONVERSION(F64ConvertF32, F64, F32, nullptr, kNoTrap)
     761          67 :       CASE_TYPE_CONVERSION(F64ReinterpretI64, F64, I64, nullptr, kNoTrap)
     762          14 :       CASE_I32_SIGN_EXTENSION(I32SExtendI8, i32_signextend_i8)
     763           8 :       CASE_I32_SIGN_EXTENSION(I32SExtendI16, i32_signextend_i16)
     764          14 :       CASE_I64_SIGN_EXTENSION(I64SExtendI8, i64_signextend_i8)
     765           8 :       CASE_I64_SIGN_EXTENSION(I64SExtendI16, i64_signextend_i16)
     766           8 :       CASE_I64_SIGN_EXTENSION(I64SExtendI32, i64_signextend_i32)
     767             :       case kExprI32Popcnt:
     768             :         EmitI32UnOpWithCFallback(&LiftoffAssembler::emit_i32_popcnt,
     769             :                                  &ExternalReference::wasm_word32_popcnt);
     770             :         break;
     771             :       case WasmOpcode::kExprI64Eqz:
     772             :         EmitUnOp<kWasmI64, kWasmI32>(
     773             :             [=](LiftoffRegister dst, LiftoffRegister src) {
     774          69 :               __ emit_i64_eqz(dst.gp(), src);
     775          69 :             });
     776             :         break;
     777             :       default:
     778         103 :         return unsupported(decoder, WasmOpcodes::OpcodeName(opcode));
     779             :     }
     780             : #undef CASE_I32_UNOP
     781             : #undef CASE_I32_SIGN_EXTENSION
     782             : #undef CASE_I64_SIGN_EXTENSION
     783             : #undef CASE_FLOAT_UNOP
     784             : #undef CASE_FLOAT_UNOP_WITH_CFALLBACK
     785             : #undef CASE_TYPE_CONVERSION
     786             :   }
     787             : 
     788             :   template <ValueType src_type, ValueType result_type, typename EmitFn,
     789             :             typename EmitFnImm>
     790      113892 :   void EmitBinOpImm(EmitFn fn, EmitFnImm fnImm) {
     791             :     static constexpr RegClass src_rc = reg_class_for(src_type);
     792             :     static constexpr RegClass result_rc = reg_class_for(result_type);
     793             : 
     794      113892 :     LiftoffAssembler::VarState rhs_slot = __ cache_state()->stack_state.back();
     795             :     // Check if the RHS is an immediate.
     796      113892 :     if (rhs_slot.loc() == LiftoffAssembler::VarState::kIntConst) {
     797             :       __ cache_state()->stack_state.pop_back();
     798             :       int32_t imm = rhs_slot.i32_const();
     799             : 
     800      112295 :       LiftoffRegister lhs = __ PopToRegister();
     801             :       LiftoffRegister dst = src_rc == result_rc
     802      224590 :                                 ? __ GetUnusedRegister(result_rc, {lhs})
     803      112295 :                                 : __ GetUnusedRegister(result_rc);
     804             : 
     805             :       fnImm(dst, lhs, imm);
     806             :       __ PushRegister(result_type, dst);
     807             :     } else {
     808             :       // The RHS was not an immediate.
     809        1597 :       LiftoffRegister rhs = __ PopToRegister();
     810        1597 :       LiftoffRegister lhs = __ PopToRegister(LiftoffRegList::ForRegs(rhs));
     811             :       LiftoffRegister dst = src_rc == result_rc
     812        3194 :                                 ? __ GetUnusedRegister(result_rc, {lhs, rhs})
     813        1597 :                                 : __ GetUnusedRegister(result_rc);
     814             :       fn(dst, lhs, rhs);
     815             :       __ PushRegister(result_type, dst);
     816             :     }
     817      113892 :   }
     818             : 
     819             :   template <ValueType src_type, ValueType result_type, typename EmitFn>
     820      399053 :   void EmitBinOp(EmitFn fn) {
     821             :     static constexpr RegClass src_rc = reg_class_for(src_type);
     822             :     static constexpr RegClass result_rc = reg_class_for(result_type);
     823      399053 :     LiftoffRegister rhs = __ PopToRegister();
     824      399116 :     LiftoffRegister lhs = __ PopToRegister(LiftoffRegList::ForRegs(rhs));
     825             :     LiftoffRegister dst = src_rc == result_rc
     826      797320 :                               ? __ GetUnusedRegister(result_rc, {lhs, rhs})
     827      398681 :                               : __ GetUnusedRegister(result_rc);
     828      136121 :     fn(dst, lhs, rhs);
     829             :     __ PushRegister(result_type, dst);
     830      399081 :   }
     831             : 
     832             :   void EmitDivOrRem64CCall(LiftoffRegister dst, LiftoffRegister lhs,
     833             :                            LiftoffRegister rhs, ExternalReference ext_ref,
     834             :                            Label* trap_by_zero,
     835             :                            Label* trap_unrepresentable = nullptr) {
     836             :     // Cannot emit native instructions, build C call.
     837             :     LiftoffRegister ret =
     838             :         __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst));
     839             :     LiftoffRegister tmp =
     840             :         __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(dst, ret));
     841             :     LiftoffRegister arg_regs[] = {lhs, rhs};
     842             :     LiftoffRegister result_regs[] = {ret, dst};
     843             :     ValueType sig_types[] = {kWasmI32, kWasmI64, kWasmI64};
     844             :     // <i64, i64> -> i32 (with i64 output argument)
     845             :     FunctionSig sig(1, 2, sig_types);
     846             :     GenerateCCall(result_regs, &sig, kWasmI64, arg_regs, ext_ref);
     847             :     __ LoadConstant(tmp, WasmValue(int32_t{0}));
     848             :     __ emit_cond_jump(kEqual, trap_by_zero, kWasmI32, ret.gp(), tmp.gp());
     849             :     if (trap_unrepresentable) {
     850             :       __ LoadConstant(tmp, WasmValue(int32_t{-1}));
     851             :       __ emit_cond_jump(kEqual, trap_unrepresentable, kWasmI32, ret.gp(),
     852             :                         tmp.gp());
     853             :     }
     854             :   }
     855             : 
     856      513192 :   void BinOp(FullDecoder* decoder, WasmOpcode opcode, const Value& lhs,
     857             :              const Value& rhs, Value* result) {
     858             : #define CASE_I32_BINOP(opcode, fn)                                           \
     859             :   case WasmOpcode::kExpr##opcode:                                            \
     860             :     return EmitBinOp<kWasmI32, kWasmI32>(                                    \
     861             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     862             :           __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp());                        \
     863             :         });
     864             : #define CASE_I32_BINOPI(opcode, fn)                                          \
     865             :   case WasmOpcode::kExpr##opcode:                                            \
     866             :     return EmitBinOpImm<kWasmI32, kWasmI32>(                                 \
     867             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     868             :           __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp());                        \
     869             :         },                                                                   \
     870             :         [=](LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) {         \
     871             :           __ emit_##fn(dst.gp(), lhs.gp(), imm);                             \
     872             :         });
     873             : #define CASE_I64_BINOP(opcode, fn)                                           \
     874             :   case WasmOpcode::kExpr##opcode:                                            \
     875             :     return EmitBinOp<kWasmI64, kWasmI64>(                                    \
     876             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     877             :           __ emit_##fn(dst, lhs, rhs);                                       \
     878             :         });
     879             : #define CASE_I64_BINOPI(opcode, fn)                                          \
     880             :   case WasmOpcode::kExpr##opcode:                                            \
     881             :     return EmitBinOpImm<kWasmI64, kWasmI64>(                                 \
     882             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     883             :           __ emit_##fn(dst, lhs, rhs);                                       \
     884             :         },                                                                   \
     885             :         [=](LiftoffRegister dst, LiftoffRegister lhs, int32_t imm) {         \
     886             :           __ emit_##fn(dst, lhs, imm);                                       \
     887             :         });
     888             : #define CASE_FLOAT_BINOP(opcode, type, fn)                                   \
     889             :   case WasmOpcode::kExpr##opcode:                                            \
     890             :     return EmitBinOp<kWasm##type, kWasm##type>(                              \
     891             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     892             :           __ emit_##fn(dst.fp(), lhs.fp(), rhs.fp());                        \
     893             :         });
     894             : #define CASE_I32_CMPOP(opcode, cond)                                         \
     895             :   case WasmOpcode::kExpr##opcode:                                            \
     896             :     return EmitBinOp<kWasmI32, kWasmI32>(                                    \
     897             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     898             :           __ emit_i32_set_cond(cond, dst.gp(), lhs.gp(), rhs.gp());          \
     899             :         });
     900             : #define CASE_I64_CMPOP(opcode, cond)                                         \
     901             :   case WasmOpcode::kExpr##opcode:                                            \
     902             :     return EmitBinOp<kWasmI64, kWasmI32>(                                    \
     903             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     904             :           __ emit_i64_set_cond(cond, dst.gp(), lhs, rhs);                    \
     905             :         });
     906             : #define CASE_F32_CMPOP(opcode, cond)                                         \
     907             :   case WasmOpcode::kExpr##opcode:                                            \
     908             :     return EmitBinOp<kWasmF32, kWasmI32>(                                    \
     909             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     910             :           __ emit_f32_set_cond(cond, dst.gp(), lhs.fp(), rhs.fp());          \
     911             :         });
     912             : #define CASE_F64_CMPOP(opcode, cond)                                         \
     913             :   case WasmOpcode::kExpr##opcode:                                            \
     914             :     return EmitBinOp<kWasmF64, kWasmI32>(                                    \
     915             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     916             :           __ emit_f64_set_cond(cond, dst.gp(), lhs.fp(), rhs.fp());          \
     917             :         });
     918             : #define CASE_I32_SHIFTOP(opcode, fn)                                         \
     919             :   case WasmOpcode::kExpr##opcode:                                            \
     920             :     return EmitBinOp<kWasmI32, kWasmI32>(                                    \
     921             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     922             :           __ emit_##fn(dst.gp(), lhs.gp(), rhs.gp(), {});                    \
     923             :         });
     924             : #define CASE_I64_SHIFTOP(opcode, fn)                                           \
     925             :   case WasmOpcode::kExpr##opcode:                                              \
     926             :     return EmitBinOp<kWasmI64, kWasmI64>([=](LiftoffRegister dst,              \
     927             :                                              LiftoffRegister src,              \
     928             :                                              LiftoffRegister amount) {         \
     929             :       __ emit_##fn(dst, src, amount.is_pair() ? amount.low_gp() : amount.gp(), \
     930             :                    {});                                                        \
     931             :     });
     932             : #define CASE_CCALL_BINOP(opcode, type, ext_ref_fn)                           \
     933             :   case WasmOpcode::kExpr##opcode:                                            \
     934             :     return EmitBinOp<kWasmI32, kWasmI32>(                                    \
     935             :         [=](LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs) { \
     936             :           LiftoffRegister args[] = {lhs, rhs};                               \
     937             :           auto ext_ref = ExternalReference::ext_ref_fn();                    \
     938             :           ValueType sig_i_ii_reps[] = {kWasmI32, kWasmI32, kWasmI32};        \
     939             :           FunctionSig sig_i_ii(1, 2, sig_i_ii_reps);                         \
     940             :           GenerateCCall(&dst, &sig_i_ii, kWasmStmt, args, ext_ref);          \
     941             :         });
     942      513192 :     switch (opcode) {
     943      226544 :       CASE_I32_BINOPI(I32Add, i32_add)
     944       36371 :       CASE_I32_BINOP(I32Sub, i32_sub)
     945       28386 :       CASE_I32_BINOP(I32Mul, i32_mul)
     946       32874 :       CASE_I32_BINOP(I32And, i32_and)
     947       27034 :       CASE_I32_BINOP(I32Ior, i32_or)
     948       26932 :       CASE_I32_BINOP(I32Xor, i32_xor)
     949        5782 :       CASE_I64_BINOP(I64And, i64_and)
     950       12819 :       CASE_I64_BINOP(I64Ior, i64_or)
     951          84 :       CASE_I64_BINOP(I64Xor, i64_xor)
     952       79872 :       CASE_I32_CMPOP(I32Eq, kEqual)
     953       27010 :       CASE_I32_CMPOP(I32Ne, kUnequal)
     954       26990 :       CASE_I32_CMPOP(I32LtS, kSignedLessThan)
     955       26992 :       CASE_I32_CMPOP(I32LtU, kUnsignedLessThan)
     956       26944 :       CASE_I32_CMPOP(I32GtS, kSignedGreaterThan)
     957       26944 :       CASE_I32_CMPOP(I32GtU, kUnsignedGreaterThan)
     958       27020 :       CASE_I32_CMPOP(I32LeS, kSignedLessEqual)
     959       26962 :       CASE_I32_CMPOP(I32LeU, kUnsignedLessEqual)
     960       26938 :       CASE_I32_CMPOP(I32GeS, kSignedGreaterEqual)
     961       26958 :       CASE_I32_CMPOP(I32GeU, kUnsignedGreaterEqual)
     962        1240 :       CASE_I64_BINOPI(I64Add, i64_add)
     963        1318 :       CASE_I64_BINOP(I64Sub, i64_sub)
     964        1313 :       CASE_I64_BINOP(I64Mul, i64_mul)
     965       26059 :       CASE_I64_CMPOP(I64Eq, kEqual)
     966          36 :       CASE_I64_CMPOP(I64Ne, kUnequal)
     967          60 :       CASE_I64_CMPOP(I64LtS, kSignedLessThan)
     968          42 :       CASE_I64_CMPOP(I64LtU, kUnsignedLessThan)
     969          42 :       CASE_I64_CMPOP(I64GtS, kSignedGreaterThan)
     970          42 :       CASE_I64_CMPOP(I64GtU, kUnsignedGreaterThan)
     971          36 :       CASE_I64_CMPOP(I64LeS, kSignedLessEqual)
     972          48 :       CASE_I64_CMPOP(I64LeU, kUnsignedLessEqual)
     973          36 :       CASE_I64_CMPOP(I64GeS, kSignedGreaterEqual)
     974          36 :       CASE_I64_CMPOP(I64GeU, kUnsignedGreaterEqual)
     975         112 :       CASE_F32_CMPOP(F32Eq, kEqual)
     976          40 :       CASE_F32_CMPOP(F32Ne, kUnequal)
     977          94 :       CASE_F32_CMPOP(F32Lt, kUnsignedLessThan)
     978         100 :       CASE_F32_CMPOP(F32Gt, kUnsignedGreaterThan)
     979          94 :       CASE_F32_CMPOP(F32Le, kUnsignedLessEqual)
     980          88 :       CASE_F32_CMPOP(F32Ge, kUnsignedGreaterEqual)
     981          70 :       CASE_F64_CMPOP(F64Eq, kEqual)
     982          40 :       CASE_F64_CMPOP(F64Ne, kUnequal)
     983         100 :       CASE_F64_CMPOP(F64Lt, kUnsignedLessThan)
     984          82 :       CASE_F64_CMPOP(F64Gt, kUnsignedGreaterThan)
     985          94 :       CASE_F64_CMPOP(F64Le, kUnsignedLessEqual)
     986          82 :       CASE_F64_CMPOP(F64Ge, kUnsignedGreaterEqual)
     987       56200 :       CASE_I32_SHIFTOP(I32Shl, i32_shl)
     988       56164 :       CASE_I32_SHIFTOP(I32ShrS, i32_sar)
     989       56164 :       CASE_I32_SHIFTOP(I32ShrU, i32_shr)
     990       27779 :       CASE_I64_SHIFTOP(I64Shl, i64_shl)
     991        2428 :       CASE_I64_SHIFTOP(I64ShrS, i64_sar)
     992        2620 :       CASE_I64_SHIFTOP(I64ShrU, i64_shr)
     993       40407 :       CASE_CCALL_BINOP(I32Rol, I32, wasm_word32_rol)
     994       40389 :       CASE_CCALL_BINOP(I32Ror, I32, wasm_word32_ror)
     995         358 :       CASE_FLOAT_BINOP(F32Add, F32, f32_add)
     996         290 :       CASE_FLOAT_BINOP(F32Sub, F32, f32_sub)
     997         346 :       CASE_FLOAT_BINOP(F32Mul, F32, f32_mul)
     998         300 :       CASE_FLOAT_BINOP(F32Div, F32, f32_div)
     999          68 :       CASE_FLOAT_BINOP(F32Min, F32, f32_min)
    1000          68 :       CASE_FLOAT_BINOP(F32Max, F32, f32_max)
    1001          26 :       CASE_FLOAT_BINOP(F32CopySign, F32, f32_copysign)
    1002         652 :       CASE_FLOAT_BINOP(F64Add, F64, f64_add)
    1003         288 :       CASE_FLOAT_BINOP(F64Sub, F64, f64_sub)
    1004         436 :       CASE_FLOAT_BINOP(F64Mul, F64, f64_mul)
    1005         348 :       CASE_FLOAT_BINOP(F64Div, F64, f64_div)
    1006          80 :       CASE_FLOAT_BINOP(F64Min, F64, f64_min)
    1007          68 :       CASE_FLOAT_BINOP(F64Max, F64, f64_max)
    1008          26 :       CASE_FLOAT_BINOP(F64CopySign, F64, f64_copysign)
    1009             :       case WasmOpcode::kExprI32DivS:
    1010             :         EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
    1011             :                                                       LiftoffRegister lhs,
    1012       56488 :                                                       LiftoffRegister rhs) {
    1013             :           WasmCodePosition position = decoder->position();
    1014             :           AddOutOfLineTrap(position, WasmCode::kThrowWasmTrapDivByZero);
    1015             :           // Adding the second trap might invalidate the pointer returned for
    1016             :           // the first one, thus get both pointers afterwards.
    1017             :           AddOutOfLineTrap(position,
    1018             :                            WasmCode::kThrowWasmTrapDivUnrepresentable);
    1019             :           Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
    1020             :           Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
    1021       14122 :           __ emit_i32_divs(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero,
    1022             :                            div_unrepresentable);
    1023       28244 :         });
    1024             :         break;
    1025             :       case WasmOpcode::kExprI32DivU:
    1026             :         EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
    1027             :                                                       LiftoffRegister lhs,
    1028       42243 :                                                       LiftoffRegister rhs) {
    1029             :           Label* div_by_zero = AddOutOfLineTrap(
    1030             :               decoder->position(), WasmCode::kThrowWasmTrapDivByZero);
    1031       14081 :           __ emit_i32_divu(dst.gp(), lhs.gp(), rhs.gp(), div_by_zero);
    1032       28162 :         });
    1033             :         break;
    1034             :       case WasmOpcode::kExprI32RemS:
    1035             :         EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
    1036             :                                                       LiftoffRegister lhs,
    1037       42189 :                                                       LiftoffRegister rhs) {
    1038             :           Label* rem_by_zero = AddOutOfLineTrap(
    1039             :               decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
    1040       14063 :           __ emit_i32_rems(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
    1041       28126 :         });
    1042             :         break;
    1043             :       case WasmOpcode::kExprI32RemU:
    1044             :         EmitBinOp<kWasmI32, kWasmI32>([this, decoder](LiftoffRegister dst,
    1045             :                                                       LiftoffRegister lhs,
    1046       42162 :                                                       LiftoffRegister rhs) {
    1047             :           Label* rem_by_zero = AddOutOfLineTrap(
    1048             :               decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
    1049       14054 :           __ emit_i32_remu(dst.gp(), lhs.gp(), rhs.gp(), rem_by_zero);
    1050       28108 :         });
    1051             :         break;
    1052             :       case WasmOpcode::kExprI64DivS:
    1053             :         EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
    1054             :                                                       LiftoffRegister lhs,
    1055        2624 :                                                       LiftoffRegister rhs) {
    1056             :           WasmCodePosition position = decoder->position();
    1057             :           AddOutOfLineTrap(position, WasmCode::kThrowWasmTrapDivByZero);
    1058             :           // Adding the second trap might invalidate the pointer returned for
    1059             :           // the first one, thus get both pointers afterwards.
    1060             :           AddOutOfLineTrap(position,
    1061             :                            WasmCode::kThrowWasmTrapDivUnrepresentable);
    1062             :           Label* div_by_zero = out_of_line_code_.end()[-2].label.get();
    1063             :           Label* div_unrepresentable = out_of_line_code_.end()[-1].label.get();
    1064         656 :           if (!__ emit_i64_divs(dst, lhs, rhs, div_by_zero,
    1065             :                                 div_unrepresentable)) {
    1066             :             ExternalReference ext_ref = ExternalReference::wasm_int64_div();
    1067             :             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero,
    1068             :                                 div_unrepresentable);
    1069             :           }
    1070        1312 :         });
    1071             :         break;
    1072             :       case WasmOpcode::kExprI64DivU:
    1073             :         EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
    1074             :                                                       LiftoffRegister lhs,
    1075        1818 :                                                       LiftoffRegister rhs) {
    1076             :           Label* div_by_zero = AddOutOfLineTrap(
    1077             :               decoder->position(), WasmCode::kThrowWasmTrapDivByZero);
    1078         606 :           if (!__ emit_i64_divu(dst, lhs, rhs, div_by_zero)) {
    1079             :             ExternalReference ext_ref = ExternalReference::wasm_uint64_div();
    1080             :             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, div_by_zero);
    1081             :           }
    1082        1212 :         });
    1083             :         break;
    1084             :       case WasmOpcode::kExprI64RemS:
    1085             :         EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
    1086             :                                                       LiftoffRegister lhs,
    1087        1800 :                                                       LiftoffRegister rhs) {
    1088             :           Label* rem_by_zero = AddOutOfLineTrap(
    1089             :               decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
    1090         600 :           if (!__ emit_i64_rems(dst, lhs, rhs, rem_by_zero)) {
    1091             :             ExternalReference ext_ref = ExternalReference::wasm_int64_mod();
    1092             :             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
    1093             :           }
    1094        1200 :         });
    1095             :         break;
    1096             :       case WasmOpcode::kExprI64RemU:
    1097             :         EmitBinOp<kWasmI64, kWasmI64>([this, decoder](LiftoffRegister dst,
    1098             :                                                       LiftoffRegister lhs,
    1099        1800 :                                                       LiftoffRegister rhs) {
    1100             :           Label* rem_by_zero = AddOutOfLineTrap(
    1101             :               decoder->position(), WasmCode::kThrowWasmTrapRemByZero);
    1102         600 :           if (!__ emit_i64_remu(dst, lhs, rhs, rem_by_zero)) {
    1103             :             ExternalReference ext_ref = ExternalReference::wasm_uint64_mod();
    1104             :             EmitDivOrRem64CCall(dst, lhs, rhs, ext_ref, rem_by_zero);
    1105             :           }
    1106        1200 :         });
    1107             :         break;
    1108             :       default:
    1109         146 :         return unsupported(decoder, WasmOpcodes::OpcodeName(opcode));
    1110             :     }
    1111             : #undef CASE_I32_BINOP
    1112             : #undef CASE_I32_BINOPI
    1113             : #undef CASE_I64_BINOP
    1114             : #undef CASE_I64_BINOPI
    1115             : #undef CASE_FLOAT_BINOP
    1116             : #undef CASE_I32_CMPOP
    1117             : #undef CASE_I64_CMPOP
    1118             : #undef CASE_F32_CMPOP
    1119             : #undef CASE_F64_CMPOP
    1120             : #undef CASE_I32_SHIFTOP
    1121             : #undef CASE_I64_SHIFTOP
    1122             : #undef CASE_CCALL_BINOP
    1123             :   }
    1124             : 
    1125             :   void I32Const(FullDecoder* decoder, Value* result, int32_t value) {
    1126     1044612 :     __ cache_state()->stack_state.emplace_back(kWasmI32, value);
    1127             :   }
    1128             : 
    1129       21141 :   void I64Const(FullDecoder* decoder, Value* result, int64_t value) {
    1130             :     // The {VarState} stores constant values as int32_t, thus we only store
    1131             :     // 64-bit constants in this field if it fits in an int32_t. Larger values
    1132             :     // cannot be used as immediate value anyway, so we can also just put them in
    1133             :     // a register immediately.
    1134       21141 :     int32_t value_i32 = static_cast<int32_t>(value);
    1135       21141 :     if (value_i32 == value) {
    1136       11847 :       __ cache_state()->stack_state.emplace_back(kWasmI64, value_i32);
    1137             :     } else {
    1138        9294 :       LiftoffRegister reg = __ GetUnusedRegister(reg_class_for(kWasmI64));
    1139        9294 :       __ LoadConstant(reg, WasmValue(value));
    1140             :       __ PushRegister(kWasmI64, reg);
    1141             :     }
    1142       21145 :   }
    1143             : 
    1144       44398 :   void F32Const(FullDecoder* decoder, Value* result, float value) {
    1145       44398 :     LiftoffRegister reg = __ GetUnusedRegister(kFpReg);
    1146       44400 :     __ LoadConstant(reg, WasmValue(value));
    1147             :     __ PushRegister(kWasmF32, reg);
    1148       44400 :   }
    1149             : 
    1150       45101 :   void F64Const(FullDecoder* decoder, Value* result, double value) {
    1151       45101 :     LiftoffRegister reg = __ GetUnusedRegister(kFpReg);
    1152       45101 :     __ LoadConstant(reg, WasmValue(value));
    1153             :     __ PushRegister(kWasmF64, reg);
    1154       45101 :   }
    1155             : 
    1156             :   void RefNull(FullDecoder* decoder, Value* result) {
    1157             :     unsupported(decoder, "ref_null");
    1158             :   }
    1159             : 
    1160             :   void Drop(FullDecoder* decoder, const Value& value) {
    1161             :     auto& slot = __ cache_state()->stack_state.back();
    1162             :     // If the dropped slot contains a register, decrement it's use count.
    1163        1204 :     if (slot.is_reg()) __ cache_state()->dec_used(slot.reg());
    1164             :     __ cache_state()->stack_state.pop_back();
    1165             :   }
    1166             : 
    1167      448810 :   void ReturnImpl(FullDecoder* decoder) {
    1168      448810 :     size_t num_returns = decoder->sig_->return_count();
    1169      448810 :     if (num_returns > 1) return unsupported(decoder, "multi-return");
    1170      448275 :     if (num_returns > 0) __ MoveToReturnRegisters(decoder->sig_);
    1171      448066 :     __ LeaveFrame(StackFrame::WASM_COMPILED);
    1172      447911 :     __ DropStackSlotsAndRet(
    1173      447911 :         static_cast<uint32_t>(descriptor_->StackParameterCount()));
    1174             :   }
    1175             : 
    1176             :   void DoReturn(FullDecoder* decoder, Vector<Value> /*values*/) {
    1177      448791 :     ReturnImpl(decoder);
    1178             :   }
    1179             : 
    1180      193520 :   void GetLocal(FullDecoder* decoder, Value* result,
    1181             :                 const LocalIndexImmediate<validate>& imm) {
    1182      193520 :     auto& slot = __ cache_state()->stack_state[imm.index];
    1183             :     DCHECK_EQ(slot.type(), imm.type);
    1184      193520 :     switch (slot.loc()) {
    1185             :       case kRegister:
    1186             :         __ PushRegister(slot.type(), slot.reg());
    1187             :         break;
    1188             :       case kIntConst:
    1189         175 :         __ cache_state()->stack_state.emplace_back(imm.type, slot.i32_const());
    1190             :         break;
    1191             :       case kStack: {
    1192       49795 :         auto rc = reg_class_for(imm.type);
    1193       49795 :         LiftoffRegister reg = __ GetUnusedRegister(rc);
    1194       49796 :         __ Fill(reg, imm.index, imm.type);
    1195             :         __ PushRegister(slot.type(), reg);
    1196             :         break;
    1197             :       }
    1198             :     }
    1199      193435 :   }
    1200             : 
    1201           0 :   void SetLocalFromStackSlot(LiftoffAssembler::VarState& dst_slot,
    1202             :                              uint32_t local_index) {
    1203             :     auto& state = *__ cache_state();
    1204             :     ValueType type = dst_slot.type();
    1205           0 :     if (dst_slot.is_reg()) {
    1206             :       LiftoffRegister slot_reg = dst_slot.reg();
    1207           0 :       if (state.get_use_count(slot_reg) == 1) {
    1208           0 :         __ Fill(dst_slot.reg(), state.stack_height() - 1, type);
    1209             :         return;
    1210             :       }
    1211             :       state.dec_used(slot_reg);
    1212             :       dst_slot.MakeStack();
    1213             :     }
    1214             :     DCHECK_EQ(type, __ local_type(local_index));
    1215             :     RegClass rc = reg_class_for(type);
    1216           0 :     LiftoffRegister dst_reg = __ GetUnusedRegister(rc);
    1217           0 :     __ Fill(dst_reg, __ cache_state()->stack_height() - 1, type);
    1218           0 :     dst_slot = LiftoffAssembler::VarState(type, dst_reg);
    1219             :     __ cache_state()->inc_used(dst_reg);
    1220             :   }
    1221             : 
    1222       68412 :   void SetLocal(uint32_t local_index, bool is_tee) {
    1223             :     auto& state = *__ cache_state();
    1224             :     auto& source_slot = state.stack_state.back();
    1225       68412 :     auto& target_slot = state.stack_state[local_index];
    1226       68412 :     switch (source_slot.loc()) {
    1227             :       case kRegister:
    1228       67863 :         if (target_slot.is_reg()) state.dec_used(target_slot.reg());
    1229       67863 :         target_slot = source_slot;
    1230       67863 :         if (is_tee) state.inc_used(target_slot.reg());
    1231             :         break;
    1232             :       case kIntConst:
    1233         552 :         if (target_slot.is_reg()) state.dec_used(target_slot.reg());
    1234         552 :         target_slot = source_slot;
    1235         552 :         break;
    1236             :       case kStack:
    1237           0 :         SetLocalFromStackSlot(target_slot, local_index);
    1238           0 :         break;
    1239             :     }
    1240       68412 :     if (!is_tee) __ cache_state()->stack_state.pop_back();
    1241       68412 :   }
    1242             : 
    1243             :   void SetLocal(FullDecoder* decoder, const Value& value,
    1244             :                 const LocalIndexImmediate<validate>& imm) {
    1245       67934 :     SetLocal(imm.index, false);
    1246             :   }
    1247             : 
    1248             :   void TeeLocal(FullDecoder* decoder, const Value& value, Value* result,
    1249             :                 const LocalIndexImmediate<validate>& imm) {
    1250         480 :     SetLocal(imm.index, true);
    1251             :   }
    1252             : 
    1253         660 :   Register GetGlobalBaseAndOffset(const WasmGlobal* global,
    1254             :                                   LiftoffRegList& pinned, uint32_t* offset) {
    1255         660 :     Register addr = pinned.set(__ GetUnusedRegister(kGpReg)).gp();
    1256         658 :     if (global->mutability && global->imported) {
    1257          63 :       LOAD_INSTANCE_FIELD(addr, ImportedMutableGlobals, kSystemPointerSize);
    1258         126 :       __ Load(LiftoffRegister(addr), addr, no_reg,
    1259         126 :               global->index * sizeof(Address), kPointerLoadType, pinned);
    1260          63 :       *offset = 0;
    1261             :     } else {
    1262         595 :       LOAD_INSTANCE_FIELD(addr, GlobalsStart, kSystemPointerSize);
    1263         600 :       *offset = global->offset;
    1264             :     }
    1265         663 :     return addr;
    1266             :   }
    1267             : 
    1268         578 :   void GetGlobal(FullDecoder* decoder, Value* result,
    1269             :                  const GlobalIndexImmediate<validate>& imm) {
    1270         578 :     const auto* global = &env_->module->globals[imm.index];
    1271         578 :     if (!CheckSupportedType(decoder, kSupportedTypes, global->type, "global"))
    1272          92 :       return;
    1273         485 :     LiftoffRegList pinned;
    1274         485 :     uint32_t offset = 0;
    1275         485 :     Register addr = GetGlobalBaseAndOffset(global, pinned, &offset);
    1276             :     LiftoffRegister value =
    1277         974 :         pinned.set(__ GetUnusedRegister(reg_class_for(global->type), pinned));
    1278         487 :     LoadType type = LoadType::ForValueType(global->type);
    1279         487 :     __ Load(value, addr, no_reg, offset, type, pinned, nullptr, true);
    1280         486 :     __ PushRegister(global->type, value);
    1281             :   }
    1282             : 
    1283         176 :   void SetGlobal(FullDecoder* decoder, const Value& value,
    1284             :                  const GlobalIndexImmediate<validate>& imm) {
    1285         176 :     auto* global = &env_->module->globals[imm.index];
    1286         176 :     if (!CheckSupportedType(decoder, kSupportedTypes, global->type, "global"))
    1287           0 :       return;
    1288         175 :     LiftoffRegList pinned;
    1289         175 :     uint32_t offset = 0;
    1290         175 :     Register addr = GetGlobalBaseAndOffset(global, pinned, &offset);
    1291         352 :     LiftoffRegister reg = pinned.set(__ PopToRegister(pinned));
    1292         176 :     StoreType type = StoreType::ForValueType(global->type);
    1293         176 :     __ Store(addr, no_reg, offset, reg, type, {}, nullptr, true);
    1294             :   }
    1295             : 
    1296             :   void GetTable(FullDecoder* decoder, const Value& index, Value* result,
    1297             :                 TableIndexImmediate<validate>& imm) {
    1298             :     unsupported(decoder, "table_get");
    1299             :   }
    1300             : 
    1301             :   void SetTable(FullDecoder* decoder, const Value& index, const Value& value,
    1302             :                 TableIndexImmediate<validate>& imm) {
    1303             :     unsupported(decoder, "table_set");
    1304             :   }
    1305             : 
    1306       39823 :   void Unreachable(FullDecoder* decoder) {
    1307             :     Label* unreachable_label = AddOutOfLineTrap(
    1308             :         decoder->position(), WasmCode::kThrowWasmTrapUnreachable);
    1309             :     __ emit_jump(unreachable_label);
    1310             :     __ AssertUnreachable(AbortReason::kUnexpectedReturnFromWasmTrap);
    1311       39822 :   }
    1312             : 
    1313         332 :   void Select(FullDecoder* decoder, const Value& cond, const Value& fval,
    1314             :               const Value& tval, Value* result) {
    1315             :     LiftoffRegList pinned;
    1316         662 :     Register condition = pinned.set(__ PopToRegister()).gp();
    1317             :     ValueType type = __ cache_state()->stack_state.end()[-1].type();
    1318             :     DCHECK_EQ(type, __ cache_state()->stack_state.end()[-2].type());
    1319         330 :     LiftoffRegister false_value = pinned.set(__ PopToRegister(pinned));
    1320         335 :     LiftoffRegister true_value = __ PopToRegister(pinned);
    1321             :     LiftoffRegister dst =
    1322         670 :         __ GetUnusedRegister(true_value.reg_class(), {true_value, false_value});
    1323             :     __ PushRegister(type, dst);
    1324             : 
    1325             :     // Now emit the actual code to move either {true_value} or {false_value}
    1326             :     // into {dst}.
    1327         334 :     Label cont;
    1328         334 :     Label case_false;
    1329         334 :     __ emit_cond_jump(kEqual, &case_false, kWasmI32, condition);
    1330         329 :     if (dst != true_value) __ Move(dst, true_value, type);
    1331             :     __ emit_jump(&cont);
    1332             : 
    1333         330 :     __ bind(&case_false);
    1334         330 :     if (dst != false_value) __ Move(dst, false_value, type);
    1335         329 :     __ bind(&cont);
    1336         332 :   }
    1337             : 
    1338       52878 :   void BrImpl(Control* target) {
    1339       52878 :     if (!target->br_merge()->reached) {
    1340      157194 :       target->label_state.InitMerge(*__ cache_state(), __ num_locals(),
    1341             :                                     target->br_merge()->arity,
    1342       52398 :                                     target->stack_depth);
    1343             :     }
    1344       52880 :     __ MergeStackWith(target->label_state, target->br_merge()->arity);
    1345       52877 :     __ jmp(target->label.get());
    1346       52876 :   }
    1347             : 
    1348        1016 :   void Br(FullDecoder* decoder, Control* target) { BrImpl(target); }
    1349             : 
    1350       51942 :   void BrOrRet(FullDecoder* decoder, uint32_t depth) {
    1351       51942 :     if (depth == decoder->control_depth() - 1) {
    1352          80 :       ReturnImpl(decoder);
    1353             :     } else {
    1354       51862 :       BrImpl(decoder->control_at(depth));
    1355             :     }
    1356       51941 :   }
    1357             : 
    1358       39793 :   void BrIf(FullDecoder* decoder, const Value& cond, uint32_t depth) {
    1359       39793 :     Label cont_false;
    1360       79586 :     Register value = __ PopToRegister().gp();
    1361       39793 :     __ emit_cond_jump(kEqual, &cont_false, kWasmI32, value);
    1362             : 
    1363       39794 :     BrOrRet(decoder, depth);
    1364       39793 :     __ bind(&cont_false);
    1365       39794 :   }
    1366             : 
    1367             :   // Generate a branch table case, potentially reusing previously generated
    1368             :   // stack transfer code.
    1369       91446 :   void GenerateBrCase(FullDecoder* decoder, uint32_t br_depth,
    1370             :                       std::map<uint32_t, MovableLabel>& br_targets) {
    1371       91446 :     MovableLabel& label = br_targets[br_depth];
    1372       91467 :     if (label.get()->is_bound()) {
    1373       79318 :       __ jmp(label.get());
    1374             :     } else {
    1375       12149 :       __ bind(label.get());
    1376       12149 :       BrOrRet(decoder, br_depth);
    1377             :     }
    1378       91469 :   }
    1379             : 
    1380             :   // Generate a branch table for input in [min, max).
    1381             :   // TODO(wasm): Generate a real branch table (like TF TableSwitch).
    1382      169258 :   void GenerateBrTable(FullDecoder* decoder, LiftoffRegister tmp,
    1383             :                        LiftoffRegister value, uint32_t min, uint32_t max,
    1384             :                        BranchTableIterator<validate>& table_iterator,
    1385             :                        std::map<uint32_t, MovableLabel>& br_targets) {
    1386             :     DCHECK_LT(min, max);
    1387             :     // Check base case.
    1388      169258 :     if (max == min + 1) {
    1389             :       DCHECK_EQ(min, table_iterator.cur_index());
    1390       86846 :       GenerateBrCase(decoder, table_iterator.next(), br_targets);
    1391       86847 :       return;
    1392             :     }
    1393             : 
    1394       82412 :     uint32_t split = min + (max - min) / 2;
    1395       82412 :     Label upper_half;
    1396       82412 :     __ LoadConstant(tmp, WasmValue(split));
    1397             :     __ emit_cond_jump(kUnsignedGreaterEqual, &upper_half, kWasmI32, value.gp(),
    1398       82412 :                       tmp.gp());
    1399             :     // Emit br table for lower half:
    1400             :     GenerateBrTable(decoder, tmp, value, min, split, table_iterator,
    1401       82411 :                     br_targets);
    1402       82413 :     __ bind(&upper_half);
    1403             :     // Emit br table for upper half:
    1404             :     GenerateBrTable(decoder, tmp, value, split, max, table_iterator,
    1405       82413 :                     br_targets);
    1406             :   }
    1407             : 
    1408        4619 :   void BrTable(FullDecoder* decoder, const BranchTableImmediate<validate>& imm,
    1409             :                const Value& key) {
    1410             :     LiftoffRegList pinned;
    1411        9239 :     LiftoffRegister value = pinned.set(__ PopToRegister());
    1412             :     BranchTableIterator<validate> table_iterator(decoder, imm);
    1413             :     std::map<uint32_t, MovableLabel> br_targets;
    1414             : 
    1415        4620 :     if (imm.table_count > 0) {
    1416             :       LiftoffRegister tmp = __ GetUnusedRegister(kGpReg, pinned);
    1417        8880 :       __ LoadConstant(tmp, WasmValue(uint32_t{imm.table_count}));
    1418        4440 :       Label case_default;
    1419             :       __ emit_cond_jump(kUnsignedGreaterEqual, &case_default, kWasmI32,
    1420        4440 :                         value.gp(), tmp.gp());
    1421             : 
    1422        4439 :       GenerateBrTable(decoder, tmp, value, 0, imm.table_count, table_iterator,
    1423        4439 :                       br_targets);
    1424             : 
    1425        4440 :       __ bind(&case_default);
    1426             :     }
    1427             : 
    1428             :     // Generate the default case.
    1429        4620 :     GenerateBrCase(decoder, table_iterator.next(), br_targets);
    1430             :     DCHECK(!table_iterator.has_next());
    1431        4620 :   }
    1432             : 
    1433         706 :   void Else(FullDecoder* decoder, Control* c) {
    1434         706 :     if (c->reachable()) {
    1435         591 :       if (!c->end_merge.reached) {
    1436        1174 :         c->label_state.InitMerge(*__ cache_state(), __ num_locals(),
    1437         587 :                                  c->end_merge.arity, c->stack_depth);
    1438             :       }
    1439         591 :       __ MergeFullStackWith(c->label_state, *__ cache_state());
    1440             :       __ emit_jump(c->label.get());
    1441             :     }
    1442         706 :     __ bind(c->else_state->label.get());
    1443        1412 :     __ cache_state()->Steal(c->else_state->state);
    1444         706 :   }
    1445             : 
    1446             :   Label* AddOutOfLineTrap(WasmCodePosition position,
    1447             :                           WasmCode::RuntimeStubId stub, uint32_t pc = 0) {
    1448             :     DCHECK(!FLAG_wasm_no_bounds_checks);
    1449             :     // The pc is needed for memory OOB trap with trap handler enabled. Other
    1450             :     // callers should not even compute it.
    1451             :     DCHECK_EQ(pc != 0, stub == WasmCode::kThrowWasmTrapMemOutOfBounds &&
    1452             :                            env_->use_trap_handler);
    1453             : 
    1454      593886 :     out_of_line_code_.push_back(OutOfLineCode::Trap(stub, position, pc));
    1455             :     return out_of_line_code_.back().label.get();
    1456             :   }
    1457             : 
    1458             :   // Returns true if the memory access is statically known to be out of bounds
    1459             :   // (a jump to the trap was generated then); return false otherwise.
    1460      180485 :   bool BoundsCheckMem(FullDecoder* decoder, uint32_t access_size,
    1461             :                       uint32_t offset, Register index, LiftoffRegList pinned) {
    1462             :     const bool statically_oob =
    1463      180485 :         !IsInBounds(offset, access_size, env_->max_memory_size);
    1464             : 
    1465      180485 :     if (!statically_oob &&
    1466      179872 :         (FLAG_wasm_no_bounds_checks || env_->use_trap_handler)) {
    1467             :       return false;
    1468             :     }
    1469             : 
    1470             :     // TODO(wasm): This adds protected instruction information for the jump
    1471             :     // instruction we are about to generate. It would be better to just not add
    1472             :     // protected instruction info when the pc is 0.
    1473        1202 :     Label* trap_label = AddOutOfLineTrap(
    1474             :         decoder->position(), WasmCode::kThrowWasmTrapMemOutOfBounds,
    1475         603 :         env_->use_trap_handler ? __ pc_offset() : 0);
    1476             : 
    1477         610 :     if (statically_oob) {
    1478             :       __ emit_jump(trap_label);
    1479             :       Control* current_block = decoder->control_at(0);
    1480         601 :       if (current_block->reachable()) {
    1481         601 :         current_block->reachability = kSpecOnlyReachable;
    1482             :       }
    1483             :       return true;
    1484             :     }
    1485             : 
    1486             :     DCHECK(!env_->use_trap_handler);
    1487             :     DCHECK(!FLAG_wasm_no_bounds_checks);
    1488             : 
    1489           1 :     uint64_t end_offset = uint64_t{offset} + access_size - 1u;
    1490             : 
    1491             :     // If the end offset is larger than the smallest memory, dynamically check
    1492             :     // the end offset against the actual memory size, which is not known at
    1493             :     // compile time. Otherwise, only one check is required (see below).
    1494             :     LiftoffRegister end_offset_reg =
    1495           1 :         pinned.set(__ GetUnusedRegister(kGpReg, pinned));
    1496             :     Register mem_size = __ GetUnusedRegister(kGpReg, pinned).gp();
    1497           1 :     LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize);
    1498             : 
    1499             :     if (kSystemPointerSize == 8) {
    1500           1 :       __ LoadConstant(end_offset_reg, WasmValue(end_offset));
    1501             :     } else {
    1502             :       __ LoadConstant(end_offset_reg,
    1503             :                       WasmValue(static_cast<uint32_t>(end_offset)));
    1504             :     }
    1505             : 
    1506           1 :     if (end_offset >= env_->min_memory_size) {
    1507             :       __ emit_cond_jump(kUnsignedGreaterEqual, trap_label,
    1508             :                         LiftoffAssembler::kWasmIntPtr, end_offset_reg.gp(),
    1509           0 :                         mem_size);
    1510             :     }
    1511             : 
    1512             :     // Just reuse the end_offset register for computing the effective size.
    1513             :     LiftoffRegister effective_size_reg = end_offset_reg;
    1514             :     __ emit_ptrsize_sub(effective_size_reg.gp(), mem_size, end_offset_reg.gp());
    1515             : 
    1516             :     __ emit_i32_to_intptr(index, index);
    1517             : 
    1518             :     __ emit_cond_jump(kUnsignedGreaterEqual, trap_label,
    1519             :                       LiftoffAssembler::kWasmIntPtr, index,
    1520           1 :                       effective_size_reg.gp());
    1521           1 :     return false;
    1522             :   }
    1523             : 
    1524          20 :   void TraceMemoryOperation(bool is_store, MachineRepresentation rep,
    1525             :                             Register index, uint32_t offset,
    1526             :                             WasmCodePosition position) {
    1527             :     // Before making the runtime call, spill all cache registers.
    1528          20 :     __ SpillAllRegisters();
    1529             : 
    1530             :     LiftoffRegList pinned = LiftoffRegList::ForRegs(index);
    1531             :     // Get one register for computing the address (offset + index).
    1532             :     LiftoffRegister address = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
    1533             :     // Compute offset+index in address.
    1534          20 :     __ LoadConstant(address, WasmValue(offset));
    1535          20 :     __ emit_i32_add(address.gp(), address.gp(), index);
    1536             : 
    1537             :     // Get a register to hold the stack slot for MemoryTracingInfo.
    1538             :     LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned));
    1539             :     // Allocate stack slot for MemoryTracingInfo.
    1540          20 :     __ AllocateStackSlot(info.gp(), sizeof(MemoryTracingInfo));
    1541             : 
    1542             :     // Now store all information into the MemoryTracingInfo struct.
    1543          20 :     __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, address), address,
    1544          20 :              StoreType::kI32Store, pinned);
    1545          40 :     __ LoadConstant(address, WasmValue(is_store ? 1 : 0));
    1546          20 :     __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), address,
    1547          20 :              StoreType::kI32Store8, pinned);
    1548          20 :     __ LoadConstant(address, WasmValue(static_cast<int>(rep)));
    1549          20 :     __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), address,
    1550          20 :              StoreType::kI32Store8, pinned);
    1551             : 
    1552          40 :     source_position_table_builder_.AddPosition(__ pc_offset(),
    1553          20 :                                                SourcePosition(position), false);
    1554             : 
    1555          20 :     Register args[] = {info.gp()};
    1556          20 :     GenerateRuntimeCall(Runtime::kWasmTraceMemory, arraysize(args), args);
    1557             :     __ DeallocateStackSlot(sizeof(MemoryTracingInfo));
    1558          20 :   }
    1559             : 
    1560          20 :   void GenerateRuntimeCall(Runtime::FunctionId runtime_function, int num_args,
    1561             :                            Register* args) {
    1562          40 :     auto call_descriptor = compiler::Linkage::GetRuntimeCallDescriptor(
    1563             :         compilation_zone_, runtime_function, num_args,
    1564          20 :         compiler::Operator::kNoProperties, compiler::CallDescriptor::kNoFlags);
    1565             :     // Currently, only one argument is supported. More arguments require some
    1566             :     // caution for the parallel register moves (reuse StackTransferRecipe).
    1567             :     DCHECK_EQ(1, num_args);
    1568             :     constexpr size_t kInputShift = 1;  // Input 0 is the call target.
    1569             :     compiler::LinkageLocation param_loc =
    1570             :         call_descriptor->GetInputLocation(kInputShift);
    1571          20 :     if (param_loc.IsRegister()) {
    1572             :       Register reg = Register::from_code(param_loc.AsRegister());
    1573           0 :       __ Move(LiftoffRegister(reg), LiftoffRegister(args[0]),
    1574           0 :               LiftoffAssembler::kWasmIntPtr);
    1575             :     } else {
    1576             :       DCHECK(param_loc.IsCallerFrameSlot());
    1577          20 :       LiftoffStackSlots stack_slots(&asm_);
    1578          20 :       stack_slots.Add(LiftoffAssembler::VarState(LiftoffAssembler::kWasmIntPtr,
    1579             :                                                  LiftoffRegister(args[0])));
    1580          20 :       stack_slots.Construct();
    1581             :     }
    1582             : 
    1583             :     // Set context to "no context" for the runtime call.
    1584          20 :     __ TurboAssembler::Move(kContextRegister,
    1585          20 :                             Smi::FromInt(Context::kNoContext));
    1586          19 :     Register centry = kJavaScriptCallCodeStartRegister;
    1587          19 :     LOAD_TAGGED_PTR_INSTANCE_FIELD(centry, CEntryStub);
    1588          20 :     __ CallRuntimeWithCEntry(runtime_function, centry);
    1589             :     safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
    1590          19 :                                              Safepoint::kNoLazyDeopt);
    1591          20 :   }
    1592             : 
    1593      179868 :   Register AddMemoryMasking(Register index, uint32_t* offset,
    1594             :                             LiftoffRegList& pinned) {
    1595      179868 :     if (!FLAG_untrusted_code_mitigations || env_->use_trap_handler) {
    1596      179868 :       return index;
    1597             :     }
    1598             :     DEBUG_CODE_COMMENT("Mask memory index");
    1599             :     // Make sure that we can overwrite {index}.
    1600           0 :     if (__ cache_state()->is_used(LiftoffRegister(index))) {
    1601             :       Register old_index = index;
    1602             :       pinned.clear(LiftoffRegister(old_index));
    1603           0 :       index = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1604           0 :       if (index != old_index) __ Move(index, old_index, kWasmI32);
    1605             :     }
    1606           0 :     Register tmp = __ GetUnusedRegister(kGpReg, pinned).gp();
    1607           0 :     __ emit_ptrsize_add(index, index, *offset);
    1608           0 :     LOAD_INSTANCE_FIELD(tmp, MemoryMask, kSystemPointerSize);
    1609             :     __ emit_ptrsize_and(index, index, tmp);
    1610           0 :     *offset = 0;
    1611           0 :     return index;
    1612             :   }
    1613             : 
    1614       74558 :   void LoadMem(FullDecoder* decoder, LoadType type,
    1615             :                const MemoryAccessImmediate<validate>& imm,
    1616             :                const Value& index_val, Value* result) {
    1617             :     ValueType value_type = type.value_type();
    1618       74558 :     if (!CheckSupportedType(decoder, kSupportedTypes, value_type, "load"))
    1619         329 :       return;
    1620       74558 :     LiftoffRegList pinned;
    1621      149119 :     Register index = pinned.set(__ PopToRegister()).gp();
    1622      149122 :     if (BoundsCheckMem(decoder, type.size(), imm.offset, index, pinned)) {
    1623             :       return;
    1624             :     }
    1625       74230 :     uint32_t offset = imm.offset;
    1626       74230 :     index = AddMemoryMasking(index, &offset, pinned);
    1627             :     DEBUG_CODE_COMMENT("Load from memory");
    1628       74229 :     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1629       74229 :     LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
    1630             :     RegClass rc = reg_class_for(value_type);
    1631             :     LiftoffRegister value = pinned.set(__ GetUnusedRegister(rc, pinned));
    1632       74239 :     uint32_t protected_load_pc = 0;
    1633       74239 :     __ Load(value, addr, index, offset, type, pinned, &protected_load_pc, true);
    1634       74232 :     if (env_->use_trap_handler) {
    1635             :       AddOutOfLineTrap(decoder->position(),
    1636             :                        WasmCode::kThrowWasmTrapMemOutOfBounds,
    1637       74231 :                        protected_load_pc);
    1638             :     }
    1639             :     __ PushRegister(value_type, value);
    1640             : 
    1641       74236 :     if (FLAG_trace_wasm_memory) {
    1642             :       TraceMemoryOperation(false, type.mem_type().representation(), index,
    1643          24 :                            offset, decoder->position());
    1644             :     }
    1645             :   }
    1646             : 
    1647      106081 :   void StoreMem(FullDecoder* decoder, StoreType type,
    1648             :                 const MemoryAccessImmediate<validate>& imm,
    1649             :                 const Value& index_val, const Value& value_val) {
    1650             :     ValueType value_type = type.value_type();
    1651      106081 :     if (!CheckSupportedType(decoder, kSupportedTypes, value_type, "store"))
    1652         284 :       return;
    1653      106047 :     LiftoffRegList pinned;
    1654      212033 :     LiftoffRegister value = pinned.set(__ PopToRegister());
    1655      105986 :     Register index = pinned.set(__ PopToRegister(pinned)).gp();
    1656      212016 :     if (BoundsCheckMem(decoder, type.size(), imm.offset, index, pinned)) {
    1657             :       return;
    1658             :     }
    1659      105718 :     uint32_t offset = imm.offset;
    1660      105718 :     index = AddMemoryMasking(index, &offset, pinned);
    1661             :     DEBUG_CODE_COMMENT("Store to memory");
    1662      105846 :     Register addr = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1663      105846 :     LOAD_INSTANCE_FIELD(addr, MemoryStart, kSystemPointerSize);
    1664      105982 :     uint32_t protected_store_pc = 0;
    1665             :     LiftoffRegList outer_pinned;
    1666      105982 :     if (FLAG_trace_wasm_memory) outer_pinned.set(index);
    1667             :     __ Store(addr, index, offset, value, type, outer_pinned,
    1668      105982 :              &protected_store_pc, true);
    1669      105936 :     if (env_->use_trap_handler) {
    1670             :       AddOutOfLineTrap(decoder->position(),
    1671             :                        WasmCode::kThrowWasmTrapMemOutOfBounds,
    1672      105966 :                        protected_store_pc);
    1673             :     }
    1674      106020 :     if (FLAG_trace_wasm_memory) {
    1675             :       TraceMemoryOperation(true, type.mem_rep(), index, offset,
    1676          16 :                            decoder->position());
    1677             :     }
    1678             :   }
    1679             : 
    1680         192 :   void CurrentMemoryPages(FullDecoder* decoder, Value* result) {
    1681         192 :     Register mem_size = __ GetUnusedRegister(kGpReg).gp();
    1682         192 :     LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize);
    1683             :     __ emit_ptrsize_shr(mem_size, mem_size, kWasmPageSizeLog2);
    1684             :     __ PushRegister(kWasmI32, LiftoffRegister(mem_size));
    1685         192 :   }
    1686             : 
    1687         513 :   void MemoryGrow(FullDecoder* decoder, const Value& value, Value* result_val) {
    1688             :     // Pop the input, then spill all cache registers to make the runtime call.
    1689             :     LiftoffRegList pinned;
    1690        1026 :     LiftoffRegister input = pinned.set(__ PopToRegister());
    1691         513 :     __ SpillAllRegisters();
    1692             : 
    1693             :     constexpr Register kGpReturnReg = kGpReturnRegisters[0];
    1694             :     static_assert(kLiftoffAssemblerGpCacheRegs & Register::bit<kGpReturnReg>(),
    1695             :                   "first return register is a cache register (needs more "
    1696             :                   "complex code here otherwise)");
    1697             :     LiftoffRegister result = pinned.set(LiftoffRegister(kGpReturnReg));
    1698             : 
    1699             :     WasmMemoryGrowDescriptor descriptor;
    1700             :     DCHECK_EQ(0, descriptor.GetStackParameterCount());
    1701             :     DCHECK_EQ(1, descriptor.GetRegisterParameterCount());
    1702             :     DCHECK_EQ(ValueTypes::MachineTypeFor(kWasmI32),
    1703             :               descriptor.GetParameterType(0));
    1704             : 
    1705             :     Register param_reg = descriptor.GetRegisterParameter(0);
    1706         513 :     if (input.gp() != param_reg) __ Move(param_reg, input.gp(), kWasmI32);
    1707             : 
    1708             :     __ CallRuntimeStub(WasmCode::kWasmMemoryGrow);
    1709             :     safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
    1710         513 :                                              Safepoint::kNoLazyDeopt);
    1711             : 
    1712             :     if (kReturnRegister0 != result.gp()) {
    1713             :       __ Move(result.gp(), kReturnRegister0, kWasmI32);
    1714             :     }
    1715             : 
    1716             :     __ PushRegister(kWasmI32, result);
    1717         513 :   }
    1718             : 
    1719       46797 :   void CallDirect(FullDecoder* decoder,
    1720             :                   const CallFunctionImmediate<validate>& imm,
    1721             :                   const Value args[], Value returns[]) {
    1722       46797 :     if (imm.sig->return_count() > 1)
    1723             :       return unsupported(decoder, "multi-return");
    1724       89999 :     if (imm.sig->return_count() == 1 &&
    1725       43733 :         !CheckSupportedType(decoder, kSupportedTypes, imm.sig->GetReturn(0),
    1726             :                             "return"))
    1727             :       return;
    1728             : 
    1729             :     auto call_descriptor =
    1730       46273 :         compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
    1731             :     call_descriptor =
    1732             :         GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
    1733             : 
    1734       46284 :     if (imm.index < env_->module->num_imported_functions) {
    1735             :       // A direct call to an imported function.
    1736             :       LiftoffRegList pinned;
    1737       42722 :       Register tmp = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1738       42722 :       Register target = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1739             : 
    1740       42722 :       Register imported_targets = tmp;
    1741       42722 :       LOAD_INSTANCE_FIELD(imported_targets, ImportedFunctionTargets,
    1742             :                           kSystemPointerSize);
    1743       85446 :       __ Load(LiftoffRegister(target), imported_targets, no_reg,
    1744       85446 :               imm.index * sizeof(Address), kPointerLoadType, pinned);
    1745             : 
    1746       42723 :       Register imported_function_refs = tmp;
    1747       42723 :       LOAD_TAGGED_PTR_INSTANCE_FIELD(imported_function_refs,
    1748             :                                      ImportedFunctionRefs);
    1749       42722 :       Register imported_function_ref = tmp;
    1750       42722 :       __ LoadTaggedPointer(
    1751             :           imported_function_ref, imported_function_refs, no_reg,
    1752       85444 :           ObjectAccess::ElementOffsetInTaggedFixedArray(imm.index), pinned);
    1753             : 
    1754             :       Register* explicit_instance = &imported_function_ref;
    1755       42723 :       __ PrepareCall(imm.sig, call_descriptor, &target, explicit_instance);
    1756       85446 :       source_position_table_builder_.AddPosition(
    1757       42723 :           __ pc_offset(), SourcePosition(decoder->position()), false);
    1758             : 
    1759       42723 :       __ CallIndirect(imm.sig, call_descriptor, target);
    1760             : 
    1761             :       safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
    1762       42723 :                                                Safepoint::kNoLazyDeopt);
    1763             : 
    1764       42723 :       __ FinishCall(imm.sig, call_descriptor);
    1765             :     } else {
    1766             :       // A direct call within this module just gets the current instance.
    1767        3562 :       __ PrepareCall(imm.sig, call_descriptor);
    1768             : 
    1769        7144 :       source_position_table_builder_.AddPosition(
    1770        3562 :           __ pc_offset(), SourcePosition(decoder->position()), false);
    1771             : 
    1772             :       // Just encode the function index. This will be patched at instantiation.
    1773        3582 :       Address addr = static_cast<Address>(imm.index);
    1774             :       __ CallNativeWasmCode(addr);
    1775             : 
    1776             :       safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
    1777        3559 :                                                Safepoint::kNoLazyDeopt);
    1778             : 
    1779        3573 :       __ FinishCall(imm.sig, call_descriptor);
    1780             :     }
    1781             :   }
    1782             : 
    1783        1449 :   void CallIndirect(FullDecoder* decoder, const Value& index_val,
    1784             :                     const CallIndirectImmediate<validate>& imm,
    1785             :                     const Value args[], Value returns[]) {
    1786        1449 :     if (imm.sig->return_count() > 1) {
    1787          18 :       return unsupported(decoder, "multi-return");
    1788             :     }
    1789        1449 :     if (imm.table_index != 0) {
    1790             :       return unsupported(decoder, "table index != 0");
    1791             :     }
    1792        2022 :     if (imm.sig->return_count() == 1 &&
    1793         591 :         !CheckSupportedType(decoder, kSupportedTypes, imm.sig->GetReturn(0),
    1794             :                             "return")) {
    1795             :       return;
    1796             :     }
    1797             : 
    1798             :     // Pop the index.
    1799        2863 :     Register index = __ PopToRegister().gp();
    1800             :     // If that register is still being used after popping, we move it to another
    1801             :     // register, because we want to modify that register.
    1802        1431 :     if (__ cache_state()->is_used(LiftoffRegister(index))) {
    1803             :       Register new_index =
    1804             :           __ GetUnusedRegister(kGpReg, LiftoffRegList::ForRegs(index)).gp();
    1805         232 :       __ Move(new_index, index, kWasmI32);
    1806             :       index = new_index;
    1807             :     }
    1808             : 
    1809             :     LiftoffRegList pinned = LiftoffRegList::ForRegs(index);
    1810             :     // Get three temporary registers.
    1811             :     Register table = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1812        1431 :     Register tmp_const = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1813             :     Register scratch = pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp();
    1814             : 
    1815             :     // Bounds check against the table size.
    1816             :     Label* invalid_func_label = AddOutOfLineTrap(
    1817             :         decoder->position(), WasmCode::kThrowWasmTrapFuncInvalid);
    1818             : 
    1819        2864 :     uint32_t canonical_sig_num = env_->module->signature_ids[imm.sig_index];
    1820             :     DCHECK_GE(canonical_sig_num, 0);
    1821             :     DCHECK_GE(kMaxInt, canonical_sig_num);
    1822             : 
    1823             :     // Compare against table size stored in
    1824             :     // {instance->indirect_function_table_size}.
    1825        1432 :     LOAD_INSTANCE_FIELD(tmp_const, IndirectFunctionTableSize, kUInt32Size);
    1826             :     __ emit_cond_jump(kUnsignedGreaterEqual, invalid_func_label, kWasmI32,
    1827        1431 :                       index, tmp_const);
    1828             : 
    1829             :     // Mask the index to prevent SSCA.
    1830        1431 :     if (FLAG_untrusted_code_mitigations) {
    1831             :       DEBUG_CODE_COMMENT("Mask indirect call index");
    1832             :       // mask = ((index - size) & ~index) >> 31
    1833             :       // Reuse allocated registers; note: size is still stored in {tmp_const}.
    1834             :       Register diff = table;
    1835           0 :       Register neg_index = tmp_const;
    1836             :       Register mask = scratch;
    1837             :       // 1) diff = index - size
    1838           0 :       __ emit_i32_sub(diff, index, tmp_const);
    1839             :       // 2) neg_index = ~index
    1840           0 :       __ LoadConstant(LiftoffRegister(neg_index), WasmValue(int32_t{-1}));
    1841             :       __ emit_i32_xor(neg_index, neg_index, index);
    1842             :       // 3) mask = diff & neg_index
    1843             :       __ emit_i32_and(mask, diff, neg_index);
    1844             :       // 4) mask = mask >> 31
    1845           0 :       __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(int32_t{31}));
    1846             :       __ emit_i32_sar(mask, mask, tmp_const, pinned);
    1847             : 
    1848             :       // Apply mask.
    1849             :       __ emit_i32_and(index, index, mask);
    1850             :     }
    1851             : 
    1852             :     DEBUG_CODE_COMMENT("Check indirect call signature");
    1853             :     // Load the signature from {instance->ift_sig_ids[key]}
    1854        1431 :     LOAD_INSTANCE_FIELD(table, IndirectFunctionTableSigIds, kSystemPointerSize);
    1855             :     // Multiply {index} by 4 to represent kInt32Size items.
    1856             :     STATIC_ASSERT(kInt32Size == 4);
    1857             :     // TODO(wasm): use a emit_i32_shli() instead of two adds.
    1858             :     // (currently cannot use shl on ia32/x64 because it clobbers %rcx).
    1859        1432 :     __ emit_i32_add(index, index, index);
    1860        1432 :     __ emit_i32_add(index, index, index);
    1861        1432 :     __ Load(LiftoffRegister(scratch), table, index, 0, LoadType::kI32Load,
    1862        1432 :             pinned);
    1863             : 
    1864             :     // Compare against expected signature.
    1865        1432 :     __ LoadConstant(LiftoffRegister(tmp_const), WasmValue(canonical_sig_num));
    1866             : 
    1867             :     Label* sig_mismatch_label = AddOutOfLineTrap(
    1868             :         decoder->position(), WasmCode::kThrowWasmTrapFuncSigMismatch);
    1869             :     __ emit_cond_jump(kUnequal, sig_mismatch_label,
    1870        1432 :                       LiftoffAssembler::kWasmIntPtr, scratch, tmp_const);
    1871             : 
    1872             :     // At this point {index} has already been multiplied by 4.
    1873             :     DEBUG_CODE_COMMENT("Execute indirect call");
    1874             :     if (kTaggedSize != kInt32Size) {
    1875             :       DCHECK_EQ(kTaggedSize, kInt32Size * 2);
    1876             :       // Multiply {index} by another 2 to represent kTaggedSize items.
    1877        1432 :       __ emit_i32_add(index, index, index);
    1878             :     }
    1879             :     // At this point {index} has already been multiplied by kTaggedSize.
    1880             : 
    1881             :     // Load the instance from {instance->ift_instances[key]}
    1882        1432 :     LOAD_TAGGED_PTR_INSTANCE_FIELD(table, IndirectFunctionTableRefs);
    1883             :     __ LoadTaggedPointer(tmp_const, table, index,
    1884             :                          ObjectAccess::ElementOffsetInTaggedFixedArray(0),
    1885        1432 :                          pinned);
    1886             : 
    1887             :     if (kTaggedSize != kSystemPointerSize) {
    1888             :       DCHECK_EQ(kSystemPointerSize, kTaggedSize * 2);
    1889             :       // Multiply {index} by another 2 to represent kSystemPointerSize items.
    1890             :       __ emit_i32_add(index, index, index);
    1891             :     }
    1892             :     // At this point {index} has already been multiplied by kSystemPointerSize.
    1893             : 
    1894             :     Register* explicit_instance = &tmp_const;
    1895             : 
    1896             :     // Load the target from {instance->ift_targets[key]}
    1897        1431 :     LOAD_INSTANCE_FIELD(table, IndirectFunctionTableTargets,
    1898             :                         kSystemPointerSize);
    1899        1432 :     __ Load(LiftoffRegister(scratch), table, index, 0, kPointerLoadType,
    1900        1431 :             pinned);
    1901             : 
    1902        2864 :     source_position_table_builder_.AddPosition(
    1903        1432 :         __ pc_offset(), SourcePosition(decoder->position()), false);
    1904             : 
    1905             :     auto call_descriptor =
    1906        1432 :         compiler::GetWasmCallDescriptor(compilation_zone_, imm.sig);
    1907             :     call_descriptor =
    1908             :         GetLoweredCallDescriptor(compilation_zone_, call_descriptor);
    1909             : 
    1910        1432 :     Register target = scratch;
    1911        1432 :     __ PrepareCall(imm.sig, call_descriptor, &target, explicit_instance);
    1912        1432 :     __ CallIndirect(imm.sig, call_descriptor, target);
    1913             : 
    1914             :     safepoint_table_builder_.DefineSafepoint(&asm_, Safepoint::kSimple,
    1915        1431 :                                              Safepoint::kNoLazyDeopt);
    1916             : 
    1917        1431 :     __ FinishCall(imm.sig, call_descriptor);
    1918             :   }
    1919             : 
    1920             :   void ReturnCall(FullDecoder* decoder,
    1921             :                   const CallFunctionImmediate<validate>& imm,
    1922             :                   const Value args[]) {
    1923             :     unsupported(decoder, "return_call");
    1924             :   }
    1925             :   void ReturnCallIndirect(FullDecoder* decoder, const Value& index_val,
    1926             :                           const CallIndirectImmediate<validate>& imm,
    1927             :                           const Value args[]) {
    1928             :     unsupported(decoder, "return_call_indirect");
    1929             :   }
    1930             :   void SimdOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args,
    1931             :               Value* result) {
    1932             :     unsupported(decoder, "simd");
    1933             :   }
    1934             :   void SimdLaneOp(FullDecoder* decoder, WasmOpcode opcode,
    1935             :                   const SimdLaneImmediate<validate>& imm,
    1936             :                   const Vector<Value> inputs, Value* result) {
    1937             :     unsupported(decoder, "simd");
    1938             :   }
    1939             :   void SimdShiftOp(FullDecoder* decoder, WasmOpcode opcode,
    1940             :                    const SimdShiftImmediate<validate>& imm, const Value& input,
    1941             :                    Value* result) {
    1942             :     unsupported(decoder, "simd");
    1943             :   }
    1944             :   void Simd8x16ShuffleOp(FullDecoder* decoder,
    1945             :                          const Simd8x16ShuffleImmediate<validate>& imm,
    1946             :                          const Value& input0, const Value& input1,
    1947             :                          Value* result) {
    1948             :     unsupported(decoder, "simd");
    1949             :   }
    1950             :   void Throw(FullDecoder* decoder, const ExceptionIndexImmediate<validate>&,
    1951             :              const Vector<Value>& args) {
    1952             :     unsupported(decoder, "throw");
    1953             :   }
    1954             :   void Rethrow(FullDecoder* decoder, const Value& exception) {
    1955             :     unsupported(decoder, "rethrow");
    1956             :   }
    1957             :   void BrOnException(FullDecoder* decoder, const Value& exception,
    1958             :                      const ExceptionIndexImmediate<validate>& imm,
    1959             :                      uint32_t depth, Vector<Value> values) {
    1960             :     unsupported(decoder, "br_on_exn");
    1961             :   }
    1962             :   void AtomicOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args,
    1963             :                 const MemoryAccessImmediate<validate>& imm, Value* result) {
    1964             :     unsupported(decoder, "atomicop");
    1965             :   }
    1966             :   void MemoryInit(FullDecoder* decoder,
    1967             :                   const MemoryInitImmediate<validate>& imm, const Value& dst,
    1968             :                   const Value& src, const Value& size) {
    1969             :     unsupported(decoder, "memory.init");
    1970             :   }
    1971             :   void DataDrop(FullDecoder* decoder, const DataDropImmediate<validate>& imm) {
    1972             :     unsupported(decoder, "data.drop");
    1973             :   }
    1974             :   void MemoryCopy(FullDecoder* decoder,
    1975             :                   const MemoryCopyImmediate<validate>& imm, const Value& dst,
    1976             :                   const Value& src, const Value& size) {
    1977             :     unsupported(decoder, "memory.copy");
    1978             :   }
    1979             :   void MemoryFill(FullDecoder* decoder,
    1980             :                   const MemoryIndexImmediate<validate>& imm, const Value& dst,
    1981             :                   const Value& value, const Value& size) {
    1982             :     unsupported(decoder, "memory.fill");
    1983             :   }
    1984             :   void TableInit(FullDecoder* decoder, const TableInitImmediate<validate>& imm,
    1985             :                  Vector<Value> args) {
    1986             :     unsupported(decoder, "table.init");
    1987             :   }
    1988             :   void ElemDrop(FullDecoder* decoder, const ElemDropImmediate<validate>& imm) {
    1989             :     unsupported(decoder, "elem.drop");
    1990             :   }
    1991             :   void TableCopy(FullDecoder* decoder, const TableCopyImmediate<validate>& imm,
    1992             :                  Vector<Value> args) {
    1993             :     unsupported(decoder, "table.copy");
    1994             :   }
    1995             : 
    1996             :  private:
    1997             :   LiftoffAssembler asm_;
    1998             :   compiler::CallDescriptor* const descriptor_;
    1999             :   CompilationEnv* const env_;
    2000             :   bool ok_ = true;
    2001             :   std::vector<OutOfLineCode> out_of_line_code_;
    2002             :   SourcePositionTableBuilder source_position_table_builder_;
    2003             :   std::vector<trap_handler::ProtectedInstructionData> protected_instructions_;
    2004             :   // Zone used to store information during compilation. The result will be
    2005             :   // stored independently, such that this zone can die together with the
    2006             :   // LiftoffCompiler after compilation.
    2007             :   Zone* compilation_zone_;
    2008             :   SafepointTableBuilder safepoint_table_builder_;
    2009             :   // The pc offset of the instructions to reserve the stack frame. Needed to
    2010             :   // patch the actually needed stack size in the end.
    2011             :   uint32_t pc_offset_stack_frame_construction_ = 0;
    2012             : 
    2013             :   void TraceCacheState(FullDecoder* decoder) const {
    2014             : #ifdef DEBUG
    2015             :     if (!FLAG_trace_liftoff || !FLAG_trace_wasm_decoder) return;
    2016             :     StdoutStream os;
    2017             :     for (int control_depth = decoder->control_depth() - 1; control_depth >= -1;
    2018             :          --control_depth) {
    2019             :       auto* cache_state =
    2020             :           control_depth == -1 ? __ cache_state()
    2021             :                               : &decoder->control_at(control_depth)
    2022             :                                      ->label_state;
    2023             :       os << PrintCollection(cache_state->stack_state);
    2024             :       if (control_depth != -1) PrintF("; ");
    2025             :     }
    2026             :     os << "\n";
    2027             : #endif
    2028             :   }
    2029             : 
    2030             :   DISALLOW_IMPLICIT_CONSTRUCTORS(LiftoffCompiler);
    2031             : };
    2032             : 
    2033             : }  // namespace
    2034             : 
    2035      461206 : WasmCompilationResult LiftoffCompilationUnit::ExecuteCompilation(
    2036             :     AccountingAllocator* allocator, CompilationEnv* env,
    2037             :     const FunctionBody& func_body, Counters* counters, WasmFeatures* detected) {
    2038     1383648 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    2039             :                "ExecuteLiftoffCompilation");
    2040             :   base::ElapsedTimer compile_timer;
    2041             :   if (FLAG_trace_wasm_decode_time) {
    2042             :     compile_timer.Start();
    2043             :   }
    2044             : 
    2045      922653 :   Zone zone(allocator, "LiftoffCompilationZone");
    2046      461028 :   const WasmModule* module = env ? env->module : nullptr;
    2047      461028 :   auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig);
    2048             :   base::Optional<TimedHistogramScope> liftoff_compile_time_scope(
    2049             :       base::in_place, counters->liftoff_compile_time());
    2050             :   std::unique_ptr<wasm::WasmInstructionBuffer> instruction_buffer =
    2051      922625 :       wasm::WasmInstructionBuffer::New();
    2052             :   WasmFullDecoder<Decoder::kValidate, LiftoffCompiler> decoder(
    2053      461252 :       &zone, module, env->enabled_features, detected, func_body,
    2054     1844176 :       call_descriptor, env, &zone, instruction_buffer->CreateView());
    2055      460810 :   decoder.Decode();
    2056             :   liftoff_compile_time_scope.reset();
    2057             :   LiftoffCompiler* compiler = &decoder.interface();
    2058      460638 :   if (decoder.failed()) {
    2059             :     compiler->OnFirstError(&decoder);
    2060       19235 :     return WasmCompilationResult{};
    2061             :   }
    2062      441403 :   if (!compiler->ok()) {
    2063             :     // Liftoff compilation failed.
    2064           0 :     counters->liftoff_unsupported_functions()->Increment();
    2065           0 :     return WasmCompilationResult{};
    2066             :   }
    2067             : 
    2068      441403 :   counters->liftoff_compiled_functions()->Increment();
    2069             : 
    2070             :   if (FLAG_trace_wasm_decode_time) {
    2071             :     double compile_ms = compile_timer.Elapsed().InMillisecondsF();
    2072             :     PrintF(
    2073             :         "wasm-compilation liftoff phase 1 ok: %u bytes, %0.3f ms decode and "
    2074             :         "compile\n",
    2075             :         static_cast<unsigned>(func_body.end - func_body.start), compile_ms);
    2076             :   }
    2077             : 
    2078      441575 :   WasmCompilationResult result;
    2079             :   compiler->GetCode(&result.code_desc);
    2080      882770 :   result.instr_buffer = instruction_buffer->ReleaseBuffer();
    2081      441582 :   result.source_positions = compiler->GetSourcePositionTable();
    2082      441492 :   result.protected_instructions = compiler->GetProtectedInstructions();
    2083      441492 :   result.frame_slot_count = compiler->GetTotalFrameSlotCount();
    2084      441492 :   result.tagged_parameter_slots = call_descriptor->GetTaggedParameterSlots();
    2085      441575 :   result.result_tier = ExecutionTier::kLiftoff;
    2086             : 
    2087             :   DCHECK(result.succeeded());
    2088             :   return result;
    2089             : }
    2090             : 
    2091             : #undef __
    2092             : #undef TRACE
    2093             : #undef WASM_INSTANCE_OBJECT_FIELD_OFFSET
    2094             : #undef WASM_INSTANCE_OBJECT_FIELD_SIZE
    2095             : #undef LOAD_INSTANCE_FIELD
    2096             : #undef LOAD_TAGGED_PTR_INSTANCE_FIELD
    2097             : #undef DEBUG_CODE_COMMENT
    2098             : 
    2099             : }  // namespace wasm
    2100             : }  // namespace internal
    2101      121996 : }  // namespace v8

Generated by: LCOV version 1.10