LCOV - code coverage report
Current view: top level - src/compiler - wasm-compiler.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 2200 2611 84.3 %
Date: 2019-03-21 Functions: 196 244 80.3 %

          Line data    Source code
       1             : // Copyright 2015 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #include "src/compiler/wasm-compiler.h"
       6             : 
       7             : #include <memory>
       8             : 
       9             : #include "src/assembler-inl.h"
      10             : #include "src/assembler.h"
      11             : #include "src/base/optional.h"
      12             : #include "src/base/platform/elapsed-timer.h"
      13             : #include "src/base/platform/platform.h"
      14             : #include "src/base/v8-fallthrough.h"
      15             : #include "src/builtins/builtins.h"
      16             : #include "src/code-factory.h"
      17             : #include "src/compiler/backend/code-generator.h"
      18             : #include "src/compiler/backend/instruction-selector.h"
      19             : #include "src/compiler/common-operator.h"
      20             : #include "src/compiler/compiler-source-position-table.h"
      21             : #include "src/compiler/diamond.h"
      22             : #include "src/compiler/graph-visualizer.h"
      23             : #include "src/compiler/graph.h"
      24             : #include "src/compiler/int64-lowering.h"
      25             : #include "src/compiler/js-graph.h"
      26             : #include "src/compiler/js-operator.h"
      27             : #include "src/compiler/linkage.h"
      28             : #include "src/compiler/machine-operator.h"
      29             : #include "src/compiler/node-matchers.h"
      30             : #include "src/compiler/node-origin-table.h"
      31             : #include "src/compiler/pipeline.h"
      32             : #include "src/compiler/simd-scalar-lowering.h"
      33             : #include "src/compiler/zone-stats.h"
      34             : #include "src/counters.h"
      35             : #include "src/heap/factory.h"
      36             : #include "src/interface-descriptors.h"
      37             : #include "src/isolate-inl.h"
      38             : #include "src/log.h"
      39             : #include "src/objects/heap-number.h"
      40             : #include "src/optimized-compilation-info.h"
      41             : #include "src/tracing/trace-event.h"
      42             : #include "src/trap-handler/trap-handler.h"
      43             : #include "src/vector.h"
      44             : #include "src/wasm/function-body-decoder-impl.h"
      45             : #include "src/wasm/function-compiler.h"
      46             : #include "src/wasm/graph-builder-interface.h"
      47             : #include "src/wasm/jump-table-assembler.h"
      48             : #include "src/wasm/memory-tracing.h"
      49             : #include "src/wasm/object-access.h"
      50             : #include "src/wasm/wasm-code-manager.h"
      51             : #include "src/wasm/wasm-limits.h"
      52             : #include "src/wasm/wasm-linkage.h"
      53             : #include "src/wasm/wasm-module.h"
      54             : #include "src/wasm/wasm-objects-inl.h"
      55             : #include "src/wasm/wasm-opcodes.h"
      56             : #include "src/wasm/wasm-text.h"
      57             : 
      58             : namespace v8 {
      59             : namespace internal {
      60             : namespace compiler {
      61             : 
      62             : namespace {
      63             : 
      64             : // TODO(titzer): pull WASM_64 up to a common header.
      65             : #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
      66             : #define WASM_64 1
      67             : #else
      68             : #define WASM_64 0
      69             : #endif
      70             : 
      71             : #define FATAL_UNSUPPORTED_OPCODE(opcode)        \
      72             :   FATAL("Unsupported opcode 0x%x:%s", (opcode), \
      73             :         wasm::WasmOpcodes::OpcodeName(opcode));
      74             : 
      75             : MachineType assert_size(int expected_size, MachineType type) {
      76             :   DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()));
      77             :   return type;
      78             : }
      79             : 
      80             : #define WASM_INSTANCE_OBJECT_SIZE(name)     \
      81             :   (WasmInstanceObject::k##name##OffsetEnd - \
      82             :    WasmInstanceObject::k##name##Offset + 1)  // NOLINT(whitespace/indent)
      83             : 
      84             : #define WASM_INSTANCE_OBJECT_OFFSET(name) \
      85             :   wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
      86             : 
      87             : #define LOAD_RAW(base_pointer, byte_offset, type)                             \
      88             :   SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), base_pointer,  \
      89             :                              mcgraph()->Int32Constant(byte_offset), Effect(), \
      90             :                              Control()))
      91             : 
      92             : #define LOAD_INSTANCE_FIELD(name, type)                             \
      93             :   LOAD_RAW(instance_node_.get(), WASM_INSTANCE_OBJECT_OFFSET(name), \
      94             :            assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type))
      95             : 
      96             : #define LOAD_TAGGED_POINTER(base_pointer, byte_offset) \
      97             :   LOAD_RAW(base_pointer, byte_offset, MachineType::TaggedPointer())
      98             : 
      99             : #define LOAD_TAGGED_ANY(base_pointer, byte_offset) \
     100             :   LOAD_RAW(base_pointer, byte_offset, MachineType::AnyTagged())
     101             : 
     102             : #define LOAD_FIXED_ARRAY_SLOT(array_node, index, type) \
     103             :   LOAD_RAW(array_node,                                 \
     104             :            wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), type)
     105             : 
     106             : #define LOAD_FIXED_ARRAY_SLOT_SMI(array_node, index) \
     107             :   LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedSigned())
     108             : 
     109             : #define LOAD_FIXED_ARRAY_SLOT_PTR(array_node, index) \
     110             :   LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedPointer())
     111             : 
     112             : #define LOAD_FIXED_ARRAY_SLOT_ANY(array_node, index) \
     113             :   LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::AnyTagged())
     114             : 
     115             : // This can be used to store tagged Smi values only.
     116             : #define STORE_FIXED_ARRAY_SLOT_SMI(array_node, index, value)           \
     117             :   SetEffect(graph()->NewNode(                                          \
     118             :       mcgraph()->machine()->Store(StoreRepresentation(                 \
     119             :           MachineRepresentation::kTaggedSigned, kNoWriteBarrier)),     \
     120             :       array_node,                                                      \
     121             :       mcgraph()->Int32Constant(                                        \
     122             :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
     123             :       value, Effect(), Control()))
     124             : 
     125             : // This can be used to store any tagged (Smi and HeapObject) value.
     126             : #define STORE_FIXED_ARRAY_SLOT_ANY(array_node, index, value)           \
     127             :   SetEffect(graph()->NewNode(                                          \
     128             :       mcgraph()->machine()->Store(StoreRepresentation(                 \
     129             :           MachineRepresentation::kTagged, kFullWriteBarrier)),         \
     130             :       array_node,                                                      \
     131             :       mcgraph()->Int32Constant(                                        \
     132             :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
     133             :       value, Effect(), Control()))
     134             : 
     135     1050184 : void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
     136             :   Graph* g = mcgraph->graph();
     137     1050184 :   if (g->end()) {
     138       36875 :     NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
     139             :   } else {
     140     1013309 :     g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
     141             :   }
     142     1051252 : }
     143             : 
     144             : bool ContainsSimd(wasm::FunctionSig* sig) {
     145     3476476 :   for (auto type : sig->all()) {
     146     1227448 :     if (type == wasm::kWasmS128) return true;
     147             :   }
     148             :   return false;
     149             : }
     150             : 
     151             : bool ContainsInt64(wasm::FunctionSig* sig) {
     152     1140858 :   for (auto type : sig->all()) {
     153      388583 :     if (type == wasm::kWasmI64) return true;
     154             :   }
     155             :   return false;
     156             : }
     157             : }  // namespace
     158             : 
     159         492 : WasmGraphBuilder::WasmGraphBuilder(
     160             :     wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
     161             :     wasm::FunctionSig* sig,
     162             :     compiler::SourcePositionTable* source_position_table)
     163             :     : zone_(zone),
     164             :       mcgraph_(mcgraph),
     165             :       env_(env),
     166             :       cur_buffer_(def_buffer_),
     167             :       cur_bufsize_(kDefaultBufferSize),
     168             :       has_simd_(ContainsSimd(sig)),
     169             :       untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
     170             :       sig_(sig),
     171     3062682 :       source_position_table_(source_position_table) {
     172             :   DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
     173             :   DCHECK_NOT_NULL(mcgraph_);
     174         492 : }
     175             : 
     176           0 : Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
     177             : 
     178     1021419 : Node* WasmGraphBuilder::Start(unsigned params) {
     179     1021419 :   Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
     180             :   graph()->SetStart(start);
     181     1021031 :   return start;
     182             : }
     183             : 
     184     1239946 : Node* WasmGraphBuilder::Param(unsigned index) {
     185     1239946 :   return graph()->NewNode(mcgraph()->common()->Parameter(index),
     186     1240567 :                           graph()->start());
     187             : }
     188             : 
     189        9191 : Node* WasmGraphBuilder::Loop(Node* entry) {
     190       18386 :   return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
     191             : }
     192             : 
     193        9193 : Node* WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
     194             :   Node* terminate =
     195        9193 :       graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
     196        9195 :   MergeControlToEnd(mcgraph(), terminate);
     197        9193 :   return terminate;
     198             : }
     199             : 
     200         980 : Node* WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
     201             :   Node* terminate =
     202         980 :       graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
     203         981 :   MergeControlToEnd(mcgraph(), terminate);
     204         980 :   return terminate;
     205             : }
     206             : 
     207     1206444 : bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
     208     2639582 :   return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
     209     1433135 :          NodeProperties::GetControlInput(phi) == merge;
     210             : }
     211             : 
     212        1287 : bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
     213             :                                        Node** if_exception) {
     214        1287 :   if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
     215             :     return false;
     216             :   }
     217             : 
     218        1326 :   *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
     219             :   *if_exception =
     220        1326 :       graph()->NewNode(mcgraph()->common()->IfException(), node, node);
     221             : 
     222         663 :   return true;
     223             : }
     224             : 
     225      213527 : void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
     226             :   DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
     227      213527 :   merge->AppendInput(mcgraph()->zone(), from);
     228             :   int new_size = merge->InputCount();
     229      213526 :   NodeProperties::ChangeOp(
     230      213527 :       merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
     231      213527 : }
     232             : 
     233      107847 : void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
     234             :   DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
     235             :   int new_size = phi->InputCount();
     236      107847 :   phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
     237      107851 :   NodeProperties::ChangeOp(
     238      107849 :       phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
     239      107848 : }
     240             : 
     241       35271 : Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
     242       35271 :   return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
     243             : }
     244             : 
     245       98078 : Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
     246             :                             Node* control) {
     247             :   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
     248       98078 :   Node** buf = Realloc(vals, count, count + 1);
     249       98085 :   buf[count] = control;
     250       98085 :   return graph()->NewNode(
     251             :       mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
     252             :                                count),
     253       98078 :       count + 1, buf);
     254             : }
     255             : 
     256       31290 : Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
     257             :                                   Node* control) {
     258             :   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
     259       31290 :   Node** buf = Realloc(effects, count, count + 1);
     260       31294 :   buf[count] = control;
     261       31294 :   return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
     262       31294 :                           buf);
     263             : }
     264             : 
     265         232 : Node* WasmGraphBuilder::RefNull() {
     266         696 :   return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
     267             : }
     268             : 
     269           0 : Node* WasmGraphBuilder::NoContextConstant() {
     270             :   // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
     271      370274 :   return mcgraph()->IntPtrConstant(0);
     272             : }
     273             : 
     274           0 : Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
     275           0 :   return mcgraph()->Uint32Constant(value);
     276             : }
     277             : 
     278     1633197 : Node* WasmGraphBuilder::Int32Constant(int32_t value) {
     279     2050713 :   return mcgraph()->Int32Constant(value);
     280             : }
     281             : 
     282       28106 : Node* WasmGraphBuilder::Int64Constant(int64_t value) {
     283       28653 :   return mcgraph()->Int64Constant(value);
     284             : }
     285             : 
     286           0 : Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
     287       37917 :   return mcgraph()->IntPtrConstant(value);
     288             : }
     289             : 
     290       23014 : void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
     291             :                                   Node** effect, Node** control) {
     292             :   DCHECK_NOT_NULL(env_);  // Wrappers don't get stack checks.
     293       23014 :   if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
     294        3192 :     return;
     295             :   }
     296       19822 :   if (effect == nullptr) effect = effect_;
     297       19822 :   if (control == nullptr) control = control_;
     298             : 
     299             :   // This instruction sequence is matched in the instruction selector to
     300             :   // load the stack pointer directly on some platforms. Hence, when modifying
     301             :   // please also fix WasmStackCheckMatcher in node-matchers.h
     302             : 
     303       39688 :   Node* limit_address = graph()->NewNode(
     304             :       mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
     305             :       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
     306             :       *effect, *control);
     307       19869 :   Node* limit = graph()->NewNode(
     308             :       mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
     309             :       mcgraph()->IntPtrConstant(0), limit_address, *control);
     310       19854 :   *effect = limit;
     311       19854 :   Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
     312             : 
     313             :   Node* check =
     314       19832 :       graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
     315             : 
     316       19845 :   Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
     317       19848 :   stack_check.Chain(*control);
     318             : 
     319       19856 :   if (stack_check_call_operator_ == nullptr) {
     320             :     // Build and cache the stack check call operator and the constant
     321             :     // representing the stack check code.
     322       12340 :     auto call_descriptor = Linkage::GetStubCallDescriptor(
     323             :         mcgraph()->zone(),                    // zone
     324             :         NoContextDescriptor{},                // descriptor
     325             :         0,                                    // stack parameter count
     326             :         CallDescriptor::kNoFlags,             // flags
     327             :         Operator::kNoProperties,              // properties
     328       12359 :         StubCallMode::kCallWasmRuntimeStub);  // stub call mode
     329             :     // A direct call to a wasm runtime stub defined in this module.
     330             :     // Just encode the stub index. This will be patched at relocation.
     331       12340 :     stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
     332             :         wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
     333       12374 :     stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
     334             :   }
     335             : 
     336       19874 :   Node* call = graph()->NewNode(stack_check_call_operator_.get(),
     337             :                                 stack_check_code_node_.get(), *effect,
     338             :                                 stack_check.if_false);
     339             : 
     340             :   SetSourcePosition(call, position);
     341             : 
     342       19877 :   Node* ephi = stack_check.EffectPhi(*effect, call);
     343             : 
     344       19842 :   *control = stack_check.merge;
     345       19842 :   *effect = ephi;
     346             : }
     347             : 
     348      495745 : void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
     349      980708 :   if (!needs_stack_check_) return;
     350             : 
     351             :   Node* start = graph()->start();
     352             :   // Place a stack check which uses a dummy node as control and effect.
     353       13819 :   Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
     354       13837 :   Node* control = dummy;
     355       13837 :   Node* effect = dummy;
     356             :   // The function-prologue stack check is associated with position 0, which
     357             :   // is never a position of any instruction in the function.
     358       13837 :   StackCheck(0, &effect, &control);
     359             : 
     360             :   // In testing, no steck checks were emitted. Nothing to rewire then.
     361       13857 :   if (effect == dummy) return;
     362             : 
     363             :   // Now patch all control uses of {start} to use {control} and all effect uses
     364             :   // to use {effect} instead. Then rewire the dummy node to use start instead.
     365       10820 :   NodeProperties::ReplaceUses(start, start, effect, control);
     366       10848 :   NodeProperties::ReplaceUses(dummy, nullptr, start, start);
     367             : }
     368             : 
     369      770903 : Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
     370             :                               wasm::WasmCodePosition position) {
     371             :   const Operator* op;
     372             :   MachineOperatorBuilder* m = mcgraph()->machine();
     373      770903 :   switch (opcode) {
     374             :     case wasm::kExprI32Add:
     375      105718 :       op = m->Int32Add();
     376      105706 :       break;
     377             :     case wasm::kExprI32Sub:
     378       27574 :       op = m->Int32Sub();
     379       27572 :       break;
     380             :     case wasm::kExprI32Mul:
     381       18089 :       op = m->Int32Mul();
     382       18089 :       break;
     383             :     case wasm::kExprI32DivS:
     384       14180 :       return BuildI32DivS(left, right, position);
     385             :     case wasm::kExprI32DivU:
     386       14124 :       return BuildI32DivU(left, right, position);
     387             :     case wasm::kExprI32RemS:
     388       14104 :       return BuildI32RemS(left, right, position);
     389             :     case wasm::kExprI32RemU:
     390       14084 :       return BuildI32RemU(left, right, position);
     391             :     case wasm::kExprI32And:
     392      111339 :       op = m->Word32And();
     393      111339 :       break;
     394             :     case wasm::kExprI32Ior:
     395       19049 :       op = m->Word32Or();
     396       19049 :       break;
     397             :     case wasm::kExprI32Xor:
     398       14367 :       op = m->Word32Xor();
     399       14367 :       break;
     400             :     case wasm::kExprI32Shl:
     401       25741 :       op = m->Word32Shl();
     402       25741 :       right = MaskShiftCount32(right);
     403       25741 :       break;
     404             :     case wasm::kExprI32ShrU:
     405       29469 :       op = m->Word32Shr();
     406       29469 :       right = MaskShiftCount32(right);
     407       29469 :       break;
     408             :     case wasm::kExprI32ShrS:
     409       15615 :       op = m->Word32Sar();
     410       15615 :       right = MaskShiftCount32(right);
     411       15615 :       break;
     412             :     case wasm::kExprI32Ror:
     413       26944 :       op = m->Word32Ror();
     414       26944 :       right = MaskShiftCount32(right);
     415       26944 :       break;
     416             :     case wasm::kExprI32Rol:
     417       13472 :       right = MaskShiftCount32(right);
     418       13472 :       return BuildI32Rol(left, right);
     419             :     case wasm::kExprI32Eq:
     420       69044 :       op = m->Word32Equal();
     421       69044 :       break;
     422             :     case wasm::kExprI32Ne:
     423       20856 :       return Invert(Binop(wasm::kExprI32Eq, left, right));
     424             :     case wasm::kExprI32LtS:
     425       15502 :       op = m->Int32LessThan();
     426       15502 :       break;
     427             :     case wasm::kExprI32LeS:
     428       14405 :       op = m->Int32LessThanOrEqual();
     429       14405 :       break;
     430             :     case wasm::kExprI32LtU:
     431       17819 :       op = m->Uint32LessThan();
     432       17819 :       break;
     433             :     case wasm::kExprI32LeU:
     434       13632 :       op = m->Uint32LessThanOrEqual();
     435       13632 :       break;
     436             :     case wasm::kExprI32GtS:
     437       15677 :       op = m->Int32LessThan();
     438             :       std::swap(left, right);
     439             :       break;
     440             :     case wasm::kExprI32GeS:
     441       14060 :       op = m->Int32LessThanOrEqual();
     442             :       std::swap(left, right);
     443             :       break;
     444             :     case wasm::kExprI32GtU:
     445       15194 :       op = m->Uint32LessThan();
     446             :       std::swap(left, right);
     447             :       break;
     448             :     case wasm::kExprI32GeU:
     449       13632 :       op = m->Uint32LessThanOrEqual();
     450             :       std::swap(left, right);
     451             :       break;
     452             :     case wasm::kExprI64And:
     453         167 :       op = m->Word64And();
     454         167 :       break;
     455             :     case wasm::kExprI64Add:
     456         675 :       op = m->Int64Add();
     457         675 :       break;
     458             :     case wasm::kExprI64Sub:
     459         830 :       op = m->Int64Sub();
     460         830 :       break;
     461             :     case wasm::kExprI64Mul:
     462         806 :       op = m->Int64Mul();
     463         806 :       break;
     464             :     case wasm::kExprI64DivS:
     465         720 :       return BuildI64DivS(left, right, position);
     466             :     case wasm::kExprI64DivU:
     467         660 :       return BuildI64DivU(left, right, position);
     468             :     case wasm::kExprI64RemS:
     469         644 :       return BuildI64RemS(left, right, position);
     470             :     case wasm::kExprI64RemU:
     471         644 :       return BuildI64RemU(left, right, position);
     472             :     case wasm::kExprI64Ior:
     473       13805 :       op = m->Word64Or();
     474       13792 :       break;
     475             :     case wasm::kExprI64Xor:
     476          56 :       op = m->Word64Xor();
     477          56 :       break;
     478             :     case wasm::kExprI64Shl:
     479       14299 :       op = m->Word64Shl();
     480       14302 :       right = MaskShiftCount64(right);
     481       14299 :       break;
     482             :     case wasm::kExprI64ShrU:
     483         684 :       op = m->Word64Shr();
     484         684 :       right = MaskShiftCount64(right);
     485         684 :       break;
     486             :     case wasm::kExprI64ShrS:
     487         628 :       op = m->Word64Sar();
     488         628 :       right = MaskShiftCount64(right);
     489         628 :       break;
     490             :     case wasm::kExprI64Eq:
     491         286 :       op = m->Word64Equal();
     492         286 :       break;
     493             :     case wasm::kExprI64Ne:
     494          68 :       return Invert(Binop(wasm::kExprI64Eq, left, right));
     495             :     case wasm::kExprI64LtS:
     496          56 :       op = m->Int64LessThan();
     497          56 :       break;
     498             :     case wasm::kExprI64LeS:
     499          32 :       op = m->Int64LessThanOrEqual();
     500          32 :       break;
     501             :     case wasm::kExprI64LtU:
     502          40 :       op = m->Uint64LessThan();
     503          40 :       break;
     504             :     case wasm::kExprI64LeU:
     505          48 :       op = m->Uint64LessThanOrEqual();
     506          48 :       break;
     507             :     case wasm::kExprI64GtS:
     508          40 :       op = m->Int64LessThan();
     509             :       std::swap(left, right);
     510             :       break;
     511             :     case wasm::kExprI64GeS:
     512          32 :       op = m->Int64LessThanOrEqual();
     513             :       std::swap(left, right);
     514             :       break;
     515             :     case wasm::kExprI64GtU:
     516          40 :       op = m->Uint64LessThan();
     517             :       std::swap(left, right);
     518             :       break;
     519             :     case wasm::kExprI64GeU:
     520          32 :       op = m->Uint64LessThanOrEqual();
     521             :       std::swap(left, right);
     522             :       break;
     523             :     case wasm::kExprI64Ror:
     524         120 :       op = m->Word64Ror();
     525         120 :       right = MaskShiftCount64(right);
     526         120 :       break;
     527             :     case wasm::kExprI64Rol:
     528          60 :       return BuildI64Rol(left, right);
     529             :     case wasm::kExprF32CopySign:
     530          32 :       return BuildF32CopySign(left, right);
     531             :     case wasm::kExprF64CopySign:
     532          32 :       return BuildF64CopySign(left, right);
     533             :     case wasm::kExprF32Add:
     534         461 :       op = m->Float32Add();
     535         461 :       break;
     536             :     case wasm::kExprF32Sub:
     537         305 :       op = m->Float32Sub();
     538         305 :       break;
     539             :     case wasm::kExprF32Mul:
     540         425 :       op = m->Float32Mul();
     541         425 :       break;
     542             :     case wasm::kExprF32Div:
     543         348 :       op = m->Float32Div();
     544         348 :       break;
     545             :     case wasm::kExprF32Eq:
     546         438 :       op = m->Float32Equal();
     547         438 :       break;
     548             :     case wasm::kExprF32Ne:
     549         333 :       return Invert(Binop(wasm::kExprF32Eq, left, right));
     550             :     case wasm::kExprF32Lt:
     551         153 :       op = m->Float32LessThan();
     552         153 :       break;
     553             :     case wasm::kExprF32Ge:
     554         105 :       op = m->Float32LessThanOrEqual();
     555             :       std::swap(left, right);
     556             :       break;
     557             :     case wasm::kExprF32Gt:
     558         129 :       op = m->Float32LessThan();
     559             :       std::swap(left, right);
     560             :       break;
     561             :     case wasm::kExprF32Le:
     562         121 :       op = m->Float32LessThanOrEqual();
     563         121 :       break;
     564             :     case wasm::kExprF64Add:
     565        4174 :       op = m->Float64Add();
     566        4173 :       break;
     567             :     case wasm::kExprF64Sub:
     568        2902 :       op = m->Float64Sub();
     569        2902 :       break;
     570             :     case wasm::kExprF64Mul:
     571        6486 :       op = m->Float64Mul();
     572        6486 :       break;
     573             :     case wasm::kExprF64Div:
     574         638 :       op = m->Float64Div();
     575         638 :       break;
     576             :     case wasm::kExprF64Eq:
     577         726 :       op = m->Float64Equal();
     578         726 :       break;
     579             :     case wasm::kExprF64Ne:
     580         410 :       return Invert(Binop(wasm::kExprF64Eq, left, right));
     581             :     case wasm::kExprF64Lt:
     582         742 :       op = m->Float64LessThan();
     583         742 :       break;
     584             :     case wasm::kExprF64Le:
     585         303 :       op = m->Float64LessThanOrEqual();
     586         303 :       break;
     587             :     case wasm::kExprF64Gt:
     588         696 :       op = m->Float64LessThan();
     589             :       std::swap(left, right);
     590             :       break;
     591             :     case wasm::kExprF64Ge:
     592         301 :       op = m->Float64LessThanOrEqual();
     593             :       std::swap(left, right);
     594             :       break;
     595             :     case wasm::kExprF32Min:
     596          70 :       op = m->Float32Min();
     597          70 :       break;
     598             :     case wasm::kExprF64Min:
     599          80 :       op = m->Float64Min();
     600          80 :       break;
     601             :     case wasm::kExprF32Max:
     602          70 :       op = m->Float32Max();
     603          70 :       break;
     604             :     case wasm::kExprF64Max:
     605          72 :       op = m->Float64Max();
     606          72 :       break;
     607             :     case wasm::kExprF64Pow:
     608           9 :       return BuildF64Pow(left, right);
     609             :     case wasm::kExprF64Atan2:
     610          17 :       op = m->Float64Atan2();
     611          17 :       break;
     612             :     case wasm::kExprF64Mod:
     613          20 :       return BuildF64Mod(left, right);
     614             :     case wasm::kExprI32AsmjsDivS:
     615         385 :       return BuildI32AsmjsDivS(left, right);
     616             :     case wasm::kExprI32AsmjsDivU:
     617         137 :       return BuildI32AsmjsDivU(left, right);
     618             :     case wasm::kExprI32AsmjsRemS:
     619         421 :       return BuildI32AsmjsRemS(left, right);
     620             :     case wasm::kExprI32AsmjsRemU:
     621         186 :       return BuildI32AsmjsRemU(left, right);
     622             :     case wasm::kExprI32AsmjsStoreMem8:
     623        3041 :       return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
     624             :     case wasm::kExprI32AsmjsStoreMem16:
     625        1385 :       return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
     626             :     case wasm::kExprI32AsmjsStoreMem:
     627       31530 :       return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
     628             :     case wasm::kExprF32AsmjsStoreMem:
     629        3299 :       return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
     630             :     case wasm::kExprF64AsmjsStoreMem:
     631         759 :       return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
     632             :     default:
     633           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
     634             :   }
     635      635204 :   return graph()->NewNode(op, left, right);
     636             : }
     637             : 
     638      133751 : Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
     639             :                              wasm::WasmCodePosition position) {
     640             :   const Operator* op;
     641             :   MachineOperatorBuilder* m = mcgraph()->machine();
     642      133751 :   switch (opcode) {
     643             :     case wasm::kExprI32Eqz:
     644       31983 :       op = m->Word32Equal();
     645       63966 :       return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
     646             :     case wasm::kExprF32Abs:
     647          62 :       op = m->Float32Abs();
     648          62 :       break;
     649             :     case wasm::kExprF32Neg: {
     650         221 :       op = m->Float32Neg();
     651         221 :       break;
     652             :     }
     653             :     case wasm::kExprF32Sqrt:
     654         174 :       op = m->Float32Sqrt();
     655         174 :       break;
     656             :     case wasm::kExprF64Abs:
     657          87 :       op = m->Float64Abs();
     658          87 :       break;
     659             :     case wasm::kExprF64Neg: {
     660        1049 :       op = m->Float64Neg();
     661        1047 :       break;
     662             :     }
     663             :     case wasm::kExprF64Sqrt:
     664         256 :       op = m->Float64Sqrt();
     665         256 :       break;
     666             :     case wasm::kExprI32SConvertF32:
     667             :     case wasm::kExprI32UConvertF32:
     668             :     case wasm::kExprI32SConvertF64:
     669             :     case wasm::kExprI32UConvertF64:
     670             :     case wasm::kExprI32SConvertSatF64:
     671             :     case wasm::kExprI32UConvertSatF64:
     672             :     case wasm::kExprI32SConvertSatF32:
     673             :     case wasm::kExprI32UConvertSatF32:
     674         352 :       return BuildIntConvertFloat(input, position, opcode);
     675             :     case wasm::kExprI32AsmjsSConvertF64:
     676         108 :       return BuildI32AsmjsSConvertF64(input);
     677             :     case wasm::kExprI32AsmjsUConvertF64:
     678           8 :       return BuildI32AsmjsUConvertF64(input);
     679             :     case wasm::kExprF32ConvertF64:
     680        3427 :       op = m->TruncateFloat64ToFloat32();
     681        3427 :       break;
     682             :     case wasm::kExprF64SConvertI32:
     683         592 :       op = m->ChangeInt32ToFloat64();
     684         592 :       break;
     685             :     case wasm::kExprF64UConvertI32:
     686         279 :       op = m->ChangeUint32ToFloat64();
     687         279 :       break;
     688             :     case wasm::kExprF32SConvertI32:
     689         188 :       op = m->RoundInt32ToFloat32();
     690         188 :       break;
     691             :     case wasm::kExprF32UConvertI32:
     692          96 :       op = m->RoundUint32ToFloat32();
     693          96 :       break;
     694             :     case wasm::kExprI32AsmjsSConvertF32:
     695           8 :       return BuildI32AsmjsSConvertF32(input);
     696             :     case wasm::kExprI32AsmjsUConvertF32:
     697           8 :       return BuildI32AsmjsUConvertF32(input);
     698             :     case wasm::kExprF64ConvertF32:
     699        5536 :       op = m->ChangeFloat32ToFloat64();
     700        5536 :       break;
     701             :     case wasm::kExprF32ReinterpretI32:
     702         183 :       op = m->BitcastInt32ToFloat32();
     703         182 :       break;
     704             :     case wasm::kExprI32ReinterpretF32:
     705         442 :       op = m->BitcastFloat32ToInt32();
     706         441 :       break;
     707             :     case wasm::kExprI32Clz:
     708         300 :       op = m->Word32Clz();
     709         300 :       break;
     710             :     case wasm::kExprI32Ctz: {
     711         340 :       if (m->Word32Ctz().IsSupported()) {
     712         340 :         op = m->Word32Ctz().op();
     713         340 :         break;
     714           0 :       } else if (m->Word32ReverseBits().IsSupported()) {
     715           0 :         Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
     716           0 :         Node* result = graph()->NewNode(m->Word32Clz(), reversed);
     717           0 :         return result;
     718             :       } else {
     719           0 :         return BuildI32Ctz(input);
     720             :       }
     721             :     }
     722             :     case wasm::kExprI32Popcnt: {
     723          64 :       if (m->Word32Popcnt().IsSupported()) {
     724          64 :         op = m->Word32Popcnt().op();
     725          64 :         break;
     726             :       } else {
     727           0 :         return BuildI32Popcnt(input);
     728             :       }
     729             :     }
     730             :     case wasm::kExprF32Floor: {
     731          50 :       if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
     732          50 :       op = m->Float32RoundDown().op();
     733          50 :       break;
     734             :     }
     735             :     case wasm::kExprF32Ceil: {
     736          50 :       if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
     737          50 :       op = m->Float32RoundUp().op();
     738          50 :       break;
     739             :     }
     740             :     case wasm::kExprF32Trunc: {
     741         204 :       if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
     742         204 :       op = m->Float32RoundTruncate().op();
     743         204 :       break;
     744             :     }
     745             :     case wasm::kExprF32NearestInt: {
     746          24 :       if (!m->Float32RoundTiesEven().IsSupported())
     747           0 :         return BuildF32NearestInt(input);
     748          24 :       op = m->Float32RoundTiesEven().op();
     749          24 :       break;
     750             :     }
     751             :     case wasm::kExprF64Floor: {
     752          98 :       if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
     753          98 :       op = m->Float64RoundDown().op();
     754          98 :       break;
     755             :     }
     756             :     case wasm::kExprF64Ceil: {
     757          66 :       if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
     758          66 :       op = m->Float64RoundUp().op();
     759          66 :       break;
     760             :     }
     761             :     case wasm::kExprF64Trunc: {
     762         196 :       if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
     763         196 :       op = m->Float64RoundTruncate().op();
     764         196 :       break;
     765             :     }
     766             :     case wasm::kExprF64NearestInt: {
     767          24 :       if (!m->Float64RoundTiesEven().IsSupported())
     768           0 :         return BuildF64NearestInt(input);
     769          24 :       op = m->Float64RoundTiesEven().op();
     770          24 :       break;
     771             :     }
     772             :     case wasm::kExprF64Acos: {
     773          21 :       return BuildF64Acos(input);
     774             :     }
     775             :     case wasm::kExprF64Asin: {
     776          21 :       return BuildF64Asin(input);
     777             :     }
     778             :     case wasm::kExprF64Atan:
     779          21 :       op = m->Float64Atan();
     780          21 :       break;
     781             :     case wasm::kExprF64Cos: {
     782         146 :       op = m->Float64Cos();
     783         146 :       break;
     784             :     }
     785             :     case wasm::kExprF64Sin: {
     786         146 :       op = m->Float64Sin();
     787         146 :       break;
     788             :     }
     789             :     case wasm::kExprF64Tan: {
     790          21 :       op = m->Float64Tan();
     791          21 :       break;
     792             :     }
     793             :     case wasm::kExprF64Exp: {
     794          21 :       op = m->Float64Exp();
     795          21 :       break;
     796             :     }
     797             :     case wasm::kExprF64Log:
     798          21 :       op = m->Float64Log();
     799          21 :       break;
     800             :     case wasm::kExprI32ConvertI64:
     801         764 :       op = m->TruncateInt64ToInt32();
     802         764 :       break;
     803             :     case wasm::kExprI64SConvertI32:
     804          44 :       op = m->ChangeInt32ToInt64();
     805          44 :       break;
     806             :     case wasm::kExprI64UConvertI32:
     807       27342 :       op = m->ChangeUint32ToUint64();
     808       27354 :       break;
     809             :     case wasm::kExprF64ReinterpretI64:
     810         159 :       op = m->BitcastInt64ToFloat64();
     811         159 :       break;
     812             :     case wasm::kExprI64ReinterpretF64:
     813         409 :       op = m->BitcastFloat64ToInt64();
     814         409 :       break;
     815             :     case wasm::kExprI64Clz:
     816          36 :       op = m->Word64Clz();
     817          36 :       break;
     818             :     case wasm::kExprI64Ctz: {
     819         188 :       OptionalOperator ctz64 = m->Word64Ctz();
     820         188 :       if (ctz64.IsSupported()) {
     821             :         op = ctz64.op();
     822         188 :         break;
     823           0 :       } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
     824             :         op = ctz64.placeholder();
     825             :         break;
     826           0 :       } else if (m->Word64ReverseBits().IsSupported()) {
     827           0 :         Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
     828           0 :         Node* result = graph()->NewNode(m->Word64Clz(), reversed);
     829           0 :         return result;
     830             :       } else {
     831           0 :         return BuildI64Ctz(input);
     832             :       }
     833             :     }
     834             :     case wasm::kExprI64Popcnt: {
     835          44 :       OptionalOperator popcnt64 = m->Word64Popcnt();
     836          44 :       if (popcnt64.IsSupported()) {
     837             :         op = popcnt64.op();
     838           0 :       } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
     839             :         op = popcnt64.placeholder();
     840             :       } else {
     841           0 :         return BuildI64Popcnt(input);
     842             :       }
     843          44 :       break;
     844             :     }
     845             :     case wasm::kExprI64Eqz:
     846         152 :       op = m->Word64Equal();
     847         304 :       return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
     848             :     case wasm::kExprF32SConvertI64:
     849          64 :       if (m->Is32()) {
     850           0 :         return BuildF32SConvertI64(input);
     851             :       }
     852          64 :       op = m->RoundInt64ToFloat32();
     853          64 :       break;
     854             :     case wasm::kExprF32UConvertI64:
     855          32 :       if (m->Is32()) {
     856           0 :         return BuildF32UConvertI64(input);
     857             :       }
     858          32 :       op = m->RoundUint64ToFloat32();
     859          32 :       break;
     860             :     case wasm::kExprF64SConvertI64:
     861          80 :       if (m->Is32()) {
     862           0 :         return BuildF64SConvertI64(input);
     863             :       }
     864          80 :       op = m->RoundInt64ToFloat64();
     865          80 :       break;
     866             :     case wasm::kExprF64UConvertI64:
     867         134 :       if (m->Is32()) {
     868           0 :         return BuildF64UConvertI64(input);
     869             :       }
     870         134 :       op = m->RoundUint64ToFloat64();
     871         133 :       break;
     872             :     case wasm::kExprI32SExtendI8:
     873           4 :       op = m->SignExtendWord8ToInt32();
     874           4 :       break;
     875             :     case wasm::kExprI32SExtendI16:
     876           4 :       op = m->SignExtendWord16ToInt32();
     877           4 :       break;
     878             :     case wasm::kExprI64SExtendI8:
     879           4 :       op = m->SignExtendWord8ToInt64();
     880           4 :       break;
     881             :     case wasm::kExprI64SExtendI16:
     882           4 :       op = m->SignExtendWord16ToInt64();
     883           4 :       break;
     884             :     case wasm::kExprI64SExtendI32:
     885           4 :       op = m->SignExtendWord32ToInt64();
     886           4 :       break;
     887             :     case wasm::kExprI64SConvertF32:
     888             :     case wasm::kExprI64UConvertF32:
     889             :     case wasm::kExprI64SConvertF64:
     890             :     case wasm::kExprI64UConvertF64:
     891             :     case wasm::kExprI64SConvertSatF32:
     892             :     case wasm::kExprI64UConvertSatF32:
     893             :     case wasm::kExprI64SConvertSatF64:
     894             :     case wasm::kExprI64UConvertSatF64:
     895             :       return mcgraph()->machine()->Is32()
     896             :                  ? BuildCcallConvertFloat(input, position, opcode)
     897         967 :                  : BuildIntConvertFloat(input, position, opcode);
     898             :     case wasm::kExprRefIsNull:
     899          64 :       return graph()->NewNode(m->WordEqual(), input, RefNull());
     900             :     case wasm::kExprI32AsmjsLoadMem8S:
     901        2804 :       return BuildAsmjsLoadMem(MachineType::Int8(), input);
     902             :     case wasm::kExprI32AsmjsLoadMem8U:
     903        2082 :       return BuildAsmjsLoadMem(MachineType::Uint8(), input);
     904             :     case wasm::kExprI32AsmjsLoadMem16S:
     905         958 :       return BuildAsmjsLoadMem(MachineType::Int16(), input);
     906             :     case wasm::kExprI32AsmjsLoadMem16U:
     907         531 :       return BuildAsmjsLoadMem(MachineType::Uint16(), input);
     908             :     case wasm::kExprI32AsmjsLoadMem:
     909       43775 :       return BuildAsmjsLoadMem(MachineType::Int32(), input);
     910             :     case wasm::kExprF32AsmjsLoadMem:
     911        5396 :       return BuildAsmjsLoadMem(MachineType::Float32(), input);
     912             :     case wasm::kExprF64AsmjsLoadMem:
     913         845 :       return BuildAsmjsLoadMem(MachineType::Float64(), input);
     914             :     default:
     915           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
     916             :   }
     917       43689 :   return graph()->NewNode(op, input);
     918             : }
     919             : 
     920        3583 : Node* WasmGraphBuilder::Float32Constant(float value) {
     921        3615 :   return mcgraph()->Float32Constant(value);
     922             : }
     923             : 
     924       10398 : Node* WasmGraphBuilder::Float64Constant(double value) {
     925       10430 :   return mcgraph()->Float64Constant(value);
     926             : }
     927             : 
     928             : namespace {
     929       80984 : Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
     930             :              Node** false_node, Node* control, BranchHint hint) {
     931             :   DCHECK_NOT_NULL(cond);
     932             :   DCHECK_NOT_NULL(control);
     933             :   Node* branch =
     934       80984 :       mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
     935      161999 :   *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
     936      162015 :   *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
     937       81013 :   return branch;
     938             : }
     939             : }  // namespace
     940             : 
     941       66083 : Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
     942             :                                      Node** false_node) {
     943             :   return Branch(mcgraph(), cond, true_node, false_node, Control(),
     944       66083 :                 BranchHint::kNone);
     945             : }
     946             : 
     947           0 : Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
     948             :                                          Node** false_node) {
     949             :   return Branch(mcgraph(), cond, true_node, false_node, Control(),
     950           0 :                 BranchHint::kTrue);
     951             : }
     952             : 
     953           0 : Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
     954             :                                           Node** false_node) {
     955             :   return Branch(mcgraph(), cond, true_node, false_node, Control(),
     956       14900 :                 BranchHint::kFalse);
     957             : }
     958             : 
     959       34400 : TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
     960             :   // TODO(wasm): "!env_" should not happen when compiling an actual wasm
     961             :   // function.
     962       34400 :   if (!env_ || !env_->runtime_exception_support) {
     963             :     // We use TrapId::kInvalid as a marker to tell the code generator
     964             :     // to generate a call to a testing c-function instead of a runtime
     965             :     // stub. This code should only be called from a cctest.
     966             :     return TrapId::kInvalid;
     967             :   }
     968             : 
     969       23737 :   switch (reason) {
     970             : #define TRAPREASON_TO_TRAPID(name)                                             \
     971             :   case wasm::k##name:                                                          \
     972             :     static_assert(                                                             \
     973             :         static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
     974             :         "trap id mismatch");                                                   \
     975             :     return TrapId::k##name;
     976       10277 :     FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
     977             : #undef TRAPREASON_TO_TRAPID
     978             :     default:
     979           0 :       UNREACHABLE();
     980             :   }
     981             : }
     982             : 
     983        5884 : Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
     984             :                                    wasm::WasmCodePosition position) {
     985        5884 :   TrapId trap_id = GetTrapIdForTrap(reason);
     986        5885 :   Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
     987             :                                            cond, Effect(), Control()));
     988             :   SetSourcePosition(node, position);
     989        5898 :   return node;
     990             : }
     991             : 
     992       28504 : Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
     993             :                                     wasm::WasmCodePosition position) {
     994       28504 :   TrapId trap_id = GetTrapIdForTrap(reason);
     995       28667 :   Node* node = SetControl(graph()->NewNode(
     996             :       mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
     997             :   SetSourcePosition(node, position);
     998       28545 :   return node;
     999             : }
    1000             : 
    1001             : // Add a check that traps if {node} is equal to {val}.
    1002       70688 : Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
    1003             :                                    int32_t val,
    1004             :                                    wasm::WasmCodePosition position) {
    1005             :   Int32Matcher m(node);
    1006      138200 :   if (m.HasValue() && !m.Is(val)) return graph()->start();
    1007        4380 :   if (val == 0) {
    1008        3424 :     return TrapIfFalse(reason, node, position);
    1009             :   } else {
    1010         956 :     return TrapIfTrue(reason,
    1011             :                       graph()->NewNode(mcgraph()->machine()->Word32Equal(),
    1012             :                                        node, mcgraph()->Int32Constant(val)),
    1013         956 :                       position);
    1014             :   }
    1015             : }
    1016             : 
    1017             : // Add a check that traps if {node} is zero.
    1018           0 : Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
    1019             :                                     wasm::WasmCodePosition position) {
    1020       56492 :   return TrapIfEq32(reason, node, 0, position);
    1021             : }
    1022             : 
    1023             : // Add a check that traps if {node} is equal to {val}.
    1024        4319 : Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
    1025             :                                    int64_t val,
    1026             :                                    wasm::WasmCodePosition position) {
    1027             :   Int64Matcher m(node);
    1028        4553 :   if (m.HasValue() && !m.Is(val)) return graph()->start();
    1029        4103 :   return TrapIfTrue(reason,
    1030             :                     graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
    1031             :                                      mcgraph()->Int64Constant(val)),
    1032        4113 :                     position);
    1033             : }
    1034             : 
    1035             : // Add a check that traps if {node} is zero.
    1036           0 : Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
    1037             :                                     wasm::WasmCodePosition position) {
    1038        3606 :   return TrapIfEq64(reason, node, 0, position);
    1039             : }
    1040             : 
    1041        4848 : Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
    1042        9695 :   return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
    1043             : }
    1044             : 
    1045      210665 : Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
    1046             :   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
    1047      421336 :   return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
    1048             : }
    1049             : 
    1050        4846 : Node* WasmGraphBuilder::IfDefault(Node* sw) {
    1051             :   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
    1052        9694 :   return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
    1053             : }
    1054             : 
    1055     1040047 : Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
    1056             :   static const int kStackAllocatedNodeBufferSize = 8;
    1057             :   Node* stack_buffer[kStackAllocatedNodeBufferSize];
    1058             :   std::vector<Node*> heap_buffer;
    1059             : 
    1060             :   Node** buf = stack_buffer;
    1061     1040047 :   if (count + 3 > kStackAllocatedNodeBufferSize) {
    1062           0 :     heap_buffer.resize(count + 3);
    1063             :     buf = heap_buffer.data();
    1064             :   }
    1065             : 
    1066     1040047 :   buf[0] = mcgraph()->Int32Constant(0);
    1067     1041180 :   if (count > 0) {
    1068     1012534 :     memcpy(buf + 1, vals, sizeof(void*) * count);
    1069             :   }
    1070     2082360 :   buf[count + 1] = Effect();
    1071     2082360 :   buf[count + 2] = Control();
    1072             :   Node* ret =
    1073     1041180 :       graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
    1074             : 
    1075     1040235 :   MergeControlToEnd(mcgraph(), ret);
    1076     1040845 :   return ret;
    1077             : }
    1078             : 
    1079        1607 : Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
    1080             : 
    1081        1526 : Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
    1082        1531 :   TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
    1083             :   ReturnVoid();
    1084        1529 :   return nullptr;
    1085             : }
    1086             : 
    1087      111241 : Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
    1088             :   static const int32_t kMask32 = 0x1F;
    1089      111241 :   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
    1090             :     // Shifts by constants are so common we pattern-match them here.
    1091             :     Int32Matcher match(node);
    1092          24 :     if (match.HasValue()) {
    1093           0 :       int32_t masked = (match.Value() & kMask32);
    1094           0 :       if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
    1095             :     } else {
    1096          24 :       node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
    1097             :                               mcgraph()->Int32Constant(kMask32));
    1098             :     }
    1099             :   }
    1100      111241 :   return node;
    1101             : }
    1102             : 
    1103       15721 : Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
    1104             :   static const int64_t kMask64 = 0x3F;
    1105       15721 :   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
    1106             :     // Shifts by constants are so common we pattern-match them here.
    1107             :     Int64Matcher match(node);
    1108          20 :     if (match.HasValue()) {
    1109           0 :       int64_t masked = (match.Value() & kMask64);
    1110           0 :       if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
    1111             :     } else {
    1112          20 :       node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
    1113             :                               mcgraph()->Int64Constant(kMask64));
    1114             :     }
    1115             :   }
    1116       15721 :   return node;
    1117             : }
    1118             : 
    1119             : static bool ReverseBytesSupported(MachineOperatorBuilder* m,
    1120             :                                   size_t size_in_bytes) {
    1121           0 :   switch (size_in_bytes) {
    1122             :     case 4:
    1123             :     case 16:
    1124             :       return true;
    1125             :     case 8:
    1126             :       return m->Is64();
    1127             :     default:
    1128             :       break;
    1129             :   }
    1130             :   return false;
    1131             : }
    1132             : 
    1133           0 : Node* WasmGraphBuilder::BuildChangeEndiannessStore(
    1134             :     Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
    1135             :   Node* result;
    1136             :   Node* value = node;
    1137             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1138           0 :   int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
    1139           0 :   int valueSizeInBits = 8 * valueSizeInBytes;
    1140             :   bool isFloat = false;
    1141             : 
    1142           0 :   switch (wasmtype) {
    1143             :     case wasm::kWasmF64:
    1144           0 :       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
    1145             :       isFloat = true;
    1146             :       V8_FALLTHROUGH;
    1147             :     case wasm::kWasmI64:
    1148           0 :       result = mcgraph()->Int64Constant(0);
    1149           0 :       break;
    1150             :     case wasm::kWasmF32:
    1151           0 :       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
    1152             :       isFloat = true;
    1153             :       V8_FALLTHROUGH;
    1154             :     case wasm::kWasmI32:
    1155           0 :       result = mcgraph()->Int32Constant(0);
    1156           0 :       break;
    1157             :     case wasm::kWasmS128:
    1158             :       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
    1159             :       break;
    1160             :     default:
    1161           0 :       UNREACHABLE();
    1162             :       break;
    1163             :   }
    1164             : 
    1165           0 :   if (mem_rep == MachineRepresentation::kWord8) {
    1166             :     // No need to change endianness for byte size, return original node
    1167             :     return node;
    1168             :   }
    1169           0 :   if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
    1170             :     // In case we store lower part of WasmI64 expression, we can truncate
    1171             :     // upper 32bits
    1172           0 :     value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
    1173             :     valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
    1174             :     valueSizeInBits = 8 * valueSizeInBytes;
    1175           0 :     if (mem_rep == MachineRepresentation::kWord16) {
    1176             :       value =
    1177           0 :           graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
    1178             :     }
    1179           0 :   } else if (wasmtype == wasm::kWasmI32 &&
    1180           0 :              mem_rep == MachineRepresentation::kWord16) {
    1181             :     value =
    1182           0 :         graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
    1183             :   }
    1184             : 
    1185             :   int i;
    1186             :   uint32_t shiftCount;
    1187             : 
    1188           0 :   if (ReverseBytesSupported(m, valueSizeInBytes)) {
    1189           0 :     switch (valueSizeInBytes) {
    1190             :       case 4:
    1191           0 :         result = graph()->NewNode(m->Word32ReverseBytes(), value);
    1192           0 :         break;
    1193             :       case 8:
    1194           0 :         result = graph()->NewNode(m->Word64ReverseBytes(), value);
    1195           0 :         break;
    1196             :       case 16: {
    1197             :         Node* byte_reversed_lanes[4];
    1198           0 :         for (int lane = 0; lane < 4; lane++) {
    1199           0 :           byte_reversed_lanes[lane] = graph()->NewNode(
    1200             :               m->Word32ReverseBytes(),
    1201             :               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
    1202           0 :                                value));
    1203             :         }
    1204             : 
    1205             :         // This is making a copy of the value.
    1206             :         result =
    1207           0 :             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
    1208             : 
    1209           0 :         for (int lane = 0; lane < 4; lane++) {
    1210             :           result =
    1211           0 :               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
    1212             :                                result, byte_reversed_lanes[lane]);
    1213             :         }
    1214             : 
    1215             :         break;
    1216             :       }
    1217             :       default:
    1218           0 :         UNREACHABLE();
    1219             :         break;
    1220             :     }
    1221             :   } else {
    1222           0 :     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
    1223             :          i += 8, shiftCount -= 16) {
    1224             :       Node* shiftLower;
    1225             :       Node* shiftHigher;
    1226             :       Node* lowerByte;
    1227             :       Node* higherByte;
    1228             : 
    1229             :       DCHECK_LT(0, shiftCount);
    1230             :       DCHECK_EQ(0, (shiftCount + 8) % 16);
    1231             : 
    1232           0 :       if (valueSizeInBits > 32) {
    1233           0 :         shiftLower = graph()->NewNode(m->Word64Shl(), value,
    1234             :                                       mcgraph()->Int64Constant(shiftCount));
    1235           0 :         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
    1236             :                                        mcgraph()->Int64Constant(shiftCount));
    1237           0 :         lowerByte = graph()->NewNode(
    1238             :             m->Word64And(), shiftLower,
    1239             :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
    1240           0 :                                      << (valueSizeInBits - 8 - i)));
    1241           0 :         higherByte = graph()->NewNode(
    1242             :             m->Word64And(), shiftHigher,
    1243           0 :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
    1244           0 :         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
    1245           0 :         result = graph()->NewNode(m->Word64Or(), result, higherByte);
    1246             :       } else {
    1247           0 :         shiftLower = graph()->NewNode(m->Word32Shl(), value,
    1248             :                                       mcgraph()->Int32Constant(shiftCount));
    1249           0 :         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
    1250             :                                        mcgraph()->Int32Constant(shiftCount));
    1251           0 :         lowerByte = graph()->NewNode(
    1252             :             m->Word32And(), shiftLower,
    1253             :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
    1254           0 :                                      << (valueSizeInBits - 8 - i)));
    1255           0 :         higherByte = graph()->NewNode(
    1256             :             m->Word32And(), shiftHigher,
    1257           0 :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
    1258           0 :         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
    1259           0 :         result = graph()->NewNode(m->Word32Or(), result, higherByte);
    1260             :       }
    1261             :     }
    1262             :   }
    1263             : 
    1264           0 :   if (isFloat) {
    1265           0 :     switch (wasmtype) {
    1266             :       case wasm::kWasmF64:
    1267           0 :         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
    1268           0 :         break;
    1269             :       case wasm::kWasmF32:
    1270           0 :         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
    1271           0 :         break;
    1272             :       default:
    1273           0 :         UNREACHABLE();
    1274             :         break;
    1275             :     }
    1276             :   }
    1277             : 
    1278             :   return result;
    1279             : }
    1280             : 
    1281           0 : Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
    1282             :                                                   MachineType memtype,
    1283             :                                                   wasm::ValueType wasmtype) {
    1284             :   Node* result;
    1285             :   Node* value = node;
    1286             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1287             :   int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
    1288           0 :   int valueSizeInBits = 8 * valueSizeInBytes;
    1289             :   bool isFloat = false;
    1290             : 
    1291           0 :   switch (memtype.representation()) {
    1292             :     case MachineRepresentation::kFloat64:
    1293           0 :       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
    1294             :       isFloat = true;
    1295             :       V8_FALLTHROUGH;
    1296             :     case MachineRepresentation::kWord64:
    1297           0 :       result = mcgraph()->Int64Constant(0);
    1298           0 :       break;
    1299             :     case MachineRepresentation::kFloat32:
    1300           0 :       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
    1301             :       isFloat = true;
    1302             :       V8_FALLTHROUGH;
    1303             :     case MachineRepresentation::kWord32:
    1304             :     case MachineRepresentation::kWord16:
    1305           0 :       result = mcgraph()->Int32Constant(0);
    1306           0 :       break;
    1307             :     case MachineRepresentation::kWord8:
    1308             :       // No need to change endianness for byte size, return original node
    1309             :       return node;
    1310             :       break;
    1311             :     case MachineRepresentation::kSimd128:
    1312             :       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
    1313             :       break;
    1314             :     default:
    1315           0 :       UNREACHABLE();
    1316             :       break;
    1317             :   }
    1318             : 
    1319             :   int i;
    1320             :   uint32_t shiftCount;
    1321             : 
    1322           0 :   if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
    1323           0 :     switch (valueSizeInBytes) {
    1324             :       case 2:
    1325             :         result =
    1326           0 :             graph()->NewNode(m->Word32ReverseBytes(),
    1327             :                              graph()->NewNode(m->Word32Shl(), value,
    1328             :                                               mcgraph()->Int32Constant(16)));
    1329           0 :         break;
    1330             :       case 4:
    1331           0 :         result = graph()->NewNode(m->Word32ReverseBytes(), value);
    1332           0 :         break;
    1333             :       case 8:
    1334           0 :         result = graph()->NewNode(m->Word64ReverseBytes(), value);
    1335           0 :         break;
    1336             :       case 16: {
    1337             :         Node* byte_reversed_lanes[4];
    1338           0 :         for (int lane = 0; lane < 4; lane++) {
    1339           0 :           byte_reversed_lanes[lane] = graph()->NewNode(
    1340             :               m->Word32ReverseBytes(),
    1341             :               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
    1342           0 :                                value));
    1343             :         }
    1344             : 
    1345             :         // This is making a copy of the value.
    1346             :         result =
    1347           0 :             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
    1348             : 
    1349           0 :         for (int lane = 0; lane < 4; lane++) {
    1350             :           result =
    1351           0 :               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
    1352             :                                result, byte_reversed_lanes[lane]);
    1353             :         }
    1354             : 
    1355             :         break;
    1356             :       }
    1357             :       default:
    1358           0 :         UNREACHABLE();
    1359             :     }
    1360             :   } else {
    1361           0 :     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
    1362             :          i += 8, shiftCount -= 16) {
    1363             :       Node* shiftLower;
    1364             :       Node* shiftHigher;
    1365             :       Node* lowerByte;
    1366             :       Node* higherByte;
    1367             : 
    1368             :       DCHECK_LT(0, shiftCount);
    1369             :       DCHECK_EQ(0, (shiftCount + 8) % 16);
    1370             : 
    1371           0 :       if (valueSizeInBits > 32) {
    1372           0 :         shiftLower = graph()->NewNode(m->Word64Shl(), value,
    1373             :                                       mcgraph()->Int64Constant(shiftCount));
    1374           0 :         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
    1375             :                                        mcgraph()->Int64Constant(shiftCount));
    1376           0 :         lowerByte = graph()->NewNode(
    1377             :             m->Word64And(), shiftLower,
    1378             :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
    1379           0 :                                      << (valueSizeInBits - 8 - i)));
    1380           0 :         higherByte = graph()->NewNode(
    1381             :             m->Word64And(), shiftHigher,
    1382           0 :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
    1383           0 :         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
    1384           0 :         result = graph()->NewNode(m->Word64Or(), result, higherByte);
    1385             :       } else {
    1386           0 :         shiftLower = graph()->NewNode(m->Word32Shl(), value,
    1387             :                                       mcgraph()->Int32Constant(shiftCount));
    1388           0 :         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
    1389             :                                        mcgraph()->Int32Constant(shiftCount));
    1390           0 :         lowerByte = graph()->NewNode(
    1391             :             m->Word32And(), shiftLower,
    1392             :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
    1393           0 :                                      << (valueSizeInBits - 8 - i)));
    1394           0 :         higherByte = graph()->NewNode(
    1395             :             m->Word32And(), shiftHigher,
    1396           0 :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
    1397           0 :         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
    1398           0 :         result = graph()->NewNode(m->Word32Or(), result, higherByte);
    1399             :       }
    1400             :     }
    1401             :   }
    1402             : 
    1403           0 :   if (isFloat) {
    1404           0 :     switch (memtype.representation()) {
    1405             :       case MachineRepresentation::kFloat64:
    1406           0 :         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
    1407           0 :         break;
    1408             :       case MachineRepresentation::kFloat32:
    1409           0 :         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
    1410           0 :         break;
    1411             :       default:
    1412           0 :         UNREACHABLE();
    1413             :         break;
    1414             :     }
    1415             :   }
    1416             : 
    1417             :   // We need to sign extend the value
    1418           0 :   if (memtype.IsSigned()) {
    1419             :     DCHECK(!isFloat);
    1420           0 :     if (valueSizeInBits < 32) {
    1421             :       Node* shiftBitCount;
    1422             :       // Perform sign extension using following trick
    1423             :       // result = (x << machine_width - type_width) >> (machine_width -
    1424             :       // type_width)
    1425           0 :       if (wasmtype == wasm::kWasmI64) {
    1426           0 :         shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
    1427           0 :         result = graph()->NewNode(
    1428             :             m->Word64Sar(),
    1429             :             graph()->NewNode(m->Word64Shl(),
    1430             :                              graph()->NewNode(m->ChangeInt32ToInt64(), result),
    1431             :                              shiftBitCount),
    1432             :             shiftBitCount);
    1433           0 :       } else if (wasmtype == wasm::kWasmI32) {
    1434           0 :         shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
    1435           0 :         result = graph()->NewNode(
    1436             :             m->Word32Sar(),
    1437             :             graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
    1438             :             shiftBitCount);
    1439             :       }
    1440             :     }
    1441             :   }
    1442             : 
    1443             :   return result;
    1444             : }
    1445             : 
    1446          32 : Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
    1447          32 :   Node* result = Unop(
    1448             :       wasm::kExprF32ReinterpretI32,
    1449             :       Binop(wasm::kExprI32Ior,
    1450             :             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
    1451             :                   mcgraph()->Int32Constant(0x7FFFFFFF)),
    1452             :             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
    1453          32 :                   mcgraph()->Int32Constant(0x80000000))));
    1454             : 
    1455          32 :   return result;
    1456             : }
    1457             : 
    1458          32 : Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
    1459             : #if WASM_64
    1460          32 :   Node* result = Unop(
    1461             :       wasm::kExprF64ReinterpretI64,
    1462             :       Binop(wasm::kExprI64Ior,
    1463             :             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
    1464             :                   mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
    1465             :             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
    1466          32 :                   mcgraph()->Int64Constant(0x8000000000000000))));
    1467             : 
    1468          32 :   return result;
    1469             : #else
    1470             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1471             : 
    1472             :   Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
    1473             :   Node* high_word_right =
    1474             :       graph()->NewNode(m->Float64ExtractHighWord32(), right);
    1475             : 
    1476             :   Node* new_high_word = Binop(wasm::kExprI32Ior,
    1477             :                               Binop(wasm::kExprI32And, high_word_left,
    1478             :                                     mcgraph()->Int32Constant(0x7FFFFFFF)),
    1479             :                               Binop(wasm::kExprI32And, high_word_right,
    1480             :                                     mcgraph()->Int32Constant(0x80000000)));
    1481             : 
    1482             :   return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
    1483             : #endif
    1484             : }
    1485             : 
    1486             : namespace {
    1487             : 
    1488        1322 : MachineType IntConvertType(wasm::WasmOpcode opcode) {
    1489        1322 :   switch (opcode) {
    1490             :     case wasm::kExprI32SConvertF32:
    1491             :     case wasm::kExprI32SConvertF64:
    1492             :     case wasm::kExprI32SConvertSatF32:
    1493             :     case wasm::kExprI32SConvertSatF64:
    1494             :       return MachineType::Int32();
    1495             :     case wasm::kExprI32UConvertF32:
    1496             :     case wasm::kExprI32UConvertF64:
    1497             :     case wasm::kExprI32UConvertSatF32:
    1498             :     case wasm::kExprI32UConvertSatF64:
    1499             :       return MachineType::Uint32();
    1500             :     case wasm::kExprI64SConvertF32:
    1501             :     case wasm::kExprI64SConvertF64:
    1502             :     case wasm::kExprI64SConvertSatF32:
    1503             :     case wasm::kExprI64SConvertSatF64:
    1504             :       return MachineType::Int64();
    1505             :     case wasm::kExprI64UConvertF32:
    1506             :     case wasm::kExprI64UConvertF64:
    1507             :     case wasm::kExprI64UConvertSatF32:
    1508             :     case wasm::kExprI64UConvertSatF64:
    1509             :       return MachineType::Uint64();
    1510             :     default:
    1511           0 :       UNREACHABLE();
    1512             :   }
    1513             : }
    1514             : 
    1515        1317 : MachineType FloatConvertType(wasm::WasmOpcode opcode) {
    1516        1317 :   switch (opcode) {
    1517             :     case wasm::kExprI32SConvertF32:
    1518             :     case wasm::kExprI32UConvertF32:
    1519             :     case wasm::kExprI32SConvertSatF32:
    1520             :     case wasm::kExprI64SConvertF32:
    1521             :     case wasm::kExprI64UConvertF32:
    1522             :     case wasm::kExprI32UConvertSatF32:
    1523             :     case wasm::kExprI64SConvertSatF32:
    1524             :     case wasm::kExprI64UConvertSatF32:
    1525             :       return MachineType::Float32();
    1526             :     case wasm::kExprI32SConvertF64:
    1527             :     case wasm::kExprI32UConvertF64:
    1528             :     case wasm::kExprI64SConvertF64:
    1529             :     case wasm::kExprI64UConvertF64:
    1530             :     case wasm::kExprI32SConvertSatF64:
    1531             :     case wasm::kExprI32UConvertSatF64:
    1532             :     case wasm::kExprI64SConvertSatF64:
    1533             :     case wasm::kExprI64UConvertSatF64:
    1534             :       return MachineType::Float64();
    1535             :     default:
    1536           0 :       UNREACHABLE();
    1537             :   }
    1538             : }
    1539             : 
    1540        1320 : const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
    1541        1320 :   switch (opcode) {
    1542             :     case wasm::kExprI32SConvertF32:
    1543             :     case wasm::kExprI32SConvertSatF32:
    1544         124 :       return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
    1545             :     case wasm::kExprI32UConvertF32:
    1546             :     case wasm::kExprI32UConvertSatF32:
    1547          56 :       return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
    1548             :     case wasm::kExprI32SConvertF64:
    1549             :     case wasm::kExprI32SConvertSatF64:
    1550         124 :       return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
    1551             :     case wasm::kExprI32UConvertF64:
    1552             :     case wasm::kExprI32UConvertSatF64:
    1553          48 :       return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
    1554             :     case wasm::kExprI64SConvertF32:
    1555             :     case wasm::kExprI64SConvertSatF32:
    1556          48 :       return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
    1557             :     case wasm::kExprI64UConvertF32:
    1558             :     case wasm::kExprI64UConvertSatF32:
    1559          48 :       return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
    1560             :     case wasm::kExprI64SConvertF64:
    1561             :     case wasm::kExprI64SConvertSatF64:
    1562         816 :       return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
    1563             :     case wasm::kExprI64UConvertF64:
    1564             :     case wasm::kExprI64UConvertSatF64:
    1565          56 :       return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
    1566             :     default:
    1567           0 :       UNREACHABLE();
    1568             :   }
    1569             : }
    1570             : 
    1571         352 : wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
    1572         352 :   switch (opcode) {
    1573             :     case wasm::kExprI32SConvertF32:
    1574             :     case wasm::kExprI32SConvertSatF32:
    1575             :       return wasm::kExprF32SConvertI32;
    1576             :     case wasm::kExprI32UConvertF32:
    1577             :     case wasm::kExprI32UConvertSatF32:
    1578          56 :       return wasm::kExprF32UConvertI32;
    1579             :     case wasm::kExprI32SConvertF64:
    1580             :     case wasm::kExprI32SConvertSatF64:
    1581         124 :       return wasm::kExprF64SConvertI32;
    1582             :     case wasm::kExprI32UConvertF64:
    1583             :     case wasm::kExprI32UConvertSatF64:
    1584          48 :       return wasm::kExprF64UConvertI32;
    1585             :     default:
    1586           0 :       UNREACHABLE();
    1587             :   }
    1588             : }
    1589             : 
    1590        1320 : bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
    1591        1320 :   switch (opcode) {
    1592             :     case wasm::kExprI32SConvertF32:
    1593             :     case wasm::kExprI32UConvertF32:
    1594             :     case wasm::kExprI32SConvertF64:
    1595             :     case wasm::kExprI32UConvertF64:
    1596             :     case wasm::kExprI64SConvertF32:
    1597             :     case wasm::kExprI64UConvertF32:
    1598             :     case wasm::kExprI64SConvertF64:
    1599             :     case wasm::kExprI64UConvertF64:
    1600             :       return true;
    1601             :     case wasm::kExprI32SConvertSatF64:
    1602             :     case wasm::kExprI32UConvertSatF64:
    1603             :     case wasm::kExprI32SConvertSatF32:
    1604             :     case wasm::kExprI32UConvertSatF32:
    1605             :     case wasm::kExprI64SConvertSatF32:
    1606             :     case wasm::kExprI64UConvertSatF32:
    1607             :     case wasm::kExprI64SConvertSatF64:
    1608             :     case wasm::kExprI64UConvertSatF64:
    1609          64 :       return false;
    1610             :     default:
    1611           0 :       UNREACHABLE();
    1612             :   }
    1613             : }
    1614             : 
    1615         128 : Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
    1616         128 :   switch (ty.representation()) {
    1617             :     case MachineRepresentation::kWord32:
    1618          32 :       return builder->Int32Constant(0);
    1619             :     case MachineRepresentation::kWord64:
    1620          32 :       return builder->Int64Constant(0);
    1621             :     case MachineRepresentation::kFloat32:
    1622          32 :       return builder->Float32Constant(0.0);
    1623             :     case MachineRepresentation::kFloat64:
    1624          32 :       return builder->Float64Constant(0.0);
    1625             :     default:
    1626           0 :       UNREACHABLE();
    1627             :   }
    1628             : }
    1629             : 
    1630          64 : Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
    1631          64 :   switch (ty.semantic()) {
    1632             :     case MachineSemantic::kInt32:
    1633          16 :       return builder->Int32Constant(std::numeric_limits<int32_t>::min());
    1634             :     case MachineSemantic::kUint32:
    1635          16 :       return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
    1636             :     case MachineSemantic::kInt64:
    1637          16 :       return builder->Int64Constant(std::numeric_limits<int64_t>::min());
    1638             :     case MachineSemantic::kUint64:
    1639          16 :       return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
    1640             :     default:
    1641           0 :       UNREACHABLE();
    1642             :   }
    1643             : }
    1644             : 
    1645          64 : Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
    1646          64 :   switch (ty.semantic()) {
    1647             :     case MachineSemantic::kInt32:
    1648          16 :       return builder->Int32Constant(std::numeric_limits<int32_t>::max());
    1649             :     case MachineSemantic::kUint32:
    1650          16 :       return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
    1651             :     case MachineSemantic::kInt64:
    1652          16 :       return builder->Int64Constant(std::numeric_limits<int64_t>::max());
    1653             :     case MachineSemantic::kUint64:
    1654          16 :       return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
    1655             :     default:
    1656           0 :       UNREACHABLE();
    1657             :   }
    1658             : }
    1659             : 
    1660         352 : wasm::WasmOpcode TruncOp(const MachineType& ty) {
    1661         352 :   switch (ty.representation()) {
    1662             :     case MachineRepresentation::kFloat32:
    1663             :       return wasm::kExprF32Trunc;
    1664             :     case MachineRepresentation::kFloat64:
    1665         172 :       return wasm::kExprF64Trunc;
    1666             :     default:
    1667           0 :       UNREACHABLE();
    1668             :   }
    1669             : }
    1670             : 
    1671         416 : wasm::WasmOpcode NeOp(const MachineType& ty) {
    1672         416 :   switch (ty.representation()) {
    1673             :     case MachineRepresentation::kFloat32:
    1674             :       return wasm::kExprF32Ne;
    1675             :     case MachineRepresentation::kFloat64:
    1676         204 :       return wasm::kExprF64Ne;
    1677             :     default:
    1678           0 :       UNREACHABLE();
    1679             :   }
    1680             : }
    1681             : 
    1682          64 : wasm::WasmOpcode LtOp(const MachineType& ty) {
    1683          64 :   switch (ty.representation()) {
    1684             :     case MachineRepresentation::kFloat32:
    1685             :       return wasm::kExprF32Lt;
    1686             :     case MachineRepresentation::kFloat64:
    1687          32 :       return wasm::kExprF64Lt;
    1688             :     default:
    1689           0 :       UNREACHABLE();
    1690             :   }
    1691             : }
    1692             : 
    1693        1321 : Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
    1694             :                       const MachineType& int_ty, const MachineType& float_ty,
    1695             :                       Node* trunc, Node* converted_value) {
    1696        1321 :   if (int_ty.representation() == MachineRepresentation::kWord32) {
    1697         352 :     Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
    1698         352 :     return builder->Binop(NeOp(float_ty), trunc, check);
    1699             :   }
    1700         969 :   return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
    1701         976 :                                    trunc, builder->graph()->start());
    1702             : }
    1703             : 
    1704          64 : Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
    1705             :                           const MachineType& int_ty,
    1706             :                           const MachineType& float_ty, Node* trunc,
    1707             :                           Node* converted_value) {
    1708             :   Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
    1709          64 :                                converted_value);
    1710          64 :   if (int_ty.representation() == MachineRepresentation::kWord64) {
    1711          32 :     test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
    1712             :   }
    1713          64 :   return test;
    1714             : }
    1715             : 
    1716             : }  // namespace
    1717             : 
    1718        1317 : Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
    1719             :                                              wasm::WasmCodePosition position,
    1720             :                                              wasm::WasmOpcode opcode) {
    1721        1317 :   const MachineType int_ty = IntConvertType(opcode);
    1722        1317 :   const MachineType float_ty = FloatConvertType(opcode);
    1723        1317 :   const Operator* conv_op = ConvertOp(this, opcode);
    1724             :   Node* trunc = nullptr;
    1725             :   Node* converted_value = nullptr;
    1726             :   const bool is_int32 =
    1727             :       int_ty.representation() == MachineRepresentation::kWord32;
    1728        1318 :   if (is_int32) {
    1729         352 :     trunc = Unop(TruncOp(float_ty), input);
    1730             :     converted_value = graph()->NewNode(conv_op, trunc);
    1731             :   } else {
    1732             :     trunc = graph()->NewNode(conv_op, input);
    1733         968 :     converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
    1734             :                                        trunc, graph()->start());
    1735             :   }
    1736        1327 :   if (IsTrappingConvertOp(opcode)) {
    1737             :     Node* test =
    1738        1257 :         ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
    1739        1258 :     if (is_int32) {
    1740         320 :       TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
    1741             :     } else {
    1742             :       ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
    1743             :     }
    1744             :     return converted_value;
    1745             :   }
    1746             :   Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
    1747          64 :                                    converted_value);
    1748          64 :   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
    1749             :   tl_d.Chain(Control());
    1750          64 :   Node* nan_test = Binop(NeOp(float_ty), input, input);
    1751          64 :   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
    1752          64 :   nan_d.Nest(tl_d, true);
    1753          64 :   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
    1754          64 :   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
    1755          64 :   sat_d.Nest(nan_d, false);
    1756             :   Node* sat_val =
    1757          64 :       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
    1758             :   Node* nan_val =
    1759          64 :       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
    1760          64 :   return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
    1761             : }
    1762             : 
    1763           8 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
    1764             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1765             :   // asm.js must use the wacky JS semantics.
    1766           8 :   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
    1767          16 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1768             : }
    1769             : 
    1770         108 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
    1771             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1772             :   // asm.js must use the wacky JS semantics.
    1773         216 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1774             : }
    1775             : 
    1776           8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
    1777             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1778             :   // asm.js must use the wacky JS semantics.
    1779           8 :   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
    1780          16 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1781             : }
    1782             : 
    1783           8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
    1784             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1785             :   // asm.js must use the wacky JS semantics.
    1786          16 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1787             : }
    1788             : 
    1789           0 : Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
    1790             :                                              MachineRepresentation input_type) {
    1791             :   Node* stack_slot_param =
    1792           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
    1793             : 
    1794           0 :   const Operator* store_op = mcgraph()->machine()->Store(
    1795           0 :       StoreRepresentation(input_type, kNoWriteBarrier));
    1796           0 :   SetEffect(graph()->NewNode(store_op, stack_slot_param,
    1797             :                              mcgraph()->Int32Constant(0), input, Effect(),
    1798             :                              Control()));
    1799             : 
    1800           0 :   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    1801             :   MachineSignature sig(1, 1, sig_types);
    1802             : 
    1803           0 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    1804             : 
    1805           0 :   return BuildCCall(&sig, function, stack_slot_param);
    1806             : }
    1807             : 
    1808           0 : Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
    1809           0 :   return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
    1810           0 :                               MachineRepresentation::kWord32);
    1811             : }
    1812             : 
    1813           0 : Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
    1814           0 :   return Unop(wasm::kExprI64UConvertI32,
    1815             :               BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
    1816           0 :                                    MachineRepresentation::kWord64));
    1817             : }
    1818             : 
    1819           0 : Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
    1820           0 :   return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
    1821           0 :                               MachineRepresentation::kWord32);
    1822             : }
    1823             : 
    1824           0 : Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
    1825           0 :   return Unop(
    1826             :       wasm::kExprI64UConvertI32,
    1827             :       BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
    1828           0 :                            MachineRepresentation::kWord64));
    1829             : }
    1830             : 
    1831           0 : Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
    1832           0 :   MachineType type = MachineType::Float32();
    1833           0 :   ExternalReference ref = ExternalReference::wasm_f32_trunc();
    1834             : 
    1835           0 :   return BuildCFuncInstruction(ref, type, input);
    1836             : }
    1837             : 
    1838           0 : Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
    1839           0 :   MachineType type = MachineType::Float32();
    1840           0 :   ExternalReference ref = ExternalReference::wasm_f32_floor();
    1841           0 :   return BuildCFuncInstruction(ref, type, input);
    1842             : }
    1843             : 
    1844           0 : Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
    1845           0 :   MachineType type = MachineType::Float32();
    1846           0 :   ExternalReference ref = ExternalReference::wasm_f32_ceil();
    1847           0 :   return BuildCFuncInstruction(ref, type, input);
    1848             : }
    1849             : 
    1850           0 : Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
    1851           0 :   MachineType type = MachineType::Float32();
    1852           0 :   ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
    1853           0 :   return BuildCFuncInstruction(ref, type, input);
    1854             : }
    1855             : 
    1856           0 : Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
    1857           0 :   MachineType type = MachineType::Float64();
    1858           0 :   ExternalReference ref = ExternalReference::wasm_f64_trunc();
    1859           0 :   return BuildCFuncInstruction(ref, type, input);
    1860             : }
    1861             : 
    1862           0 : Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
    1863           0 :   MachineType type = MachineType::Float64();
    1864           0 :   ExternalReference ref = ExternalReference::wasm_f64_floor();
    1865           0 :   return BuildCFuncInstruction(ref, type, input);
    1866             : }
    1867             : 
    1868           0 : Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
    1869           0 :   MachineType type = MachineType::Float64();
    1870           0 :   ExternalReference ref = ExternalReference::wasm_f64_ceil();
    1871           0 :   return BuildCFuncInstruction(ref, type, input);
    1872             : }
    1873             : 
    1874           0 : Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
    1875           0 :   MachineType type = MachineType::Float64();
    1876           0 :   ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
    1877           0 :   return BuildCFuncInstruction(ref, type, input);
    1878             : }
    1879             : 
    1880          21 : Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
    1881          21 :   MachineType type = MachineType::Float64();
    1882          21 :   ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
    1883          21 :   return BuildCFuncInstruction(ref, type, input);
    1884             : }
    1885             : 
    1886          21 : Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
    1887          21 :   MachineType type = MachineType::Float64();
    1888          21 :   ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
    1889          21 :   return BuildCFuncInstruction(ref, type, input);
    1890             : }
    1891             : 
    1892           9 : Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
    1893           9 :   MachineType type = MachineType::Float64();
    1894           9 :   ExternalReference ref = ExternalReference::wasm_float64_pow();
    1895           9 :   return BuildCFuncInstruction(ref, type, left, right);
    1896             : }
    1897             : 
    1898          20 : Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
    1899          20 :   MachineType type = MachineType::Float64();
    1900          20 :   ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
    1901          20 :   return BuildCFuncInstruction(ref, type, left, right);
    1902             : }
    1903             : 
    1904          71 : Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
    1905             :                                               MachineType type, Node* input0,
    1906             :                                               Node* input1) {
    1907             :   // We do truncation by calling a C function which calculates the result.
    1908             :   // The input is passed to the C function as a byte buffer holding the two
    1909             :   // input doubles. We reserve this byte buffer as a stack slot, store the
    1910             :   // parameters in this buffer slots, pass a pointer to the buffer to the C
    1911             :   // function, and after calling the C function we collect the return value from
    1912             :   // the buffer.
    1913             : 
    1914             :   const int type_size = ElementSizeInBytes(type.representation());
    1915          71 :   const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
    1916             :   Node* stack_slot =
    1917          71 :       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
    1918             : 
    1919          71 :   const Operator* store_op = mcgraph()->machine()->Store(
    1920          71 :       StoreRepresentation(type.representation(), kNoWriteBarrier));
    1921          71 :   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
    1922             :                              input0, Effect(), Control()));
    1923             : 
    1924          71 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    1925             : 
    1926          71 :   if (input1 != nullptr) {
    1927          29 :     SetEffect(graph()->NewNode(store_op, stack_slot,
    1928             :                                mcgraph()->Int32Constant(type_size), input1,
    1929             :                                Effect(), Control()));
    1930             :   }
    1931             : 
    1932          71 :   MachineType sig_types[] = {MachineType::Pointer()};
    1933             :   MachineSignature sig(0, 1, sig_types);
    1934          71 :   BuildCCall(&sig, function, stack_slot);
    1935             : 
    1936          71 :   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
    1937             :                                     stack_slot, mcgraph()->Int32Constant(0),
    1938          71 :                                     Effect(), Control()));
    1939             : }
    1940             : 
    1941           0 : Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
    1942             :   // TODO(titzer/bradnelson): Check handlng of asm.js case.
    1943           0 :   return BuildIntToFloatConversionInstruction(
    1944             :       input, ExternalReference::wasm_int64_to_float32(),
    1945           0 :       MachineRepresentation::kWord64, MachineType::Float32());
    1946             : }
    1947           0 : Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
    1948             :   // TODO(titzer/bradnelson): Check handlng of asm.js case.
    1949           0 :   return BuildIntToFloatConversionInstruction(
    1950             :       input, ExternalReference::wasm_uint64_to_float32(),
    1951           0 :       MachineRepresentation::kWord64, MachineType::Float32());
    1952             : }
    1953           0 : Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
    1954           0 :   return BuildIntToFloatConversionInstruction(
    1955             :       input, ExternalReference::wasm_int64_to_float64(),
    1956           0 :       MachineRepresentation::kWord64, MachineType::Float64());
    1957             : }
    1958           0 : Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
    1959           0 :   return BuildIntToFloatConversionInstruction(
    1960             :       input, ExternalReference::wasm_uint64_to_float64(),
    1961           0 :       MachineRepresentation::kWord64, MachineType::Float64());
    1962             : }
    1963             : 
    1964           0 : Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
    1965             :     Node* input, ExternalReference ref,
    1966             :     MachineRepresentation parameter_representation,
    1967             :     const MachineType result_type) {
    1968             :   int stack_slot_size =
    1969             :       std::max(ElementSizeInBytes(parameter_representation),
    1970           0 :                ElementSizeInBytes(result_type.representation()));
    1971             :   Node* stack_slot =
    1972           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
    1973           0 :   const Operator* store_op = mcgraph()->machine()->Store(
    1974           0 :       StoreRepresentation(parameter_representation, kNoWriteBarrier));
    1975           0 :   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
    1976             :                              input, Effect(), Control()));
    1977           0 :   MachineType sig_types[] = {MachineType::Pointer()};
    1978             :   MachineSignature sig(0, 1, sig_types);
    1979           0 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    1980           0 :   BuildCCall(&sig, function, stack_slot);
    1981           0 :   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
    1982             :                                     stack_slot, mcgraph()->Int32Constant(0),
    1983           0 :                                     Effect(), Control()));
    1984             : }
    1985             : 
    1986             : namespace {
    1987             : 
    1988           0 : ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
    1989             :                                     wasm::WasmOpcode opcode) {
    1990           0 :   switch (opcode) {
    1991             :     case wasm::kExprI64SConvertF32:
    1992             :     case wasm::kExprI64SConvertSatF32:
    1993           0 :       return ExternalReference::wasm_float32_to_int64();
    1994             :     case wasm::kExprI64UConvertF32:
    1995             :     case wasm::kExprI64UConvertSatF32:
    1996           0 :       return ExternalReference::wasm_float32_to_uint64();
    1997             :     case wasm::kExprI64SConvertF64:
    1998             :     case wasm::kExprI64SConvertSatF64:
    1999           0 :       return ExternalReference::wasm_float64_to_int64();
    2000             :     case wasm::kExprI64UConvertF64:
    2001             :     case wasm::kExprI64UConvertSatF64:
    2002           0 :       return ExternalReference::wasm_float64_to_uint64();
    2003             :     default:
    2004           0 :       UNREACHABLE();
    2005             :   }
    2006             : }
    2007             : 
    2008             : }  // namespace
    2009             : 
    2010           0 : Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
    2011             :                                                wasm::WasmCodePosition position,
    2012             :                                                wasm::WasmOpcode opcode) {
    2013           0 :   const MachineType int_ty = IntConvertType(opcode);
    2014           0 :   const MachineType float_ty = FloatConvertType(opcode);
    2015           0 :   ExternalReference call_ref = convert_ccall_ref(this, opcode);
    2016             :   int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
    2017           0 :                                  ElementSizeInBytes(float_ty.representation()));
    2018             :   Node* stack_slot =
    2019           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
    2020           0 :   const Operator* store_op = mcgraph()->machine()->Store(
    2021           0 :       StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
    2022             :   SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
    2023             :                              Effect(), Control()));
    2024           0 :   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    2025             :   MachineSignature sig(1, 1, sig_types);
    2026             :   Node* function =
    2027           0 :       graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
    2028           0 :   Node* overflow = BuildCCall(&sig, function, stack_slot);
    2029           0 :   if (IsTrappingConvertOp(opcode)) {
    2030             :     ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
    2031           0 :     return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
    2032             :                                       stack_slot, Int32Constant(0), Effect(),
    2033           0 :                                       Control()));
    2034             :   }
    2035           0 :   Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
    2036           0 :   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
    2037             :   tl_d.Chain(Control());
    2038           0 :   Node* nan_test = Binop(NeOp(float_ty), input, input);
    2039           0 :   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
    2040           0 :   nan_d.Nest(tl_d, true);
    2041           0 :   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
    2042           0 :   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
    2043           0 :   sat_d.Nest(nan_d, false);
    2044             :   Node* sat_val =
    2045           0 :       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
    2046             :   Node* load =
    2047           0 :       SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
    2048             :                                  Int32Constant(0), Effect(), Control()));
    2049             :   Node* nan_val =
    2050           0 :       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
    2051           0 :   return tl_d.Phi(int_ty.representation(), nan_val, load);
    2052             : }
    2053             : 
    2054        1132 : Node* WasmGraphBuilder::MemoryGrow(Node* input) {
    2055        1132 :   needs_stack_check_ = true;
    2056             : 
    2057             :   WasmMemoryGrowDescriptor interface_descriptor;
    2058        1131 :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    2059             :       mcgraph()->zone(),                              // zone
    2060             :       interface_descriptor,                           // descriptor
    2061             :       interface_descriptor.GetStackParameterCount(),  // stack parameter count
    2062             :       CallDescriptor::kNoFlags,                       // flags
    2063             :       Operator::kNoProperties,                        // properties
    2064        1132 :       StubCallMode::kCallWasmRuntimeStub);            // stub call mode
    2065             :   // A direct call to a wasm runtime stub defined in this module.
    2066             :   // Just encode the stub index. This will be patched at relocation.
    2067             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    2068        1131 :       wasm::WasmCode::kWasmMemoryGrow, RelocInfo::WASM_STUB_CALL);
    2069        1132 :   return SetEffect(
    2070             :       SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    2071        1132 :                                   call_target, input, Effect(), Control())));
    2072             : }
    2073             : 
    2074         534 : Node* WasmGraphBuilder::Throw(uint32_t exception_index,
    2075             :                               const wasm::WasmException* exception,
    2076             :                               const Vector<Node*> values) {
    2077         534 :   needs_stack_check_ = true;
    2078         534 :   uint32_t encoded_size = WasmExceptionPackage::GetEncodedSize(exception);
    2079             :   Node* create_parameters[] = {
    2080         535 :       LoadExceptionTagFromTable(exception_index),
    2081        1071 :       BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
    2082             :   Node* except_obj =
    2083         535 :       BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
    2084         536 :                          arraysize(create_parameters));
    2085             :   Node* values_array =
    2086         536 :       BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
    2087         536 :   uint32_t index = 0;
    2088         536 :   const wasm::WasmExceptionSig* sig = exception->sig;
    2089             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2090        1080 :   for (size_t i = 0; i < sig->parameter_count(); ++i) {
    2091         272 :     Node* value = values[i];
    2092         272 :     switch (sig->GetParam(i)) {
    2093             :       case wasm::kWasmF32:
    2094          16 :         value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
    2095             :         V8_FALLTHROUGH;
    2096             :       case wasm::kWasmI32:
    2097         192 :         BuildEncodeException32BitValue(values_array, &index, value);
    2098         192 :         break;
    2099             :       case wasm::kWasmF64:
    2100          16 :         value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
    2101             :         V8_FALLTHROUGH;
    2102             :       case wasm::kWasmI64: {
    2103          32 :         Node* upper32 = graph()->NewNode(
    2104             :             m->TruncateInt64ToInt32(),
    2105             :             Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
    2106          32 :         BuildEncodeException32BitValue(values_array, &index, upper32);
    2107          32 :         Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
    2108          32 :         BuildEncodeException32BitValue(values_array, &index, lower32);
    2109          32 :         break;
    2110             :       }
    2111             :       case wasm::kWasmS128:
    2112          16 :         BuildEncodeException32BitValue(
    2113             :             values_array, &index,
    2114          16 :             graph()->NewNode(m->I32x4ExtractLane(0), value));
    2115          16 :         BuildEncodeException32BitValue(
    2116             :             values_array, &index,
    2117          16 :             graph()->NewNode(m->I32x4ExtractLane(1), value));
    2118          16 :         BuildEncodeException32BitValue(
    2119             :             values_array, &index,
    2120          16 :             graph()->NewNode(m->I32x4ExtractLane(2), value));
    2121          16 :         BuildEncodeException32BitValue(
    2122             :             values_array, &index,
    2123          16 :             graph()->NewNode(m->I32x4ExtractLane(3), value));
    2124          16 :         break;
    2125             :       case wasm::kWasmAnyRef:
    2126         128 :         STORE_FIXED_ARRAY_SLOT_ANY(values_array, index, value);
    2127          32 :         ++index;
    2128          32 :         break;
    2129             :       default:
    2130           0 :         UNREACHABLE();
    2131             :     }
    2132             :   }
    2133             :   DCHECK_EQ(encoded_size, index);
    2134             :   WasmThrowDescriptor interface_descriptor;
    2135         534 :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    2136             :       mcgraph()->zone(), interface_descriptor,
    2137             :       interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
    2138         536 :       Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
    2139             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    2140         534 :       wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
    2141         534 :   return SetEffect(SetControl(
    2142             :       graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
    2143         534 :                        except_obj, Effect(), Control())));
    2144             : }
    2145             : 
    2146         320 : void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
    2147             :                                                       uint32_t* index,
    2148             :                                                       Node* value) {
    2149             :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2150         320 :   Node* upper_halfword_as_smi = BuildChangeUint31ToSmi(
    2151         320 :       graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16)));
    2152        1280 :   STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, upper_halfword_as_smi);
    2153         320 :   ++(*index);
    2154         320 :   Node* lower_halfword_as_smi = BuildChangeUint31ToSmi(
    2155         320 :       graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu)));
    2156        1280 :   STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, lower_halfword_as_smi);
    2157         320 :   ++(*index);
    2158         320 : }
    2159             : 
    2160         272 : Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
    2161             :                                                        uint32_t* index) {
    2162             :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2163             :   Node* upper =
    2164         816 :       BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
    2165         272 :   (*index)++;
    2166         272 :   upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
    2167             :   Node* lower =
    2168         816 :       BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
    2169         272 :   (*index)++;
    2170         272 :   Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
    2171         272 :   return value;
    2172             : }
    2173             : 
    2174          16 : Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
    2175             :                                                        uint32_t* index) {
    2176          16 :   Node* upper = Binop(wasm::kExprI64Shl,
    2177             :                       Unop(wasm::kExprI64UConvertI32,
    2178             :                            BuildDecodeException32BitValue(values_array, index)),
    2179          16 :                       Int64Constant(32));
    2180          16 :   Node* lower = Unop(wasm::kExprI64UConvertI32,
    2181          16 :                      BuildDecodeException32BitValue(values_array, index));
    2182          16 :   return Binop(wasm::kExprI64Ior, upper, lower);
    2183             : }
    2184             : 
    2185         447 : Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
    2186         447 :   needs_stack_check_ = true;
    2187             :   WasmThrowDescriptor interface_descriptor;
    2188         447 :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    2189             :       mcgraph()->zone(), interface_descriptor,
    2190             :       interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
    2191         447 :       Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
    2192             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    2193         447 :       wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
    2194         446 :   return SetEffect(SetControl(
    2195             :       graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
    2196         447 :                        except_obj, Effect(), Control())));
    2197             : }
    2198             : 
    2199         376 : Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
    2200             :                                           Node* expected_tag) {
    2201             :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2202         752 :   return graph()->NewNode(machine->WordEqual(), caught_tag, expected_tag);
    2203             : }
    2204             : 
    2205         910 : Node* WasmGraphBuilder::LoadExceptionTagFromTable(uint32_t exception_index) {
    2206             :   Node* exceptions_table =
    2207        1820 :       LOAD_INSTANCE_FIELD(ExceptionsTable, MachineType::TaggedPointer());
    2208        1824 :   Node* tag = LOAD_FIXED_ARRAY_SLOT_PTR(exceptions_table, exception_index);
    2209         912 :   return tag;
    2210             : }
    2211             : 
    2212         376 : Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
    2213         376 :   needs_stack_check_ = true;
    2214         376 :   return BuildCallToRuntime(Runtime::kWasmExceptionGetTag, &except_obj, 1);
    2215             : }
    2216             : 
    2217         376 : Node** WasmGraphBuilder::GetExceptionValues(
    2218             :     Node* except_obj, const wasm::WasmException* exception) {
    2219             :   Node* values_array =
    2220         376 :       BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
    2221         376 :   uint32_t index = 0;
    2222         376 :   const wasm::WasmExceptionSig* sig = exception->sig;
    2223         376 :   Node** values = Buffer(sig->parameter_count());
    2224         872 :   for (size_t i = 0; i < sig->parameter_count(); ++i) {
    2225             :     Node* value;
    2226         248 :     switch (sig->GetParam(i)) {
    2227             :       case wasm::kWasmI32:
    2228         200 :         value = BuildDecodeException32BitValue(values_array, &index);
    2229         200 :         break;
    2230             :       case wasm::kWasmI64:
    2231           8 :         value = BuildDecodeException64BitValue(values_array, &index);
    2232           8 :         break;
    2233             :       case wasm::kWasmF32: {
    2234           8 :         value = Unop(wasm::kExprF32ReinterpretI32,
    2235           8 :                      BuildDecodeException32BitValue(values_array, &index));
    2236           8 :         break;
    2237             :       }
    2238             :       case wasm::kWasmF64: {
    2239           8 :         value = Unop(wasm::kExprF64ReinterpretI64,
    2240           8 :                      BuildDecodeException64BitValue(values_array, &index));
    2241           8 :         break;
    2242             :       }
    2243             :       case wasm::kWasmS128:
    2244           8 :         value = graph()->NewNode(
    2245             :             mcgraph()->machine()->I32x4Splat(),
    2246             :             BuildDecodeException32BitValue(values_array, &index));
    2247           8 :         value = graph()->NewNode(
    2248             :             mcgraph()->machine()->I32x4ReplaceLane(1), value,
    2249             :             BuildDecodeException32BitValue(values_array, &index));
    2250           8 :         value = graph()->NewNode(
    2251             :             mcgraph()->machine()->I32x4ReplaceLane(2), value,
    2252             :             BuildDecodeException32BitValue(values_array, &index));
    2253           8 :         value = graph()->NewNode(
    2254             :             mcgraph()->machine()->I32x4ReplaceLane(3), value,
    2255             :             BuildDecodeException32BitValue(values_array, &index));
    2256           8 :         break;
    2257             :       case wasm::kWasmAnyRef:
    2258          32 :         value = LOAD_FIXED_ARRAY_SLOT_ANY(values_array, index);
    2259          16 :         ++index;
    2260          16 :         break;
    2261             :       default:
    2262           0 :         UNREACHABLE();
    2263             :     }
    2264         248 :     values[i] = value;
    2265             :   }
    2266             :   DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(exception));
    2267         376 :   return values;
    2268             : }
    2269             : 
    2270       14180 : Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
    2271             :                                      wasm::WasmCodePosition position) {
    2272             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2273             :   ZeroCheck32(wasm::kTrapDivByZero, right, position);
    2274             :   Node* before = Control();
    2275             :   Node* denom_is_m1;
    2276             :   Node* denom_is_not_m1;
    2277       14180 :   BranchExpectFalse(
    2278             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
    2279             :       &denom_is_m1, &denom_is_not_m1);
    2280       14180 :   SetControl(denom_is_m1);
    2281       14180 :   TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
    2282       14180 :   if (Control() != denom_is_m1) {
    2283         956 :     SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
    2284             :                                 Control()));
    2285             :   } else {
    2286             :     SetControl(before);
    2287             :   }
    2288       28360 :   return graph()->NewNode(m->Int32Div(), left, right, Control());
    2289             : }
    2290             : 
    2291       14104 : Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
    2292             :                                      wasm::WasmCodePosition position) {
    2293             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2294             : 
    2295             :   ZeroCheck32(wasm::kTrapRemByZero, right, position);
    2296             : 
    2297             :   Diamond d(
    2298             :       graph(), mcgraph()->common(),
    2299             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
    2300       28208 :       BranchHint::kFalse);
    2301             :   d.Chain(Control());
    2302             : 
    2303       28208 :   return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
    2304       14104 :                graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
    2305             : }
    2306             : 
    2307       14124 : Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
    2308             :                                      wasm::WasmCodePosition position) {
    2309             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2310       14124 :   return graph()->NewNode(m->Uint32Div(), left, right,
    2311       14124 :                           ZeroCheck32(wasm::kTrapDivByZero, right, position));
    2312             : }
    2313             : 
    2314       14084 : Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
    2315             :                                      wasm::WasmCodePosition position) {
    2316             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2317       14084 :   return graph()->NewNode(m->Uint32Mod(), left, right,
    2318       14084 :                           ZeroCheck32(wasm::kTrapRemByZero, right, position));
    2319             : }
    2320             : 
    2321         385 : Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
    2322             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2323             : 
    2324             :   Int32Matcher mr(right);
    2325         385 :   if (mr.HasValue()) {
    2326         317 :     if (mr.Value() == 0) {
    2327          13 :       return mcgraph()->Int32Constant(0);
    2328         304 :     } else if (mr.Value() == -1) {
    2329             :       // The result is the negation of the left input.
    2330          26 :       return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
    2331             :     }
    2332         582 :     return graph()->NewNode(m->Int32Div(), left, right, Control());
    2333             :   }
    2334             : 
    2335             :   // asm.js semantics return 0 on divide or mod by zero.
    2336          68 :   if (m->Int32DivIsSafe()) {
    2337             :     // The hardware instruction does the right thing (e.g. arm).
    2338           0 :     return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
    2339             :   }
    2340             : 
    2341             :   // Check denominator for zero.
    2342             :   Diamond z(
    2343             :       graph(), mcgraph()->common(),
    2344             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
    2345         136 :       BranchHint::kFalse);
    2346             : 
    2347             :   // Check numerator for -1. (avoid minint / -1 case).
    2348             :   Diamond n(
    2349             :       graph(), mcgraph()->common(),
    2350             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
    2351         136 :       BranchHint::kFalse);
    2352             : 
    2353          68 :   Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
    2354             :   Node* neg =
    2355          68 :       graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
    2356             : 
    2357          68 :   return n.Phi(
    2358             :       MachineRepresentation::kWord32, neg,
    2359          68 :       z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
    2360             : }
    2361             : 
    2362         421 : Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
    2363             :   CommonOperatorBuilder* c = mcgraph()->common();
    2364             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2365         421 :   Node* const zero = mcgraph()->Int32Constant(0);
    2366             : 
    2367             :   Int32Matcher mr(right);
    2368         421 :   if (mr.HasValue()) {
    2369         360 :     if (mr.Value() == 0 || mr.Value() == -1) {
    2370             :       return zero;
    2371             :     }
    2372         668 :     return graph()->NewNode(m->Int32Mod(), left, right, Control());
    2373             :   }
    2374             : 
    2375             :   // General case for signed integer modulus, with optimization for (unknown)
    2376             :   // power of 2 right hand side.
    2377             :   //
    2378             :   //   if 0 < right then
    2379             :   //     msk = right - 1
    2380             :   //     if right & msk != 0 then
    2381             :   //       left % right
    2382             :   //     else
    2383             :   //       if left < 0 then
    2384             :   //         -(-left & msk)
    2385             :   //       else
    2386             :   //         left & msk
    2387             :   //   else
    2388             :   //     if right < -1 then
    2389             :   //       left % right
    2390             :   //     else
    2391             :   //       zero
    2392             :   //
    2393             :   // Note: We do not use the Diamond helper class here, because it really hurts
    2394             :   // readability with nested diamonds.
    2395          61 :   Node* const minus_one = mcgraph()->Int32Constant(-1);
    2396             : 
    2397          61 :   const Operator* const merge_op = c->Merge(2);
    2398          61 :   const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
    2399             : 
    2400          61 :   Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
    2401             :   Node* branch0 =
    2402          61 :       graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
    2403             : 
    2404          61 :   Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
    2405             :   Node* true0;
    2406             :   {
    2407          61 :     Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
    2408             : 
    2409          61 :     Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
    2410          61 :     Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
    2411             : 
    2412          61 :     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
    2413          61 :     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
    2414             : 
    2415          61 :     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
    2416             :     Node* false1;
    2417             :     {
    2418          61 :       Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
    2419             :       Node* branch2 =
    2420          61 :           graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
    2421             : 
    2422          61 :       Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
    2423         183 :       Node* true2 = graph()->NewNode(
    2424             :           m->Int32Sub(), zero,
    2425             :           graph()->NewNode(m->Word32And(),
    2426             :                            graph()->NewNode(m->Int32Sub(), zero, left), msk));
    2427             : 
    2428          61 :       Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
    2429          61 :       Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
    2430             : 
    2431             :       if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
    2432             :       false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
    2433             :     }
    2434             : 
    2435             :     if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
    2436             :     true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
    2437             :   }
    2438             : 
    2439          61 :   Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
    2440             :   Node* false0;
    2441             :   {
    2442          61 :     Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
    2443             :     Node* branch1 =
    2444          61 :         graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
    2445             : 
    2446          61 :     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
    2447          61 :     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
    2448             : 
    2449          61 :     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
    2450             :     Node* false1 = zero;
    2451             : 
    2452             :     if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
    2453             :     false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
    2454             :   }
    2455             : 
    2456             :   Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
    2457          61 :   return graph()->NewNode(phi_op, true0, false0, merge0);
    2458             : }
    2459             : 
    2460         137 : Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
    2461             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2462             :   // asm.js semantics return 0 on divide or mod by zero.
    2463         137 :   if (m->Uint32DivIsSafe()) {
    2464             :     // The hardware instruction does the right thing (e.g. arm).
    2465           0 :     return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
    2466             :   }
    2467             : 
    2468             :   // Explicit check for x % 0.
    2469             :   Diamond z(
    2470             :       graph(), mcgraph()->common(),
    2471             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
    2472         274 :       BranchHint::kFalse);
    2473             : 
    2474         274 :   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
    2475             :                graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
    2476         137 :                                 z.if_false));
    2477             : }
    2478             : 
    2479         186 : Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
    2480             :   MachineOperatorBuilder* m = mcgraph()->machine();
    2481             :   // asm.js semantics return 0 on divide or mod by zero.
    2482             :   // Explicit check for x % 0.
    2483             :   Diamond z(
    2484             :       graph(), mcgraph()->common(),
    2485             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
    2486         372 :       BranchHint::kFalse);
    2487             : 
    2488         186 :   Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
    2489             :                                z.if_false);
    2490         186 :   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
    2491         186 :                rem);
    2492             : }
    2493             : 
    2494         720 : Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
    2495             :                                      wasm::WasmCodePosition position) {
    2496         720 :   if (mcgraph()->machine()->Is32()) {
    2497           0 :     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
    2498           0 :                           MachineType::Int64(), wasm::kTrapDivByZero, position);
    2499             :   }
    2500             :   ZeroCheck64(wasm::kTrapDivByZero, right, position);
    2501             :   Node* before = Control();
    2502             :   Node* denom_is_m1;
    2503             :   Node* denom_is_not_m1;
    2504         720 :   BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
    2505             :                                      mcgraph()->Int64Constant(-1)),
    2506             :                     &denom_is_m1, &denom_is_not_m1);
    2507         720 :   SetControl(denom_is_m1);
    2508             :   TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
    2509         720 :              std::numeric_limits<int64_t>::min(), position);
    2510         720 :   if (Control() != denom_is_m1) {
    2511         716 :     SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
    2512             :                                 Control()));
    2513             :   } else {
    2514             :     SetControl(before);
    2515             :   }
    2516         720 :   return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
    2517         720 :                           Control());
    2518             : }
    2519             : 
    2520         644 : Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
    2521             :                                      wasm::WasmCodePosition position) {
    2522         644 :   if (mcgraph()->machine()->Is32()) {
    2523           0 :     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
    2524           0 :                           MachineType::Int64(), wasm::kTrapRemByZero, position);
    2525             :   }
    2526             :   ZeroCheck64(wasm::kTrapRemByZero, right, position);
    2527             :   Diamond d(mcgraph()->graph(), mcgraph()->common(),
    2528             :             graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
    2529        1288 :                              mcgraph()->Int64Constant(-1)));
    2530             : 
    2531             :   d.Chain(Control());
    2532             : 
    2533         644 :   Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
    2534             :                                d.if_false);
    2535             : 
    2536         644 :   return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
    2537         644 :                rem);
    2538             : }
    2539             : 
    2540         660 : Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
    2541             :                                      wasm::WasmCodePosition position) {
    2542         660 :   if (mcgraph()->machine()->Is32()) {
    2543           0 :     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
    2544           0 :                           MachineType::Int64(), wasm::kTrapDivByZero, position);
    2545             :   }
    2546         660 :   return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
    2547         660 :                           ZeroCheck64(wasm::kTrapDivByZero, right, position));
    2548             : }
    2549         644 : Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
    2550             :                                      wasm::WasmCodePosition position) {
    2551         644 :   if (mcgraph()->machine()->Is32()) {
    2552           0 :     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
    2553           0 :                           MachineType::Int64(), wasm::kTrapRemByZero, position);
    2554             :   }
    2555         644 :   return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
    2556         644 :                           ZeroCheck64(wasm::kTrapRemByZero, right, position));
    2557             : }
    2558             : 
    2559           0 : Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
    2560             :                                        ExternalReference ref,
    2561             :                                        MachineType result_type,
    2562             :                                        wasm::TrapReason trap_zero,
    2563             :                                        wasm::WasmCodePosition position) {
    2564             :   Node* stack_slot =
    2565           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
    2566             : 
    2567           0 :   const Operator* store_op = mcgraph()->machine()->Store(
    2568           0 :       StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
    2569           0 :   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
    2570             :                              left, Effect(), Control()));
    2571           0 :   SetEffect(graph()->NewNode(store_op, stack_slot,
    2572             :                              mcgraph()->Int32Constant(sizeof(double)), right,
    2573             :                              Effect(), Control()));
    2574             : 
    2575           0 :   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    2576             :   MachineSignature sig(1, 1, sig_types);
    2577             : 
    2578           0 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    2579           0 :   Node* call = BuildCCall(&sig, function, stack_slot);
    2580             : 
    2581             :   ZeroCheck32(trap_zero, call, position);
    2582           0 :   TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
    2583           0 :   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
    2584             :                                     stack_slot, mcgraph()->Int32Constant(0),
    2585           0 :                                     Effect(), Control()));
    2586             : }
    2587             : 
    2588             : template <typename... Args>
    2589         199 : Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
    2590             :                                    Args... args) {
    2591             :   DCHECK_LE(sig->return_count(), 1);
    2592             :   DCHECK_EQ(sizeof...(args), sig->parameter_count());
    2593         597 :   Node* const call_args[] = {function, args..., Effect(), Control()};
    2594             : 
    2595             :   auto call_descriptor =
    2596         199 :       Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
    2597             : 
    2598         199 :   const Operator* op = mcgraph()->common()->Call(call_descriptor);
    2599         398 :   return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
    2600             : }
    2601             : 
    2602      172398 : Node* WasmGraphBuilder::BuildCallNode(wasm::FunctionSig* sig, Node** args,
    2603             :                                       wasm::WasmCodePosition position,
    2604             :                                       Node* instance_node, const Operator* op) {
    2605      172398 :   if (instance_node == nullptr) {
    2606             :     DCHECK_NOT_NULL(instance_node_);
    2607             :     instance_node = instance_node_.get();
    2608             :   }
    2609      172398 :   needs_stack_check_ = true;
    2610             :   const size_t params = sig->parameter_count();
    2611             :   const size_t extra = 3;  // instance_node, effect, and control.
    2612      172398 :   const size_t count = 1 + params + extra;
    2613             : 
    2614             :   // Reallocate the buffer to make space for extra inputs.
    2615      172398 :   args = Realloc(args, 1 + params, count);
    2616             : 
    2617             :   // Make room for the instance_node parameter at index 1, just after code.
    2618      172420 :   memmove(&args[2], &args[1], params * sizeof(Node*));
    2619      172420 :   args[1] = instance_node;
    2620             : 
    2621             :   // Add effect and control inputs.
    2622      344840 :   args[params + 2] = Effect();
    2623      344840 :   args[params + 3] = Control();
    2624             : 
    2625      172420 :   Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
    2626             :   DCHECK(position == wasm::kNoCodePosition || position > 0);
    2627      172440 :   if (position > 0) SetSourcePosition(call, position);
    2628             : 
    2629      172441 :   return call;
    2630             : }
    2631             : 
    2632      172156 : Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
    2633             :                                       Node*** rets,
    2634             :                                       wasm::WasmCodePosition position,
    2635             :                                       Node* instance_node,
    2636             :                                       UseRetpoline use_retpoline) {
    2637             :   auto call_descriptor =
    2638      172156 :       GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
    2639      172196 :   const Operator* op = mcgraph()->common()->Call(call_descriptor);
    2640      172187 :   Node* call = BuildCallNode(sig, args, position, instance_node, op);
    2641             : 
    2642             :   size_t ret_count = sig->return_count();
    2643      172201 :   if (ret_count == 0) return call;  // No return value.
    2644             : 
    2645       39067 :   *rets = Buffer(ret_count);
    2646       39068 :   if (ret_count == 1) {
    2647             :     // Only a single return value.
    2648       37932 :     (*rets)[0] = call;
    2649             :   } else {
    2650             :     // Create projections for all return values.
    2651        5712 :     for (size_t i = 0; i < ret_count; i++) {
    2652        2288 :       (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
    2653        2288 :                                     graph()->start());
    2654             :     }
    2655             :   }
    2656             :   return call;
    2657             : }
    2658             : 
    2659         248 : Node* WasmGraphBuilder::BuildWasmReturnCall(wasm::FunctionSig* sig, Node** args,
    2660             :                                             wasm::WasmCodePosition position,
    2661             :                                             Node* instance_node,
    2662             :                                             UseRetpoline use_retpoline) {
    2663             :   auto call_descriptor =
    2664         248 :       GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
    2665         248 :   const Operator* op = mcgraph()->common()->TailCall(call_descriptor);
    2666         248 :   Node* call = BuildCallNode(sig, args, position, instance_node, op);
    2667             : 
    2668         248 :   MergeControlToEnd(mcgraph(), call);
    2669             : 
    2670         248 :   return call;
    2671             : }
    2672             : 
    2673        6969 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
    2674             :                                         Node*** rets,
    2675             :                                         wasm::WasmCodePosition position,
    2676             :                                         int func_index,
    2677             :                                         IsReturnCall continuation) {
    2678             :   // Load the imported function refs array from the instance.
    2679             :   Node* imported_function_refs =
    2680       13938 :       LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
    2681             :   Node* ref_node =
    2682        6967 :       LOAD_FIXED_ARRAY_SLOT_PTR(imported_function_refs, func_index);
    2683             : 
    2684             :   // Load the target from the imported_targets array at a known offset.
    2685             :   Node* imported_targets =
    2686       13937 :       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
    2687        6969 :   Node* target_node = SetEffect(graph()->NewNode(
    2688             :       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
    2689             :       mcgraph()->Int32Constant(func_index * kSystemPointerSize), Effect(),
    2690             :       Control()));
    2691        6969 :   args[0] = target_node;
    2692             :   const UseRetpoline use_retpoline =
    2693        6969 :       untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
    2694             : 
    2695        6969 :   switch (continuation) {
    2696             :     case kCallContinues:
    2697        6953 :       return BuildWasmCall(sig, args, rets, position, ref_node, use_retpoline);
    2698             :     case kReturnCall:
    2699             :       DCHECK_NULL(rets);
    2700          16 :       return BuildWasmReturnCall(sig, args, position, ref_node, use_retpoline);
    2701             :   }
    2702             : }
    2703             : 
    2704         456 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
    2705             :                                         Node*** rets,
    2706             :                                         wasm::WasmCodePosition position,
    2707             :                                         Node* func_index,
    2708             :                                         IsReturnCall continuation) {
    2709             :   // Load the imported function refs array from the instance.
    2710             :   Node* imported_function_refs =
    2711         912 :       LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
    2712             :   // Access fixed array at {header_size - tag + func_index * kTaggedSize}.
    2713         456 :   Node* imported_instances_data = graph()->NewNode(
    2714             :       mcgraph()->machine()->IntAdd(), imported_function_refs,
    2715             :       mcgraph()->IntPtrConstant(
    2716             :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
    2717         456 :   Node* func_index_times_tagged_size = graph()->NewNode(
    2718             :       mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
    2719             :       mcgraph()->Int32Constant(kTaggedSize));
    2720         456 :   Node* ref_node = SetEffect(
    2721             :       graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    2722             :                        imported_instances_data, func_index_times_tagged_size,
    2723             :                        Effect(), Control()));
    2724             : 
    2725             :   // Load the target from the imported_targets array at the offset of
    2726             :   // {func_index}.
    2727             :   Node* func_index_times_pointersize;
    2728             :   if (kSystemPointerSize == kTaggedSize) {
    2729             :     func_index_times_pointersize = func_index_times_tagged_size;
    2730             : 
    2731             :   } else {
    2732             :     DCHECK_EQ(kSystemPointerSize, kTaggedSize + kTaggedSize);
    2733         456 :     func_index_times_pointersize = graph()->NewNode(
    2734             :         mcgraph()->machine()->Int32Add(), func_index_times_tagged_size,
    2735             :         func_index_times_tagged_size);
    2736             :   }
    2737             :   Node* imported_targets =
    2738         912 :       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
    2739         456 :   Node* target_node = SetEffect(graph()->NewNode(
    2740             :       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
    2741             :       func_index_times_pointersize, Effect(), Control()));
    2742         456 :   args[0] = target_node;
    2743             :   const UseRetpoline use_retpoline =
    2744         456 :       untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
    2745             : 
    2746         456 :   switch (continuation) {
    2747             :     case kCallContinues:
    2748         456 :       return BuildWasmCall(sig, args, rets, position, ref_node, use_retpoline);
    2749             :     case kReturnCall:
    2750             :       DCHECK_NULL(rets);
    2751           0 :       return BuildWasmReturnCall(sig, args, position, ref_node, use_retpoline);
    2752             :   }
    2753             : }
    2754             : 
    2755       29225 : Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
    2756             :                                    wasm::WasmCodePosition position) {
    2757             :   DCHECK_NULL(args[0]);
    2758       58450 :   wasm::FunctionSig* sig = env_->module->functions[index].sig;
    2759             : 
    2760       29225 :   if (env_ && index < env_->module->num_imported_functions) {
    2761             :     // Call to an imported function.
    2762        6953 :     return BuildImportCall(sig, args, rets, position, index, kCallContinues);
    2763             :   }
    2764             : 
    2765             :   // A direct call to a wasm function defined in this module.
    2766             :   // Just encode the function index. This will be patched at instantiation.
    2767             :   Address code = static_cast<Address>(index);
    2768       22272 :   args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
    2769             : 
    2770       22294 :   return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
    2771             : }
    2772             : 
    2773        3353 : Node* WasmGraphBuilder::CallIndirect(uint32_t table_index, uint32_t sig_index,
    2774             :                                      Node** args, Node*** rets,
    2775             :                                      wasm::WasmCodePosition position) {
    2776        3353 :   if (table_index == 0) {
    2777        3257 :     return BuildIndirectCall(sig_index, args, rets, position, kCallContinues);
    2778             :   }
    2779             :   return BuildIndirectCall(table_index, sig_index, args, rets, position,
    2780          96 :                            kCallContinues);
    2781             : }
    2782             : 
    2783        3298 : Node* WasmGraphBuilder::BuildIndirectCall(uint32_t sig_index, Node** args,
    2784             :                                           Node*** rets,
    2785             :                                           wasm::WasmCodePosition position,
    2786             :                                           IsReturnCall continuation) {
    2787             :   DCHECK_NOT_NULL(args[0]);
    2788             :   DCHECK_NOT_NULL(env_);
    2789             : 
    2790             :   // Assume only one table for now.
    2791        6596 :   wasm::FunctionSig* sig = env_->module->signatures[sig_index];
    2792             : 
    2793             :   Node* ift_size =
    2794        6598 :       LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
    2795             : 
    2796             :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2797        3300 :   Node* key = args[0];
    2798             : 
    2799             :   // Bounds check against the table size.
    2800        3300 :   Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
    2801        3299 :   TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
    2802             : 
    2803             :   // Mask the key to prevent SSCA.
    2804        3300 :   if (untrusted_code_mitigations_) {
    2805             :     // mask = ((key - size) & ~key) >> 31
    2806             :     Node* neg_key =
    2807           0 :         graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
    2808           0 :     Node* masked_diff = graph()->NewNode(
    2809             :         machine->Word32And(),
    2810             :         graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
    2811             :     Node* mask =
    2812           0 :         graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
    2813           0 :     key = graph()->NewNode(machine->Word32And(), key, mask);
    2814             :   }
    2815             : 
    2816             :   // Load signature from the table and check.
    2817             :   Node* ift_sig_ids =
    2818        6597 :       LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
    2819             : 
    2820        6600 :   int32_t expected_sig_id = env_->module->signature_ids[sig_index];
    2821        3299 :   Node* int32_scaled_key = Uint32ToUintptr(
    2822        3299 :       graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
    2823             : 
    2824        3299 :   Node* loaded_sig = SetEffect(
    2825             :       graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
    2826             :                        int32_scaled_key, Effect(), Control()));
    2827        3299 :   Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
    2828             :                                      Int32Constant(expected_sig_id));
    2829             : 
    2830        3299 :   TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
    2831             : 
    2832             :   Node* ift_targets =
    2833        6600 :       LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
    2834        6600 :   Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableRefs,
    2835             :                                             MachineType::TaggedPointer());
    2836             : 
    2837             :   Node* tagged_scaled_key;
    2838             :   if (kTaggedSize == kInt32Size) {
    2839             :     tagged_scaled_key = int32_scaled_key;
    2840             :   } else {
    2841             :     DCHECK_EQ(kTaggedSize, kInt32Size * 2);
    2842             :     tagged_scaled_key = graph()->NewNode(machine->Int32Add(), int32_scaled_key,
    2843             :                                          int32_scaled_key);
    2844             :   }
    2845             : 
    2846        6600 :   Node* target_instance = SetEffect(graph()->NewNode(
    2847             :       machine->Load(MachineType::TaggedPointer()),
    2848             :       graph()->NewNode(machine->IntAdd(), ift_instances, tagged_scaled_key),
    2849             :       Int32Constant(wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)),
    2850             :       Effect(), Control()));
    2851             : 
    2852             :   Node* intptr_scaled_key;
    2853             :   if (kSystemPointerSize == kTaggedSize) {
    2854             :     intptr_scaled_key = tagged_scaled_key;
    2855             :   } else {
    2856             :     DCHECK_EQ(kSystemPointerSize, kTaggedSize + kTaggedSize);
    2857        3299 :     intptr_scaled_key = graph()->NewNode(machine->Int32Add(), tagged_scaled_key,
    2858             :                                          tagged_scaled_key);
    2859             :   }
    2860             : 
    2861        3298 :   Node* target = SetEffect(
    2862             :       graph()->NewNode(machine->Load(MachineType::Pointer()), ift_targets,
    2863             :                        intptr_scaled_key, Effect(), Control()));
    2864             : 
    2865        3299 :   args[0] = target;
    2866             :   const UseRetpoline use_retpoline =
    2867        3299 :       untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
    2868             : 
    2869        3299 :   switch (continuation) {
    2870             :     case kCallContinues:
    2871        3259 :       return BuildWasmCall(sig, args, rets, position, target_instance,
    2872        3259 :                            use_retpoline);
    2873             :     case kReturnCall:
    2874          40 :       return BuildWasmReturnCall(sig, args, position, target_instance,
    2875          40 :                                  use_retpoline);
    2876             :   }
    2877             : }
    2878             : 
    2879         144 : Node* WasmGraphBuilder::BuildIndirectCall(uint32_t table_index,
    2880             :                                           uint32_t sig_index, Node** args,
    2881             :                                           Node*** rets,
    2882             :                                           wasm::WasmCodePosition position,
    2883             :                                           IsReturnCall continuation) {
    2884             :   DCHECK_NOT_NULL(args[0]);
    2885         144 :   Node* entry_index = args[0];
    2886             :   DCHECK_NOT_NULL(env_);
    2887             :   BoundsCheckTable(table_index, entry_index, position, wasm::kTrapFuncInvalid,
    2888         144 :                    nullptr);
    2889             : 
    2890             :   DCHECK(Smi::IsValid(table_index));
    2891             :   DCHECK(Smi::IsValid(sig_index));
    2892             :   Node* runtime_args[]{
    2893         143 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
    2894         143 :       BuildChangeUint31ToSmi(entry_index),
    2895         429 :       graph()->NewNode(mcgraph()->common()->NumberConstant(sig_index))};
    2896             : 
    2897             :   Node* target_instance = BuildCallToRuntime(
    2898             :       Runtime::kWasmIndirectCallCheckSignatureAndGetTargetInstance,
    2899         143 :       runtime_args, arraysize(runtime_args));
    2900             : 
    2901             :   // We reuse the runtime_args array here, even though we only need the first
    2902             :   // two arguments.
    2903             :   Node* call_target = BuildCallToRuntime(
    2904         143 :       Runtime::kWasmIndirectCallGetTargetAddress, runtime_args, 2);
    2905             : 
    2906         288 :   wasm::FunctionSig* sig = env_->module->signatures[sig_index];
    2907         144 :   args[0] = call_target;
    2908             :   const UseRetpoline use_retpoline =
    2909         144 :       untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline;
    2910             : 
    2911         144 :   switch (continuation) {
    2912             :     case kCallContinues:
    2913          96 :       return BuildWasmCall(sig, args, rets, position, target_instance,
    2914          96 :                            use_retpoline);
    2915             :     case kReturnCall:
    2916          48 :       return BuildWasmReturnCall(sig, args, position, target_instance,
    2917          48 :                                  use_retpoline);
    2918             :   }
    2919             : }
    2920             : 
    2921         160 : Node* WasmGraphBuilder::ReturnCall(uint32_t index, Node** args,
    2922             :                                    wasm::WasmCodePosition position) {
    2923             :   DCHECK_NULL(args[0]);
    2924         320 :   wasm::FunctionSig* sig = env_->module->functions[index].sig;
    2925             : 
    2926         160 :   if (env_ && index < env_->module->num_imported_functions) {
    2927             :     // Return Call to an imported function.
    2928          16 :     return BuildImportCall(sig, args, nullptr, position, index, kReturnCall);
    2929             :   }
    2930             : 
    2931             :   // A direct tail call to a wasm function defined in this module.
    2932             :   // Just encode the function index. This will be patched during code
    2933             :   // generation.
    2934             :   Address code = static_cast<Address>(index);
    2935         144 :   args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
    2936             : 
    2937         144 :   return BuildWasmReturnCall(sig, args, position, nullptr, kNoRetpoline);
    2938             : }
    2939             : 
    2940          88 : Node* WasmGraphBuilder::ReturnCallIndirect(uint32_t table_index,
    2941             :                                            uint32_t sig_index, Node** args,
    2942             :                                            wasm::WasmCodePosition position) {
    2943          88 :   if (table_index == 0) {
    2944          40 :     return BuildIndirectCall(sig_index, args, nullptr, position, kReturnCall);
    2945             :   }
    2946             :   return BuildIndirectCall(table_index, sig_index, args, nullptr, position,
    2947          48 :                            kReturnCall);
    2948             : }
    2949             : 
    2950       13472 : Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
    2951             :   // Implement Rol by Ror since TurboFan does not have Rol opcode.
    2952             :   // TODO(weiliang): support Word32Rol opcode in TurboFan.
    2953             :   Int32Matcher m(right);
    2954       13472 :   if (m.HasValue()) {
    2955       13456 :     return Binop(wasm::kExprI32Ror, left,
    2956       26912 :                  mcgraph()->Int32Constant(32 - (m.Value() & 0x1F)));
    2957             :   } else {
    2958          16 :     return Binop(wasm::kExprI32Ror, left,
    2959          16 :                  Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
    2960             :   }
    2961             : }
    2962             : 
    2963          60 : Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
    2964             :   // Implement Rol by Ror since TurboFan does not have Rol opcode.
    2965             :   // TODO(weiliang): support Word64Rol opcode in TurboFan.
    2966             :   Int64Matcher m(right);
    2967          60 :   if (m.HasValue()) {
    2968          16 :     return Binop(wasm::kExprI64Ror, left,
    2969          32 :                  mcgraph()->Int64Constant(64 - (m.Value() & 0x3F)));
    2970             :   } else {
    2971          44 :     return Binop(wasm::kExprI64Ror, left,
    2972          44 :                  Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
    2973             :   }
    2974             : }
    2975             : 
    2976       21667 : Node* WasmGraphBuilder::Invert(Node* node) {
    2977       21667 :   return Unop(wasm::kExprI32Eqz, node);
    2978             : }
    2979             : 
    2980           0 : bool CanCover(Node* value, IrOpcode::Value opcode) {
    2981           0 :   if (value->opcode() != opcode) return false;
    2982             :   bool first = true;
    2983           0 :   for (Edge const edge : value->use_edges()) {
    2984           0 :     if (NodeProperties::IsControlEdge(edge)) continue;
    2985           0 :     if (NodeProperties::IsEffectEdge(edge)) continue;
    2986             :     DCHECK(NodeProperties::IsValueEdge(edge));
    2987           0 :     if (!first) return false;
    2988             :     first = false;
    2989             :   }
    2990           0 :   return true;
    2991             : }
    2992             : 
    2993       64630 : Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
    2994       64630 :   if (mcgraph()->machine()->Is64()) {
    2995       64631 :     value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
    2996             :   }
    2997       64634 :   return value;
    2998             : }
    2999             : 
    3000           0 : Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
    3001           0 :   value = BuildChangeInt32ToIntPtr(value);
    3002           0 :   return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
    3003           0 :                           BuildSmiShiftBitsConstant());
    3004             : }
    3005             : 
    3006        1581 : Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
    3007        1582 :   return graph()->NewNode(mcgraph()->machine()->WordShl(),
    3008        1582 :                           Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
    3009             : }
    3010             : 
    3011           0 : Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
    3012      225706 :   return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
    3013             : }
    3014             : 
    3015      224125 : Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
    3016      224129 :   value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
    3017             :                            BuildSmiShiftBitsConstant());
    3018      224132 :   if (mcgraph()->machine()->Is64()) {
    3019             :     value =
    3020      224133 :         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
    3021             :   }
    3022      224131 :   return value;
    3023             : }
    3024             : 
    3025         264 : Node* WasmGraphBuilder::BuildConvertUint32ToSmiWithSaturation(Node* value,
    3026             :                                                               uint32_t maxval) {
    3027             :   DCHECK(Smi::IsValid(maxval));
    3028             :   Node* max = Uint32Constant(maxval);
    3029         264 :   Node* check = graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(),
    3030             :                                  value, max);
    3031         264 :   Node* valsmi = BuildChangeUint31ToSmi(value);
    3032         264 :   Node* maxsmi = graph()->NewNode(mcgraph()->common()->NumberConstant(maxval));
    3033         264 :   Diamond d(graph(), mcgraph()->common(), check, BranchHint::kTrue);
    3034             :   d.Chain(Control());
    3035         264 :   return d.Phi(MachineRepresentation::kTagged, valsmi, maxsmi);
    3036             : }
    3037             : 
    3038      537951 : void WasmGraphBuilder::InitInstanceCache(
    3039             :     WasmInstanceCacheNodes* instance_cache) {
    3040             :   DCHECK_NOT_NULL(instance_node_);
    3041             : 
    3042             :   // Load the memory start.
    3043     1076422 :   instance_cache->mem_start = SetEffect(graph()->NewNode(
    3044             :       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
    3045             :       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
    3046      538456 :       Effect(), Control()));
    3047             : 
    3048             :   // Load the memory size.
    3049     1076324 :   instance_cache->mem_size = SetEffect(graph()->NewNode(
    3050             :       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
    3051             :       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
    3052      538165 :       Effect(), Control()));
    3053             : 
    3054      538165 :   if (untrusted_code_mitigations_) {
    3055             :     // Load the memory mask.
    3056             :     instance_cache->mem_mask =
    3057           0 :         LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr());
    3058             :   } else {
    3059             :     // Explicitly set to nullptr to ensure a SEGV when we try to use it.
    3060      538165 :     instance_cache->mem_mask = nullptr;
    3061             :   }
    3062      538165 : }
    3063             : 
    3064        4299 : void WasmGraphBuilder::PrepareInstanceCacheForLoop(
    3065             :     WasmInstanceCacheNodes* instance_cache, Node* control) {
    3066             : #define INTRODUCE_PHI(field, rep)                                            \
    3067             :   instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
    3068             :                                            instance_cache->field, control);
    3069             : 
    3070        8599 :   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
    3071        8600 :   INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
    3072        4300 :   if (untrusted_code_mitigations_) {
    3073           0 :     INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation());
    3074             :   }
    3075             : 
    3076             : #undef INTRODUCE_PHI
    3077        4300 : }
    3078             : 
    3079       34461 : void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
    3080             :                                              WasmInstanceCacheNodes* from,
    3081             :                                              Node* merge) {
    3082             : #define INTRODUCE_PHI(field, rep)                                            \
    3083             :   if (to->field != from->field) {                                            \
    3084             :     Node* vals[] = {to->field, from->field, merge};                          \
    3085             :     to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
    3086             :   }
    3087             : 
    3088       34461 :   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
    3089       34459 :   INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
    3090       34459 :   if (untrusted_code_mitigations_) {
    3091           0 :     INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
    3092             :   }
    3093             : 
    3094             : #undef INTRODUCE_PHI
    3095       34459 : }
    3096             : 
    3097      213526 : void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
    3098             :                                               WasmInstanceCacheNodes* from,
    3099             :                                               Node* merge) {
    3100      213526 :   to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
    3101      213527 :                                       merge, to->mem_size, from->mem_size);
    3102      213527 :   to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
    3103      213529 :                                        merge, to->mem_start, from->mem_start);
    3104      213529 :   if (untrusted_code_mitigations_) {
    3105           0 :     to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
    3106           0 :                                         merge, to->mem_mask, from->mem_mask);
    3107             :   }
    3108      213529 : }
    3109             : 
    3110      992919 : Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
    3111             :                                              Node* merge, Node* tnode,
    3112             :                                              Node* fnode) {
    3113      992919 :   if (IsPhiWithMerge(tnode, merge)) {
    3114       94401 :     AppendToPhi(tnode, fnode);
    3115      898537 :   } else if (tnode != fnode) {
    3116       13298 :     uint32_t count = merge->InputCount();
    3117             :     // + 1 for the merge node.
    3118       13298 :     Node** vals = Buffer(count + 1);
    3119       40031 :     for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
    3120       13300 :     vals[count - 1] = fnode;
    3121       13300 :     vals[count] = merge;
    3122       13300 :     return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
    3123       13301 :                             vals);
    3124             :   }
    3125             :   return tnode;
    3126             : }
    3127             : 
    3128      213526 : Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
    3129             :                                                    Node* fnode) {
    3130      213526 :   if (IsPhiWithMerge(tnode, merge)) {
    3131       13446 :     AppendToPhi(tnode, fnode);
    3132      200081 :   } else if (tnode != fnode) {
    3133         128 :     uint32_t count = merge->InputCount();
    3134         128 :     Node** effects = Buffer(count);
    3135         858 :     for (uint32_t j = 0; j < count - 1; j++) {
    3136         365 :       effects[j] = tnode;
    3137             :     }
    3138         128 :     effects[count - 1] = fnode;
    3139         128 :     tnode = EffectPhi(count, effects, merge);
    3140             :   }
    3141      213529 :   return tnode;
    3142             : }
    3143             : 
    3144         327 : Node* WasmGraphBuilder::GetImportedMutableGlobals() {
    3145         327 :   if (imported_mutable_globals_ == nullptr) {
    3146             :     // Load imported_mutable_globals_ from the instance object at runtime.
    3147         590 :     imported_mutable_globals_ = graph()->NewNode(
    3148             :         mcgraph()->machine()->Load(MachineType::UintPtr()),
    3149             :         instance_node_.get(),
    3150             :         mcgraph()->Int32Constant(
    3151             :             WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
    3152             :         graph()->start(), graph()->start());
    3153             :   }
    3154         327 :   return imported_mutable_globals_.get();
    3155             : }
    3156             : 
    3157       53694 : void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
    3158             :                                               const wasm::WasmGlobal& global,
    3159             :                                               Node** base_node,
    3160             :                                               Node** offset_node) {
    3161             :   DCHECK_NOT_NULL(instance_node_);
    3162       53694 :   if (global.mutability && global.imported) {
    3163         136 :     *base_node = SetEffect(graph()->NewNode(
    3164             :         mcgraph()->machine()->Load(MachineType::UintPtr()),
    3165             :         GetImportedMutableGlobals(),
    3166         136 :         mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
    3167         135 :         Control()));
    3168         135 :     *offset_node = mcgraph()->Int32Constant(0);
    3169             :   } else {
    3170       53558 :     if (globals_start_ == nullptr) {
    3171             :       // Load globals_start from the instance object at runtime.
    3172             :       // TODO(wasm): we currently generate only one load of the {globals_start}
    3173             :       // start per graph, which means it can be placed anywhere by the
    3174             :       // scheduler. This is legal because the globals_start should never change.
    3175             :       // However, in some cases (e.g. if the instance object is already in a
    3176             :       // register), it is slightly more efficient to reload this value from the
    3177             :       // instance object. Since this depends on register allocation, it is not
    3178             :       // possible to express in the graph, and would essentially constitute a
    3179             :       // "mem2reg" optimization in TurboFan.
    3180       23807 :       globals_start_ = graph()->NewNode(
    3181             :           mcgraph()->machine()->Load(MachineType::UintPtr()),
    3182             :           instance_node_.get(),
    3183             :           mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
    3184             :           graph()->start(), graph()->start());
    3185             :     }
    3186       53561 :     *base_node = globals_start_.get();
    3187       53561 :     *offset_node = mcgraph()->Int32Constant(global.offset);
    3188             : 
    3189       53559 :     if (mem_type == MachineType::Simd128() && global.offset != 0) {
    3190             :       // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
    3191       10568 :       *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
    3192       10568 :                                     *offset_node);
    3193       10568 :       *offset_node = mcgraph()->Int32Constant(0);
    3194             :     }
    3195             :   }
    3196       53693 : }
    3197             : 
    3198         192 : void WasmGraphBuilder::GetBaseAndOffsetForImportedMutableAnyRefGlobal(
    3199             :     const wasm::WasmGlobal& global, Node** base, Node** offset) {
    3200             :   // Load the base from the ImportedMutableGlobalsBuffer of the instance.
    3201         384 :   Node* buffers = LOAD_INSTANCE_FIELD(ImportedMutableGlobalsBuffers,
    3202             :                                       MachineType::TaggedPointer());
    3203         576 :   *base = LOAD_FIXED_ARRAY_SLOT_ANY(buffers, global.index);
    3204             : 
    3205             :   // For the offset we need the index of the global in the buffer, and then
    3206             :   // calculate the actual offset from the index. Load the index from the
    3207             :   // ImportedMutableGlobals array of the instance.
    3208         192 :   Node* index = SetEffect(
    3209             :       graph()->NewNode(mcgraph()->machine()->Load(MachineType::UintPtr()),
    3210             :                        GetImportedMutableGlobals(),
    3211         192 :                        mcgraph()->Int32Constant(global.index * sizeof(Address)),
    3212             :                        Effect(), Control()));
    3213             : 
    3214             :   // From the index, calculate the actual offset in the FixeArray. This
    3215             :   // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
    3216             :   // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
    3217             :   Node* index_times_tagged_size =
    3218         192 :       graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
    3219             :                        mcgraph()->Int32Constant(kTaggedSize));
    3220         192 :   *offset = graph()->NewNode(
    3221             :       mcgraph()->machine()->IntAdd(), index_times_tagged_size,
    3222             :       mcgraph()->IntPtrConstant(
    3223         192 :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
    3224         192 : }
    3225             : 
    3226      274183 : Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
    3227             :   DCHECK_NOT_NULL(instance_cache_);
    3228      274351 :   Node* mem_start = instance_cache_->mem_start;
    3229             :   DCHECK_NOT_NULL(mem_start);
    3230      274183 :   if (offset == 0) return mem_start;
    3231       44535 :   return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
    3232       44501 :                           mcgraph()->IntPtrConstant(offset));
    3233             : }
    3234             : 
    3235         376 : Node* WasmGraphBuilder::CurrentMemoryPages() {
    3236             :   // CurrentMemoryPages can not be called from asm.js.
    3237             :   DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
    3238             :   DCHECK_NOT_NULL(instance_cache_);
    3239         376 :   Node* mem_size = instance_cache_->mem_size;
    3240             :   DCHECK_NOT_NULL(mem_size);
    3241             :   Node* result =
    3242         376 :       graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
    3243             :                        mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
    3244         376 :   if (mcgraph()->machine()->Is64()) {
    3245             :     result =
    3246         376 :         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
    3247             :   }
    3248         376 :   return result;
    3249             : }
    3250             : 
    3251         638 : Node* WasmGraphBuilder::BuildLoadBuiltinFromInstance(int builtin_index) {
    3252             :   DCHECK(Builtins::IsBuiltinId(builtin_index));
    3253        1276 :   Node* isolate_root = LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
    3254        1276 :   return LOAD_TAGGED_POINTER(isolate_root,
    3255             :                              IsolateData::builtin_slot_offset(builtin_index));
    3256             : }
    3257             : 
    3258             : // Only call this function for code which is not reused across instantiations,
    3259             : // as we do not patch the embedded js_context.
    3260      371328 : Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(
    3261             :     Runtime::FunctionId f, Node* js_context, Node** parameters,
    3262             :     int parameter_count, Node** effect, Node* control) {
    3263      371328 :   const Runtime::Function* fun = Runtime::FunctionForId(f);
    3264      742648 :   auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
    3265      371323 :       mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
    3266      371323 :       CallDescriptor::kNoFlags);
    3267             :   // The CEntryStub is loaded from the instance_node so that generated code is
    3268             :   // Isolate independent. At the moment this is only done for CEntryStub(1).
    3269             :   DCHECK_EQ(1, fun->result_size);
    3270             :   Node* centry_stub =
    3271      742650 :       LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
    3272             :   // TODO(titzer): allow arbitrary number of runtime arguments
    3273             :   // At the moment we only allow 5 parameters. If more parameters are needed,
    3274             :   // increase this constant accordingly.
    3275             :   static const int kMaxParams = 5;
    3276             :   DCHECK_GE(kMaxParams, parameter_count);
    3277             :   Node* inputs[kMaxParams + 6];
    3278             :   int count = 0;
    3279      371327 :   inputs[count++] = centry_stub;
    3280     1850837 :   for (int i = 0; i < parameter_count; i++) {
    3281      739755 :     inputs[count++] = parameters[i];
    3282             :   }
    3283      371327 :   inputs[count++] =
    3284      371327 :       mcgraph()->ExternalConstant(ExternalReference::Create(f));  // ref
    3285      371313 :   inputs[count++] = mcgraph()->Int32Constant(fun->nargs);         // arity
    3286      371323 :   inputs[count++] = js_context;                                   // js_context
    3287      371323 :   inputs[count++] = *effect;
    3288      371323 :   inputs[count++] = control;
    3289             : 
    3290      371323 :   Node* call = mcgraph()->graph()->NewNode(
    3291      371320 :       mcgraph()->common()->Call(call_descriptor), count, inputs);
    3292      371329 :   *effect = call;
    3293      371329 :   return call;
    3294             : }
    3295             : 
    3296      370258 : Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
    3297             :                                            Node** parameters,
    3298             :                                            int parameter_count) {
    3299      370258 :   return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
    3300      370262 :                                        parameter_count, effect_, Control());
    3301             : }
    3302             : 
    3303       36668 : Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
    3304       36668 :   const wasm::WasmGlobal& global = env_->module->globals[index];
    3305       73336 :   if (wasm::ValueTypes::IsReferenceType(global.type)) {
    3306         379 :     if (global.mutability && global.imported) {
    3307         128 :       Node* base = nullptr;
    3308         128 :       Node* offset = nullptr;
    3309         128 :       GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
    3310         128 :       return SetEffect(
    3311             :           graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
    3312             :                            base, offset, Effect(), Control()));
    3313             :     }
    3314             :     Node* globals_buffer =
    3315         503 :         LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
    3316         764 :     return LOAD_FIXED_ARRAY_SLOT_ANY(globals_buffer, global.offset);
    3317             :   }
    3318             : 
    3319             :   MachineType mem_type =
    3320       36289 :       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
    3321       36289 :   Node* base = nullptr;
    3322       36289 :   Node* offset = nullptr;
    3323       36289 :   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
    3324       36289 :                          &offset);
    3325       36289 :   Node* result = SetEffect(graph()->NewNode(
    3326             :       mcgraph()->machine()->Load(mem_type), base, offset, Effect(), Control()));
    3327             : #if defined(V8_TARGET_BIG_ENDIAN)
    3328             :   result = BuildChangeEndiannessLoad(result, mem_type,
    3329             :                                      env_->module->globals[index].type);
    3330             : #endif
    3331       36291 :   return result;
    3332             : }
    3333             : 
    3334       17653 : Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
    3335       17653 :   const wasm::WasmGlobal& global = env_->module->globals[index];
    3336       35306 :   if (wasm::ValueTypes::IsReferenceType(global.type)) {
    3337         245 :     if (global.mutability && global.imported) {
    3338          64 :       Node* base = nullptr;
    3339          64 :       Node* offset = nullptr;
    3340          64 :       GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
    3341             : 
    3342         192 :       return SetEffect(graph()->NewNode(
    3343             :           mcgraph()->machine()->Store(StoreRepresentation(
    3344             :               MachineRepresentation::kTagged, kFullWriteBarrier)),
    3345             :           base, offset, val, Effect(), Control()));
    3346             :     }
    3347             :     Node* globals_buffer =
    3348         360 :         LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
    3349         910 :     return STORE_FIXED_ARRAY_SLOT_ANY(globals_buffer,
    3350             :                                       env_->module->globals[index].offset, val);
    3351             :   }
    3352             : 
    3353             :   MachineType mem_type =
    3354       17408 :       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
    3355       17408 :   Node* base = nullptr;
    3356       17408 :   Node* offset = nullptr;
    3357       17408 :   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
    3358       17408 :                          &offset);
    3359       17408 :   const Operator* op = mcgraph()->machine()->Store(
    3360       17408 :       StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
    3361             : #if defined(V8_TARGET_BIG_ENDIAN)
    3362             :   val = BuildChangeEndiannessStore(val, mem_type.representation(),
    3363             :                                    env_->module->globals[index].type);
    3364             : #endif
    3365       17408 :   return SetEffect(
    3366       17408 :       graph()->NewNode(op, base, offset, val, Effect(), Control()));
    3367             : }
    3368             : 
    3369         207 : void WasmGraphBuilder::BoundsCheckTable(uint32_t table_index, Node* entry_index,
    3370             :                                         wasm::WasmCodePosition position,
    3371             :                                         wasm::TrapReason trap_reason,
    3372             :                                         Node** base_node) {
    3373         414 :   Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
    3374         416 :   Node* table = LOAD_FIXED_ARRAY_SLOT_ANY(tables, table_index);
    3375             : 
    3376             :   int storage_field_size = WasmTableObject::kElementsOffsetEnd -
    3377             :                            WasmTableObject::kElementsOffset + 1;
    3378         208 :   Node* storage = LOAD_RAW(
    3379             :       table, wasm::ObjectAccess::ToTagged(WasmTableObject::kElementsOffset),
    3380             :       assert_size(storage_field_size, MachineType::TaggedPointer()));
    3381             : 
    3382             :   int length_field_size =
    3383             :       FixedArray::kLengthOffsetEnd - FixedArray::kLengthOffset + 1;
    3384             :   Node* storage_size =
    3385         208 :       LOAD_RAW(storage, wasm::ObjectAccess::ToTagged(FixedArray::kLengthOffset),
    3386             :                assert_size(length_field_size, MachineType::TaggedSigned()));
    3387             : 
    3388         208 :   storage_size = BuildChangeSmiToInt32(storage_size);
    3389             :   // Bounds check against the table size.
    3390         208 :   Node* in_bounds = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
    3391             :                                      entry_index, storage_size);
    3392         208 :   TrapIfFalse(trap_reason, in_bounds, position);
    3393             : 
    3394         207 :   if (base_node) {
    3395          64 :     *base_node = storage;
    3396             :   }
    3397         207 : }
    3398             : 
    3399          64 : void WasmGraphBuilder::GetTableBaseAndOffset(uint32_t table_index,
    3400             :                                              Node* entry_index,
    3401             :                                              wasm::WasmCodePosition position,
    3402             :                                              Node** base_node,
    3403             :                                              Node** offset_node) {
    3404             :   BoundsCheckTable(table_index, entry_index, position,
    3405          64 :                    wasm::kTrapTableOutOfBounds, base_node);
    3406             :   // From the index, calculate the actual offset in the FixeArray. This
    3407             :   // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
    3408             :   // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
    3409          64 :   Node* index_times_tagged_size = graph()->NewNode(
    3410             :       mcgraph()->machine()->IntMul(), Uint32ToUintptr(entry_index),
    3411             :       mcgraph()->Int32Constant(kTaggedSize));
    3412             : 
    3413          64 :   *offset_node = graph()->NewNode(
    3414             :       mcgraph()->machine()->IntAdd(), index_times_tagged_size,
    3415             :       mcgraph()->IntPtrConstant(
    3416          64 :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
    3417          64 : }
    3418             : 
    3419          96 : Node* WasmGraphBuilder::GetTable(uint32_t table_index, Node* index,
    3420             :                                  wasm::WasmCodePosition position) {
    3421         192 :   if (env_->module->tables[table_index].type == wasm::kWasmAnyRef) {
    3422          32 :     Node* base = nullptr;
    3423          32 :     Node* offset = nullptr;
    3424          32 :     GetTableBaseAndOffset(table_index, index, position, &base, &offset);
    3425          32 :     return SetEffect(
    3426             :         graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
    3427             :                          base, offset, Effect(), Control()));
    3428             :   }
    3429             :   // We access anyfunc tables through runtime calls.
    3430             :   WasmTableGetDescriptor interface_descriptor;
    3431          64 :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    3432             :       mcgraph()->zone(),                              // zone
    3433             :       interface_descriptor,                           // descriptor
    3434             :       interface_descriptor.GetStackParameterCount(),  // stack parameter count
    3435             :       CallDescriptor::kNoFlags,                       // flags
    3436             :       Operator::kNoProperties,                        // properties
    3437          64 :       StubCallMode::kCallWasmRuntimeStub);            // stub call mode
    3438             :   // A direct call to a wasm runtime stub defined in this module.
    3439             :   // Just encode the stub index. This will be patched at relocation.
    3440             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    3441          64 :       wasm::WasmCode::kWasmTableGet, RelocInfo::WASM_STUB_CALL);
    3442             : 
    3443         127 :   return SetEffect(SetControl(graph()->NewNode(
    3444             :       mcgraph()->common()->Call(call_descriptor), call_target,
    3445             :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)), index,
    3446             :       Effect(), Control())));
    3447             : }
    3448             : 
    3449          64 : Node* WasmGraphBuilder::SetTable(uint32_t table_index, Node* index, Node* val,
    3450             :                                  wasm::WasmCodePosition position) {
    3451         128 :   if (env_->module->tables[table_index].type == wasm::kWasmAnyRef) {
    3452          32 :     Node* base = nullptr;
    3453          32 :     Node* offset = nullptr;
    3454          32 :     GetTableBaseAndOffset(table_index, index, position, &base, &offset);
    3455             : 
    3456          32 :     const Operator* op = mcgraph()->machine()->Store(
    3457          32 :         StoreRepresentation(MachineRepresentation::kTagged, kFullWriteBarrier));
    3458             : 
    3459          32 :     Node* store = graph()->NewNode(op, base, offset, val, Effect(), Control());
    3460             :     return SetEffect(store);
    3461             :   } else {
    3462             :     // We access anyfunc tables through runtime calls.
    3463             :     WasmTableSetDescriptor interface_descriptor;
    3464          32 :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    3465             :         mcgraph()->zone(),                              // zone
    3466             :         interface_descriptor,                           // descriptor
    3467             :         interface_descriptor.GetStackParameterCount(),  // stack parameter count
    3468             :         CallDescriptor::kNoFlags,                       // flags
    3469             :         Operator::kNoProperties,                        // properties
    3470          32 :         StubCallMode::kCallWasmRuntimeStub);            // stub call mode
    3471             :     // A direct call to a wasm runtime stub defined in this module.
    3472             :     // Just encode the stub index. This will be patched at relocation.
    3473             :     Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    3474          32 :         wasm::WasmCode::kWasmTableSet, RelocInfo::WASM_STUB_CALL);
    3475             : 
    3476          64 :     return SetEffect(SetControl(graph()->NewNode(
    3477             :         mcgraph()->common()->Call(call_descriptor), call_target,
    3478             :         graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
    3479             :         index, val, Effect(), Control())));
    3480             :   }
    3481             : }
    3482             : 
    3483       32436 : Node* WasmGraphBuilder::CheckBoundsAndAlignment(
    3484             :     uint8_t access_size, Node* index, uint32_t offset,
    3485             :     wasm::WasmCodePosition position) {
    3486             :   // Atomic operations need bounds checks until the backend can emit protected
    3487             :   // loads.
    3488             :   index =
    3489       32436 :       BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
    3490             : 
    3491       32643 :   const uintptr_t align_mask = access_size - 1;
    3492             : 
    3493             :   // Don't emit an alignment check if the index is a constant.
    3494             :   // TODO(wasm): a constant match is also done above in {BoundsCheckMem}.
    3495             :   UintPtrMatcher match(index);
    3496       32643 :   if (match.HasValue()) {
    3497       27422 :     uintptr_t effective_offset = match.Value() + offset;
    3498       27422 :     if ((effective_offset & align_mask) != 0) {
    3499             :       // statically known to be unaligned; trap.
    3500           0 :       TrapIfEq32(wasm::kTrapUnalignedAccess, Int32Constant(0), 0, position);
    3501             :     }
    3502             :     return index;
    3503             :   }
    3504             : 
    3505             :   // Unlike regular memory accesses, atomic memory accesses should trap if
    3506             :   // the effective offset is misaligned.
    3507             :   // TODO(wasm): this addition is redundant with one inserted by {MemBuffer}.
    3508        5221 :   Node* effective_offset = graph()->NewNode(mcgraph()->machine()->IntAdd(),
    3509             :                                             MemBuffer(offset), index);
    3510             : 
    3511       10494 :   Node* cond = graph()->NewNode(mcgraph()->machine()->WordAnd(),
    3512             :                                 effective_offset, IntPtrConstant(align_mask));
    3513        5242 :   TrapIfFalse(wasm::kTrapUnalignedAccess,
    3514             :               graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
    3515             :                                mcgraph()->Int32Constant(0)),
    3516        5201 :               position);
    3517        5232 :   return index;
    3518             : }
    3519             : 
    3520             : // Insert code to bounds check a memory access if necessary. Return the
    3521             : // bounds-checked index, which is guaranteed to have (the equivalent of)
    3522             : // {uintptr_t} representation.
    3523      269644 : Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
    3524             :                                        uint32_t offset,
    3525             :                                        wasm::WasmCodePosition position,
    3526             :                                        EnforceBoundsCheck enforce_check) {
    3527             :   DCHECK_LE(1, access_size);
    3528      269644 :   index = Uint32ToUintptr(index);
    3529      270099 :   if (FLAG_wasm_no_bounds_checks) return index;
    3530             : 
    3531      270134 :   if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
    3532             :     return index;
    3533             :   }
    3534             : 
    3535       65352 :   if (!IsInBounds(offset, access_size, env_->max_memory_size)) {
    3536             :     // The access will be out of bounds, even for the largest memory.
    3537          16 :     TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
    3538          16 :     return mcgraph()->IntPtrConstant(0);
    3539             :   }
    3540       32660 :   uint64_t end_offset = uint64_t{offset} + access_size - 1u;
    3541       32660 :   Node* end_offset_node = IntPtrConstant(end_offset);
    3542             : 
    3543             :   // The accessed memory is [index + offset, index + end_offset].
    3544             :   // Check that the last read byte (at {index + end_offset}) is in bounds.
    3545             :   // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
    3546             :   //    compute {effective_size} as {mem_size - end_offset)}.
    3547             :   //    {effective_size} is >= 1 if condition 1) holds.
    3548             :   // 2) Check that {index + end_offset < mem_size} by
    3549             :   //    - computing {effective_size} as {mem_size - end_offset} and
    3550             :   //    - checking that {index < effective_size}.
    3551             : 
    3552             :   auto m = mcgraph()->machine();
    3553       32519 :   Node* mem_size = instance_cache_->mem_size;
    3554       32519 :   if (end_offset >= env_->min_memory_size) {
    3555             :     // The end offset is larger than the smallest memory.
    3556             :     // Dynamically check the end offset against the dynamic memory size.
    3557        5843 :     Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
    3558        5861 :     TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
    3559             :   } else {
    3560             :     // The end offset is smaller than the smallest memory, so only one check is
    3561             :     // required. Check to see if the index is also a constant.
    3562             :     UintPtrMatcher match(index);
    3563       26676 :     if (match.HasValue()) {
    3564             :       uintptr_t index_val = match.Value();
    3565       26758 :       if (index_val < env_->min_memory_size - end_offset) {
    3566             :         // The input index is a constant and everything is statically within
    3567             :         // bounds of the smallest possible memory.
    3568             :         return index;
    3569             :       }
    3570             :     }
    3571             :   }
    3572             : 
    3573             :   // This produces a positive number, since {end_offset < min_size <= mem_size}.
    3574             :   Node* effective_size =
    3575        5921 :       graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
    3576             : 
    3577             :   // Introduce the actual bounds check.
    3578        5908 :   Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
    3579        5950 :   TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
    3580             : 
    3581        5990 :   if (untrusted_code_mitigations_) {
    3582             :     // In the fallthrough case, condition the index with the memory mask.
    3583           0 :     Node* mem_mask = instance_cache_->mem_mask;
    3584             :     DCHECK_NOT_NULL(mem_mask);
    3585           0 :     index = graph()->NewNode(m->WordAnd(), index, mem_mask);
    3586             :   }
    3587             :   return index;
    3588             : }
    3589             : 
    3590         216 : Node* WasmGraphBuilder::BoundsCheckRange(Node* start, Node** size, Node* max,
    3591             :                                          wasm::WasmCodePosition position) {
    3592             :   auto m = mcgraph()->machine();
    3593             :   // The region we are trying to access is [start, start+size). If
    3594             :   // {start} > {max}, none of this region is valid, so we trap. Otherwise,
    3595             :   // there may be a subset of the region that is valid. {max - start} is the
    3596             :   // maximum valid size, so if {max - start < size}, then the region is
    3597             :   // partially out-of-bounds.
    3598         216 :   TrapIfTrue(wasm::kTrapMemOutOfBounds,
    3599         216 :              graph()->NewNode(m->Uint32LessThan(), max, start), position);
    3600         216 :   Node* sub = graph()->NewNode(m->Int32Sub(), max, start);
    3601         216 :   Node* fail = graph()->NewNode(m->Uint32LessThan(), sub, *size);
    3602         216 :   Diamond d(graph(), mcgraph()->common(), fail, BranchHint::kFalse);
    3603             :   d.Chain(Control());
    3604         216 :   *size = d.Phi(MachineRepresentation::kWord32, sub, *size);
    3605         216 :   return fail;
    3606             : }
    3607             : 
    3608         168 : Node* WasmGraphBuilder::BoundsCheckMemRange(Node** start, Node** size,
    3609             :                                             wasm::WasmCodePosition position) {
    3610             :   // TODO(binji): Support trap handler and no bounds check mode.
    3611             :   Node* fail =
    3612         168 :       BoundsCheckRange(*start, size, instance_cache_->mem_size, position);
    3613         336 :   *start = graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(0),
    3614         168 :                             Uint32ToUintptr(*start));
    3615         168 :   return fail;
    3616             : }
    3617             : 
    3618        4728 : const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
    3619             :                                                       wasm::ValueType type) {
    3620        4728 :   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
    3621        4728 :   MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
    3622        4732 :   if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
    3623             :                             wasm::ValueTypes::MachineRepresentationFor(type))) {
    3624        4728 :     return mcgraph()->machine()->Load(mach_type);
    3625             :   }
    3626           0 :   return mcgraph()->machine()->UnalignedLoad(mach_type);
    3627             : }
    3628             : 
    3629       30373 : const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
    3630             :                                                        wasm::ValueType type) {
    3631       30373 :   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
    3632       30375 :   MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
    3633       31101 :   if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
    3634             :     StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
    3635       30373 :     return mcgraph()->machine()->Store(store_rep);
    3636             :   }
    3637             :   UnalignedStoreRepresentation store_rep(rep);
    3638           0 :   return mcgraph()->machine()->UnalignedStore(store_rep);
    3639             : }
    3640             : 
    3641          20 : Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
    3642             :                                              MachineRepresentation rep,
    3643             :                                              Node* index, uint32_t offset,
    3644             :                                              wasm::WasmCodePosition position) {
    3645             :   int kAlign = 4;  // Ensure that the LSB is 0, such that this looks like a Smi.
    3646          20 :   Node* info = graph()->NewNode(
    3647          20 :       mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
    3648             : 
    3649          39 :   Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    3650             :                                    Int32Constant(offset), index);
    3651          56 :   auto store = [&](int offset, MachineRepresentation rep, Node* data) {
    3652         342 :     SetEffect(graph()->NewNode(
    3653             :         mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
    3654          56 :         info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
    3655          78 :   };
    3656             :   // Store address, is_store, and mem_rep.
    3657             :   store(offsetof(wasm::MemoryTracingInfo, address),
    3658          19 :         MachineRepresentation::kWord32, address);
    3659          19 :   store(offsetof(wasm::MemoryTracingInfo, is_store),
    3660             :         MachineRepresentation::kWord8,
    3661          19 :         mcgraph()->Int32Constant(is_store ? 1 : 0));
    3662          20 :   store(offsetof(wasm::MemoryTracingInfo, mem_rep),
    3663             :         MachineRepresentation::kWord8,
    3664          19 :         mcgraph()->Int32Constant(static_cast<int>(rep)));
    3665             : 
    3666          20 :   Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
    3667             :   SetSourcePosition(call, position);
    3668          20 :   return call;
    3669             : }
    3670             : 
    3671       93693 : Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
    3672             :                                 Node* index, uint32_t offset,
    3673             :                                 uint32_t alignment,
    3674             :                                 wasm::WasmCodePosition position) {
    3675             :   Node* load;
    3676             : 
    3677             :   // Wasm semantics throw on OOB. Introduce explicit bounds check and
    3678             :   // conditioning when not using the trap handler.
    3679             :   index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
    3680       93701 :                          position, kCanOmitBoundsCheck);
    3681             : 
    3682      186374 :   if (memtype.representation() == MachineRepresentation::kWord8 ||
    3683             :       mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
    3684       93774 :     if (use_trap_handler()) {
    3685       93773 :       load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
    3686             :                               MemBuffer(offset), index, Effect(), Control());
    3687             :       SetSourcePosition(load, position);
    3688             :     } else {
    3689           1 :       load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
    3690             :                               MemBuffer(offset), index, Effect(), Control());
    3691             :     }
    3692             :   } else {
    3693             :     // TODO(eholk): Support unaligned loads with trap handlers.
    3694             :     DCHECK(!use_trap_handler());
    3695           0 :     load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
    3696             :                             MemBuffer(offset), index, Effect(), Control());
    3697             :   }
    3698             : 
    3699             :   SetEffect(load);
    3700             : 
    3701             : #if defined(V8_TARGET_BIG_ENDIAN)
    3702             :   load = BuildChangeEndiannessLoad(load, memtype, type);
    3703             : #endif
    3704             : 
    3705      135028 :   if (type == wasm::kWasmI64 &&
    3706             :       ElementSizeInBytes(memtype.representation()) < 8) {
    3707             :     // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
    3708         688 :     if (memtype.IsSigned()) {
    3709             :       // sign extend
    3710         352 :       load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
    3711             :     } else {
    3712             :       // zero extend
    3713             :       load =
    3714         336 :           graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
    3715             :     }
    3716             :   }
    3717             : 
    3718       93779 :   if (FLAG_trace_wasm_memory) {
    3719             :     TraceMemoryOperation(false, memtype.representation(), index, offset,
    3720          12 :                          position);
    3721             :   }
    3722             : 
    3723       93779 :   return load;
    3724             : }
    3725             : 
    3726      143747 : Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
    3727             :                                  uint32_t offset, uint32_t alignment, Node* val,
    3728             :                                  wasm::WasmCodePosition position,
    3729             :                                  wasm::ValueType type) {
    3730             :   Node* store;
    3731             : 
    3732      143797 :   index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
    3733      143797 :                          position, kCanOmitBoundsCheck);
    3734             : 
    3735             : #if defined(V8_TARGET_BIG_ENDIAN)
    3736             :   val = BuildChangeEndiannessStore(val, mem_rep, type);
    3737             : #endif
    3738             : 
    3739      286856 :   if (mem_rep == MachineRepresentation::kWord8 ||
    3740             :       mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
    3741      143909 :     if (use_trap_handler()) {
    3742             :       store =
    3743      143909 :           graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
    3744             :                            MemBuffer(offset), index, val, Effect(), Control());
    3745             :       SetSourcePosition(store, position);
    3746             :     } else {
    3747             :       StoreRepresentation rep(mem_rep, kNoWriteBarrier);
    3748             :       store =
    3749           0 :           graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
    3750             :                            index, val, Effect(), Control());
    3751             :     }
    3752             :   } else {
    3753             :     // TODO(eholk): Support unaligned stores with trap handlers.
    3754             :     DCHECK(!use_trap_handler());
    3755             :     UnalignedStoreRepresentation rep(mem_rep);
    3756             :     store =
    3757           0 :         graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
    3758             :                          MemBuffer(offset), index, val, Effect(), Control());
    3759             :   }
    3760             : 
    3761             :   SetEffect(store);
    3762             : 
    3763      144106 :   if (FLAG_trace_wasm_memory) {
    3764           8 :     TraceMemoryOperation(true, mem_rep, index, offset, position);
    3765             :   }
    3766             : 
    3767      144106 :   return store;
    3768             : }
    3769             : 
    3770             : namespace {
    3771       56391 : Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
    3772       56391 :   switch (rep) {
    3773             :     case MachineRepresentation::kWord8:
    3774             :     case MachineRepresentation::kWord16:
    3775             :     case MachineRepresentation::kWord32:
    3776       50150 :       return mcgraph->Int32Constant(0);
    3777             :     case MachineRepresentation::kWord64:
    3778           0 :       return mcgraph->Int64Constant(0);
    3779             :     case MachineRepresentation::kFloat32:
    3780        5396 :       return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
    3781             :     case MachineRepresentation::kFloat64:
    3782         845 :       return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
    3783             :     default:
    3784           0 :       UNREACHABLE();
    3785             :   }
    3786             : }
    3787             : }  // namespace
    3788             : 
    3789       56391 : Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
    3790             :   DCHECK_NOT_NULL(instance_cache_);
    3791       56391 :   Node* mem_start = instance_cache_->mem_start;
    3792       56391 :   Node* mem_size = instance_cache_->mem_size;
    3793             :   DCHECK_NOT_NULL(mem_start);
    3794             :   DCHECK_NOT_NULL(mem_size);
    3795             : 
    3796             :   // Asm.js semantics are defined in terms of typed arrays, hence OOB
    3797             :   // reads return {undefined} coerced to the result type (0 for integers, NaN
    3798             :   // for float and double).
    3799             :   // Note that we check against the memory size ignoring the size of the
    3800             :   // stored value, which is conservative if misaligned. Technically, asm.js
    3801             :   // should never have misaligned accesses.
    3802       56391 :   index = Uint32ToUintptr(index);
    3803             :   Diamond bounds_check(
    3804             :       graph(), mcgraph()->common(),
    3805             :       graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
    3806      112782 :       BranchHint::kTrue);
    3807             :   bounds_check.Chain(Control());
    3808             : 
    3809       56391 :   if (untrusted_code_mitigations_) {
    3810             :     // Condition the index with the memory mask.
    3811           0 :     Node* mem_mask = instance_cache_->mem_mask;
    3812             :     DCHECK_NOT_NULL(mem_mask);
    3813           0 :     index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
    3814             :   }
    3815             : 
    3816      112782 :   Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
    3817             :                                 index, Effect(), bounds_check.if_true);
    3818       56391 :   SetEffect(bounds_check.EffectPhi(load, Effect()));
    3819       56391 :   SetControl(bounds_check.merge);
    3820       56391 :   return bounds_check.Phi(type.representation(), load,
    3821       56391 :                           GetAsmJsOOBValue(type.representation(), mcgraph()));
    3822             : }
    3823             : 
    3824      371871 : Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
    3825      371871 :   if (mcgraph()->machine()->Is32()) return node;
    3826             :   // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
    3827             :   Uint32Matcher matcher(node);
    3828      372047 :   if (matcher.HasValue()) {
    3829             :     uintptr_t value = matcher.Value();
    3830      257521 :     return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
    3831             :   }
    3832      229095 :   return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
    3833             : }
    3834             : 
    3835       40014 : Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
    3836             :                                            Node* val) {
    3837             :   DCHECK_NOT_NULL(instance_cache_);
    3838       40014 :   Node* mem_start = instance_cache_->mem_start;
    3839       40014 :   Node* mem_size = instance_cache_->mem_size;
    3840             :   DCHECK_NOT_NULL(mem_start);
    3841             :   DCHECK_NOT_NULL(mem_size);
    3842             : 
    3843             :   // Asm.js semantics are to ignore OOB writes.
    3844             :   // Note that we check against the memory size ignoring the size of the
    3845             :   // stored value, which is conservative if misaligned. Technically, asm.js
    3846             :   // should never have misaligned accesses.
    3847             :   Diamond bounds_check(
    3848             :       graph(), mcgraph()->common(),
    3849             :       graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
    3850       80028 :       BranchHint::kTrue);
    3851             :   bounds_check.Chain(Control());
    3852             : 
    3853       40014 :   if (untrusted_code_mitigations_) {
    3854             :     // Condition the index with the memory mask.
    3855           0 :     Node* mem_mask = instance_cache_->mem_mask;
    3856             :     DCHECK_NOT_NULL(mem_mask);
    3857             :     index =
    3858           0 :         graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
    3859             :   }
    3860             : 
    3861       40014 :   index = Uint32ToUintptr(index);
    3862       40014 :   const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
    3863       40014 :       type.representation(), WriteBarrierKind::kNoWriteBarrier));
    3864       40014 :   Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
    3865             :                                  bounds_check.if_true);
    3866       40014 :   SetEffect(bounds_check.EffectPhi(store, Effect()));
    3867       40014 :   SetControl(bounds_check.merge);
    3868       40014 :   return val;
    3869             : }
    3870             : 
    3871           0 : void WasmGraphBuilder::PrintDebugName(Node* node) {
    3872           0 :   PrintF("#%d:%s", node->id(), node->op()->mnemonic());
    3873           0 : }
    3874             : 
    3875           0 : Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
    3876             : 
    3877             : namespace {
    3878        4068 : Signature<MachineRepresentation>* CreateMachineSignature(
    3879             :     Zone* zone, wasm::FunctionSig* sig) {
    3880             :   Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
    3881             :                                                     sig->parameter_count());
    3882       12204 :   for (auto ret : sig->returns()) {
    3883        4068 :     builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
    3884             :   }
    3885             : 
    3886        6684 :   for (auto param : sig->parameters()) {
    3887        1308 :     builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
    3888             :   }
    3889        4068 :   return builder.Build();
    3890             : }
    3891             : }  // namespace
    3892             : 
    3893      497934 : void WasmGraphBuilder::LowerInt64() {
    3894      995868 :   if (mcgraph()->machine()->Is64()) return;
    3895             :   Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
    3896             :                   mcgraph()->zone(),
    3897           0 :                   CreateMachineSignature(mcgraph()->zone(), sig_));
    3898           0 :   r.LowerGraph();
    3899             : }
    3900             : 
    3901           0 : void WasmGraphBuilder::SimdScalarLoweringForTesting() {
    3902           0 :   SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
    3903           0 :       .LowerGraph();
    3904           0 : }
    3905             : 
    3906           0 : void WasmGraphBuilder::SetSourcePosition(Node* node,
    3907             :                                          wasm::WasmCodePosition position) {
    3908             :   DCHECK_NE(position, wasm::kNoCodePosition);
    3909      414984 :   if (source_position_table_)
    3910      325677 :     source_position_table_->SetSourcePosition(node, SourcePosition(position));
    3911           0 : }
    3912             : 
    3913        2124 : Node* WasmGraphBuilder::S128Zero() {
    3914        2124 :   has_simd_ = true;
    3915        4248 :   return graph()->NewNode(mcgraph()->machine()->S128Zero());
    3916             : }
    3917             : 
    3918        4148 : Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
    3919        4148 :   has_simd_ = true;
    3920        4148 :   switch (opcode) {
    3921             :     case wasm::kExprF32x4Splat:
    3922         560 :       return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
    3923             :     case wasm::kExprF32x4SConvertI32x4:
    3924           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
    3925           8 :                               inputs[0]);
    3926             :     case wasm::kExprF32x4UConvertI32x4:
    3927           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
    3928           8 :                               inputs[0]);
    3929             :     case wasm::kExprF32x4Abs:
    3930          16 :       return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
    3931             :     case wasm::kExprF32x4Neg:
    3932          16 :       return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
    3933             :     case wasm::kExprF32x4RecipApprox:
    3934           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
    3935           8 :                               inputs[0]);
    3936             :     case wasm::kExprF32x4RecipSqrtApprox:
    3937           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
    3938           8 :                               inputs[0]);
    3939             :     case wasm::kExprF32x4Add:
    3940          24 :       return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
    3941          24 :                               inputs[1]);
    3942             :     case wasm::kExprF32x4AddHoriz:
    3943           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
    3944           8 :                               inputs[1]);
    3945             :     case wasm::kExprF32x4Sub:
    3946           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
    3947           8 :                               inputs[1]);
    3948             :     case wasm::kExprF32x4Mul:
    3949           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
    3950           8 :                               inputs[1]);
    3951             :     case wasm::kExprF32x4Min:
    3952           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
    3953           8 :                               inputs[1]);
    3954             :     case wasm::kExprF32x4Max:
    3955           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
    3956           8 :                               inputs[1]);
    3957             :     case wasm::kExprF32x4Eq:
    3958           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
    3959           8 :                               inputs[1]);
    3960             :     case wasm::kExprF32x4Ne:
    3961           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
    3962           8 :                               inputs[1]);
    3963             :     case wasm::kExprF32x4Lt:
    3964           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
    3965           8 :                               inputs[1]);
    3966             :     case wasm::kExprF32x4Le:
    3967           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
    3968           8 :                               inputs[1]);
    3969             :     case wasm::kExprF32x4Gt:
    3970           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
    3971           8 :                               inputs[0]);
    3972             :     case wasm::kExprF32x4Ge:
    3973           8 :       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
    3974           8 :                               inputs[0]);
    3975             :     case wasm::kExprI32x4Splat:
    3976        2472 :       return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
    3977             :     case wasm::kExprI32x4SConvertF32x4:
    3978           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
    3979           8 :                               inputs[0]);
    3980             :     case wasm::kExprI32x4UConvertF32x4:
    3981           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
    3982           8 :                               inputs[0]);
    3983             :     case wasm::kExprI32x4SConvertI16x8Low:
    3984           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
    3985           8 :                               inputs[0]);
    3986             :     case wasm::kExprI32x4SConvertI16x8High:
    3987           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
    3988           8 :                               inputs[0]);
    3989             :     case wasm::kExprI32x4Neg:
    3990          16 :       return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
    3991             :     case wasm::kExprI32x4Add:
    3992          24 :       return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
    3993          24 :                               inputs[1]);
    3994             :     case wasm::kExprI32x4AddHoriz:
    3995           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
    3996           8 :                               inputs[1]);
    3997             :     case wasm::kExprI32x4Sub:
    3998           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
    3999           8 :                               inputs[1]);
    4000             :     case wasm::kExprI32x4Mul:
    4001           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
    4002           8 :                               inputs[1]);
    4003             :     case wasm::kExprI32x4MinS:
    4004           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
    4005           8 :                               inputs[1]);
    4006             :     case wasm::kExprI32x4MaxS:
    4007           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
    4008           8 :                               inputs[1]);
    4009             :     case wasm::kExprI32x4Eq:
    4010          40 :       return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
    4011          40 :                               inputs[1]);
    4012             :     case wasm::kExprI32x4Ne:
    4013          44 :       return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
    4014          44 :                               inputs[1]);
    4015             :     case wasm::kExprI32x4LtS:
    4016           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
    4017           8 :                               inputs[0]);
    4018             :     case wasm::kExprI32x4LeS:
    4019           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
    4020           8 :                               inputs[0]);
    4021             :     case wasm::kExprI32x4GtS:
    4022           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
    4023           8 :                               inputs[1]);
    4024             :     case wasm::kExprI32x4GeS:
    4025           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
    4026           8 :                               inputs[1]);
    4027             :     case wasm::kExprI32x4UConvertI16x8Low:
    4028           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
    4029           8 :                               inputs[0]);
    4030             :     case wasm::kExprI32x4UConvertI16x8High:
    4031           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
    4032           8 :                               inputs[0]);
    4033             :     case wasm::kExprI32x4MinU:
    4034           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
    4035           8 :                               inputs[1]);
    4036             :     case wasm::kExprI32x4MaxU:
    4037           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
    4038           8 :                               inputs[1]);
    4039             :     case wasm::kExprI32x4LtU:
    4040           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
    4041           8 :                               inputs[0]);
    4042             :     case wasm::kExprI32x4LeU:
    4043           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
    4044           8 :                               inputs[0]);
    4045             :     case wasm::kExprI32x4GtU:
    4046           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
    4047           8 :                               inputs[1]);
    4048             :     case wasm::kExprI32x4GeU:
    4049           8 :       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
    4050           8 :                               inputs[1]);
    4051             :     case wasm::kExprI16x8Splat:
    4052        1552 :       return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
    4053             :     case wasm::kExprI16x8SConvertI8x16Low:
    4054           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
    4055           8 :                               inputs[0]);
    4056             :     case wasm::kExprI16x8SConvertI8x16High:
    4057           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
    4058           8 :                               inputs[0]);
    4059             :     case wasm::kExprI16x8Neg:
    4060          16 :       return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
    4061             :     case wasm::kExprI16x8SConvertI32x4:
    4062           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
    4063           8 :                               inputs[0], inputs[1]);
    4064             :     case wasm::kExprI16x8Add:
    4065           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
    4066           8 :                               inputs[1]);
    4067             :     case wasm::kExprI16x8AddSaturateS:
    4068           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
    4069           8 :                               inputs[0], inputs[1]);
    4070             :     case wasm::kExprI16x8AddHoriz:
    4071           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
    4072           8 :                               inputs[1]);
    4073             :     case wasm::kExprI16x8Sub:
    4074           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
    4075           8 :                               inputs[1]);
    4076             :     case wasm::kExprI16x8SubSaturateS:
    4077           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
    4078           8 :                               inputs[0], inputs[1]);
    4079             :     case wasm::kExprI16x8Mul:
    4080           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
    4081           8 :                               inputs[1]);
    4082             :     case wasm::kExprI16x8MinS:
    4083           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
    4084           8 :                               inputs[1]);
    4085             :     case wasm::kExprI16x8MaxS:
    4086           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
    4087           8 :                               inputs[1]);
    4088             :     case wasm::kExprI16x8Eq:
    4089          40 :       return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
    4090          40 :                               inputs[1]);
    4091             :     case wasm::kExprI16x8Ne:
    4092          44 :       return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
    4093          44 :                               inputs[1]);
    4094             :     case wasm::kExprI16x8LtS:
    4095           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
    4096           8 :                               inputs[0]);
    4097             :     case wasm::kExprI16x8LeS:
    4098           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
    4099           8 :                               inputs[0]);
    4100             :     case wasm::kExprI16x8GtS:
    4101           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
    4102           8 :                               inputs[1]);
    4103             :     case wasm::kExprI16x8GeS:
    4104           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
    4105           8 :                               inputs[1]);
    4106             :     case wasm::kExprI16x8UConvertI8x16Low:
    4107           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
    4108           8 :                               inputs[0]);
    4109             :     case wasm::kExprI16x8UConvertI8x16High:
    4110           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
    4111           8 :                               inputs[0]);
    4112             :     case wasm::kExprI16x8UConvertI32x4:
    4113           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
    4114           8 :                               inputs[0], inputs[1]);
    4115             :     case wasm::kExprI16x8AddSaturateU:
    4116           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
    4117           8 :                               inputs[0], inputs[1]);
    4118             :     case wasm::kExprI16x8SubSaturateU:
    4119           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
    4120           8 :                               inputs[0], inputs[1]);
    4121             :     case wasm::kExprI16x8MinU:
    4122           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
    4123           8 :                               inputs[1]);
    4124             :     case wasm::kExprI16x8MaxU:
    4125           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
    4126           8 :                               inputs[1]);
    4127             :     case wasm::kExprI16x8LtU:
    4128           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
    4129           8 :                               inputs[0]);
    4130             :     case wasm::kExprI16x8LeU:
    4131           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
    4132           8 :                               inputs[0]);
    4133             :     case wasm::kExprI16x8GtU:
    4134           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
    4135           8 :                               inputs[1]);
    4136             :     case wasm::kExprI16x8GeU:
    4137           8 :       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
    4138           8 :                               inputs[1]);
    4139             :     case wasm::kExprI8x16Splat:
    4140        1152 :       return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
    4141             :     case wasm::kExprI8x16Neg:
    4142          16 :       return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
    4143             :     case wasm::kExprI8x16SConvertI16x8:
    4144           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
    4145           8 :                               inputs[0], inputs[1]);
    4146             :     case wasm::kExprI8x16Add:
    4147           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
    4148           8 :                               inputs[1]);
    4149             :     case wasm::kExprI8x16AddSaturateS:
    4150           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
    4151           8 :                               inputs[0], inputs[1]);
    4152             :     case wasm::kExprI8x16Sub:
    4153           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
    4154           8 :                               inputs[1]);
    4155             :     case wasm::kExprI8x16SubSaturateS:
    4156           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
    4157           8 :                               inputs[0], inputs[1]);
    4158             :     case wasm::kExprI8x16Mul:
    4159           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
    4160           8 :                               inputs[1]);
    4161             :     case wasm::kExprI8x16MinS:
    4162           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
    4163           8 :                               inputs[1]);
    4164             :     case wasm::kExprI8x16MaxS:
    4165           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
    4166           8 :                               inputs[1]);
    4167             :     case wasm::kExprI8x16Eq:
    4168          40 :       return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
    4169          40 :                               inputs[1]);
    4170             :     case wasm::kExprI8x16Ne:
    4171          44 :       return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
    4172          44 :                               inputs[1]);
    4173             :     case wasm::kExprI8x16LtS:
    4174           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
    4175           8 :                               inputs[0]);
    4176             :     case wasm::kExprI8x16LeS:
    4177           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
    4178           8 :                               inputs[0]);
    4179             :     case wasm::kExprI8x16GtS:
    4180           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
    4181           8 :                               inputs[1]);
    4182             :     case wasm::kExprI8x16GeS:
    4183           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
    4184           8 :                               inputs[1]);
    4185             :     case wasm::kExprI8x16UConvertI16x8:
    4186           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
    4187           8 :                               inputs[0], inputs[1]);
    4188             :     case wasm::kExprI8x16AddSaturateU:
    4189           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
    4190           8 :                               inputs[0], inputs[1]);
    4191             :     case wasm::kExprI8x16SubSaturateU:
    4192           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
    4193           8 :                               inputs[0], inputs[1]);
    4194             :     case wasm::kExprI8x16MinU:
    4195           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
    4196           8 :                               inputs[1]);
    4197             :     case wasm::kExprI8x16MaxU:
    4198           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
    4199           8 :                               inputs[1]);
    4200             :     case wasm::kExprI8x16LtU:
    4201           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
    4202           8 :                               inputs[0]);
    4203             :     case wasm::kExprI8x16LeU:
    4204           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
    4205           8 :                               inputs[0]);
    4206             :     case wasm::kExprI8x16GtU:
    4207           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
    4208           8 :                               inputs[1]);
    4209             :     case wasm::kExprI8x16GeU:
    4210           8 :       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
    4211           8 :                               inputs[1]);
    4212             :     case wasm::kExprS128And:
    4213           8 :       return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
    4214           8 :                               inputs[1]);
    4215             :     case wasm::kExprS128Or:
    4216           8 :       return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
    4217           8 :                               inputs[1]);
    4218             :     case wasm::kExprS128Xor:
    4219           8 :       return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
    4220           8 :                               inputs[1]);
    4221             :     case wasm::kExprS128Not:
    4222          16 :       return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
    4223             :     case wasm::kExprS128Select:
    4224          28 :       return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[2],
    4225          28 :                               inputs[0], inputs[1]);
    4226             :     case wasm::kExprS1x4AnyTrue:
    4227          72 :       return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
    4228             :     case wasm::kExprS1x4AllTrue:
    4229          72 :       return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
    4230             :     case wasm::kExprS1x8AnyTrue:
    4231          72 :       return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
    4232             :     case wasm::kExprS1x8AllTrue:
    4233          72 :       return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
    4234             :     case wasm::kExprS1x16AnyTrue:
    4235          72 :       return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
    4236             :     case wasm::kExprS1x16AllTrue:
    4237          72 :       return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
    4238             :     default:
    4239           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4240             :   }
    4241             : }
    4242             : 
    4243         676 : Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
    4244             :                                    Node* const* inputs) {
    4245         676 :   has_simd_ = true;
    4246         676 :   switch (opcode) {
    4247             :     case wasm::kExprF32x4ExtractLane:
    4248          64 :       return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
    4249          64 :                               inputs[0]);
    4250             :     case wasm::kExprF32x4ReplaceLane:
    4251          64 :       return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
    4252          64 :                               inputs[0], inputs[1]);
    4253             :     case wasm::kExprI32x4ExtractLane:
    4254         148 :       return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
    4255         148 :                               inputs[0]);
    4256             :     case wasm::kExprI32x4ReplaceLane:
    4257          96 :       return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
    4258          96 :                               inputs[0], inputs[1]);
    4259             :     case wasm::kExprI16x8ExtractLane:
    4260          32 :       return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
    4261          32 :                               inputs[0]);
    4262             :     case wasm::kExprI16x8ReplaceLane:
    4263          88 :       return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
    4264          88 :                               inputs[0], inputs[1]);
    4265             :     case wasm::kExprI8x16ExtractLane:
    4266          32 :       return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
    4267          32 :                               inputs[0]);
    4268             :     case wasm::kExprI8x16ReplaceLane:
    4269         152 :       return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
    4270         152 :                               inputs[0], inputs[1]);
    4271             :     default:
    4272           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4273             :   }
    4274             : }
    4275             : 
    4276        1272 : Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
    4277             :                                     Node* const* inputs) {
    4278        1272 :   has_simd_ = true;
    4279        1272 :   switch (opcode) {
    4280             :     case wasm::kExprI32x4Shl:
    4281         496 :       return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
    4282             :     case wasm::kExprI32x4ShrS:
    4283         248 :       return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
    4284         248 :                               inputs[0]);
    4285             :     case wasm::kExprI32x4ShrU:
    4286         248 :       return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
    4287         248 :                               inputs[0]);
    4288             :     case wasm::kExprI16x8Shl:
    4289         240 :       return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
    4290             :     case wasm::kExprI16x8ShrS:
    4291         120 :       return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
    4292         120 :                               inputs[0]);
    4293             :     case wasm::kExprI16x8ShrU:
    4294         120 :       return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
    4295         120 :                               inputs[0]);
    4296             :     case wasm::kExprI8x16Shl:
    4297         112 :       return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
    4298             :     case wasm::kExprI8x16ShrS:
    4299          56 :       return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
    4300          56 :                               inputs[0]);
    4301             :     case wasm::kExprI8x16ShrU:
    4302          56 :       return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
    4303          56 :                               inputs[0]);
    4304             :     default:
    4305           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4306             :   }
    4307             : }
    4308             : 
    4309       14800 : Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
    4310             :                                           Node* const* inputs) {
    4311       14800 :   has_simd_ = true;
    4312       14800 :   return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
    4313       14800 :                           inputs[0], inputs[1]);
    4314             : }
    4315             : 
    4316             : #define ATOMIC_BINOP_LIST(V)                        \
    4317             :   V(I32AtomicAdd, Add, Uint32, Word32)              \
    4318             :   V(I64AtomicAdd, Add, Uint64, Word64)              \
    4319             :   V(I32AtomicAdd8U, Add, Uint8, Word32)             \
    4320             :   V(I32AtomicAdd16U, Add, Uint16, Word32)           \
    4321             :   V(I64AtomicAdd8U, Add, Uint8, Word64)             \
    4322             :   V(I64AtomicAdd16U, Add, Uint16, Word64)           \
    4323             :   V(I64AtomicAdd32U, Add, Uint32, Word64)           \
    4324             :   V(I32AtomicSub, Sub, Uint32, Word32)              \
    4325             :   V(I64AtomicSub, Sub, Uint64, Word64)              \
    4326             :   V(I32AtomicSub8U, Sub, Uint8, Word32)             \
    4327             :   V(I32AtomicSub16U, Sub, Uint16, Word32)           \
    4328             :   V(I64AtomicSub8U, Sub, Uint8, Word64)             \
    4329             :   V(I64AtomicSub16U, Sub, Uint16, Word64)           \
    4330             :   V(I64AtomicSub32U, Sub, Uint32, Word64)           \
    4331             :   V(I32AtomicAnd, And, Uint32, Word32)              \
    4332             :   V(I64AtomicAnd, And, Uint64, Word64)              \
    4333             :   V(I32AtomicAnd8U, And, Uint8, Word32)             \
    4334             :   V(I64AtomicAnd16U, And, Uint16, Word64)           \
    4335             :   V(I32AtomicAnd16U, And, Uint16, Word32)           \
    4336             :   V(I64AtomicAnd8U, And, Uint8, Word64)             \
    4337             :   V(I64AtomicAnd32U, And, Uint32, Word64)           \
    4338             :   V(I32AtomicOr, Or, Uint32, Word32)                \
    4339             :   V(I64AtomicOr, Or, Uint64, Word64)                \
    4340             :   V(I32AtomicOr8U, Or, Uint8, Word32)               \
    4341             :   V(I32AtomicOr16U, Or, Uint16, Word32)             \
    4342             :   V(I64AtomicOr8U, Or, Uint8, Word64)               \
    4343             :   V(I64AtomicOr16U, Or, Uint16, Word64)             \
    4344             :   V(I64AtomicOr32U, Or, Uint32, Word64)             \
    4345             :   V(I32AtomicXor, Xor, Uint32, Word32)              \
    4346             :   V(I64AtomicXor, Xor, Uint64, Word64)              \
    4347             :   V(I32AtomicXor8U, Xor, Uint8, Word32)             \
    4348             :   V(I32AtomicXor16U, Xor, Uint16, Word32)           \
    4349             :   V(I64AtomicXor8U, Xor, Uint8, Word64)             \
    4350             :   V(I64AtomicXor16U, Xor, Uint16, Word64)           \
    4351             :   V(I64AtomicXor32U, Xor, Uint32, Word64)           \
    4352             :   V(I32AtomicExchange, Exchange, Uint32, Word32)    \
    4353             :   V(I64AtomicExchange, Exchange, Uint64, Word64)    \
    4354             :   V(I32AtomicExchange8U, Exchange, Uint8, Word32)   \
    4355             :   V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
    4356             :   V(I64AtomicExchange8U, Exchange, Uint8, Word64)   \
    4357             :   V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
    4358             :   V(I64AtomicExchange32U, Exchange, Uint32, Word64)
    4359             : 
    4360             : #define ATOMIC_CMP_EXCHG_LIST(V)                 \
    4361             :   V(I32AtomicCompareExchange, Uint32, Word32)    \
    4362             :   V(I64AtomicCompareExchange, Uint64, Word64)    \
    4363             :   V(I32AtomicCompareExchange8U, Uint8, Word32)   \
    4364             :   V(I32AtomicCompareExchange16U, Uint16, Word32) \
    4365             :   V(I64AtomicCompareExchange8U, Uint8, Word64)   \
    4366             :   V(I64AtomicCompareExchange16U, Uint16, Word64) \
    4367             :   V(I64AtomicCompareExchange32U, Uint32, Word64)
    4368             : 
    4369             : #define ATOMIC_LOAD_LIST(V)           \
    4370             :   V(I32AtomicLoad, Uint32, Word32)    \
    4371             :   V(I64AtomicLoad, Uint64, Word64)    \
    4372             :   V(I32AtomicLoad8U, Uint8, Word32)   \
    4373             :   V(I32AtomicLoad16U, Uint16, Word32) \
    4374             :   V(I64AtomicLoad8U, Uint8, Word64)   \
    4375             :   V(I64AtomicLoad16U, Uint16, Word64) \
    4376             :   V(I64AtomicLoad32U, Uint32, Word64)
    4377             : 
    4378             : #define ATOMIC_STORE_LIST(V)                    \
    4379             :   V(I32AtomicStore, Uint32, kWord32, Word32)    \
    4380             :   V(I64AtomicStore, Uint64, kWord64, Word64)    \
    4381             :   V(I32AtomicStore8U, Uint8, kWord8, Word32)    \
    4382             :   V(I32AtomicStore16U, Uint16, kWord16, Word32) \
    4383             :   V(I64AtomicStore8U, Uint8, kWord8, Word64)    \
    4384             :   V(I64AtomicStore16U, Uint16, kWord16, Word64) \
    4385             :   V(I64AtomicStore32U, Uint32, kWord32, Word64)
    4386             : 
    4387       32913 : Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
    4388             :                                  uint32_t alignment, uint32_t offset,
    4389             :                                  wasm::WasmCodePosition position) {
    4390             :   Node* node;
    4391       32913 :   switch (opcode) {
    4392             : #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix)                     \
    4393             :   case wasm::kExpr##Name: {                                                   \
    4394             :     Node* index = CheckBoundsAndAlignment(                                    \
    4395             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,    \
    4396             :         position);                                                            \
    4397             :     node = graph()->NewNode(                                                  \
    4398             :         mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
    4399             :         MemBuffer(offset), index, inputs[1], Effect(), Control());            \
    4400             :     break;                                                                    \
    4401             :   }
    4402       75121 :     ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
    4403             : #undef BUILD_ATOMIC_BINOP
    4404             : 
    4405             : #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix)                            \
    4406             :   case wasm::kExpr##Name: {                                                   \
    4407             :     Node* index = CheckBoundsAndAlignment(                                    \
    4408             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,    \
    4409             :         position);                                                            \
    4410             :     node = graph()->NewNode(                                                  \
    4411             :         mcgraph()->machine()->Prefix##AtomicCompareExchange(                  \
    4412             :             MachineType::Type()),                                             \
    4413             :         MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
    4414             :     break;                                                                    \
    4415             :   }
    4416         558 :     ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
    4417             : #undef BUILD_ATOMIC_CMP_EXCHG
    4418             : 
    4419             : #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix)                           \
    4420             :   case wasm::kExpr##Name: {                                                \
    4421             :     Node* index = CheckBoundsAndAlignment(                                 \
    4422             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
    4423             :         position);                                                         \
    4424             :     node = graph()->NewNode(                                               \
    4425             :         mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()),     \
    4426             :         MemBuffer(offset), index, Effect(), Control());                    \
    4427             :     break;                                                                 \
    4428             :   }
    4429       10544 :     ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
    4430             : #undef BUILD_ATOMIC_LOAD_OP
    4431             : 
    4432             : #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix)                         \
    4433             :   case wasm::kExpr##Name: {                                                    \
    4434             :     Node* index = CheckBoundsAndAlignment(                                     \
    4435             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,     \
    4436             :         position);                                                             \
    4437             :     node = graph()->NewNode(                                                   \
    4438             :         mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
    4439             :         MemBuffer(offset), index, inputs[1], Effect(), Control());             \
    4440             :     break;                                                                     \
    4441             :   }
    4442       11493 :     ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
    4443             : #undef BUILD_ATOMIC_STORE_OP
    4444             :     case wasm::kExprAtomicNotify: {
    4445         208 :       Node* index = CheckBoundsAndAlignment(
    4446             :           wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
    4447         208 :           position);
    4448             :       // Now that we've bounds-checked, compute the effective address.
    4449         208 :       Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    4450             :                                        Uint32Constant(offset), index);
    4451             :       WasmAtomicNotifyDescriptor interface_descriptor;
    4452         208 :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4453             :           mcgraph()->zone(), interface_descriptor,
    4454             :           interface_descriptor.GetStackParameterCount(),
    4455             :           CallDescriptor::kNoFlags, Operator::kNoProperties,
    4456         208 :           StubCallMode::kCallWasmRuntimeStub);
    4457             :       Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    4458         208 :           wasm::WasmCode::kWasmAtomicNotify, RelocInfo::WASM_STUB_CALL);
    4459         208 :       node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4460             :                               call_target, address, inputs[1], Effect(),
    4461             :                               Control());
    4462             :       break;
    4463             :     }
    4464             : 
    4465             :     case wasm::kExprI32AtomicWait: {
    4466         304 :       Node* index = CheckBoundsAndAlignment(
    4467             :           wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
    4468         305 :           position);
    4469             :       // Now that we've bounds-checked, compute the effective address.
    4470         307 :       Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    4471             :                                        Uint32Constant(offset), index);
    4472             :       Node* timeout;
    4473         304 :       if (mcgraph()->machine()->Is32()) {
    4474           0 :         timeout = BuildF64SConvertI64(inputs[2]);
    4475             :       } else {
    4476         304 :         timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
    4477             :                                    inputs[2]);
    4478             :       }
    4479             :       WasmI32AtomicWaitDescriptor interface_descriptor;
    4480         306 :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4481             :           mcgraph()->zone(), interface_descriptor,
    4482             :           interface_descriptor.GetStackParameterCount(),
    4483             :           CallDescriptor::kNoFlags, Operator::kNoProperties,
    4484         308 :           StubCallMode::kCallWasmRuntimeStub);
    4485             :       Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    4486         306 :           wasm::WasmCode::kWasmI32AtomicWait, RelocInfo::WASM_STUB_CALL);
    4487         306 :       node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4488             :                               call_target, address, inputs[1], timeout,
    4489             :                               Effect(), Control());
    4490             :       break;
    4491             :     }
    4492             : 
    4493             :     case wasm::kExprI64AtomicWait: {
    4494         362 :       Node* index = CheckBoundsAndAlignment(
    4495             :           wasm::ValueTypes::MemSize(MachineType::Uint64()), inputs[0], offset,
    4496         364 :           position);
    4497             :       // Now that we've bounds-checked, compute the effective address.
    4498         372 :       Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    4499             :                                        Uint32Constant(offset), index);
    4500             :       Node* timeout;
    4501         369 :       if (mcgraph()->machine()->Is32()) {
    4502           0 :         timeout = BuildF64SConvertI64(inputs[2]);
    4503             :       } else {
    4504         369 :         timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
    4505             :                                    inputs[2]);
    4506             :       }
    4507         368 :       Node* expected_value_low = graph()->NewNode(
    4508             :           mcgraph()->machine()->TruncateInt64ToInt32(), inputs[1]);
    4509         372 :       Node* tmp = graph()->NewNode(mcgraph()->machine()->Word64Shr(), inputs[1],
    4510             :                                    Int64Constant(32));
    4511             :       Node* expected_value_high =
    4512         370 :           graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), tmp);
    4513             :       WasmI64AtomicWaitDescriptor interface_descriptor;
    4514         363 :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4515             :           mcgraph()->zone(), interface_descriptor,
    4516             :           interface_descriptor.GetStackParameterCount(),
    4517             :           CallDescriptor::kNoFlags, Operator::kNoProperties,
    4518         371 :           StubCallMode::kCallWasmRuntimeStub);
    4519             :       Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    4520         363 :           wasm::WasmCode::kWasmI64AtomicWait, RelocInfo::WASM_STUB_CALL);
    4521         372 :       node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4522             :                               call_target, address, expected_value_high,
    4523             :                               expected_value_low, timeout, Effect(), Control());
    4524             :       break;
    4525             :     }
    4526             : 
    4527             :     default:
    4528           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4529             :   }
    4530       32940 :   return SetEffect(node);
    4531             : }
    4532             : 
    4533             : #undef ATOMIC_BINOP_LIST
    4534             : #undef ATOMIC_CMP_EXCHG_LIST
    4535             : #undef ATOMIC_LOAD_LIST
    4536             : #undef ATOMIC_STORE_LIST
    4537             : 
    4538          72 : Node* WasmGraphBuilder::CheckDataSegmentIsPassiveAndNotDropped(
    4539             :     uint32_t data_segment_index, wasm::WasmCodePosition position) {
    4540             :   // The data segment index must be in bounds since it is required by
    4541             :   // validation.
    4542             :   DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments);
    4543             : 
    4544             :   Node* dropped_data_segments =
    4545         144 :       LOAD_INSTANCE_FIELD(DroppedDataSegments, MachineType::Pointer());
    4546          72 :   Node* is_segment_dropped = SetEffect(graph()->NewNode(
    4547             :       mcgraph()->machine()->Load(MachineType::Uint8()), dropped_data_segments,
    4548             :       mcgraph()->IntPtrConstant(data_segment_index), Effect(), Control()));
    4549          72 :   TrapIfTrue(wasm::kTrapDataSegmentDropped, is_segment_dropped, position);
    4550          72 :   return dropped_data_segments;
    4551             : }
    4552             : 
    4553          48 : Node* WasmGraphBuilder::MemoryInit(uint32_t data_segment_index, Node* dst,
    4554             :                                    Node* src, Node* size,
    4555             :                                    wasm::WasmCodePosition position) {
    4556          48 :   CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
    4557          48 :   Node* dst_fail = BoundsCheckMemRange(&dst, &size, position);
    4558             :   auto m = mcgraph()->machine();
    4559             : 
    4560             :   Node* seg_index = Uint32Constant(data_segment_index);
    4561             :   Node* src_fail;
    4562             : 
    4563             :   {
    4564             :     // Load segment size from WasmInstanceObject::data_segment_sizes.
    4565             :     Node* seg_size_array =
    4566          96 :         LOAD_INSTANCE_FIELD(DataSegmentSizes, MachineType::Pointer());
    4567             :     STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2);
    4568          48 :     Node* scaled_index = Uint32ToUintptr(
    4569          48 :         graph()->NewNode(m->Word32Shl(), seg_index, Int32Constant(2)));
    4570          48 :     Node* seg_size = SetEffect(graph()->NewNode(m->Load(MachineType::Uint32()),
    4571             :                                                 seg_size_array, scaled_index,
    4572             :                                                 Effect(), Control()));
    4573             : 
    4574             :     // Bounds check the src index against the segment size.
    4575          48 :     src_fail = BoundsCheckRange(src, &size, seg_size, position);
    4576             :   }
    4577             : 
    4578             :   {
    4579             :     // Load segment's base pointer from WasmInstanceObject::data_segment_starts.
    4580             :     Node* seg_start_array =
    4581          96 :         LOAD_INSTANCE_FIELD(DataSegmentStarts, MachineType::Pointer());
    4582             :     STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <=
    4583             :                   kMaxUInt32 / kSystemPointerSize);
    4584          48 :     Node* scaled_index = Uint32ToUintptr(graph()->NewNode(
    4585          48 :         m->Word32Shl(), seg_index, Int32Constant(kSystemPointerSizeLog2)));
    4586          48 :     Node* seg_start = SetEffect(
    4587             :         graph()->NewNode(m->Load(MachineType::Pointer()), seg_start_array,
    4588             :                          scaled_index, Effect(), Control()));
    4589             : 
    4590             :     // Convert src index to pointer.
    4591          48 :     src = graph()->NewNode(m->IntAdd(), seg_start, Uint32ToUintptr(src));
    4592             :   }
    4593             : 
    4594          48 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
    4595          96 :       ExternalReference::wasm_memory_copy()));
    4596             :   MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
    4597          48 :                              MachineType::Uint32()};
    4598             :   MachineSignature sig(0, 3, sig_types);
    4599          48 :   BuildCCall(&sig, function, dst, src, size);
    4600          48 :   return TrapIfTrue(wasm::kTrapMemOutOfBounds,
    4601             :                     graph()->NewNode(m->Word32Or(), dst_fail, src_fail),
    4602          48 :                     position);
    4603             : }
    4604             : 
    4605          24 : Node* WasmGraphBuilder::DataDrop(uint32_t data_segment_index,
    4606             :                                  wasm::WasmCodePosition position) {
    4607             :   Node* dropped_data_segments =
    4608          24 :       CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
    4609          24 :   const Operator* store_op = mcgraph()->machine()->Store(
    4610          24 :       StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
    4611          24 :   return SetEffect(
    4612             :       graph()->NewNode(store_op, dropped_data_segments,
    4613             :                        mcgraph()->IntPtrConstant(data_segment_index),
    4614          24 :                        mcgraph()->Int32Constant(1), Effect(), Control()));
    4615             : }
    4616             : 
    4617          40 : Node* WasmGraphBuilder::MemoryCopy(Node* dst, Node* src, Node* size,
    4618             :                                    wasm::WasmCodePosition position) {
    4619             :   auto m = mcgraph()->machine();
    4620             :   // The data must be copied backward if the regions overlap and src < dst. The
    4621             :   // regions overlap if {src + size > dst && dst + size > src}. Since we already
    4622             :   // test that {src < dst}, we know that {dst + size > src}, so this simplifies
    4623             :   // to just {src + size > dst}. That sum can overflow, but if we subtract
    4624             :   // {size} from both sides of the inequality we get the equivalent test
    4625             :   // {size > dst - src}.
    4626         160 :   Node* copy_backward = graph()->NewNode(
    4627             :       m->Word32And(), graph()->NewNode(m->Uint32LessThan(), src, dst),
    4628             :       graph()->NewNode(m->Uint32LessThan(),
    4629             :                        graph()->NewNode(m->Int32Sub(), dst, src), size));
    4630             : 
    4631          40 :   Node* dst_fail = BoundsCheckMemRange(&dst, &size, position);
    4632             : 
    4633             :   // Trap without copying any bytes if we are copying backward and the copy is
    4634             :   // partially out-of-bounds. We only need to check that the dst region is
    4635             :   // out-of-bounds, because we know that {src < dst}, so the src region is
    4636             :   // always out of bounds if the dst region is.
    4637          40 :   TrapIfTrue(wasm::kTrapMemOutOfBounds,
    4638             :              graph()->NewNode(m->Word32And(), dst_fail, copy_backward),
    4639          40 :              position);
    4640             : 
    4641          40 :   Node* src_fail = BoundsCheckMemRange(&src, &size, position);
    4642             : 
    4643          40 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
    4644          80 :       ExternalReference::wasm_memory_copy()));
    4645             :   MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
    4646          40 :                              MachineType::Uint32()};
    4647             :   MachineSignature sig(0, 3, sig_types);
    4648          40 :   BuildCCall(&sig, function, dst, src, size);
    4649          40 :   return TrapIfTrue(wasm::kTrapMemOutOfBounds,
    4650             :                     graph()->NewNode(m->Word32Or(), dst_fail, src_fail),
    4651          40 :                     position);
    4652             : }
    4653             : 
    4654          40 : Node* WasmGraphBuilder::MemoryFill(Node* dst, Node* value, Node* size,
    4655             :                                    wasm::WasmCodePosition position) {
    4656          40 :   Node* fail = BoundsCheckMemRange(&dst, &size, position);
    4657          40 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
    4658          80 :       ExternalReference::wasm_memory_fill()));
    4659             :   MachineType sig_types[] = {MachineType::Pointer(), MachineType::Uint32(),
    4660          40 :                              MachineType::Uint32()};
    4661             :   MachineSignature sig(0, 3, sig_types);
    4662          40 :   BuildCCall(&sig, function, dst, value, size);
    4663          40 :   return TrapIfTrue(wasm::kTrapMemOutOfBounds, fail, position);
    4664             : }
    4665             : 
    4666          48 : Node* WasmGraphBuilder::CheckElemSegmentIsPassiveAndNotDropped(
    4667             :     uint32_t elem_segment_index, wasm::WasmCodePosition position) {
    4668             :   // The elem segment index must be in bounds since it is required by
    4669             :   // validation.
    4670             :   DCHECK_LT(elem_segment_index, env_->module->elem_segments.size());
    4671             : 
    4672             :   Node* dropped_elem_segments =
    4673          96 :       LOAD_INSTANCE_FIELD(DroppedElemSegments, MachineType::Pointer());
    4674          48 :   Node* is_segment_dropped = SetEffect(graph()->NewNode(
    4675             :       mcgraph()->machine()->Load(MachineType::Uint8()), dropped_elem_segments,
    4676             :       mcgraph()->IntPtrConstant(elem_segment_index), Effect(), Control()));
    4677          48 :   TrapIfTrue(wasm::kTrapElemSegmentDropped, is_segment_dropped, position);
    4678          48 :   return dropped_elem_segments;
    4679             : }
    4680             : 
    4681          32 : Node* WasmGraphBuilder::TableInit(uint32_t table_index,
    4682             :                                   uint32_t elem_segment_index, Node* dst,
    4683             :                                   Node* src, Node* size,
    4684             :                                   wasm::WasmCodePosition position) {
    4685          32 :   CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
    4686             :   Node* args[] = {
    4687          32 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
    4688          32 :       graph()->NewNode(mcgraph()->common()->NumberConstant(elem_segment_index)),
    4689          32 :       BuildConvertUint32ToSmiWithSaturation(dst, FLAG_wasm_max_table_size),
    4690          32 :       BuildConvertUint32ToSmiWithSaturation(src, FLAG_wasm_max_table_size),
    4691         128 :       BuildConvertUint32ToSmiWithSaturation(size, FLAG_wasm_max_table_size)};
    4692             :   Node* result =
    4693          32 :       BuildCallToRuntime(Runtime::kWasmTableInit, args, arraysize(args));
    4694             : 
    4695          32 :   return result;
    4696             : }
    4697             : 
    4698          16 : Node* WasmGraphBuilder::ElemDrop(uint32_t elem_segment_index,
    4699             :                                  wasm::WasmCodePosition position) {
    4700             :   Node* dropped_elem_segments =
    4701          16 :       CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
    4702          16 :   const Operator* store_op = mcgraph()->machine()->Store(
    4703          16 :       StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
    4704          16 :   return SetEffect(
    4705             :       graph()->NewNode(store_op, dropped_elem_segments,
    4706             :                        mcgraph()->IntPtrConstant(elem_segment_index),
    4707          16 :                        mcgraph()->Int32Constant(1), Effect(), Control()));
    4708             : }
    4709             : 
    4710          56 : Node* WasmGraphBuilder::TableCopy(uint32_t table_src_index,
    4711             :                                   uint32_t table_dst_index, Node* dst,
    4712             :                                   Node* src, Node* size,
    4713             :                                   wasm::WasmCodePosition position) {
    4714             :   Node* args[] = {
    4715          56 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_src_index)),
    4716          56 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_dst_index)),
    4717          56 :       BuildConvertUint32ToSmiWithSaturation(dst, FLAG_wasm_max_table_size),
    4718          56 :       BuildConvertUint32ToSmiWithSaturation(src, FLAG_wasm_max_table_size),
    4719         224 :       BuildConvertUint32ToSmiWithSaturation(size, FLAG_wasm_max_table_size)};
    4720             :   Node* result =
    4721          56 :       BuildCallToRuntime(Runtime::kWasmTableCopy, args, arraysize(args));
    4722             : 
    4723          56 :   return result;
    4724             : }
    4725             : 
    4726           0 : class WasmDecorator final : public GraphDecorator {
    4727             :  public:
    4728             :   explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
    4729           0 :       : origins_(origins), decoder_(decoder) {}
    4730             : 
    4731           0 :   void Decorate(Node* node) final {
    4732           0 :     origins_->SetNodeOrigin(
    4733             :         node, NodeOrigin("wasm graph creation", "n/a",
    4734           0 :                          NodeOrigin::kWasmBytecode, decoder_->position()));
    4735           0 :   }
    4736             : 
    4737             :  private:
    4738             :   compiler::NodeOriginTable* origins_;
    4739             :   wasm::Decoder* decoder_;
    4740             : };
    4741             : 
    4742           0 : void WasmGraphBuilder::AddBytecodePositionDecorator(
    4743             :     NodeOriginTable* node_origins, wasm::Decoder* decoder) {
    4744             :   DCHECK_NULL(decorator_);
    4745           0 :   decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
    4746           0 :   graph()->AddDecorator(decorator_);
    4747           0 : }
    4748             : 
    4749           0 : void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
    4750             :   DCHECK_NOT_NULL(decorator_);
    4751           0 :   graph()->RemoveDecorator(decorator_);
    4752           0 :   decorator_ = nullptr;
    4753           0 : }
    4754             : 
    4755             : namespace {
    4756             : bool must_record_function_compilation(Isolate* isolate) {
    4757      140317 :   return isolate->logger()->is_listening_to_code_events() ||
    4758             :          isolate->is_profiling();
    4759             : }
    4760             : 
    4761             : PRINTF_FORMAT(4, 5)
    4762           4 : void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
    4763             :                                Isolate* isolate, Handle<Code> code,
    4764             :                                const char* format, ...) {
    4765             :   DCHECK(must_record_function_compilation(isolate));
    4766             : 
    4767             :   ScopedVector<char> buffer(128);
    4768             :   va_list arguments;
    4769           4 :   va_start(arguments, format);
    4770           4 :   int len = VSNPrintF(buffer, format, arguments);
    4771           4 :   CHECK_LT(0, len);
    4772           4 :   va_end(arguments);
    4773             :   Handle<String> name_str =
    4774           4 :       isolate->factory()->NewStringFromAsciiChecked(buffer.start());
    4775           8 :   PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
    4776           4 : }
    4777             : 
    4778             : class WasmWrapperGraphBuilder : public WasmGraphBuilder {
    4779             :  public:
    4780      516230 :   WasmWrapperGraphBuilder(Zone* zone, JSGraph* jsgraph, wasm::FunctionSig* sig,
    4781             :                           compiler::SourcePositionTable* spt,
    4782             :                           StubCallMode stub_mode, wasm::WasmFeatures features)
    4783             :       : WasmGraphBuilder(nullptr, zone, jsgraph, sig, spt),
    4784             :         isolate_(jsgraph->isolate()),
    4785             :         jsgraph_(jsgraph),
    4786             :         stub_mode_(stub_mode),
    4787     1032460 :         enabled_features_(features) {}
    4788             : 
    4789       64636 :   Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
    4790             :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4791             :     CommonOperatorBuilder* common = mcgraph()->common();
    4792       64636 :     Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    4793             :                        ? mcgraph()->RelocatableIntPtrConstant(
    4794             :                              wasm::WasmCode::kWasmAllocateHeapNumber,
    4795             :                              RelocInfo::WASM_STUB_CALL)
    4796       21553 :                        : jsgraph()->HeapConstant(
    4797       86189 :                              BUILTIN_CODE(isolate_, AllocateHeapNumber));
    4798       64636 :     if (!allocate_heap_number_operator_.is_set()) {
    4799       53303 :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4800             :           mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
    4801       26652 :           CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
    4802       26651 :       allocate_heap_number_operator_.set(common->Call(call_descriptor));
    4803             :     }
    4804             :     Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
    4805             :                                          target, Effect(), control);
    4806      129272 :     SetEffect(
    4807             :         graph()->NewNode(machine->Store(StoreRepresentation(
    4808             :                              MachineRepresentation::kFloat64, kNoWriteBarrier)),
    4809             :                          heap_number, BuildHeapNumberValueIndexConstant(),
    4810             :                          value, heap_number, control));
    4811       64636 :     return heap_number;
    4812             :   }
    4813             : 
    4814       83672 :   Node* BuildChangeSmiToFloat64(Node* value) {
    4815       83672 :     return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
    4816       83677 :                             BuildChangeSmiToInt32(value));
    4817             :   }
    4818             : 
    4819       83674 :   Node* BuildTestHeapObject(Node* value) {
    4820       83674 :     return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
    4821       83678 :                             mcgraph()->IntPtrConstant(kHeapObjectTag));
    4822             :   }
    4823             : 
    4824       83679 :   Node* BuildLoadHeapNumberValue(Node* value) {
    4825       83678 :     return SetEffect(graph()->NewNode(
    4826             :         mcgraph()->machine()->Load(MachineType::Float64()), value,
    4827       83668 :         BuildHeapNumberValueIndexConstant(), Effect(), Control()));
    4828             :   }
    4829             : 
    4830             :   Node* BuildHeapNumberValueIndexConstant() {
    4831      148310 :     return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
    4832             :   }
    4833             : 
    4834       25174 :   Node* BuildChangeInt32ToTagged(Node* value) {
    4835             :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4836             :     CommonOperatorBuilder* common = mcgraph()->common();
    4837             : 
    4838             :     if (SmiValuesAre32Bits()) {
    4839             :       return BuildChangeInt32ToSmi(value);
    4840             :     }
    4841             :     DCHECK(SmiValuesAre31Bits());
    4842             : 
    4843             :     Node* effect = Effect();
    4844             :     Node* control = Control();
    4845       25174 :     Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
    4846             :                                  graph()->start());
    4847             : 
    4848       25178 :     Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
    4849             :     Node* branch =
    4850       25181 :         graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
    4851             : 
    4852       25182 :     Node* if_true = graph()->NewNode(common->IfTrue(), branch);
    4853       25177 :     Node* vtrue = BuildAllocateHeapNumberWithValue(
    4854       25182 :         graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
    4855             :     Node* etrue = Effect();
    4856             : 
    4857       25182 :     Node* if_false = graph()->NewNode(common->IfFalse(), branch);
    4858       25181 :     Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
    4859       25182 :     vfalse = BuildChangeInt32ToIntPtr(vfalse);
    4860             : 
    4861             :     Node* merge =
    4862       25179 :         SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
    4863       25183 :     SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
    4864       25183 :     return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
    4865             :                             vtrue, vfalse, merge);
    4866             :   }
    4867             : 
    4868       39454 :   Node* BuildChangeFloat64ToTagged(Node* value) {
    4869             :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4870             :     CommonOperatorBuilder* common = mcgraph()->common();
    4871             : 
    4872             :     // Check several conditions:
    4873             :     //  i32?
    4874             :     //  ├─ true: zero?
    4875             :     //  │        ├─ true: negative?
    4876             :     //  │        │        ├─ true: box
    4877             :     //  │        │        └─ false: potentially Smi
    4878             :     //  │        └─ false: potentially Smi
    4879             :     //  └─ false: box
    4880             :     // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
    4881             :     // still need to check whether the value fits in a Smi.
    4882             : 
    4883             :     Node* effect = Effect();
    4884             :     Node* control = Control();
    4885       39454 :     Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
    4886       78908 :     Node* check_i32 = graph()->NewNode(
    4887             :         machine->Float64Equal(), value,
    4888             :         graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
    4889       39454 :     Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
    4890             : 
    4891       39454 :     Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
    4892       39454 :     Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
    4893             : 
    4894             :     // We only need to check for -0 if the {value} can potentially contain -0.
    4895       39454 :     Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
    4896             :                                         mcgraph()->Int32Constant(0));
    4897       39454 :     Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
    4898             :                                          check_zero, if_i32);
    4899             : 
    4900       39454 :     Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
    4901       39454 :     Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
    4902             : 
    4903             :     // In case of 0, we need to check the high bits for the IEEE -0 pattern.
    4904       78908 :     Node* check_negative = graph()->NewNode(
    4905             :         machine->Int32LessThan(),
    4906             :         graph()->NewNode(machine->Float64ExtractHighWord32(), value),
    4907             :         mcgraph()->Int32Constant(0));
    4908       39454 :     Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
    4909             :                                              check_negative, if_zero);
    4910             : 
    4911       39454 :     Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
    4912             :     Node* if_not_negative =
    4913       39454 :         graph()->NewNode(common->IfFalse(), branch_negative);
    4914             : 
    4915             :     // We need to create a box for negative 0.
    4916             :     Node* if_smi =
    4917       39454 :         graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
    4918       39454 :     Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
    4919             : 
    4920             :     // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
    4921             :     // 32-bit machines we need to deal with potential overflow and fallback to
    4922             :     // boxing.
    4923             :     Node* vsmi;
    4924             :     if (SmiValuesAre32Bits()) {
    4925             :       vsmi = BuildChangeInt32ToSmi(value32);
    4926             :     } else {
    4927             :       DCHECK(SmiValuesAre31Bits());
    4928       39454 :       Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
    4929             :                                        value32, if_smi);
    4930             : 
    4931             :       Node* check_ovf =
    4932       39454 :           graph()->NewNode(common->Projection(1), smi_tag, if_smi);
    4933       39454 :       Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
    4934             :                                           check_ovf, if_smi);
    4935             : 
    4936       39454 :       Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
    4937       39454 :       if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
    4938             : 
    4939       39454 :       if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
    4940       39454 :       vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
    4941       39454 :       vsmi = BuildChangeInt32ToIntPtr(vsmi);
    4942             :     }
    4943             : 
    4944             :     // Allocate the box for the {value}.
    4945       39454 :     Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
    4946             :     Node* ebox = Effect();
    4947             : 
    4948             :     Node* merge =
    4949       39454 :         SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
    4950       39454 :     SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
    4951       39454 :     return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
    4952       39454 :                             vsmi, vbox, merge);
    4953             :   }
    4954             : 
    4955        6414 :   int AddArgumentNodes(Node** args, int pos, int param_count,
    4956             :                        wasm::FunctionSig* sig) {
    4957             :     // Convert wasm numbers to JS values.
    4958       92932 :     for (int i = 0; i < param_count; ++i) {
    4959             :       Node* param =
    4960       43259 :           Param(i + 1);  // Start from index 1 to drop the instance_node.
    4961       86518 :       args[pos++] = ToJS(param, sig->GetParam(i));
    4962             :     }
    4963        6414 :     return pos;
    4964             :   }
    4965             : 
    4966       83676 :   Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
    4967      167354 :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    4968             :         mcgraph()->zone(), TypeConversionDescriptor{}, 0,
    4969       83676 :         CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
    4970             :     Node* stub_code =
    4971       83678 :         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    4972             :             ? mcgraph()->RelocatableIntPtrConstant(
    4973             :                   wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
    4974      244016 :             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
    4975             : 
    4976       83676 :     Node* result = SetEffect(
    4977             :         graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
    4978             :                          node, js_context, Effect(), Control()));
    4979             : 
    4980             :     SetSourcePosition(result, 1);
    4981             : 
    4982       83679 :     return result;
    4983             :   }
    4984             : 
    4985       83674 :   Node* BuildChangeTaggedToFloat64(Node* value) {
    4986             :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4987             :     CommonOperatorBuilder* common = mcgraph()->common();
    4988             : 
    4989             :     // Implement the following decision tree:
    4990             :     //  heap object?
    4991             :     //  ├─ true: undefined?
    4992             :     //  │        ├─ true: f64 const
    4993             :     //  │        └─ false: load heap number value
    4994             :     //  └─ false: smi to float64
    4995             : 
    4996       83674 :     Node* check_heap_object = BuildTestHeapObject(value);
    4997             :     Diamond is_heap_object(graph(), common, check_heap_object,
    4998       83678 :                            BranchHint::kFalse);
    4999             :     is_heap_object.Chain(Control());
    5000             : 
    5001       83676 :     SetControl(is_heap_object.if_true);
    5002             :     Node* orig_effect = Effect();
    5003             : 
    5004             :     Node* undefined_node =
    5005      167350 :         LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
    5006             :     Node* check_undefined =
    5007       83678 :         graph()->NewNode(machine->WordEqual(), value, undefined_node);
    5008             :     Node* effect_tagged = Effect();
    5009             : 
    5010       83679 :     Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
    5011       83678 :     is_undefined.Nest(is_heap_object, true);
    5012             : 
    5013       83678 :     SetControl(is_undefined.if_false);
    5014       83678 :     Node* vheap_number = BuildLoadHeapNumberValue(value);
    5015             :     Node* effect_undefined = Effect();
    5016             : 
    5017       83672 :     SetControl(is_undefined.merge);
    5018             :     Node* vundefined =
    5019       83672 :         mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
    5020             :     Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
    5021       83673 :                                      vundefined, vheap_number);
    5022             : 
    5023       83677 :     effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
    5024             : 
    5025             :     // If input is Smi: just convert to float64.
    5026       83674 :     Node* vfrom_smi = BuildChangeSmiToFloat64(value);
    5027             : 
    5028       83677 :     SetControl(is_heap_object.merge);
    5029       83677 :     SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
    5030             :     return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
    5031       83676 :                               vfrom_smi);
    5032             :   }
    5033             : 
    5034       65212 :   Node* ToJS(Node* node, wasm::ValueType type) {
    5035       65212 :     switch (type) {
    5036             :       case wasm::kWasmI32:
    5037       25175 :         return BuildChangeInt32ToTagged(node);
    5038             :       case wasm::kWasmS128:
    5039           0 :         UNREACHABLE();
    5040             :       case wasm::kWasmI64: {
    5041             :         DCHECK(enabled_features_.bigint);
    5042          32 :         return BuildChangeInt64ToBigInt(node);
    5043             :       }
    5044             :       case wasm::kWasmF32:
    5045       11025 :         node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
    5046             :                                 node);
    5047       11025 :         return BuildChangeFloat64ToTagged(node);
    5048             :       case wasm::kWasmF64:
    5049       28429 :         return BuildChangeFloat64ToTagged(node);
    5050             :       case wasm::kWasmAnyRef:
    5051             :       case wasm::kWasmAnyFunc:
    5052             :       case wasm::kWasmExceptRef:
    5053             :         return node;
    5054             :       default:
    5055           0 :         UNREACHABLE();
    5056             :     }
    5057             :   }
    5058             : 
    5059          32 :   Node* BuildChangeInt64ToBigInt(Node* input) {
    5060             :     I64ToBigIntDescriptor interface_descriptor;
    5061             : 
    5062          64 :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    5063             :         mcgraph()->zone(),                              // zone
    5064             :         interface_descriptor,                           // descriptor
    5065             :         interface_descriptor.GetStackParameterCount(),  // stack parameter count
    5066             :         CallDescriptor::kNoFlags,                       // flags
    5067             :         Operator::kNoProperties,                        // properties
    5068          32 :         stub_mode_);                                    // stub call mode
    5069             : 
    5070             :     Node* target =
    5071          32 :         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    5072             :             ? mcgraph()->RelocatableIntPtrConstant(
    5073             :                   wasm::WasmCode::kWasmI64ToBigInt, RelocInfo::WASM_STUB_CALL)
    5074          80 :             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, I64ToBigInt));
    5075             : 
    5076          32 :     return SetEffect(
    5077             :         SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    5078          32 :                                     target, input, Effect(), Control())));
    5079             :   }
    5080             : 
    5081          16 :   Node* BuildChangeBigIntToInt64(Node* input, Node* context) {
    5082             :     BigIntToI64Descriptor interface_descriptor;
    5083             : 
    5084          32 :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    5085             :         mcgraph()->zone(),                              // zone
    5086             :         interface_descriptor,                           // descriptor
    5087             :         interface_descriptor.GetStackParameterCount(),  // stack parameter count
    5088             :         CallDescriptor::kNoFlags,                       // flags
    5089             :         Operator::kNoProperties,                        // properties
    5090          16 :         stub_mode_);                                    // stub call mode
    5091             : 
    5092             :     Node* target =
    5093          16 :         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    5094             :             ? mcgraph()->RelocatableIntPtrConstant(
    5095             :                   wasm::WasmCode::kWasmBigIntToI64, RelocInfo::WASM_STUB_CALL)
    5096          32 :             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, BigIntToI64));
    5097             : 
    5098          16 :     return SetEffect(SetControl(
    5099             :         graph()->NewNode(mcgraph()->common()->Call(call_descriptor), target,
    5100          16 :                          input, context, Effect(), Control())));
    5101             :   }
    5102             : 
    5103       85788 :   Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
    5104             :     DCHECK_NE(wasm::kWasmStmt, type);
    5105             : 
    5106             :     // The parameter is of type anyref or except_ref, we take it as is.
    5107       85788 :     if (type == wasm::kWasmAnyRef || type == wasm::kWasmExceptRef) {
    5108        1976 :       return node;
    5109             :     }
    5110             : 
    5111       83812 :     if (type == wasm::kWasmAnyFunc) {
    5112             :       Node* check =
    5113         120 :           BuildChangeSmiToInt32(SetEffect(BuildCallToRuntimeWithContext(
    5114             :               Runtime::kWasmIsValidAnyFuncValue, js_context, &node, 1, effect_,
    5115         120 :               Control())));
    5116             : 
    5117             :       Diamond type_check(graph(), mcgraph()->common(), check,
    5118         120 :                          BranchHint::kTrue);
    5119             :       type_check.Chain(Control());
    5120             : 
    5121         120 :       Node* effect = Effect();
    5122         120 :       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
    5123         120 :                                     nullptr, 0, &effect, type_check.if_false);
    5124             : 
    5125         240 :       SetEffect(type_check.EffectPhi(Effect(), effect));
    5126             : 
    5127         120 :       SetControl(type_check.merge);
    5128             : 
    5129         120 :       return node;
    5130             :     }
    5131             :     Node* num = nullptr;
    5132             : 
    5133       83692 :     if (type != wasm::kWasmI64) {
    5134             :       // Do a JavaScript ToNumber.
    5135       83676 :       num = BuildJavaScriptToNumber(node, js_context);
    5136             : 
    5137             :       // Change representation.
    5138       83679 :       num = BuildChangeTaggedToFloat64(num);
    5139             :     }
    5140             : 
    5141       83691 :     switch (type) {
    5142             :       case wasm::kWasmI32: {
    5143       30690 :         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
    5144             :                                num);
    5145       30692 :         break;
    5146             :       }
    5147             :       case wasm::kWasmI64: {
    5148             :         DCHECK(enabled_features_.bigint);
    5149          16 :         num = BuildChangeBigIntToInt64(node, js_context);
    5150          16 :         break;
    5151             :       }
    5152             :       case wasm::kWasmF32:
    5153       12038 :         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
    5154             :                                num);
    5155       12038 :         break;
    5156             :       case wasm::kWasmF64:
    5157             :         break;
    5158             :       case wasm::kWasmS128:
    5159           0 :         UNREACHABLE();
    5160             :       default:
    5161           0 :         UNREACHABLE();
    5162             :     }
    5163             :     DCHECK_NOT_NULL(num);
    5164             : 
    5165             :     return num;
    5166             :   }
    5167             : 
    5168      291995 :   void BuildModifyThreadInWasmFlag(bool new_value) {
    5169      291995 :     if (!trap_handler::IsTrapHandlerEnabled()) return;
    5170             :     Node* isolate_root =
    5171      583516 :         LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
    5172             : 
    5173             :     Node* thread_in_wasm_flag_address =
    5174      291760 :         LOAD_RAW(isolate_root, Isolate::thread_in_wasm_flag_address_offset(),
    5175             :                  MachineType::Pointer());
    5176             : 
    5177      291759 :     if (FLAG_debug_code) {
    5178          16 :       Node* flag_value = SetEffect(
    5179             :           graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
    5180             :                            thread_in_wasm_flag_address,
    5181             :                            mcgraph()->Int32Constant(0), Effect(), Control()));
    5182             :       Node* check =
    5183          16 :           graph()->NewNode(mcgraph()->machine()->Word32Equal(), flag_value,
    5184             :                            mcgraph()->Int32Constant(new_value ? 0 : 1));
    5185             : 
    5186             :       Diamond flag_check(graph(), mcgraph()->common(), check,
    5187          16 :                          BranchHint::kTrue);
    5188             :       flag_check.Chain(Control());
    5189          16 :       Node* message_id = jsgraph()->SmiConstant(static_cast<int32_t>(
    5190             :           new_value ? AbortReason::kUnexpectedThreadInWasmSet
    5191          16 :                     : AbortReason::kUnexpectedThreadInWasmUnset));
    5192             : 
    5193          16 :       Node* effect = Effect();
    5194          32 :       BuildCallToRuntimeWithContext(Runtime::kAbort, NoContextConstant(),
    5195             :                                     &message_id, 1, &effect,
    5196          16 :                                     flag_check.if_false);
    5197             : 
    5198          32 :       SetEffect(flag_check.EffectPhi(Effect(), effect));
    5199             : 
    5200          16 :       SetControl(flag_check.merge);
    5201             :     }
    5202             : 
    5203      875275 :     SetEffect(graph()->NewNode(
    5204             :         mcgraph()->machine()->Store(StoreRepresentation(
    5205             :             MachineRepresentation::kWord32, kNoWriteBarrier)),
    5206             :         thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
    5207             :         mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
    5208             :   }
    5209             : 
    5210      140320 :   Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
    5211      140320 :     Node* shared = SetEffect(graph()->NewNode(
    5212             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
    5213             :         jsgraph()->Int32Constant(
    5214             :             wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()),
    5215             :         Effect(), Control()));
    5216      140320 :     return SetEffect(graph()->NewNode(
    5217             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
    5218             :         jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
    5219             :                                  kHeapObjectTag),
    5220      140319 :         Effect(), Control()));
    5221             :   }
    5222             : 
    5223      140320 :   Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
    5224      140320 :     return SetEffect(graph()->NewNode(
    5225             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
    5226             :         jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
    5227             :                                  kHeapObjectTag),
    5228      140321 :         Effect(), Control()));
    5229             :   }
    5230             : 
    5231         456 :   Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
    5232         456 :     Node* function_index_smi = SetEffect(graph()->NewNode(
    5233             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
    5234             :         jsgraph()->Int32Constant(
    5235             :             WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
    5236             :         Effect(), Control()));
    5237         456 :     Node* function_index = BuildChangeSmiToInt32(function_index_smi);
    5238         456 :     return function_index;
    5239             :   }
    5240             : 
    5241      139124 :   Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
    5242      139124 :     Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
    5243             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
    5244             :         jsgraph()->Int32Constant(
    5245             :             WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
    5246             :         Effect(), Control()));
    5247      139128 :     Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
    5248      139128 :     return jump_table_offset;
    5249             :   }
    5250             : 
    5251      140321 :   void BuildJSToWasmWrapper(bool is_import) {
    5252      140321 :     const int wasm_count = static_cast<int>(sig_->parameter_count());
    5253             : 
    5254             :     // Build the start and the JS parameter nodes.
    5255      140321 :     SetEffect(SetControl(Start(wasm_count + 5)));
    5256             : 
    5257             :     // Create the js_closure and js_context parameters.
    5258             :     Node* js_closure =
    5259      140321 :         graph()->NewNode(jsgraph()->common()->Parameter(
    5260             :                              Linkage::kJSCallClosureParamIndex, "%closure"),
    5261             :                          graph()->start());
    5262      140321 :     Node* js_context = graph()->NewNode(
    5263             :         mcgraph()->common()->Parameter(
    5264             :             Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
    5265             :         graph()->start());
    5266             : 
    5267             :     // Create the instance_node node to pass as parameter. It is loaded from
    5268             :     // an actual reference to an instance or a placeholder reference,
    5269             :     // called {WasmExportedFunction} via the {WasmExportedFunctionData}
    5270             :     // structure.
    5271      140321 :     Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
    5272      140319 :     instance_node_.set(
    5273             :         BuildLoadInstanceFromExportedFunctionData(function_data));
    5274             : 
    5275      140321 :     if (!wasm::IsJSCompatibleSignature(sig_, enabled_features_.bigint)) {
    5276             :       // Throw a TypeError. Use the js_context of the calling javascript
    5277             :       // function (passed as a parameter), such that the generated code is
    5278             :       // js_context independent.
    5279         736 :       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
    5280         736 :                                     nullptr, 0, effect_, Control());
    5281             :       Return(jsgraph()->SmiConstant(0));
    5282         736 :       return;
    5283             :     }
    5284             : 
    5285      139583 :     const int args_count = wasm_count + 1;  // +1 for wasm_code.
    5286      139583 :     Node** args = Buffer(args_count);
    5287             :     Node** rets;
    5288             : 
    5289             :     // Convert JS parameters to wasm numbers.
    5290      304124 :     for (int i = 0; i < wasm_count; ++i) {
    5291       82270 :       Node* param = Param(i + 1);
    5292      164546 :       Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
    5293       82270 :       args[i + 1] = wasm_param;
    5294             :     }
    5295             : 
    5296             :     // Set the ThreadInWasm flag before we do the actual call.
    5297      139584 :     BuildModifyThreadInWasmFlag(true);
    5298             : 
    5299      139580 :     if (is_import) {
    5300             :       // Call to an imported function.
    5301             :       // Load function index from {WasmExportedFunctionData}.
    5302             :       Node* function_index =
    5303         456 :           BuildLoadFunctionIndexFromExportedFunctionData(function_data);
    5304         456 :       BuildImportCall(sig_, args, &rets, wasm::kNoCodePosition, function_index,
    5305         456 :                       kCallContinues);
    5306             :     } else {
    5307             :       // Call to a wasm function defined in this module.
    5308             :       // The call target is the jump table slot for that function.
    5309             :       Node* jump_table_start =
    5310      278252 :           LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
    5311             :       Node* jump_table_offset =
    5312      139127 :           BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
    5313      139126 :       Node* jump_table_slot = graph()->NewNode(
    5314             :           mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
    5315      139128 :       args[0] = jump_table_slot;
    5316             : 
    5317      139128 :       BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
    5318      139128 :                     kNoRetpoline);
    5319             :     }
    5320             : 
    5321             :     // Clear the ThreadInWasm flag.
    5322      139585 :     BuildModifyThreadInWasmFlag(false);
    5323             : 
    5324      139582 :     Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
    5325      161541 :                                             : ToJS(rets[0], sig_->GetReturn());
    5326             :     Return(jsval);
    5327             :   }
    5328             : 
    5329        6490 :   bool BuildWasmImportCallWrapper(WasmImportCallKind kind) {
    5330        6490 :     int wasm_count = static_cast<int>(sig_->parameter_count());
    5331             : 
    5332             :     // Build the start and the parameter nodes.
    5333        6490 :     SetEffect(SetControl(Start(wasm_count + 4)));
    5334             : 
    5335        6490 :     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
    5336             : 
    5337             :     Node* native_context =
    5338       12980 :         LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
    5339             : 
    5340        6490 :     if (kind == WasmImportCallKind::kRuntimeTypeError) {
    5341             :       // =======================================================================
    5342             :       // === Runtime TypeError =================================================
    5343             :       // =======================================================================
    5344          76 :       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
    5345             :                                     native_context, nullptr, 0, effect_,
    5346          76 :                                     Control());
    5347             :       // We don't need to return a value here, as the runtime call will not
    5348             :       // return anyway (the c entry stub will trigger stack unwinding).
    5349             :       ReturnVoid();
    5350          76 :       return false;
    5351             :     }
    5352             : 
    5353             :     // The callable is passed as the last parameter, after WASM arguments.
    5354        6414 :     Node* callable_node = Param(wasm_count + 1);
    5355             : 
    5356             :     Node* undefined_node =
    5357       12828 :         LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
    5358             : 
    5359             :     Node* call = nullptr;
    5360             :     bool sloppy_receiver = true;
    5361             : 
    5362             :     // Clear the ThreadInWasm flag.
    5363        6414 :     BuildModifyThreadInWasmFlag(false);
    5364             : 
    5365        6414 :     switch (kind) {
    5366             :       // =======================================================================
    5367             :       // === JS Functions with matching arity ==================================
    5368             :       // =======================================================================
    5369             :       case WasmImportCallKind::kJSFunctionArityMatch:
    5370             :         sloppy_receiver = false;
    5371             :         V8_FALLTHROUGH;  // fallthru
    5372             :       case WasmImportCallKind::kJSFunctionArityMatchSloppy: {
    5373        4056 :         Node** args = Buffer(wasm_count + 9);
    5374             :         int pos = 0;
    5375        4056 :         Node* function_context = SetEffect(graph()->NewNode(
    5376             :             mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    5377             :             callable_node,
    5378             :             mcgraph()->Int32Constant(
    5379             :                 wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
    5380             :             Effect(), Control()));
    5381        4056 :         args[pos++] = callable_node;  // target callable.
    5382             :         // Receiver.
    5383        4056 :         if (sloppy_receiver) {
    5384        3968 :           Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
    5385             :               native_context, Context::GLOBAL_PROXY_INDEX);
    5386        3968 :           args[pos++] = global_proxy;
    5387             :         } else {
    5388          88 :           args[pos++] = undefined_node;
    5389             :         }
    5390             : 
    5391        4056 :         auto call_descriptor = Linkage::GetJSCallDescriptor(
    5392        4056 :             graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
    5393             : 
    5394             :         // Convert wasm numbers to JS values.
    5395        4056 :         pos = AddArgumentNodes(args, pos, wasm_count, sig_);
    5396             : 
    5397        4056 :         args[pos++] = undefined_node;                        // new target
    5398        4056 :         args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
    5399        4056 :         args[pos++] = function_context;
    5400        8112 :         args[pos++] = Effect();
    5401        8112 :         args[pos++] = Control();
    5402             : 
    5403        4056 :         call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
    5404        4056 :                                 args);
    5405        4056 :         break;
    5406             :       }
    5407             :       // =======================================================================
    5408             :       // === JS Functions with arguments adapter ===============================
    5409             :       // =======================================================================
    5410             :       case WasmImportCallKind::kJSFunctionArityMismatch:
    5411             :         sloppy_receiver = false;
    5412             :         V8_FALLTHROUGH;  // fallthru
    5413             :       case WasmImportCallKind::kJSFunctionArityMismatchSloppy: {
    5414         638 :         Node** args = Buffer(wasm_count + 9);
    5415             :         int pos = 0;
    5416         638 :         Node* function_context = SetEffect(graph()->NewNode(
    5417             :             mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    5418             :             callable_node,
    5419             :             mcgraph()->Int32Constant(
    5420             :                 wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
    5421             :             Effect(), Control()));
    5422             :         args[pos++] =
    5423         638 :             BuildLoadBuiltinFromInstance(Builtins::kArgumentsAdaptorTrampoline);
    5424         638 :         args[pos++] = callable_node;                         // target callable
    5425         638 :         args[pos++] = undefined_node;                        // new target
    5426         638 :         args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
    5427             : 
    5428             :         // Load shared function info, and then the formal parameter count.
    5429         638 :         Node* shared_function_info = SetEffect(graph()->NewNode(
    5430             :             mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    5431             :             callable_node,
    5432             :             mcgraph()->Int32Constant(
    5433             :                 wasm::ObjectAccess::
    5434             :                     SharedFunctionInfoOffsetInTaggedJSFunction()),
    5435             :             Effect(), Control()));
    5436         638 :         Node* formal_param_count = SetEffect(graph()->NewNode(
    5437             :             mcgraph()->machine()->Load(MachineType::Uint16()),
    5438             :             shared_function_info,
    5439             :             mcgraph()->Int32Constant(
    5440             :                 wasm::ObjectAccess::
    5441             :                     FormalParameterCountOffsetInSharedFunctionInfo()),
    5442             :             Effect(), Control()));
    5443         638 :         args[pos++] = formal_param_count;
    5444             : 
    5445             :         // Receiver.
    5446         638 :         if (sloppy_receiver) {
    5447         588 :           Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
    5448             :               native_context, Context::GLOBAL_PROXY_INDEX);
    5449         588 :           args[pos++] = global_proxy;
    5450             :         } else {
    5451          50 :           args[pos++] = undefined_node;
    5452             :         }
    5453             : 
    5454             : #ifdef V8_TARGET_ARCH_IA32
    5455             :         // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative
    5456             :         // call instead once builtins are embedded in every build configuration.
    5457             :         CallDescriptor::Flags flags = CallDescriptor::kAllowCallThroughSlot;
    5458             : #else
    5459             :         CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
    5460             : #endif
    5461         638 :         auto call_descriptor = Linkage::GetStubCallDescriptor(
    5462             :             mcgraph()->zone(), ArgumentsAdaptorDescriptor{}, 1 + wasm_count,
    5463         638 :             flags, Operator::kNoProperties);
    5464             : 
    5465             :         // Convert wasm numbers to JS values.
    5466         638 :         pos = AddArgumentNodes(args, pos, wasm_count, sig_);
    5467         638 :         args[pos++] = function_context;
    5468        1276 :         args[pos++] = Effect();
    5469        1276 :         args[pos++] = Control();
    5470         638 :         call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
    5471         638 :                                 args);
    5472             :         break;
    5473             :       }
    5474             :       // =======================================================================
    5475             :       // === General case of unknown callable ==================================
    5476             :       // =======================================================================
    5477             :       case WasmImportCallKind::kUseCallBuiltin: {
    5478        1720 :         Node** args = Buffer(wasm_count + 9);
    5479             :         int pos = 0;
    5480        1720 :         args[pos++] = mcgraph()->RelocatableIntPtrConstant(
    5481        1720 :             wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
    5482        1720 :         args[pos++] = callable_node;
    5483        1720 :         args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
    5484        1720 :         args[pos++] = undefined_node;                        // receiver
    5485             : 
    5486        1720 :         auto call_descriptor = Linkage::GetStubCallDescriptor(
    5487             :             graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
    5488             :             CallDescriptor::kNoFlags, Operator::kNoProperties,
    5489        1720 :             StubCallMode::kCallWasmRuntimeStub);
    5490             : 
    5491             :         // Convert wasm numbers to JS values.
    5492        1720 :         pos = AddArgumentNodes(args, pos, wasm_count, sig_);
    5493             : 
    5494             :         // The native_context is sufficient here, because all kind of callables
    5495             :         // which depend on the context provide their own context. The context
    5496             :         // here is only needed if the target is a constructor to throw a
    5497             :         // TypeError, if the target is a native function, or if the target is a
    5498             :         // callable JSObject, which can only be constructed by the runtime.
    5499        1720 :         args[pos++] = native_context;
    5500        3440 :         args[pos++] = Effect();
    5501        3440 :         args[pos++] = Control();
    5502             : 
    5503        1720 :         call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
    5504        1720 :                                 args);
    5505        1720 :         break;
    5506             :       }
    5507             :       default:
    5508           0 :         UNREACHABLE();
    5509             :     }
    5510             :     DCHECK_NOT_NULL(call);
    5511             : 
    5512             :     SetEffect(call);
    5513             :     SetSourcePosition(call, 0);
    5514             : 
    5515             :     // Convert the return value back.
    5516        6414 :     Node* val = sig_->return_count() == 0
    5517             :                     ? mcgraph()->Int32Constant(0)
    5518        9931 :                     : FromJS(call, native_context, sig_->GetReturn());
    5519             : 
    5520             :     // Set the ThreadInWasm flag again.
    5521        6414 :     BuildModifyThreadInWasmFlag(true);
    5522             : 
    5523             :     Return(val);
    5524        6414 :     return true;
    5525             :   }
    5526             : 
    5527      368043 :   void BuildWasmInterpreterEntry(uint32_t func_index) {
    5528      368043 :     int param_count = static_cast<int>(sig_->parameter_count());
    5529             : 
    5530             :     // Build the start and the parameter nodes.
    5531      368043 :     SetEffect(SetControl(Start(param_count + 3)));
    5532             : 
    5533             :     // Create the instance_node from the passed parameter.
    5534      368042 :     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
    5535             : 
    5536             :     // Compute size for the argument buffer.
    5537      368038 :     int args_size_bytes = 0;
    5538      796822 :     for (wasm::ValueType type : sig_->parameters()) {
    5539       30373 :       args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
    5540             :     }
    5541             : 
    5542             :     // The return value is also passed via this buffer:
    5543      368038 :     int return_size_bytes = 0;
    5544     1101422 :     for (wasm::ValueType type : sig_->returns()) {
    5545      366692 :       return_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
    5546             :     }
    5547             : 
    5548             :     // Get a stack slot for the arguments.
    5549             :     Node* arg_buffer =
    5550      368038 :         args_size_bytes == 0 && return_size_bytes == 0
    5551             :             ? mcgraph()->IntPtrConstant(0)
    5552      366432 :             : graph()->NewNode(mcgraph()->machine()->StackSlot(
    5553      368038 :                   std::max(args_size_bytes, return_size_bytes), 8));
    5554             : 
    5555             :     // Now store all our arguments to the buffer.
    5556             :     int offset = 0;
    5557             : 
    5558      428787 :     for (int i = 0; i < param_count; ++i) {
    5559       30373 :       wasm::ValueType type = sig_->GetParam(i);
    5560             :       // Start from the parameter with index 1 to drop the instance_node.
    5561       60748 :       SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
    5562       30373 :                                  Int32Constant(offset), Param(i + 1), Effect(),
    5563             :                                  Control()));
    5564       30373 :       offset += wasm::ValueTypes::ElementSizeInBytes(type);
    5565             :     }
    5566             :     DCHECK_EQ(args_size_bytes, offset);
    5567             : 
    5568             :     // We are passing the raw arg_buffer here. To the GC and other parts, it
    5569             :     // looks like a Smi (lowest bit not set). In the runtime function however,
    5570             :     // don't call Smi::value on it, but just cast it to a byte pointer.
    5571      736082 :     Node* parameters[] = {jsgraph()->SmiConstant(func_index), arg_buffer};
    5572             :     BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
    5573      368041 :                        arraysize(parameters));
    5574             : 
    5575             :     // Read back the return value.
    5576             :     DCHECK_LT(sig_->return_count(), wasm::kV8MaxWasmFunctionMultiReturns);
    5577      368042 :     unsigned return_count = static_cast<unsigned>(sig_->return_count());
    5578      368042 :     if (return_count == 0) {
    5579             :       Return(Int32Constant(0));
    5580             :     } else {
    5581      366175 :       Node** returns = Buffer(return_count);
    5582             :       offset = 0;
    5583     1099553 :       for (size_t i = 0; i < return_count; ++i) {
    5584      366691 :         wasm::ValueType type = sig_->GetReturn(i);
    5585      366694 :         Node* val = SetEffect(graph()->NewNode(
    5586             :             mcgraph()->machine()->Load(wasm::ValueTypes::MachineTypeFor(type)),
    5587             :             arg_buffer, Int32Constant(offset), Effect(), Control()));
    5588      366695 :         returns[i] = val;
    5589      366695 :         offset += wasm::ValueTypes::ElementSizeInBytes(type);
    5590             :       }
    5591      366171 :       Return(return_count, returns);
    5592             :     }
    5593             : 
    5594      736080 :     if (ContainsInt64(sig_)) LowerInt64();
    5595      368040 :   }
    5596             : 
    5597        1376 :   void BuildCWasmEntry() {
    5598             :     // Build the start and the JS parameter nodes.
    5599        1376 :     SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
    5600             : 
    5601             :     // Create parameter nodes (offset by 1 for the receiver parameter).
    5602        1376 :     Node* code_entry = Param(CWasmEntryParameters::kCodeEntry + 1);
    5603        1376 :     Node* object_ref_node = Param(CWasmEntryParameters::kObjectRef + 1);
    5604        1376 :     Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
    5605             : 
    5606        1376 :     int wasm_arg_count = static_cast<int>(sig_->parameter_count());
    5607             :     int arg_count =
    5608        1376 :         wasm_arg_count + 4;  // code, object_ref_node, control, effect
    5609        1376 :     Node** args = Buffer(arg_count);
    5610             : 
    5611             :     int pos = 0;
    5612        1376 :     args[pos++] = code_entry;
    5613        1376 :     args[pos++] = object_ref_node;
    5614             : 
    5615             :     int offset = 0;
    5616       12208 :     for (wasm::ValueType type : sig_->parameters()) {
    5617        4728 :       Node* arg_load = SetEffect(
    5618             :           graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
    5619             :                            Int32Constant(offset), Effect(), Control()));
    5620        4728 :       args[pos++] = arg_load;
    5621        4728 :       offset += wasm::ValueTypes::ElementSizeInBytes(type);
    5622             :     }
    5623             : 
    5624        2752 :     args[pos++] = Effect();
    5625        2752 :     args[pos++] = Control();
    5626             :     DCHECK_EQ(arg_count, pos);
    5627             : 
    5628             :     // Call the wasm code.
    5629        1376 :     auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
    5630             : 
    5631        1376 :     Node* call = SetEffect(graph()->NewNode(
    5632             :         mcgraph()->common()->Call(call_descriptor), arg_count, args));
    5633             : 
    5634             :     // Store the return value.
    5635             :     DCHECK_GE(1, sig_->return_count());
    5636        1376 :     if (sig_->return_count() == 1) {
    5637             :       StoreRepresentation store_rep(
    5638             :           wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
    5639        1324 :           kNoWriteBarrier);
    5640        1324 :       SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
    5641             :                                  arg_buffer, Int32Constant(0), call, Effect(),
    5642             :                                  Control()));
    5643             :     }
    5644             :     Return(jsgraph()->SmiConstant(0));
    5645             : 
    5646        1376 :     if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
    5647             :       MachineRepresentation sig_reps[] = {
    5648             :           MachineRepresentation::kWord32,  // return value
    5649             :           MachineRepresentation::kTagged,  // receiver
    5650             :           MachineRepresentation::kTagged,  // arg0 (code)
    5651             :           MachineRepresentation::kTagged   // arg1 (buffer)
    5652           0 :       };
    5653             :       Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
    5654             :       Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
    5655           0 :                       mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
    5656           0 :       r.LowerGraph();
    5657             :     }
    5658        1376 :   }
    5659             : 
    5660             :   JSGraph* jsgraph() { return jsgraph_; }
    5661             : 
    5662             :  private:
    5663             :   Isolate* const isolate_;
    5664             :   JSGraph* jsgraph_;
    5665             :   StubCallMode stub_mode_;
    5666             :   SetOncePointer<const Operator> allocate_heap_number_operator_;
    5667             :   wasm::WasmFeatures enabled_features_;
    5668             : };
    5669             : 
    5670      141694 : void AppendSignature(char* buffer, size_t max_name_len,
    5671             :                      wasm::FunctionSig* sig) {
    5672      141694 :   size_t name_len = strlen(buffer);
    5673             :   auto append_name_char = [&](char c) {
    5674      253554 :     if (name_len + 1 < max_name_len) buffer[name_len++] = c;
    5675             :   };
    5676      317552 :   for (wasm::ValueType t : sig->parameters()) {
    5677             :     append_name_char(wasm::ValueTypes::ShortNameOf(t));
    5678             :   }
    5679             :   append_name_char(':');
    5680      189556 :   for (wasm::ValueType t : sig->returns()) {
    5681             :     append_name_char(wasm::ValueTypes::ShortNameOf(t));
    5682             :   }
    5683      141694 :   buffer[name_len] = '\0';
    5684      141694 : }
    5685             : 
    5686             : }  // namespace
    5687             : 
    5688      140321 : MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
    5689             :                                          wasm::FunctionSig* sig,
    5690             :                                          bool is_import) {
    5691      420963 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5692             :                "CompileJSToWasmWrapper");
    5693             :   //----------------------------------------------------------------------------
    5694             :   // Create the Graph.
    5695             :   //----------------------------------------------------------------------------
    5696      280641 :   Zone zone(isolate->allocator(), ZONE_NAME);
    5697      140321 :   Graph graph(&zone);
    5698      140321 :   CommonOperatorBuilder common(&zone);
    5699             :   MachineOperatorBuilder machine(
    5700             :       &zone, MachineType::PointerRepresentation(),
    5701             :       InstructionSelector::SupportedMachineOperatorFlags(),
    5702      140321 :       InstructionSelector::AlignmentRequirements());
    5703      140321 :   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
    5704             : 
    5705      140321 :   Node* control = nullptr;
    5706      140321 :   Node* effect = nullptr;
    5707             : 
    5708             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
    5709             :                                   StubCallMode::kCallCodeObject,
    5710      140321 :                                   wasm::WasmFeaturesFromIsolate(isolate));
    5711             :   builder.set_control_ptr(&control);
    5712             :   builder.set_effect_ptr(&effect);
    5713      140321 :   builder.BuildJSToWasmWrapper(is_import);
    5714             : 
    5715             :   //----------------------------------------------------------------------------
    5716             :   // Run the compilation pipeline.
    5717             :   //----------------------------------------------------------------------------
    5718             :   static constexpr size_t kMaxNameLen = 128;
    5719      140314 :   char debug_name[kMaxNameLen] = "js_to_wasm:";
    5720      140314 :   AppendSignature(debug_name, kMaxNameLen, sig);
    5721             : 
    5722             :   // Schedule and compile to machine code.
    5723      140318 :   int params = static_cast<int>(sig->parameter_count());
    5724      280637 :   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
    5725      140318 :       &zone, false, params + 1, CallDescriptor::kNoFlags);
    5726             : 
    5727             :   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
    5728             :       isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
    5729      140319 :       WasmAssemblerOptions());
    5730             :   Handle<Code> code;
    5731      140317 :   if (!maybe_code.ToHandle(&code)) {
    5732           0 :     return maybe_code;
    5733             :   }
    5734             : #ifdef ENABLE_DISASSEMBLER
    5735             :   if (FLAG_print_opt_code) {
    5736             :     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
    5737             :     OFStream os(tracing_scope.file());
    5738             :     code->Disassemble(debug_name, os);
    5739             :   }
    5740             : #endif
    5741             : 
    5742      140317 :   if (must_record_function_compilation(isolate)) {
    5743             :     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
    5744           4 :                               debug_name);
    5745             :   }
    5746             : 
    5747      140320 :   return code;
    5748             : }
    5749             : 
    5750      119709 : WasmImportCallKind GetWasmImportCallKind(Handle<JSReceiver> target,
    5751             :                                          wasm::FunctionSig* expected_sig,
    5752             :                                          bool has_bigint_feature) {
    5753      119709 :   if (WasmExportedFunction::IsWasmExportedFunction(*target)) {
    5754      107320 :     auto imported_function = WasmExportedFunction::cast(*target);
    5755      107320 :     auto func_index = imported_function->function_index();
    5756      107320 :     auto module = imported_function->instance()->module();
    5757      214640 :     wasm::FunctionSig* imported_sig = module->functions[func_index].sig;
    5758      107320 :     if (*imported_sig != *expected_sig) {
    5759             :       return WasmImportCallKind::kLinkError;
    5760             :     }
    5761      107160 :     if (static_cast<uint32_t>(func_index) < module->num_imported_functions) {
    5762             :       // TODO(wasm): this redirects all imported-reexported functions
    5763             :       // through the call builtin. Fall through to JS function cases below?
    5764             :       return WasmImportCallKind::kUseCallBuiltin;
    5765             :     }
    5766      107096 :     return WasmImportCallKind::kWasmToWasm;
    5767             :   }
    5768             :   // Assuming we are calling to JS, check whether this would be a runtime error.
    5769       12389 :   if (!wasm::IsJSCompatibleSignature(expected_sig, has_bigint_feature)) {
    5770             :     return WasmImportCallKind::kRuntimeTypeError;
    5771             :   }
    5772             :   // For JavaScript calls, determine whether the target has an arity match
    5773             :   // and whether it has a sloppy receiver.
    5774       12313 :   if (target->IsJSFunction()) {
    5775             :     Handle<JSFunction> function = Handle<JSFunction>::cast(target);
    5776             :     SharedFunctionInfo shared = function->shared();
    5777             : 
    5778             : // Check for math intrinsics.
    5779             : #define COMPARE_SIG_FOR_BUILTIN(name)                                         \
    5780             :   {                                                                           \
    5781             :     wasm::FunctionSig* sig = wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
    5782             :     if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name);     \
    5783             :     DCHECK_NOT_NULL(sig);                                                     \
    5784             :     if (*expected_sig == *sig) return WasmImportCallKind::k##name;            \
    5785             :   }
    5786             : #define COMPARE_SIG_FOR_BUILTIN_F64(name) \
    5787             :   case Builtins::kMath##name:             \
    5788             :     COMPARE_SIG_FOR_BUILTIN(F64##name);   \
    5789             :     break;
    5790             : #define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
    5791             :   case Builtins::kMath##name:                 \
    5792             :     COMPARE_SIG_FOR_BUILTIN(F64##name);       \
    5793             :     COMPARE_SIG_FOR_BUILTIN(F32##name);       \
    5794             :     break;
    5795             : 
    5796       21138 :     if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
    5797         520 :       switch (shared->builtin_id()) {
    5798          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Acos);
    5799          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Asin);
    5800          17 :         COMPARE_SIG_FOR_BUILTIN_F64(Atan);
    5801          27 :         COMPARE_SIG_FOR_BUILTIN_F64(Cos);
    5802          27 :         COMPARE_SIG_FOR_BUILTIN_F64(Sin);
    5803          17 :         COMPARE_SIG_FOR_BUILTIN_F64(Tan);
    5804          17 :         COMPARE_SIG_FOR_BUILTIN_F64(Exp);
    5805          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Log);
    5806          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
    5807             :         //===========================================================
    5808             :         // TODO(8505): Math.pow for wasm does not match JS.
    5809             :         //        COMPARE_SIG_FOR_BUILTIN_F64(Pow);
    5810             :         //===========================================================
    5811         106 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
    5812          24 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
    5813          34 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
    5814          34 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
    5815          43 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
    5816          52 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
    5817             :         case Builtins::kMathFround:
    5818           8 :           COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
    5819             :           break;
    5820             :         default:
    5821             :           break;
    5822             :       }
    5823             :     }
    5824             : 
    5825             : #undef COMPARE_SIG_FOR_BUILTIN
    5826             : #undef COMPARE_SIG_FOR_BUILTIN_F64
    5827             : #undef COMPARE_SIG_FOR_BUILTIN_F32_F64
    5828             : 
    5829       10345 :     if (IsClassConstructor(shared->kind())) {
    5830             :       // Class constructor will throw anyway.
    5831             :       return WasmImportCallKind::kUseCallBuiltin;
    5832             :     }
    5833       20330 :     bool sloppy = is_sloppy(shared->language_mode()) && !shared->native();
    5834       10329 :     if (shared->internal_formal_parameter_count() ==
    5835             :         expected_sig->parameter_count()) {
    5836             :       return sloppy ? WasmImportCallKind::kJSFunctionArityMatchSloppy
    5837        7735 :                     : WasmImportCallKind::kJSFunctionArityMatch;
    5838             :     }
    5839             :     return sloppy ? WasmImportCallKind::kJSFunctionArityMismatchSloppy
    5840        2594 :                   : WasmImportCallKind::kJSFunctionArityMismatch;
    5841             :   }
    5842             :   // Unknown case. Use the call builtin.
    5843             :   return WasmImportCallKind::kUseCallBuiltin;
    5844             : }
    5845             : 
    5846         224 : wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
    5847             :                                         const char** name_ptr) {
    5848             : #define CASE(name)                          \
    5849             :   case WasmImportCallKind::k##name:         \
    5850             :     *name_ptr = "WasmMathIntrinsic:" #name; \
    5851             :     return wasm::kExpr##name
    5852         224 :   switch (kind) {
    5853           8 :     CASE(F64Acos);
    5854           8 :     CASE(F64Asin);
    5855           8 :     CASE(F64Atan);
    5856           8 :     CASE(F64Cos);
    5857           8 :     CASE(F64Sin);
    5858           8 :     CASE(F64Tan);
    5859           8 :     CASE(F64Exp);
    5860           8 :     CASE(F64Log);
    5861           8 :     CASE(F64Atan2);
    5862           0 :     CASE(F64Pow);
    5863          16 :     CASE(F64Ceil);
    5864          16 :     CASE(F64Floor);
    5865          16 :     CASE(F64Sqrt);
    5866          16 :     CASE(F64Min);
    5867          16 :     CASE(F64Max);
    5868          16 :     CASE(F64Abs);
    5869           8 :     CASE(F32Min);
    5870           8 :     CASE(F32Max);
    5871           8 :     CASE(F32Abs);
    5872           8 :     CASE(F32Ceil);
    5873           8 :     CASE(F32Floor);
    5874           8 :     CASE(F32Sqrt);
    5875           8 :     CASE(F32ConvertF64);
    5876             :     default:
    5877           0 :       UNREACHABLE();
    5878             :       return wasm::kExprUnreachable;
    5879             :   }
    5880             : #undef CASE
    5881             : }
    5882             : 
    5883         224 : wasm::WasmCode* CompileWasmMathIntrinsic(wasm::WasmEngine* wasm_engine,
    5884             :                                          wasm::NativeModule* native_module,
    5885             :                                          WasmImportCallKind kind,
    5886             :                                          wasm::FunctionSig* sig) {
    5887             :   DCHECK_EQ(1, sig->return_count());
    5888             : 
    5889         672 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5890             :                "CompileWasmMathIntrinsic");
    5891             : 
    5892         448 :   Zone zone(wasm_engine->allocator(), ZONE_NAME);
    5893             : 
    5894             :   // Compile a WASM function with a single bytecode and let TurboFan
    5895             :   // generate either inlined machine code or a call to a helper.
    5896             :   SourcePositionTable* source_positions = nullptr;
    5897             :   MachineGraph* mcgraph = new (&zone) MachineGraph(
    5898         448 :       new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
    5899             :       new (&zone) MachineOperatorBuilder(
    5900             :           &zone, MachineType::PointerRepresentation(),
    5901         448 :           InstructionSelector::SupportedMachineOperatorFlags(),
    5902         672 :           InstructionSelector::AlignmentRequirements()));
    5903             : 
    5904             :   wasm::CompilationEnv env(
    5905             :       native_module->module(), wasm::UseTrapHandler::kNoTrapHandler,
    5906             :       wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
    5907             :       wasm::kAllWasmFeatures, wasm::LowerSimd::kNoLowerSimd);
    5908             : 
    5909             :   WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
    5910             :                            source_positions);
    5911             : 
    5912             :   // Set up the graph start.
    5913         224 :   Node* start = builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
    5914         224 :   Node* effect = start;
    5915         224 :   Node* control = start;
    5916             :   builder.set_effect_ptr(&effect);
    5917             :   builder.set_control_ptr(&control);
    5918         224 :   builder.set_instance_node(builder.Param(wasm::kWasmInstanceParameterIndex));
    5919             : 
    5920             :   // Generate either a unop or a binop.
    5921             :   Node* node = nullptr;
    5922         224 :   const char* debug_name = "WasmMathIntrinsic";
    5923         224 :   auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
    5924         224 :   switch (sig->parameter_count()) {
    5925             :     case 1:
    5926         168 :       node = builder.Unop(opcode, builder.Param(1));
    5927         168 :       break;
    5928             :     case 2:
    5929          56 :       node = builder.Binop(opcode, builder.Param(1), builder.Param(2));
    5930          56 :       break;
    5931             :     default:
    5932           0 :       UNREACHABLE();
    5933             :       break;
    5934             :   }
    5935             : 
    5936             :   builder.Return(node);
    5937             : 
    5938             :   // Run the compiler pipeline to generate machine code.
    5939         224 :   auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
    5940         224 :   if (mcgraph->machine()->Is32()) {
    5941             :     call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
    5942             :   }
    5943             : 
    5944             :   wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
    5945             :       wasm_engine, call_descriptor, mcgraph, Code::WASM_FUNCTION,
    5946         224 :       wasm::WasmCode::kFunction, debug_name, WasmStubAssemblerOptions(),
    5947         448 :       source_positions);
    5948             :   std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode(
    5949             :       wasm::WasmCode::kAnonymousFuncIndex, result.code_desc,
    5950             :       result.frame_slot_count, result.tagged_parameter_slots,
    5951             :       std::move(result.protected_instructions),
    5952             :       std::move(result.source_positions), wasm::WasmCode::kFunction,
    5953         896 :       wasm::WasmCode::kOther);
    5954             :   // TODO(titzer): add counters for math intrinsic code size / allocation
    5955         448 :   return native_module->PublishCode(std::move(wasm_code));
    5956             : }
    5957             : 
    5958        6714 : wasm::WasmCode* CompileWasmImportCallWrapper(wasm::WasmEngine* wasm_engine,
    5959             :                                              wasm::NativeModule* native_module,
    5960             :                                              WasmImportCallKind kind,
    5961             :                                              wasm::FunctionSig* sig,
    5962             :                                              bool source_positions) {
    5963             :   DCHECK_NE(WasmImportCallKind::kLinkError, kind);
    5964             :   DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind);
    5965             : 
    5966             :   // Check for math intrinsics first.
    5967        6714 :   if (FLAG_wasm_math_intrinsics &&
    5968        6714 :       kind >= WasmImportCallKind::kFirstMathIntrinsic &&
    5969             :       kind <= WasmImportCallKind::kLastMathIntrinsic) {
    5970         224 :     return CompileWasmMathIntrinsic(wasm_engine, native_module, kind, sig);
    5971             :   }
    5972             : 
    5973       19470 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5974             :                "CompileWasmImportCallWrapper");
    5975             :   //----------------------------------------------------------------------------
    5976             :   // Create the Graph
    5977             :   //----------------------------------------------------------------------------
    5978       12980 :   Zone zone(wasm_engine->allocator(), ZONE_NAME);
    5979        6490 :   Graph graph(&zone);
    5980        6490 :   CommonOperatorBuilder common(&zone);
    5981             :   MachineOperatorBuilder machine(
    5982             :       &zone, MachineType::PointerRepresentation(),
    5983             :       InstructionSelector::SupportedMachineOperatorFlags(),
    5984        6490 :       InstructionSelector::AlignmentRequirements());
    5985        6490 :   JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
    5986             : 
    5987        6490 :   Node* control = nullptr;
    5988        6490 :   Node* effect = nullptr;
    5989             : 
    5990             :   SourcePositionTable* source_position_table =
    5991        6980 :       source_positions ? new (&zone) SourcePositionTable(&graph) : nullptr;
    5992             : 
    5993             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, source_position_table,
    5994             :                                   StubCallMode::kCallWasmRuntimeStub,
    5995        6490 :                                   native_module->enabled_features());
    5996             :   builder.set_control_ptr(&control);
    5997             :   builder.set_effect_ptr(&effect);
    5998        6490 :   builder.BuildWasmImportCallWrapper(kind);
    5999             : 
    6000             :   const char* func_name = "wasm-to-js";
    6001             : 
    6002             :   // Schedule and compile to machine code.
    6003             :   CallDescriptor* incoming =
    6004             :       GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline,
    6005        6490 :                             WasmGraphBuilder::kExtraCallableParam);
    6006        6490 :   if (machine.Is32()) {
    6007             :     incoming = GetI32WasmCallDescriptor(&zone, incoming);
    6008             :   }
    6009             :   wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
    6010             :       wasm_engine, incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
    6011        6490 :       wasm::WasmCode::kWasmToJsWrapper, func_name, WasmStubAssemblerOptions(),
    6012       12980 :       source_position_table);
    6013             :   std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode(
    6014             :       wasm::WasmCode::kAnonymousFuncIndex, result.code_desc,
    6015             :       result.frame_slot_count, result.tagged_parameter_slots,
    6016             :       std::move(result.protected_instructions),
    6017             :       std::move(result.source_positions), wasm::WasmCode::kWasmToJsWrapper,
    6018       25960 :       wasm::WasmCode::kOther);
    6019        6490 :   return native_module->PublishCode(std::move(wasm_code));
    6020             : }
    6021             : 
    6022      368044 : wasm::WasmCompilationResult CompileWasmInterpreterEntry(
    6023             :     wasm::WasmEngine* wasm_engine, const wasm::WasmFeatures& enabled_features,
    6024             :     uint32_t func_index, wasm::FunctionSig* sig) {
    6025             :   //----------------------------------------------------------------------------
    6026             :   // Create the Graph
    6027             :   //----------------------------------------------------------------------------
    6028      736082 :   Zone zone(wasm_engine->allocator(), ZONE_NAME);
    6029      368044 :   Graph graph(&zone);
    6030      368043 :   CommonOperatorBuilder common(&zone);
    6031             :   MachineOperatorBuilder machine(
    6032             :       &zone, MachineType::PointerRepresentation(),
    6033             :       InstructionSelector::SupportedMachineOperatorFlags(),
    6034      368044 :       InstructionSelector::AlignmentRequirements());
    6035      368043 :   JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
    6036             : 
    6037      368043 :   Node* control = nullptr;
    6038      368043 :   Node* effect = nullptr;
    6039             : 
    6040             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
    6041             :                                   StubCallMode::kCallWasmRuntimeStub,
    6042      368043 :                                   enabled_features);
    6043             :   builder.set_control_ptr(&control);
    6044             :   builder.set_effect_ptr(&effect);
    6045      368043 :   builder.BuildWasmInterpreterEntry(func_index);
    6046             : 
    6047             :   // Schedule and compile to machine code.
    6048      368038 :   CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
    6049      368040 :   if (machine.Is32()) {
    6050             :     incoming = GetI32WasmCallDescriptor(&zone, incoming);
    6051             :   }
    6052             : 
    6053             :   EmbeddedVector<char, 32> func_name;
    6054      368042 :   func_name.Truncate(
    6055      368040 :       SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
    6056             : 
    6057             :   wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub(
    6058             :       wasm_engine, incoming, &jsgraph, Code::WASM_INTERPRETER_ENTRY,
    6059             :       wasm::WasmCode::kInterpreterEntry, func_name.start(),
    6060      368042 :       WasmStubAssemblerOptions());
    6061      368038 :   result.result_tier = wasm::ExecutionTier::kInterpreter;
    6062             : 
    6063      368043 :   return result;
    6064             : }
    6065             : 
    6066        1376 : MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
    6067        2752 :   Zone zone(isolate->allocator(), ZONE_NAME);
    6068        1376 :   Graph graph(&zone);
    6069        1376 :   CommonOperatorBuilder common(&zone);
    6070             :   MachineOperatorBuilder machine(
    6071             :       &zone, MachineType::PointerRepresentation(),
    6072             :       InstructionSelector::SupportedMachineOperatorFlags(),
    6073        1376 :       InstructionSelector::AlignmentRequirements());
    6074        1376 :   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
    6075             : 
    6076        1376 :   Node* control = nullptr;
    6077        1376 :   Node* effect = nullptr;
    6078             : 
    6079             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
    6080             :                                   StubCallMode::kCallCodeObject,
    6081        1376 :                                   wasm::WasmFeaturesFromIsolate(isolate));
    6082             :   builder.set_control_ptr(&control);
    6083             :   builder.set_effect_ptr(&effect);
    6084        1376 :   builder.BuildCWasmEntry();
    6085             : 
    6086             :   // Schedule and compile to machine code.
    6087        1376 :   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
    6088             :       &zone, false, CWasmEntryParameters::kNumParameters + 1,
    6089        1376 :       CallDescriptor::kNoFlags);
    6090             : 
    6091             :   // Build a name in the form "c-wasm-entry:<params>:<returns>".
    6092             :   static constexpr size_t kMaxNameLen = 128;
    6093        1376 :   char debug_name[kMaxNameLen] = "c-wasm-entry:";
    6094        1376 :   AppendSignature(debug_name, kMaxNameLen, sig);
    6095             : 
    6096             :   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
    6097             :       isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
    6098        1376 :       AssemblerOptions::Default(isolate));
    6099             :   Handle<Code> code;
    6100        1376 :   if (!maybe_code.ToHandle(&code)) {
    6101           0 :     return maybe_code;
    6102             :   }
    6103             : #ifdef ENABLE_DISASSEMBLER
    6104             :   if (FLAG_print_opt_code) {
    6105             :     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
    6106             :     OFStream os(tracing_scope.file());
    6107             :     code->Disassemble(debug_name, os);
    6108             :   }
    6109             : #endif
    6110             : 
    6111        1376 :   return code;
    6112             : }
    6113             : 
    6114      506558 : TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
    6115             :     wasm::WasmCompilationUnit* wasm_unit)
    6116      506558 :     : wasm_unit_(wasm_unit) {}
    6117             : 
    6118             : // Clears unique_ptrs, but (part of) the type is forward declared in the header.
    6119             : TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
    6120             : 
    6121      503948 : bool TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
    6122             :     wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
    6123             :     wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph,
    6124             :     NodeOriginTable* node_origins, SourcePositionTable* source_positions) {
    6125             :   base::ElapsedTimer decode_timer;
    6126             :   if (FLAG_trace_wasm_decode_time) {
    6127             :     decode_timer.Start();
    6128             :   }
    6129             : 
    6130             :   // Create a TF graph during decoding.
    6131      503948 :   WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
    6132             :                            source_positions);
    6133     1512645 :   wasm::VoidResult graph_construction_result = wasm::BuildTFGraph(
    6134     1007896 :       wasm_unit_->wasm_engine_->allocator(), env->enabled_features, env->module,
    6135             :       &builder, detected, func_body, node_origins);
    6136      504749 :   if (graph_construction_result.failed()) {
    6137             :     if (FLAG_trace_wasm_compiler) {
    6138             :       StdoutStream{} << "Compilation failed: "
    6139             :                      << graph_construction_result.error().message()
    6140             :                      << std::endl;
    6141             :     }
    6142             :     return false;
    6143             :   }
    6144             : 
    6145      495546 :   builder.LowerInt64();
    6146             : 
    6147      998930 :   if (builder.has_simd() &&
    6148        8204 :       (!CpuFeatures::SupportsWasmSimd128() || env->lower_simd)) {
    6149             :     SimdScalarLowering(mcgraph,
    6150        4068 :                        CreateMachineSignature(mcgraph->zone(), func_body.sig))
    6151        8136 :         .LowerGraph();
    6152             :   }
    6153             : 
    6154      990820 :   if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
    6155      495457 :       wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
    6156           0 :     PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(), func_body,
    6157           0 :                      env->module, wasm::kPrintLocals);
    6158             :   }
    6159             :   if (FLAG_trace_wasm_decode_time) {
    6160             :     *decode_ms = decode_timer.Elapsed().InMillisecondsF();
    6161             :   }
    6162             :   return true;
    6163             : }
    6164             : 
    6165             : namespace {
    6166      504485 : Vector<const char> GetDebugName(Zone* zone, int index) {
    6167             :   // TODO(herhut): Use name from module if available.
    6168             :   constexpr int kBufferLength = 24;
    6169             : 
    6170             :   EmbeddedVector<char, kBufferLength> name_vector;
    6171      504485 :   int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
    6172             :   DCHECK(name_len > 0 && name_len < name_vector.length());
    6173             : 
    6174      505038 :   char* index_name = zone->NewArray<char>(name_len);
    6175             :   memcpy(index_name, name_vector.start(), name_len);
    6176      505126 :   return Vector<const char>(index_name, name_len);
    6177             : }
    6178             : }  // namespace
    6179             : 
    6180      504851 : wasm::WasmCompilationResult TurbofanWasmCompilationUnit::ExecuteCompilation(
    6181             :     wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
    6182             :     Counters* counters, wasm::WasmFeatures* detected) {
    6183     1514798 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    6184             :                "ExecuteTurbofanCompilation");
    6185      504861 :   double decode_ms = 0;
    6186             :   size_t node_count = 0;
    6187             : 
    6188     1512976 :   Zone zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
    6189             :   MachineGraph* mcgraph = new (&zone) MachineGraph(
    6190     1009832 :       new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
    6191             :       new (&zone) MachineOperatorBuilder(
    6192             :           &zone, MachineType::PointerRepresentation(),
    6193     1009397 :           InstructionSelector::SupportedMachineOperatorFlags(),
    6194     1514437 :           InstructionSelector::AlignmentRequirements()));
    6195             : 
    6196      504697 :   OptimizedCompilationInfo info(GetDebugName(&zone, wasm_unit_->func_index_),
    6197     1009197 :                                 &zone, Code::WASM_FUNCTION);
    6198      504130 :   if (env->runtime_exception_support) {
    6199             :     info.SetWasmRuntimeExceptionSupport();
    6200             :   }
    6201             : 
    6202      504130 :   if (info.trace_turbo_json_enabled()) {
    6203           0 :     TurboCfgFile tcf;
    6204           0 :     tcf << AsC1VCompilation(&info);
    6205             :   }
    6206             : 
    6207             :   NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
    6208             :                                       ? new (&zone)
    6209           0 :                                             NodeOriginTable(mcgraph->graph())
    6210      504130 :                                       : nullptr;
    6211             :   SourcePositionTable* source_positions =
    6212      504138 :       new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
    6213      503929 :   if (!BuildGraphForWasmFunction(env, func_body, detected, &decode_ms, mcgraph,
    6214             :                                  node_origins, source_positions)) {
    6215        7686 :     return wasm::WasmCompilationResult{};
    6216             :   }
    6217             : 
    6218      495665 :   if (node_origins) {
    6219           0 :     node_origins->AddDecorator();
    6220             :   }
    6221             : 
    6222             :   base::ElapsedTimer pipeline_timer;
    6223             :   if (FLAG_trace_wasm_decode_time) {
    6224             :     node_count = mcgraph->graph()->NodeCount();
    6225             :     pipeline_timer.Start();
    6226             :   }
    6227             : 
    6228             :   // Run the compiler pipeline to generate machine code.
    6229      495665 :   auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
    6230      496308 :   if (mcgraph->machine()->Is32()) {
    6231             :     call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
    6232             :   }
    6233             : 
    6234             :   Pipeline::GenerateCodeForWasmFunction(
    6235      496308 :       &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
    6236      496308 :       source_positions, node_origins, func_body, env->module,
    6237      992616 :       wasm_unit_->func_index_);
    6238             : 
    6239             :   if (FLAG_trace_wasm_decode_time) {
    6240             :     double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
    6241             :     PrintF(
    6242             :         "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
    6243             :         "%0.3f ms pipeline\n",
    6244             :         static_cast<unsigned>(func_body.end - func_body.start), decode_ms,
    6245             :         node_count, pipeline_ms);
    6246             :   }
    6247             :   // TODO(bradnelson): Improve histogram handling of size_t.
    6248      496827 :   counters->wasm_compile_function_peak_memory_bytes()->AddSample(
    6249      496827 :       static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
    6250      992490 :   return std::move(*info.ReleaseWasmCompilationResult());
    6251             : }
    6252             : 
    6253         448 : wasm::WasmCompilationResult InterpreterCompilationUnit::ExecuteCompilation(
    6254             :     wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
    6255             :     Counters* counters, wasm::WasmFeatures* detected) {
    6256        1342 :   Zone zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
    6257         449 :   const wasm::WasmModule* module = env ? env->module : nullptr;
    6258             :   wasm::WasmFullDecoder<wasm::Decoder::kValidate, wasm::EmptyInterface> decoder(
    6259             :       &zone, module, env->enabled_features, detected, func_body);
    6260         449 :   decoder.Decode();
    6261         452 :   if (decoder.failed()) return wasm::WasmCompilationResult{};
    6262             : 
    6263             :   wasm::WasmCompilationResult result = CompileWasmInterpreterEntry(
    6264         904 :       wasm_unit_->wasm_engine_, env->enabled_features, wasm_unit_->func_index_,
    6265        1806 :       func_body.sig);
    6266             :   DCHECK(result.succeeded());
    6267             : 
    6268             :   return result;
    6269             : }
    6270             : 
    6271             : namespace {
    6272             : // Helper for allocating either an GP or FP reg, or the next stack slot.
    6273             : class LinkageLocationAllocator {
    6274             :  public:
    6275             :   template <size_t kNumGpRegs, size_t kNumFpRegs>
    6276             :   constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
    6277             :                                      const DoubleRegister (&fp)[kNumFpRegs])
    6278             :       : allocator_(wasm::LinkageAllocator(gp, fp)) {}
    6279             : 
    6280     5583905 :   LinkageLocation Next(MachineRepresentation rep) {
    6281     5583905 :     MachineType type = MachineType::TypeForRepresentation(rep);
    6282     5584070 :     if (IsFloatingPoint(rep)) {
    6283      517389 :       if (allocator_.CanAllocateFP(rep)) {
    6284             :         int reg_code = allocator_.NextFpReg(rep);
    6285             :         return LinkageLocation::ForRegister(reg_code, type);
    6286             :       }
    6287     5066681 :     } else if (allocator_.CanAllocateGP()) {
    6288             :       int reg_code = allocator_.NextGpReg();
    6289             :       return LinkageLocation::ForRegister(reg_code, type);
    6290             :     }
    6291             :     // Cannot use register; use stack slot.
    6292      158070 :     int index = -1 - allocator_.NextStackSlot(rep);
    6293             :     return LinkageLocation::ForCallerFrameSlot(index, type);
    6294             :   }
    6295             : 
    6296             :   void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
    6297             :   int NumStackSlots() const { return allocator_.NumStackSlots(); }
    6298             : 
    6299             :  private:
    6300             :   wasm::LinkageAllocator allocator_;
    6301             : };
    6302             : }  // namespace
    6303             : 
    6304             : // General code uses the above configuration data.
    6305     2492304 : CallDescriptor* GetWasmCallDescriptor(
    6306             :     Zone* zone, wasm::FunctionSig* fsig,
    6307             :     WasmGraphBuilder::UseRetpoline use_retpoline,
    6308             :     WasmGraphBuilder::ExtraCallableParam extra_callable_param) {
    6309             :   // The extra here is to accomodate the instance object as first parameter
    6310             :   // and, in the case of an import wrapper, the additional callable.
    6311     2492304 :   int extra_params = extra_callable_param ? 2 : 1;
    6312             :   LocationSignature::Builder locations(zone, fsig->return_count(),
    6313     2492304 :                                        fsig->parameter_count() + extra_params);
    6314             : 
    6315             :   // Add register and/or stack parameter(s).
    6316             :   LinkageLocationAllocator params(wasm::kGpParamRegisters,
    6317     2493747 :                                   wasm::kFpParamRegisters);
    6318             : 
    6319             :   // The instance object.
    6320     2493747 :   locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
    6321             :   const size_t param_offset = 1;  // Actual params start here.
    6322             : 
    6323             :   // Parameters are separated into two groups (first all untagged, then all
    6324             :   // tagged parameters). This allows for easy iteration of tagged parameters
    6325             :   // during frame iteration.
    6326             :   const size_t parameter_count = fsig->parameter_count();
    6327     4254104 :   for (size_t i = 0; i < parameter_count; i++) {
    6328             :     MachineRepresentation param =
    6329      879226 :         wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
    6330             :     // Skip tagged parameters (e.g. any-ref).
    6331      887958 :     if (IsAnyTagged(param)) continue;
    6332      873958 :     auto l = params.Next(param);
    6333      871821 :     locations.AddParamAt(i + param_offset, l);
    6334             :   }
    6335     4257407 :   for (size_t i = 0; i < parameter_count; i++) {
    6336             :     MachineRepresentation param =
    6337      880844 :         wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
    6338             :     // Skip untagged parameters.
    6339     1756185 :     if (!IsAnyTagged(param)) continue;
    6340        6999 :     auto l = params.Next(param);
    6341        6082 :     locations.AddParamAt(i + param_offset, l);
    6342             :   }
    6343             : 
    6344             :   // Import call wrappers have an additional (implicit) parameter, the callable.
    6345             :   // For consistency with JS, we use the JSFunction register.
    6346     2495888 :   if (extra_callable_param) {
    6347             :     locations.AddParam(LinkageLocation::ForRegister(
    6348             :         kJSFunctionRegister.code(), MachineType::TaggedPointer()));
    6349             :   }
    6350             : 
    6351             :   // Add return location(s).
    6352             :   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
    6353     2495888 :                                 wasm::kFpReturnRegisters);
    6354             : 
    6355             :   int parameter_slots = params.NumStackSlots();
    6356             :   if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
    6357             : 
    6358             :   rets.SetStackOffset(parameter_slots);
    6359             : 
    6360     2495888 :   const int return_count = static_cast<int>(locations.return_count_);
    6361     6914916 :   for (int i = 0; i < return_count; i++) {
    6362             :     MachineRepresentation ret =
    6363     4421458 :         wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
    6364     2211463 :     auto l = rets.Next(ret);
    6365             :     locations.AddReturn(l);
    6366             :   }
    6367             : 
    6368             :   const RegList kCalleeSaveRegisters = 0;
    6369             :   const RegList kCalleeSaveFPRegisters = 0;
    6370             : 
    6371             :   // The target for wasm calls is always a code object.
    6372             :   MachineType target_type = MachineType::Pointer();
    6373             :   LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
    6374             : 
    6375             :   CallDescriptor::Kind kind = extra_callable_param
    6376             :                                   ? CallDescriptor::kCallWasmImportWrapper
    6377     2494673 :                                   : CallDescriptor::kCallWasmFunction;
    6378             : 
    6379             :   CallDescriptor::Flags flags =
    6380     2494673 :       use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
    6381             :   return new (zone) CallDescriptor(             // --
    6382             :       kind,                                     // kind
    6383             :       target_type,                              // target MachineType
    6384             :       target_loc,                               // target location
    6385             :       locations.Build(),                        // location_sig
    6386             :       parameter_slots,                          // stack_parameter_count
    6387             :       compiler::Operator::kNoProperties,        // properties
    6388             :       kCalleeSaveRegisters,                     // callee-saved registers
    6389             :       kCalleeSaveFPRegisters,                   // callee-saved fp regs
    6390             :       flags,                                    // flags
    6391             :       "wasm-call",                              // debug name
    6392             :       0,                                        // allocatable registers
    6393     7482728 :       rets.NumStackSlots() - parameter_slots);  // stack_return_count
    6394             : }
    6395             : 
    6396             : namespace {
    6397           0 : CallDescriptor* ReplaceTypeInCallDescriptorWith(
    6398             :     Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
    6399             :     MachineType input_type, MachineRepresentation output_type) {
    6400             :   size_t parameter_count = call_descriptor->ParameterCount();
    6401             :   size_t return_count = call_descriptor->ReturnCount();
    6402           0 :   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
    6403           0 :     if (call_descriptor->GetParameterType(i) == input_type) {
    6404           0 :       parameter_count += num_replacements - 1;
    6405             :     }
    6406             :   }
    6407           0 :   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
    6408           0 :     if (call_descriptor->GetReturnType(i) == input_type) {
    6409           0 :       return_count += num_replacements - 1;
    6410             :     }
    6411             :   }
    6412           0 :   if (parameter_count == call_descriptor->ParameterCount() &&
    6413             :       return_count == call_descriptor->ReturnCount()) {
    6414             :     return call_descriptor;
    6415             :   }
    6416             : 
    6417             :   LocationSignature::Builder locations(zone, return_count, parameter_count);
    6418             : 
    6419             :   LinkageLocationAllocator params(wasm::kGpParamRegisters,
    6420           0 :                                   wasm::kFpParamRegisters);
    6421           0 :   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
    6422           0 :     if (call_descriptor->GetParameterType(i) == input_type) {
    6423           0 :       for (size_t j = 0; j < num_replacements; j++) {
    6424           0 :         locations.AddParam(params.Next(output_type));
    6425             :       }
    6426             :     } else {
    6427           0 :       locations.AddParam(
    6428             :           params.Next(call_descriptor->GetParameterType(i).representation()));
    6429             :     }
    6430             :   }
    6431             : 
    6432             :   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
    6433           0 :                                 wasm::kFpReturnRegisters);
    6434             :   rets.SetStackOffset(params.NumStackSlots());
    6435           0 :   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
    6436           0 :     if (call_descriptor->GetReturnType(i) == input_type) {
    6437           0 :       for (size_t j = 0; j < num_replacements; j++) {
    6438           0 :         locations.AddReturn(rets.Next(output_type));
    6439             :       }
    6440             :     } else {
    6441           0 :       locations.AddReturn(
    6442             :           rets.Next(call_descriptor->GetReturnType(i).representation()));
    6443             :     }
    6444             :   }
    6445             : 
    6446             :   return new (zone) CallDescriptor(                    // --
    6447             :       call_descriptor->kind(),                         // kind
    6448             :       call_descriptor->GetInputType(0),                // target MachineType
    6449             :       call_descriptor->GetInputLocation(0),            // target location
    6450             :       locations.Build(),                               // location_sig
    6451             :       params.NumStackSlots(),                          // stack_parameter_count
    6452             :       call_descriptor->properties(),                   // properties
    6453             :       call_descriptor->CalleeSavedRegisters(),         // callee-saved registers
    6454             :       call_descriptor->CalleeSavedFPRegisters(),       // callee-saved fp regs
    6455             :       call_descriptor->flags(),                        // flags
    6456             :       call_descriptor->debug_name(),                   // debug name
    6457             :       call_descriptor->AllocatableRegisters(),         // allocatable registers
    6458           0 :       rets.NumStackSlots() - params.NumStackSlots());  // stack_return_count
    6459             : }
    6460             : }  // namespace
    6461             : 
    6462           0 : CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
    6463             :                                          CallDescriptor* call_descriptor) {
    6464             :   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
    6465             :                                          MachineType::Int64(),
    6466           0 :                                          MachineRepresentation::kWord32);
    6467             : }
    6468             : 
    6469           0 : CallDescriptor* GetI32WasmCallDescriptorForSimd(
    6470             :     Zone* zone, CallDescriptor* call_descriptor) {
    6471             :   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
    6472             :                                          MachineType::Simd128(),
    6473           0 :                                          MachineRepresentation::kWord32);
    6474             : }
    6475             : 
    6476      496341 : AssemblerOptions WasmAssemblerOptions() {
    6477      636660 :   AssemblerOptions options;
    6478             :   // Relocation info required to serialize {WasmCode} for proper functions.
    6479             :   options.record_reloc_info_for_serialization = true;
    6480             :   options.enable_root_array_delta_access = false;
    6481      496341 :   return options;
    6482             : }
    6483             : 
    6484           0 : AssemblerOptions WasmStubAssemblerOptions() {
    6485      374756 :   AssemblerOptions options;
    6486             :   // Relocation info not necessary because stubs are not serialized.
    6487      374756 :   options.record_reloc_info_for_serialization = false;
    6488             :   options.enable_root_array_delta_access = false;
    6489           0 :   return options;
    6490             : }
    6491             : 
    6492             : #undef WASM_64
    6493             : #undef FATAL_UNSUPPORTED_OPCODE
    6494             : #undef WASM_INSTANCE_OBJECT_SIZE
    6495             : #undef WASM_INSTANCE_OBJECT_OFFSET
    6496             : #undef LOAD_RAW
    6497             : #undef LOAD_INSTANCE_FIELD
    6498             : #undef LOAD_TAGGED_POINTER
    6499             : #undef LOAD_TAGGED_ANY
    6500             : #undef LOAD_FIXED_ARRAY_SLOT
    6501             : #undef LOAD_FIXED_ARRAY_SLOT_SMI
    6502             : #undef LOAD_FIXED_ARRAY_SLOT_PTR
    6503             : #undef LOAD_FIXED_ARRAY_SLOT_ANY
    6504             : #undef STORE_FIXED_ARRAY_SLOT_SMI
    6505             : #undef STORE_FIXED_ARRAY_SLOT_ANY
    6506             : 
    6507             : }  // namespace compiler
    6508             : }  // namespace internal
    6509      120216 : }  // namespace v8

Generated by: LCOV version 1.10