LCOV - code coverage report
Current view: top level - src/compiler - wasm-compiler.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 1970 2388 82.5 %
Date: 2019-02-19 Functions: 185 235 78.7 %

          Line data    Source code
       1             : // Copyright 2015 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #include "src/compiler/wasm-compiler.h"
       6             : 
       7             : #include <memory>
       8             : 
       9             : #include "src/assembler-inl.h"
      10             : #include "src/assembler.h"
      11             : #include "src/base/optional.h"
      12             : #include "src/base/platform/elapsed-timer.h"
      13             : #include "src/base/platform/platform.h"
      14             : #include "src/base/v8-fallthrough.h"
      15             : #include "src/builtins/builtins.h"
      16             : #include "src/code-factory.h"
      17             : #include "src/compiler/backend/code-generator.h"
      18             : #include "src/compiler/backend/instruction-selector.h"
      19             : #include "src/compiler/common-operator.h"
      20             : #include "src/compiler/compiler-source-position-table.h"
      21             : #include "src/compiler/diamond.h"
      22             : #include "src/compiler/graph-visualizer.h"
      23             : #include "src/compiler/graph.h"
      24             : #include "src/compiler/int64-lowering.h"
      25             : #include "src/compiler/js-graph.h"
      26             : #include "src/compiler/js-operator.h"
      27             : #include "src/compiler/linkage.h"
      28             : #include "src/compiler/machine-operator.h"
      29             : #include "src/compiler/node-matchers.h"
      30             : #include "src/compiler/node-origin-table.h"
      31             : #include "src/compiler/pipeline.h"
      32             : #include "src/compiler/simd-scalar-lowering.h"
      33             : #include "src/compiler/zone-stats.h"
      34             : #include "src/counters.h"
      35             : #include "src/heap/factory.h"
      36             : #include "src/interface-descriptors.h"
      37             : #include "src/isolate-inl.h"
      38             : #include "src/log.h"
      39             : #include "src/objects/heap-number.h"
      40             : #include "src/optimized-compilation-info.h"
      41             : #include "src/tracing/trace-event.h"
      42             : #include "src/trap-handler/trap-handler.h"
      43             : #include "src/wasm/function-body-decoder.h"
      44             : #include "src/wasm/function-compiler.h"
      45             : #include "src/wasm/graph-builder-interface.h"
      46             : #include "src/wasm/jump-table-assembler.h"
      47             : #include "src/wasm/memory-tracing.h"
      48             : #include "src/wasm/object-access.h"
      49             : #include "src/wasm/wasm-code-manager.h"
      50             : #include "src/wasm/wasm-limits.h"
      51             : #include "src/wasm/wasm-linkage.h"
      52             : #include "src/wasm/wasm-module.h"
      53             : #include "src/wasm/wasm-objects-inl.h"
      54             : #include "src/wasm/wasm-opcodes.h"
      55             : #include "src/wasm/wasm-text.h"
      56             : 
      57             : namespace v8 {
      58             : namespace internal {
      59             : namespace compiler {
      60             : 
      61             : namespace {
      62             : 
      63             : // TODO(titzer): pull WASM_64 up to a common header.
      64             : #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
      65             : #define WASM_64 1
      66             : #else
      67             : #define WASM_64 0
      68             : #endif
      69             : 
      70             : #define FATAL_UNSUPPORTED_OPCODE(opcode)        \
      71             :   FATAL("Unsupported opcode 0x%x:%s", (opcode), \
      72             :         wasm::WasmOpcodes::OpcodeName(opcode));
      73             : 
      74             : MachineType assert_size(int expected_size, MachineType type) {
      75             :   DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()));
      76             :   return type;
      77             : }
      78             : 
      79             : #define WASM_INSTANCE_OBJECT_SIZE(name)     \
      80             :   (WasmInstanceObject::k##name##OffsetEnd - \
      81             :    WasmInstanceObject::k##name##Offset + 1)  // NOLINT(whitespace/indent)
      82             : 
      83             : #define WASM_INSTANCE_OBJECT_OFFSET(name) \
      84             :   wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
      85             : 
      86             : #define LOAD_RAW(base_pointer, byte_offset, type)                             \
      87             :   SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), base_pointer,  \
      88             :                              mcgraph()->Int32Constant(byte_offset), Effect(), \
      89             :                              Control()))
      90             : 
      91             : #define LOAD_INSTANCE_FIELD(name, type)                             \
      92             :   LOAD_RAW(instance_node_.get(), WASM_INSTANCE_OBJECT_OFFSET(name), \
      93             :            assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type))
      94             : 
      95             : #define LOAD_TAGGED_POINTER(base_pointer, byte_offset) \
      96             :   LOAD_RAW(base_pointer, byte_offset, MachineType::TaggedPointer())
      97             : 
      98             : #define LOAD_TAGGED_ANY(base_pointer, byte_offset) \
      99             :   LOAD_RAW(base_pointer, byte_offset, MachineType::AnyTagged())
     100             : 
     101             : #define LOAD_FIXED_ARRAY_SLOT(array_node, index, type) \
     102             :   LOAD_RAW(array_node,                                 \
     103             :            wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), type)
     104             : 
     105             : #define LOAD_FIXED_ARRAY_SLOT_SMI(array_node, index) \
     106             :   LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedSigned())
     107             : 
     108             : #define LOAD_FIXED_ARRAY_SLOT_PTR(array_node, index) \
     109             :   LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedPointer())
     110             : 
     111             : #define LOAD_FIXED_ARRAY_SLOT_ANY(array_node, index) \
     112             :   LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::AnyTagged())
     113             : 
     114             : // This can be used to store tagged Smi values only.
     115             : #define STORE_FIXED_ARRAY_SLOT_SMI(array_node, index, value)           \
     116             :   SetEffect(graph()->NewNode(                                          \
     117             :       mcgraph()->machine()->Store(StoreRepresentation(                 \
     118             :           MachineRepresentation::kTaggedSigned, kNoWriteBarrier)),     \
     119             :       array_node,                                                      \
     120             :       mcgraph()->Int32Constant(                                        \
     121             :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
     122             :       value, Effect(), Control()))
     123             : 
     124             : // This can be used to store any tagged (Smi and HeapObject) value.
     125             : #define STORE_FIXED_ARRAY_SLOT_ANY(array_node, index, value)           \
     126             :   SetEffect(graph()->NewNode(                                          \
     127             :       mcgraph()->machine()->Store(StoreRepresentation(                 \
     128             :           MachineRepresentation::kTagged, kFullWriteBarrier)),         \
     129             :       array_node,                                                      \
     130             :       mcgraph()->Int32Constant(                                        \
     131             :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
     132             :       value, Effect(), Control()))
     133             : 
     134     1404362 : void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
     135      702181 :   Graph* g = mcgraph->graph();
     136      702181 :   if (g->end()) {
     137       54742 :     NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
     138             :   } else {
     139      647439 :     g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
     140             :   }
     141      702101 : }
     142             : 
     143             : bool ContainsSimd(wasm::FunctionSig* sig) {
     144     1674360 :   for (auto type : sig->all()) {
     145      837159 :     if (type == wasm::kWasmS128) return true;
     146             :   }
     147             :   return false;
     148             : }
     149             : 
     150             : bool ContainsInt64(wasm::FunctionSig* sig) {
     151       22524 :   for (auto type : sig->all()) {
     152       11280 :     if (type == wasm::kWasmI64) return true;
     153             :   }
     154             :   return false;
     155             : }
     156             : }  // namespace
     157             : 
     158      653067 : WasmGraphBuilder::WasmGraphBuilder(
     159             :     wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
     160             :     wasm::FunctionSig* sig,
     161             :     compiler::SourcePositionTable* source_position_table)
     162             :     : zone_(zone),
     163             :       mcgraph_(mcgraph),
     164             :       env_(env),
     165             :       cur_buffer_(def_buffer_),
     166             :       cur_bufsize_(kDefaultBufferSize),
     167             :       has_simd_(ContainsSimd(sig)),
     168             :       untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
     169             :       sig_(sig),
     170     1306134 :       source_position_table_(source_position_table) {
     171             :   DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
     172             :   DCHECK_NOT_NULL(mcgraph_);
     173      653067 : }
     174             : 
     175           0 : Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
     176             : 
     177     1306380 : Node* WasmGraphBuilder::Start(unsigned params) {
     178     1306380 :   Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
     179             :   graph()->SetStart(start);
     180      653116 :   return start;
     181             : }
     182             : 
     183      849643 : Node* WasmGraphBuilder::Param(unsigned index) {
     184             :   return graph()->NewNode(mcgraph()->common()->Parameter(index),
     185     1699323 :                           graph()->start());
     186             : }
     187             : 
     188        9091 : Node* WasmGraphBuilder::Loop(Node* entry) {
     189       18180 :   return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
     190             : }
     191             : 
     192       18181 : Node* WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
     193             :   Node* terminate =
     194        9091 :       graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
     195        9090 :   MergeControlToEnd(mcgraph(), terminate);
     196        9090 :   return terminate;
     197             : }
     198             : 
     199        1853 : Node* WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
     200             :   Node* terminate =
     201         927 :       graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
     202         926 :   MergeControlToEnd(mcgraph(), terminate);
     203         926 :   return terminate;
     204             : }
     205             : 
     206     2412941 : bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
     207     2639633 :   return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
     208     1433162 :          NodeProperties::GetControlInput(phi) == merge;
     209             : }
     210             : 
     211        1271 : bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
     212        1295 :                                        Node** if_exception) {
     213        1271 :   if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
     214             :     return false;
     215             :   }
     216             : 
     217        1295 :   *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
     218             :   *if_exception =
     219        1295 :       graph()->NewNode(mcgraph()->common()->IfException(), node, node);
     220             : 
     221         648 :   return true;
     222             : }
     223             : 
     224      427059 : void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
     225             :   DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
     226      213529 :   merge->AppendInput(mcgraph()->zone(), from);
     227             :   int new_size = merge->InputCount();
     228             :   NodeProperties::ChangeOp(
     229      213530 :       merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
     230      213530 : }
     231             : 
     232      323543 : void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
     233             :   DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
     234             :   int new_size = phi->InputCount();
     235      215694 :   phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
     236             :   NodeProperties::ChangeOp(
     237      107849 :       phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
     238      107847 : }
     239             : 
     240       70572 : Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
     241      105858 :   return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
     242             : }
     243             : 
     244       98091 : Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
     245       98089 :                             Node* control) {
     246             :   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
     247       98091 :   Node** buf = Realloc(vals, count, count + 1);
     248       98090 :   buf[count] = control;
     249             :   return graph()->NewNode(
     250             :       mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
     251             :                                count),
     252      294268 :       count + 1, buf);
     253             : }
     254             : 
     255       31184 : Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
     256       31184 :                                   Node* control) {
     257             :   DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
     258       31184 :   Node** buf = Realloc(effects, count, count + 1);
     259       31184 :   buf[count] = control;
     260             :   return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
     261       93551 :                           buf);
     262             : }
     263             : 
     264        1160 : Node* WasmGraphBuilder::RefNull() {
     265         696 :   return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
     266             : }
     267             : 
     268        4356 : Node* WasmGraphBuilder::NoContextConstant() {
     269             :   // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
     270        4356 :   return mcgraph()->IntPtrConstant(0);
     271             : }
     272             : 
     273        1463 : Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
     274           0 :   return mcgraph()->Uint32Constant(value);
     275             : }
     276             : 
     277     1685366 : Node* WasmGraphBuilder::Int32Constant(int32_t value) {
     278     1685678 :   return mcgraph()->Int32Constant(value);
     279             : }
     280             : 
     281       27500 : Node* WasmGraphBuilder::Int64Constant(int64_t value) {
     282       27500 :   return mcgraph()->Int64Constant(value);
     283             : }
     284             : 
     285       37817 : Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
     286       37817 :   return mcgraph()->IntPtrConstant(value);
     287             : }
     288             : 
     289       22427 : void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
     290      171317 :                                   Node** effect, Node** control) {
     291             :   DCHECK_NOT_NULL(env_);  // Wrappers don't get stack checks.
     292       22427 :   if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
     293        3056 :     return;
     294             :   }
     295       19371 :   if (effect == nullptr) effect = effect_;
     296       19371 :   if (control == nullptr) control = control_;
     297             : 
     298             :   // This instruction sequence is matched in the instruction selector to
     299             :   // load the stack pointer directly on some platforms. Hence, when modifying
     300             :   // please also fix WasmStackCheckMatcher in node-matchers.h
     301             : 
     302             :   Node* limit_address = graph()->NewNode(
     303             :       mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
     304             :       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
     305       58116 :       *effect, *control);
     306             :   Node* limit = graph()->NewNode(
     307             :       mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
     308       58136 :       mcgraph()->IntPtrConstant(0), limit_address, *control);
     309       19378 :   *effect = limit;
     310       19378 :   Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
     311             : 
     312             :   Node* check =
     313       19377 :       graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
     314             : 
     315       38754 :   Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
     316       19378 :   stack_check.Chain(*control);
     317             : 
     318       19371 :   if (stack_check_call_operator_ == nullptr) {
     319             :     // Build and cache the stack check call operator and the constant
     320             :     // representing the stack check code.
     321             :     auto call_descriptor = Linkage::GetStubCallDescriptor(
     322             :         mcgraph()->zone(),                    // zone
     323             :         NoContextDescriptor{},                // descriptor
     324             :         0,                                    // stack parameter count
     325             :         CallDescriptor::kNoFlags,             // flags
     326             :         Operator::kNoProperties,              // properties
     327       23784 :         StubCallMode::kCallWasmRuntimeStub);  // stub call mode
     328             :     // A direct call to a wasm runtime stub defined in this module.
     329             :     // Just encode the stub index. This will be patched at relocation.
     330             :     stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
     331       11892 :         wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
     332       11899 :     stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
     333             :   }
     334             : 
     335             :   Node* call = graph()->NewNode(stack_check_call_operator_.get(),
     336             :                                 stack_check_code_node_.get(), *effect,
     337       19379 :                                 stack_check.if_false);
     338             : 
     339       19380 :   SetSourcePosition(call, position);
     340             : 
     341       19379 :   Node* ephi = stack_check.EffectPhi(*effect, call);
     342             : 
     343       19377 :   *control = stack_check.merge;
     344       19377 :   *effect = ephi;
     345             : }
     346             : 
     347      496799 : void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
     348      983153 :   if (!needs_stack_check_) return;
     349             : 
     350       13346 :   Node* start = graph()->start();
     351             :   // Place a stack check which uses a dummy node as control and effect.
     352       13346 :   Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
     353       13351 :   Node* control = dummy;
     354       13351 :   Node* effect = dummy;
     355             :   // The function-prologue stack check is associated with position 0, which
     356             :   // is never a position of any instruction in the function.
     357       13351 :   StackCheck(0, &effect, &control);
     358             : 
     359             :   // In testing, no steck checks were emitted. Nothing to rewire then.
     360       13353 :   if (effect == dummy) return;
     361             : 
     362             :   // Now patch all control uses of {start} to use {control} and all effect uses
     363             :   // to use {effect} instead. Then rewire the dummy node to use start instead.
     364       10452 :   NodeProperties::ReplaceUses(start, start, effect, control);
     365       10453 :   NodeProperties::ReplaceUses(dummy, nullptr, start, start);
     366             : }
     367             : 
     368      804958 : Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
     369      804958 :                               wasm::WasmCodePosition position) {
     370             :   const Operator* op;
     371      804958 :   MachineOperatorBuilder* m = mcgraph()->machine();
     372      804958 :   switch (opcode) {
     373             :     case wasm::kExprI32Add:
     374      105960 :       op = m->Int32Add();
     375      105960 :       break;
     376             :     case wasm::kExprI32Sub:
     377       27910 :       op = m->Int32Sub();
     378       27908 :       break;
     379             :     case wasm::kExprI32Mul:
     380       18030 :       op = m->Int32Mul();
     381       18030 :       break;
     382             :     case wasm::kExprI32DivS:
     383       14180 :       return BuildI32DivS(left, right, position);
     384             :     case wasm::kExprI32DivU:
     385       14124 :       return BuildI32DivU(left, right, position);
     386             :     case wasm::kExprI32RemS:
     387       14104 :       return BuildI32RemS(left, right, position);
     388             :     case wasm::kExprI32RemU:
     389       14084 :       return BuildI32RemU(left, right, position);
     390             :     case wasm::kExprI32And:
     391      111339 :       op = m->Word32And();
     392      111339 :       break;
     393             :     case wasm::kExprI32Ior:
     394       19051 :       op = m->Word32Or();
     395       19051 :       break;
     396             :     case wasm::kExprI32Xor:
     397       14367 :       op = m->Word32Xor();
     398       14367 :       break;
     399             :     case wasm::kExprI32Shl:
     400       25741 :       op = m->Word32Shl();
     401       25741 :       right = MaskShiftCount32(right);
     402       25741 :       break;
     403             :     case wasm::kExprI32ShrU:
     404       29468 :       op = m->Word32Shr();
     405       29468 :       right = MaskShiftCount32(right);
     406       29468 :       break;
     407             :     case wasm::kExprI32ShrS:
     408       15615 :       op = m->Word32Sar();
     409       15615 :       right = MaskShiftCount32(right);
     410       15615 :       break;
     411             :     case wasm::kExprI32Ror:
     412       26944 :       op = m->Word32Ror();
     413       26944 :       right = MaskShiftCount32(right);
     414       26944 :       break;
     415             :     case wasm::kExprI32Rol:
     416       13472 :       right = MaskShiftCount32(right);
     417       13472 :       return BuildI32Rol(left, right);
     418             :     case wasm::kExprI32Eq:
     419       86387 :       op = m->Word32Equal();
     420       86387 :       break;
     421             :     case wasm::kExprI32Ne:
     422       38200 :       return Invert(Binop(wasm::kExprI32Eq, left, right));
     423             :     case wasm::kExprI32LtS:
     424       15470 :       op = m->Int32LessThan();
     425       15470 :       break;
     426             :     case wasm::kExprI32LeS:
     427       14337 :       op = m->Int32LessThanOrEqual();
     428       14337 :       break;
     429             :     case wasm::kExprI32LtU:
     430       17819 :       op = m->Uint32LessThan();
     431       17819 :       break;
     432             :     case wasm::kExprI32LeU:
     433       13632 :       op = m->Uint32LessThanOrEqual();
     434       13632 :       break;
     435             :     case wasm::kExprI32GtS:
     436       15677 :       op = m->Int32LessThan();
     437             :       std::swap(left, right);
     438             :       break;
     439             :     case wasm::kExprI32GeS:
     440       14060 :       op = m->Int32LessThanOrEqual();
     441             :       std::swap(left, right);
     442             :       break;
     443             :     case wasm::kExprI32GtU:
     444       15198 :       op = m->Uint32LessThan();
     445             :       std::swap(left, right);
     446             :       break;
     447             :     case wasm::kExprI32GeU:
     448       13632 :       op = m->Uint32LessThanOrEqual();
     449             :       std::swap(left, right);
     450             :       break;
     451             :     case wasm::kExprI64And:
     452         167 :       op = m->Word64And();
     453         167 :       break;
     454             :     case wasm::kExprI64Add:
     455         676 :       op = m->Int64Add();
     456         676 :       break;
     457             :     case wasm::kExprI64Sub:
     458         831 :       op = m->Int64Sub();
     459         831 :       break;
     460             :     case wasm::kExprI64Mul:
     461         807 :       op = m->Int64Mul();
     462         807 :       break;
     463             :     case wasm::kExprI64DivS:
     464         720 :       return BuildI64DivS(left, right, position);
     465             :     case wasm::kExprI64DivU:
     466         660 :       return BuildI64DivU(left, right, position);
     467             :     case wasm::kExprI64RemS:
     468         644 :       return BuildI64RemS(left, right, position);
     469             :     case wasm::kExprI64RemU:
     470         644 :       return BuildI64RemU(left, right, position);
     471             :     case wasm::kExprI64Ior:
     472       12706 :       op = m->Word64Or();
     473       12670 :       break;
     474             :     case wasm::kExprI64Xor:
     475          56 :       op = m->Word64Xor();
     476          56 :       break;
     477             :     case wasm::kExprI64Shl:
     478       13312 :       op = m->Word64Shl();
     479       13316 :       right = MaskShiftCount64(right);
     480       13318 :       break;
     481             :     case wasm::kExprI64ShrU:
     482         684 :       op = m->Word64Shr();
     483         684 :       right = MaskShiftCount64(right);
     484         684 :       break;
     485             :     case wasm::kExprI64ShrS:
     486         628 :       op = m->Word64Sar();
     487         628 :       right = MaskShiftCount64(right);
     488         628 :       break;
     489             :     case wasm::kExprI64Eq:
     490         287 :       op = m->Word64Equal();
     491         287 :       break;
     492             :     case wasm::kExprI64Ne:
     493          68 :       return Invert(Binop(wasm::kExprI64Eq, left, right));
     494             :     case wasm::kExprI64LtS:
     495          56 :       op = m->Int64LessThan();
     496          56 :       break;
     497             :     case wasm::kExprI64LeS:
     498          32 :       op = m->Int64LessThanOrEqual();
     499          32 :       break;
     500             :     case wasm::kExprI64LtU:
     501          40 :       op = m->Uint64LessThan();
     502          40 :       break;
     503             :     case wasm::kExprI64LeU:
     504          48 :       op = m->Uint64LessThanOrEqual();
     505          48 :       break;
     506             :     case wasm::kExprI64GtS:
     507          40 :       op = m->Int64LessThan();
     508             :       std::swap(left, right);
     509             :       break;
     510             :     case wasm::kExprI64GeS:
     511          32 :       op = m->Int64LessThanOrEqual();
     512             :       std::swap(left, right);
     513             :       break;
     514             :     case wasm::kExprI64GtU:
     515          40 :       op = m->Uint64LessThan();
     516             :       std::swap(left, right);
     517             :       break;
     518             :     case wasm::kExprI64GeU:
     519          32 :       op = m->Uint64LessThanOrEqual();
     520             :       std::swap(left, right);
     521             :       break;
     522             :     case wasm::kExprI64Ror:
     523         120 :       op = m->Word64Ror();
     524         120 :       right = MaskShiftCount64(right);
     525         120 :       break;
     526             :     case wasm::kExprI64Rol:
     527          60 :       return BuildI64Rol(left, right);
     528             :     case wasm::kExprF32CopySign:
     529          32 :       return BuildF32CopySign(left, right);
     530             :     case wasm::kExprF64CopySign:
     531          32 :       return BuildF64CopySign(left, right);
     532             :     case wasm::kExprF32Add:
     533         461 :       op = m->Float32Add();
     534         461 :       break;
     535             :     case wasm::kExprF32Sub:
     536         305 :       op = m->Float32Sub();
     537         305 :       break;
     538             :     case wasm::kExprF32Mul:
     539         425 :       op = m->Float32Mul();
     540         425 :       break;
     541             :     case wasm::kExprF32Div:
     542         349 :       op = m->Float32Div();
     543         349 :       break;
     544             :     case wasm::kExprF32Eq:
     545         821 :       op = m->Float32Equal();
     546         821 :       break;
     547             :     case wasm::kExprF32Ne:
     548         717 :       return Invert(Binop(wasm::kExprF32Eq, left, right));
     549             :     case wasm::kExprF32Lt:
     550         281 :       op = m->Float32LessThan();
     551         281 :       break;
     552             :     case wasm::kExprF32Ge:
     553         105 :       op = m->Float32LessThanOrEqual();
     554             :       std::swap(left, right);
     555             :       break;
     556             :     case wasm::kExprF32Gt:
     557         257 :       op = m->Float32LessThan();
     558             :       std::swap(left, right);
     559             :       break;
     560             :     case wasm::kExprF32Le:
     561         121 :       op = m->Float32LessThanOrEqual();
     562         121 :       break;
     563             :     case wasm::kExprF64Add:
     564        4173 :       op = m->Float64Add();
     565        4173 :       break;
     566             :     case wasm::kExprF64Sub:
     567        2902 :       op = m->Float64Sub();
     568        2902 :       break;
     569             :     case wasm::kExprF64Mul:
     570        6488 :       op = m->Float64Mul();
     571        6488 :       break;
     572             :     case wasm::kExprF64Div:
     573         639 :       op = m->Float64Div();
     574         639 :       break;
     575             :     case wasm::kExprF64Eq:
     576         727 :       op = m->Float64Equal();
     577         726 :       break;
     578             :     case wasm::kExprF64Ne:
     579         409 :       return Invert(Binop(wasm::kExprF64Eq, left, right));
     580             :     case wasm::kExprF64Lt:
     581         742 :       op = m->Float64LessThan();
     582         742 :       break;
     583             :     case wasm::kExprF64Le:
     584         303 :       op = m->Float64LessThanOrEqual();
     585         303 :       break;
     586             :     case wasm::kExprF64Gt:
     587         696 :       op = m->Float64LessThan();
     588             :       std::swap(left, right);
     589             :       break;
     590             :     case wasm::kExprF64Ge:
     591         301 :       op = m->Float64LessThanOrEqual();
     592             :       std::swap(left, right);
     593             :       break;
     594             :     case wasm::kExprF32Min:
     595          70 :       op = m->Float32Min();
     596          70 :       break;
     597             :     case wasm::kExprF64Min:
     598          80 :       op = m->Float64Min();
     599          80 :       break;
     600             :     case wasm::kExprF32Max:
     601          70 :       op = m->Float32Max();
     602          70 :       break;
     603             :     case wasm::kExprF64Max:
     604          72 :       op = m->Float64Max();
     605          72 :       break;
     606             :     case wasm::kExprF64Pow:
     607           9 :       return BuildF64Pow(left, right);
     608             :     case wasm::kExprF64Atan2:
     609          17 :       op = m->Float64Atan2();
     610          17 :       break;
     611             :     case wasm::kExprF64Mod:
     612          20 :       return BuildF64Mod(left, right);
     613             :     case wasm::kExprI32AsmjsDivS:
     614         385 :       return BuildI32AsmjsDivS(left, right);
     615             :     case wasm::kExprI32AsmjsDivU:
     616         137 :       return BuildI32AsmjsDivU(left, right);
     617             :     case wasm::kExprI32AsmjsRemS:
     618         421 :       return BuildI32AsmjsRemS(left, right);
     619             :     case wasm::kExprI32AsmjsRemU:
     620         186 :       return BuildI32AsmjsRemU(left, right);
     621             :     case wasm::kExprI32AsmjsStoreMem8:
     622        3041 :       return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
     623             :     case wasm::kExprI32AsmjsStoreMem16:
     624        1385 :       return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
     625             :     case wasm::kExprI32AsmjsStoreMem:
     626       31530 :       return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
     627             :     case wasm::kExprF32AsmjsStoreMem:
     628        3299 :       return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
     629             :     case wasm::kExprF64AsmjsStoreMem:
     630         759 :       return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
     631             :     default:
     632           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
     633             :   }
     634      651584 :   return graph()->NewNode(op, left, right);
     635             : }
     636             : 
     637      149303 : Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
     638      199168 :                              wasm::WasmCodePosition position) {
     639             :   const Operator* op;
     640      149303 :   MachineOperatorBuilder* m = mcgraph()->machine();
     641      149303 :   switch (opcode) {
     642             :     case wasm::kExprI32Eqz:
     643       49713 :       op = m->Word32Equal();
     644       99426 :       return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
     645             :     case wasm::kExprF32Abs:
     646          62 :       op = m->Float32Abs();
     647          62 :       break;
     648             :     case wasm::kExprF32Neg: {
     649         221 :       op = m->Float32Neg();
     650         221 :       break;
     651             :     }
     652             :     case wasm::kExprF32Sqrt:
     653         174 :       op = m->Float32Sqrt();
     654         174 :       break;
     655             :     case wasm::kExprF64Abs:
     656          87 :       op = m->Float64Abs();
     657          87 :       break;
     658             :     case wasm::kExprF64Neg: {
     659        1051 :       op = m->Float64Neg();
     660        1051 :       break;
     661             :     }
     662             :     case wasm::kExprF64Sqrt:
     663         256 :       op = m->Float64Sqrt();
     664         256 :       break;
     665             :     case wasm::kExprI32SConvertF32:
     666             :     case wasm::kExprI32UConvertF32:
     667             :     case wasm::kExprI32SConvertF64:
     668             :     case wasm::kExprI32UConvertF64:
     669             :     case wasm::kExprI32SConvertSatF64:
     670             :     case wasm::kExprI32UConvertSatF64:
     671             :     case wasm::kExprI32SConvertSatF32:
     672             :     case wasm::kExprI32UConvertSatF32:
     673         352 :       return BuildIntConvertFloat(input, position, opcode);
     674             :     case wasm::kExprI32AsmjsSConvertF64:
     675         108 :       return BuildI32AsmjsSConvertF64(input);
     676             :     case wasm::kExprI32AsmjsUConvertF64:
     677           8 :       return BuildI32AsmjsUConvertF64(input);
     678             :     case wasm::kExprF32ConvertF64:
     679        3435 :       op = m->TruncateFloat64ToFloat32();
     680        3435 :       break;
     681             :     case wasm::kExprF64SConvertI32:
     682         592 :       op = m->ChangeInt32ToFloat64();
     683         592 :       break;
     684             :     case wasm::kExprF64UConvertI32:
     685         278 :       op = m->ChangeUint32ToFloat64();
     686         278 :       break;
     687             :     case wasm::kExprF32SConvertI32:
     688         188 :       op = m->RoundInt32ToFloat32();
     689         188 :       break;
     690             :     case wasm::kExprF32UConvertI32:
     691          96 :       op = m->RoundUint32ToFloat32();
     692          96 :       break;
     693             :     case wasm::kExprI32AsmjsSConvertF32:
     694           8 :       return BuildI32AsmjsSConvertF32(input);
     695             :     case wasm::kExprI32AsmjsUConvertF32:
     696           8 :       return BuildI32AsmjsUConvertF32(input);
     697             :     case wasm::kExprF64ConvertF32:
     698        5544 :       op = m->ChangeFloat32ToFloat64();
     699        5544 :       break;
     700             :     case wasm::kExprF32ReinterpretI32:
     701         184 :       op = m->BitcastInt32ToFloat32();
     702         184 :       break;
     703             :     case wasm::kExprI32ReinterpretF32:
     704         444 :       op = m->BitcastFloat32ToInt32();
     705         444 :       break;
     706             :     case wasm::kExprI32Clz:
     707         300 :       op = m->Word32Clz();
     708         300 :       break;
     709             :     case wasm::kExprI32Ctz: {
     710         340 :       if (m->Word32Ctz().IsSupported()) {
     711         340 :         op = m->Word32Ctz().op();
     712         340 :         break;
     713           0 :       } else if (m->Word32ReverseBits().IsSupported()) {
     714           0 :         Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
     715           0 :         Node* result = graph()->NewNode(m->Word32Clz(), reversed);
     716           0 :         return result;
     717             :       } else {
     718           0 :         return BuildI32Ctz(input);
     719             :       }
     720             :     }
     721             :     case wasm::kExprI32Popcnt: {
     722          64 :       if (m->Word32Popcnt().IsSupported()) {
     723          64 :         op = m->Word32Popcnt().op();
     724          64 :         break;
     725             :       } else {
     726           0 :         return BuildI32Popcnt(input);
     727             :       }
     728             :     }
     729             :     case wasm::kExprF32Floor: {
     730          50 :       if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
     731          50 :       op = m->Float32RoundDown().op();
     732          50 :       break;
     733             :     }
     734             :     case wasm::kExprF32Ceil: {
     735          50 :       if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
     736          50 :       op = m->Float32RoundUp().op();
     737          50 :       break;
     738             :     }
     739             :     case wasm::kExprF32Trunc: {
     740         204 :       if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
     741         204 :       op = m->Float32RoundTruncate().op();
     742         204 :       break;
     743             :     }
     744             :     case wasm::kExprF32NearestInt: {
     745          24 :       if (!m->Float32RoundTiesEven().IsSupported())
     746           0 :         return BuildF32NearestInt(input);
     747          24 :       op = m->Float32RoundTiesEven().op();
     748          24 :       break;
     749             :     }
     750             :     case wasm::kExprF64Floor: {
     751          98 :       if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
     752          98 :       op = m->Float64RoundDown().op();
     753          98 :       break;
     754             :     }
     755             :     case wasm::kExprF64Ceil: {
     756          66 :       if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
     757          66 :       op = m->Float64RoundUp().op();
     758          66 :       break;
     759             :     }
     760             :     case wasm::kExprF64Trunc: {
     761         196 :       if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
     762         195 :       op = m->Float64RoundTruncate().op();
     763         195 :       break;
     764             :     }
     765             :     case wasm::kExprF64NearestInt: {
     766          24 :       if (!m->Float64RoundTiesEven().IsSupported())
     767           0 :         return BuildF64NearestInt(input);
     768          24 :       op = m->Float64RoundTiesEven().op();
     769          24 :       break;
     770             :     }
     771             :     case wasm::kExprF64Acos: {
     772          21 :       return BuildF64Acos(input);
     773             :     }
     774             :     case wasm::kExprF64Asin: {
     775          21 :       return BuildF64Asin(input);
     776             :     }
     777             :     case wasm::kExprF64Atan:
     778          21 :       op = m->Float64Atan();
     779          21 :       break;
     780             :     case wasm::kExprF64Cos: {
     781         146 :       op = m->Float64Cos();
     782         146 :       break;
     783             :     }
     784             :     case wasm::kExprF64Sin: {
     785         146 :       op = m->Float64Sin();
     786         146 :       break;
     787             :     }
     788             :     case wasm::kExprF64Tan: {
     789          21 :       op = m->Float64Tan();
     790          21 :       break;
     791             :     }
     792             :     case wasm::kExprF64Exp: {
     793          21 :       op = m->Float64Exp();
     794          21 :       break;
     795             :     }
     796             :     case wasm::kExprF64Log:
     797          21 :       op = m->Float64Log();
     798          21 :       break;
     799             :     case wasm::kExprI32ConvertI64:
     800         764 :       op = m->TruncateInt64ToInt32();
     801         764 :       break;
     802             :     case wasm::kExprI64SConvertI32:
     803          44 :       op = m->ChangeInt32ToInt64();
     804          44 :       break;
     805             :     case wasm::kExprI64UConvertI32:
     806       25289 :       op = m->ChangeUint32ToUint64();
     807       25292 :       break;
     808             :     case wasm::kExprF64ReinterpretI64:
     809         160 :       op = m->BitcastInt64ToFloat64();
     810         160 :       break;
     811             :     case wasm::kExprI64ReinterpretF64:
     812         412 :       op = m->BitcastFloat64ToInt64();
     813         412 :       break;
     814             :     case wasm::kExprI64Clz:
     815          36 :       op = m->Word64Clz();
     816          36 :       break;
     817             :     case wasm::kExprI64Ctz: {
     818         188 :       OptionalOperator ctz64 = m->Word64Ctz();
     819         188 :       if (ctz64.IsSupported()) {
     820             :         op = ctz64.op();
     821         188 :         break;
     822           0 :       } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
     823             :         op = ctz64.placeholder();
     824             :         break;
     825           0 :       } else if (m->Word64ReverseBits().IsSupported()) {
     826           0 :         Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
     827           0 :         Node* result = graph()->NewNode(m->Word64Clz(), reversed);
     828           0 :         return result;
     829             :       } else {
     830           0 :         return BuildI64Ctz(input);
     831             :       }
     832             :     }
     833             :     case wasm::kExprI64Popcnt: {
     834          44 :       OptionalOperator popcnt64 = m->Word64Popcnt();
     835          44 :       if (popcnt64.IsSupported()) {
     836             :         op = popcnt64.op();
     837           0 :       } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
     838             :         op = popcnt64.placeholder();
     839             :       } else {
     840           0 :         return BuildI64Popcnt(input);
     841             :       }
     842          44 :       break;
     843             :     }
     844             :     case wasm::kExprI64Eqz:
     845         152 :       op = m->Word64Equal();
     846         304 :       return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
     847             :     case wasm::kExprF32SConvertI64:
     848          64 :       if (m->Is32()) {
     849           0 :         return BuildF32SConvertI64(input);
     850             :       }
     851          64 :       op = m->RoundInt64ToFloat32();
     852          64 :       break;
     853             :     case wasm::kExprF32UConvertI64:
     854          32 :       if (m->Is32()) {
     855           0 :         return BuildF32UConvertI64(input);
     856             :       }
     857          32 :       op = m->RoundUint64ToFloat32();
     858          32 :       break;
     859             :     case wasm::kExprF64SConvertI64:
     860          87 :       if (m->Is32()) {
     861           0 :         return BuildF64SConvertI64(input);
     862             :       }
     863          87 :       op = m->RoundInt64ToFloat64();
     864          87 :       break;
     865             :     case wasm::kExprF64UConvertI64:
     866         135 :       if (m->Is32()) {
     867           0 :         return BuildF64UConvertI64(input);
     868             :       }
     869         135 :       op = m->RoundUint64ToFloat64();
     870         135 :       break;
     871             :     case wasm::kExprI32SExtendI8:
     872           4 :       op = m->SignExtendWord8ToInt32();
     873           4 :       break;
     874             :     case wasm::kExprI32SExtendI16:
     875           4 :       op = m->SignExtendWord16ToInt32();
     876           4 :       break;
     877             :     case wasm::kExprI64SExtendI8:
     878           4 :       op = m->SignExtendWord8ToInt64();
     879           4 :       break;
     880             :     case wasm::kExprI64SExtendI16:
     881           4 :       op = m->SignExtendWord16ToInt64();
     882           4 :       break;
     883             :     case wasm::kExprI64SExtendI32:
     884           4 :       op = m->SignExtendWord32ToInt64();
     885           4 :       break;
     886             :     case wasm::kExprI64SConvertF32:
     887             :     case wasm::kExprI64UConvertF32:
     888             :     case wasm::kExprI64SConvertF64:
     889             :     case wasm::kExprI64UConvertF64:
     890             :     case wasm::kExprI64SConvertSatF32:
     891             :     case wasm::kExprI64UConvertSatF32:
     892             :     case wasm::kExprI64SConvertSatF64:
     893             :     case wasm::kExprI64UConvertSatF64:
     894             :       return mcgraph()->machine()->Is32()
     895             :                  ? BuildCcallConvertFloat(input, position, opcode)
     896         811 :                  : BuildIntConvertFloat(input, position, opcode);
     897             :     case wasm::kExprRefIsNull:
     898          64 :       return graph()->NewNode(m->WordEqual(), input, RefNull());
     899             :     case wasm::kExprI32AsmjsLoadMem8S:
     900        2804 :       return BuildAsmjsLoadMem(MachineType::Int8(), input);
     901             :     case wasm::kExprI32AsmjsLoadMem8U:
     902        2082 :       return BuildAsmjsLoadMem(MachineType::Uint8(), input);
     903             :     case wasm::kExprI32AsmjsLoadMem16S:
     904         958 :       return BuildAsmjsLoadMem(MachineType::Int16(), input);
     905             :     case wasm::kExprI32AsmjsLoadMem16U:
     906         531 :       return BuildAsmjsLoadMem(MachineType::Uint16(), input);
     907             :     case wasm::kExprI32AsmjsLoadMem:
     908       43774 :       return BuildAsmjsLoadMem(MachineType::Int32(), input);
     909             :     case wasm::kExprF32AsmjsLoadMem:
     910        5396 :       return BuildAsmjsLoadMem(MachineType::Float32(), input);
     911             :     case wasm::kExprF64AsmjsLoadMem:
     912         845 :       return BuildAsmjsLoadMem(MachineType::Float64(), input);
     913             :     default:
     914           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
     915             :   }
     916       41667 :   return graph()->NewNode(op, input);
     917             : }
     918             : 
     919        3651 : Node* WasmGraphBuilder::Float32Constant(float value) {
     920        3651 :   return mcgraph()->Float32Constant(value);
     921             : }
     922             : 
     923       10443 : Node* WasmGraphBuilder::Float64Constant(double value) {
     924       10443 :   return mcgraph()->Float64Constant(value);
     925             : }
     926             : 
     927             : namespace {
     928      592714 : Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
     929             :              Node** false_node, Node* control, BranchHint hint) {
     930             :   DCHECK_NOT_NULL(cond);
     931             :   DCHECK_NOT_NULL(control);
     932             :   Node* branch =
     933       98783 :       mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
     934      197574 :   *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
     935      197575 :   *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
     936       98788 :   return branch;
     937             : }
     938             : }  // namespace
     939             : 
     940       83881 : Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
     941      167762 :                                      Node** false_node) {
     942             :   return Branch(mcgraph(), cond, true_node, false_node, Control(),
     943       83881 :                 BranchHint::kNone);
     944             : }
     945             : 
     946           0 : Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
     947           0 :                                          Node** false_node) {
     948             :   return Branch(mcgraph(), cond, true_node, false_node, Control(),
     949           0 :                 BranchHint::kTrue);
     950             : }
     951             : 
     952           0 : Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
     953       29800 :                                           Node** false_node) {
     954             :   return Branch(mcgraph(), cond, true_node, false_node, Control(),
     955       14900 :                 BranchHint::kFalse);
     956             : }
     957             : 
     958       34166 : TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
     959             :   // TODO(wasm): "!env_" should not happen when compiling an actual wasm
     960             :   // function.
     961       34166 :   if (!env_ || !env_->runtime_exception_support) {
     962             :     // We use TrapId::kInvalid as a marker to tell the code generator
     963             :     // to generate a call to a testing c-function instead of a runtime
     964             :     // stub. This code should only be called from a cctest.
     965             :     return TrapId::kInvalid;
     966             :   }
     967             : 
     968       23860 :   switch (reason) {
     969             : #define TRAPREASON_TO_TRAPID(name)                                             \
     970             :   case wasm::k##name:                                                          \
     971             :     static_assert(                                                             \
     972             :         static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
     973             :         "trap id mismatch");                                                   \
     974             :     return TrapId::k##name;
     975       10509 :     FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
     976             : #undef TRAPREASON_TO_TRAPID
     977             :     default:
     978           0 :       UNREACHABLE();
     979             :   }
     980             : }
     981             : 
     982        5338 : Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
     983       21353 :                                    wasm::WasmCodePosition position) {
     984        5338 :   TrapId trap_id = GetTrapIdForTrap(reason);
     985             :   Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
     986        5338 :                                            cond, Effect(), Control()));
     987        5339 :   SetSourcePosition(node, position);
     988        5338 :   return node;
     989             : }
     990             : 
     991       28828 : Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
     992      115307 :                                     wasm::WasmCodePosition position) {
     993       28828 :   TrapId trap_id = GetTrapIdForTrap(reason);
     994             :   Node* node = SetControl(graph()->NewNode(
     995       28827 :       mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
     996       28826 :   SetSourcePosition(node, position);
     997       28828 :   return node;
     998             : }
     999             : 
    1000             : // Add a check that traps if {node} is equal to {val}.
    1001       70688 : Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
    1002             :                                    int32_t val,
    1003        1912 :                                    wasm::WasmCodePosition position) {
    1004             :   Int32Matcher m(node);
    1005      204508 :   if (m.HasValue() && !m.Is(val)) return graph()->start();
    1006        4380 :   if (val == 0) {
    1007        3424 :     return TrapIfFalse(reason, node, position);
    1008             :   } else {
    1009             :     return TrapIfTrue(reason,
    1010             :                       graph()->NewNode(mcgraph()->machine()->Word32Equal(),
    1011             :                                        node, mcgraph()->Int32Constant(val)),
    1012        2868 :                       position);
    1013             :   }
    1014             : }
    1015             : 
    1016             : // Add a check that traps if {node} is zero.
    1017           0 : Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
    1018             :                                     wasm::WasmCodePosition position) {
    1019       56492 :   return TrapIfEq32(reason, node, 0, position);
    1020             : }
    1021             : 
    1022             : // Add a check that traps if {node} is equal to {val}.
    1023        4168 : Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
    1024             :                                    int64_t val,
    1025        7904 :                                    wasm::WasmCodePosition position) {
    1026             :   Int64Matcher m(node);
    1027        4620 :   if (m.HasValue() && !m.Is(val)) return graph()->start();
    1028             :   return TrapIfTrue(reason,
    1029             :                     graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
    1030             :                                      mcgraph()->Int64Constant(val)),
    1031       11857 :                     position);
    1032             : }
    1033             : 
    1034             : // Add a check that traps if {node} is zero.
    1035           0 : Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
    1036             :                                     wasm::WasmCodePosition position) {
    1037        3451 :   return TrapIfEq64(reason, node, 0, position);
    1038             : }
    1039             : 
    1040        9704 : Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
    1041       14556 :   return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
    1042             : }
    1043             : 
    1044      210672 : Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
    1045             :   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
    1046      421345 :   return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
    1047             : }
    1048             : 
    1049        4852 : Node* WasmGraphBuilder::IfDefault(Node* sw) {
    1050             :   DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
    1051        9704 :   return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
    1052             : }
    1053             : 
    1054     4152617 : Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
    1055             :   static const int kStackAllocatedNodeBufferSize = 8;
    1056             :   Node* stack_buffer[kStackAllocatedNodeBufferSize];
    1057             :   std::vector<Node*> heap_buffer;
    1058             : 
    1059             :   Node** buf = stack_buffer;
    1060      692026 :   if (count + 3 > kStackAllocatedNodeBufferSize) {
    1061           0 :     heap_buffer.resize(count + 3);
    1062             :     buf = heap_buffer.data();
    1063             :   }
    1064             : 
    1065      692026 :   buf[0] = mcgraph()->Int32Constant(0);
    1066      692126 :   if (count > 0) {
    1067      664860 :     memcpy(buf + 1, vals, sizeof(void*) * count);
    1068             :   }
    1069     1384252 :   buf[count + 1] = Effect();
    1070     1384252 :   buf[count + 2] = Control();
    1071             :   Node* ret =
    1072     2076303 :       graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
    1073             : 
    1074      692187 :   MergeControlToEnd(mcgraph(), ret);
    1075      692088 :   return ret;
    1076             : }
    1077             : 
    1078        1617 : Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
    1079             : 
    1080        1541 : Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
    1081        1541 :   TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
    1082             :   ReturnVoid();
    1083        1541 :   return nullptr;
    1084             : }
    1085             : 
    1086      111264 : Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
    1087             :   static const int32_t kMask32 = 0x1F;
    1088      222480 :   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
    1089             :     // Shifts by constants are so common we pattern-match them here.
    1090             :     Int32Matcher match(node);
    1091          24 :     if (match.HasValue()) {
    1092           0 :       int32_t masked = (match.Value() & kMask32);
    1093           0 :       if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
    1094             :     } else {
    1095             :       node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
    1096          48 :                               mcgraph()->Int32Constant(kMask32));
    1097             :     }
    1098             :   }
    1099      111240 :   return node;
    1100             : }
    1101             : 
    1102       14763 : Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
    1103             :   static const int64_t kMask64 = 0x3F;
    1104       29486 :   if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
    1105             :     // Shifts by constants are so common we pattern-match them here.
    1106             :     Int64Matcher match(node);
    1107          20 :     if (match.HasValue()) {
    1108           0 :       int64_t masked = (match.Value() & kMask64);
    1109           0 :       if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
    1110             :     } else {
    1111             :       node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
    1112          40 :                               mcgraph()->Int64Constant(kMask64));
    1113             :     }
    1114             :   }
    1115       14743 :   return node;
    1116             : }
    1117             : 
    1118             : static bool ReverseBytesSupported(MachineOperatorBuilder* m,
    1119             :                                   size_t size_in_bytes) {
    1120           0 :   switch (size_in_bytes) {
    1121             :     case 4:
    1122             :     case 16:
    1123             :       return true;
    1124             :     case 8:
    1125             :       return m->Is64();
    1126             :     default:
    1127             :       break;
    1128             :   }
    1129             :   return false;
    1130             : }
    1131             : 
    1132           0 : Node* WasmGraphBuilder::BuildChangeEndiannessStore(
    1133           0 :     Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
    1134             :   Node* result;
    1135             :   Node* value = node;
    1136           0 :   MachineOperatorBuilder* m = mcgraph()->machine();
    1137           0 :   int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
    1138           0 :   int valueSizeInBits = 8 * valueSizeInBytes;
    1139             :   bool isFloat = false;
    1140             : 
    1141           0 :   switch (wasmtype) {
    1142             :     case wasm::kWasmF64:
    1143           0 :       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
    1144             :       isFloat = true;
    1145             :       V8_FALLTHROUGH;
    1146             :     case wasm::kWasmI64:
    1147           0 :       result = mcgraph()->Int64Constant(0);
    1148           0 :       break;
    1149             :     case wasm::kWasmF32:
    1150           0 :       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
    1151             :       isFloat = true;
    1152             :       V8_FALLTHROUGH;
    1153             :     case wasm::kWasmI32:
    1154           0 :       result = mcgraph()->Int32Constant(0);
    1155           0 :       break;
    1156             :     case wasm::kWasmS128:
    1157             :       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
    1158             :       break;
    1159             :     default:
    1160           0 :       UNREACHABLE();
    1161             :       break;
    1162             :   }
    1163             : 
    1164           0 :   if (mem_rep == MachineRepresentation::kWord8) {
    1165             :     // No need to change endianness for byte size, return original node
    1166             :     return node;
    1167             :   }
    1168           0 :   if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
    1169             :     // In case we store lower part of WasmI64 expression, we can truncate
    1170             :     // upper 32bits
    1171           0 :     value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
    1172           0 :     valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
    1173           0 :     valueSizeInBits = 8 * valueSizeInBytes;
    1174           0 :     if (mem_rep == MachineRepresentation::kWord16) {
    1175             :       value =
    1176           0 :           graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
    1177             :     }
    1178           0 :   } else if (wasmtype == wasm::kWasmI32 &&
    1179           0 :              mem_rep == MachineRepresentation::kWord16) {
    1180             :     value =
    1181           0 :         graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
    1182             :   }
    1183             : 
    1184             :   int i;
    1185             :   uint32_t shiftCount;
    1186             : 
    1187           0 :   if (ReverseBytesSupported(m, valueSizeInBytes)) {
    1188           0 :     switch (valueSizeInBytes) {
    1189             :       case 4:
    1190           0 :         result = graph()->NewNode(m->Word32ReverseBytes(), value);
    1191           0 :         break;
    1192             :       case 8:
    1193           0 :         result = graph()->NewNode(m->Word64ReverseBytes(), value);
    1194           0 :         break;
    1195             :       case 16: {
    1196             :         Node* byte_reversed_lanes[4];
    1197           0 :         for (int lane = 0; lane < 4; lane++) {
    1198             :           byte_reversed_lanes[lane] = graph()->NewNode(
    1199             :               m->Word32ReverseBytes(),
    1200             :               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
    1201           0 :                                value));
    1202             :         }
    1203             : 
    1204             :         // This is making a copy of the value.
    1205             :         result =
    1206           0 :             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
    1207             : 
    1208           0 :         for (int lane = 0; lane < 4; lane++) {
    1209             :           result =
    1210             :               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
    1211           0 :                                result, byte_reversed_lanes[lane]);
    1212             :         }
    1213             : 
    1214             :         break;
    1215             :       }
    1216             :       default:
    1217           0 :         UNREACHABLE();
    1218             :         break;
    1219             :     }
    1220             :   } else {
    1221           0 :     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
    1222             :          i += 8, shiftCount -= 16) {
    1223             :       Node* shiftLower;
    1224             :       Node* shiftHigher;
    1225             :       Node* lowerByte;
    1226             :       Node* higherByte;
    1227             : 
    1228             :       DCHECK_LT(0, shiftCount);
    1229             :       DCHECK_EQ(0, (shiftCount + 8) % 16);
    1230             : 
    1231           0 :       if (valueSizeInBits > 32) {
    1232             :         shiftLower = graph()->NewNode(m->Word64Shl(), value,
    1233           0 :                                       mcgraph()->Int64Constant(shiftCount));
    1234             :         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
    1235           0 :                                        mcgraph()->Int64Constant(shiftCount));
    1236             :         lowerByte = graph()->NewNode(
    1237             :             m->Word64And(), shiftLower,
    1238             :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
    1239           0 :                                      << (valueSizeInBits - 8 - i)));
    1240             :         higherByte = graph()->NewNode(
    1241             :             m->Word64And(), shiftHigher,
    1242           0 :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
    1243           0 :         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
    1244           0 :         result = graph()->NewNode(m->Word64Or(), result, higherByte);
    1245             :       } else {
    1246             :         shiftLower = graph()->NewNode(m->Word32Shl(), value,
    1247           0 :                                       mcgraph()->Int32Constant(shiftCount));
    1248             :         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
    1249           0 :                                        mcgraph()->Int32Constant(shiftCount));
    1250             :         lowerByte = graph()->NewNode(
    1251             :             m->Word32And(), shiftLower,
    1252             :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
    1253           0 :                                      << (valueSizeInBits - 8 - i)));
    1254             :         higherByte = graph()->NewNode(
    1255             :             m->Word32And(), shiftHigher,
    1256           0 :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
    1257           0 :         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
    1258           0 :         result = graph()->NewNode(m->Word32Or(), result, higherByte);
    1259             :       }
    1260             :     }
    1261             :   }
    1262             : 
    1263           0 :   if (isFloat) {
    1264           0 :     switch (wasmtype) {
    1265             :       case wasm::kWasmF64:
    1266           0 :         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
    1267           0 :         break;
    1268             :       case wasm::kWasmF32:
    1269           0 :         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
    1270           0 :         break;
    1271             :       default:
    1272           0 :         UNREACHABLE();
    1273             :         break;
    1274             :     }
    1275             :   }
    1276             : 
    1277           0 :   return result;
    1278             : }
    1279             : 
    1280           0 : Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
    1281             :                                                   MachineType memtype,
    1282           0 :                                                   wasm::ValueType wasmtype) {
    1283             :   Node* result;
    1284             :   Node* value = node;
    1285           0 :   MachineOperatorBuilder* m = mcgraph()->machine();
    1286             :   int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
    1287           0 :   int valueSizeInBits = 8 * valueSizeInBytes;
    1288             :   bool isFloat = false;
    1289             : 
    1290           0 :   switch (memtype.representation()) {
    1291             :     case MachineRepresentation::kFloat64:
    1292           0 :       value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
    1293             :       isFloat = true;
    1294             :       V8_FALLTHROUGH;
    1295             :     case MachineRepresentation::kWord64:
    1296           0 :       result = mcgraph()->Int64Constant(0);
    1297           0 :       break;
    1298             :     case MachineRepresentation::kFloat32:
    1299           0 :       value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
    1300             :       isFloat = true;
    1301             :       V8_FALLTHROUGH;
    1302             :     case MachineRepresentation::kWord32:
    1303             :     case MachineRepresentation::kWord16:
    1304           0 :       result = mcgraph()->Int32Constant(0);
    1305           0 :       break;
    1306             :     case MachineRepresentation::kWord8:
    1307             :       // No need to change endianness for byte size, return original node
    1308             :       return node;
    1309             :       break;
    1310             :     case MachineRepresentation::kSimd128:
    1311             :       DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
    1312             :       break;
    1313             :     default:
    1314           0 :       UNREACHABLE();
    1315             :       break;
    1316             :   }
    1317             : 
    1318             :   int i;
    1319             :   uint32_t shiftCount;
    1320             : 
    1321           0 :   if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
    1322           0 :     switch (valueSizeInBytes) {
    1323             :       case 2:
    1324             :         result =
    1325             :             graph()->NewNode(m->Word32ReverseBytes(),
    1326             :                              graph()->NewNode(m->Word32Shl(), value,
    1327           0 :                                               mcgraph()->Int32Constant(16)));
    1328           0 :         break;
    1329             :       case 4:
    1330           0 :         result = graph()->NewNode(m->Word32ReverseBytes(), value);
    1331           0 :         break;
    1332             :       case 8:
    1333           0 :         result = graph()->NewNode(m->Word64ReverseBytes(), value);
    1334           0 :         break;
    1335             :       case 16: {
    1336             :         Node* byte_reversed_lanes[4];
    1337           0 :         for (int lane = 0; lane < 4; lane++) {
    1338             :           byte_reversed_lanes[lane] = graph()->NewNode(
    1339             :               m->Word32ReverseBytes(),
    1340             :               graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
    1341           0 :                                value));
    1342             :         }
    1343             : 
    1344             :         // This is making a copy of the value.
    1345             :         result =
    1346           0 :             graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
    1347             : 
    1348           0 :         for (int lane = 0; lane < 4; lane++) {
    1349             :           result =
    1350             :               graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
    1351           0 :                                result, byte_reversed_lanes[lane]);
    1352             :         }
    1353             : 
    1354             :         break;
    1355             :       }
    1356             :       default:
    1357           0 :         UNREACHABLE();
    1358             :     }
    1359             :   } else {
    1360           0 :     for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
    1361             :          i += 8, shiftCount -= 16) {
    1362             :       Node* shiftLower;
    1363             :       Node* shiftHigher;
    1364             :       Node* lowerByte;
    1365             :       Node* higherByte;
    1366             : 
    1367             :       DCHECK_LT(0, shiftCount);
    1368             :       DCHECK_EQ(0, (shiftCount + 8) % 16);
    1369             : 
    1370           0 :       if (valueSizeInBits > 32) {
    1371             :         shiftLower = graph()->NewNode(m->Word64Shl(), value,
    1372           0 :                                       mcgraph()->Int64Constant(shiftCount));
    1373             :         shiftHigher = graph()->NewNode(m->Word64Shr(), value,
    1374           0 :                                        mcgraph()->Int64Constant(shiftCount));
    1375             :         lowerByte = graph()->NewNode(
    1376             :             m->Word64And(), shiftLower,
    1377             :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
    1378           0 :                                      << (valueSizeInBits - 8 - i)));
    1379             :         higherByte = graph()->NewNode(
    1380             :             m->Word64And(), shiftHigher,
    1381           0 :             mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
    1382           0 :         result = graph()->NewNode(m->Word64Or(), result, lowerByte);
    1383           0 :         result = graph()->NewNode(m->Word64Or(), result, higherByte);
    1384             :       } else {
    1385             :         shiftLower = graph()->NewNode(m->Word32Shl(), value,
    1386           0 :                                       mcgraph()->Int32Constant(shiftCount));
    1387             :         shiftHigher = graph()->NewNode(m->Word32Shr(), value,
    1388           0 :                                        mcgraph()->Int32Constant(shiftCount));
    1389             :         lowerByte = graph()->NewNode(
    1390             :             m->Word32And(), shiftLower,
    1391             :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
    1392           0 :                                      << (valueSizeInBits - 8 - i)));
    1393             :         higherByte = graph()->NewNode(
    1394             :             m->Word32And(), shiftHigher,
    1395           0 :             mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
    1396           0 :         result = graph()->NewNode(m->Word32Or(), result, lowerByte);
    1397           0 :         result = graph()->NewNode(m->Word32Or(), result, higherByte);
    1398             :       }
    1399             :     }
    1400             :   }
    1401             : 
    1402           0 :   if (isFloat) {
    1403           0 :     switch (memtype.representation()) {
    1404             :       case MachineRepresentation::kFloat64:
    1405           0 :         result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
    1406           0 :         break;
    1407             :       case MachineRepresentation::kFloat32:
    1408           0 :         result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
    1409           0 :         break;
    1410             :       default:
    1411           0 :         UNREACHABLE();
    1412             :         break;
    1413             :     }
    1414             :   }
    1415             : 
    1416             :   // We need to sign extend the value
    1417           0 :   if (memtype.IsSigned()) {
    1418             :     DCHECK(!isFloat);
    1419           0 :     if (valueSizeInBits < 32) {
    1420             :       Node* shiftBitCount;
    1421             :       // Perform sign extension using following trick
    1422             :       // result = (x << machine_width - type_width) >> (machine_width -
    1423             :       // type_width)
    1424           0 :       if (wasmtype == wasm::kWasmI64) {
    1425           0 :         shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
    1426             :         result = graph()->NewNode(
    1427             :             m->Word64Sar(),
    1428             :             graph()->NewNode(m->Word64Shl(),
    1429             :                              graph()->NewNode(m->ChangeInt32ToInt64(), result),
    1430             :                              shiftBitCount),
    1431           0 :             shiftBitCount);
    1432           0 :       } else if (wasmtype == wasm::kWasmI32) {
    1433           0 :         shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
    1434             :         result = graph()->NewNode(
    1435             :             m->Word32Sar(),
    1436             :             graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
    1437           0 :             shiftBitCount);
    1438             :       }
    1439             :     }
    1440             :   }
    1441             : 
    1442           0 :   return result;
    1443             : }
    1444             : 
    1445          64 : Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
    1446             :   Node* result = Unop(
    1447             :       wasm::kExprF32ReinterpretI32,
    1448             :       Binop(wasm::kExprI32Ior,
    1449             :             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
    1450             :                   mcgraph()->Int32Constant(0x7FFFFFFF)),
    1451             :             Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
    1452          64 :                   mcgraph()->Int32Constant(0x80000000))));
    1453             : 
    1454          32 :   return result;
    1455             : }
    1456             : 
    1457          64 : Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
    1458             : #if WASM_64
    1459             :   Node* result = Unop(
    1460             :       wasm::kExprF64ReinterpretI64,
    1461             :       Binop(wasm::kExprI64Ior,
    1462             :             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
    1463             :                   mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
    1464             :             Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
    1465          64 :                   mcgraph()->Int64Constant(0x8000000000000000))));
    1466             : 
    1467          32 :   return result;
    1468             : #else
    1469             :   MachineOperatorBuilder* m = mcgraph()->machine();
    1470             : 
    1471             :   Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
    1472             :   Node* high_word_right =
    1473             :       graph()->NewNode(m->Float64ExtractHighWord32(), right);
    1474             : 
    1475             :   Node* new_high_word = Binop(wasm::kExprI32Ior,
    1476             :                               Binop(wasm::kExprI32And, high_word_left,
    1477             :                                     mcgraph()->Int32Constant(0x7FFFFFFF)),
    1478             :                               Binop(wasm::kExprI32And, high_word_right,
    1479             :                                     mcgraph()->Int32Constant(0x80000000)));
    1480             : 
    1481             :   return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
    1482             : #endif
    1483             : }
    1484             : 
    1485             : namespace {
    1486             : 
    1487        1164 : MachineType IntConvertType(wasm::WasmOpcode opcode) {
    1488        1164 :   switch (opcode) {
    1489             :     case wasm::kExprI32SConvertF32:
    1490             :     case wasm::kExprI32SConvertF64:
    1491             :     case wasm::kExprI32SConvertSatF32:
    1492             :     case wasm::kExprI32SConvertSatF64:
    1493             :       return MachineType::Int32();
    1494             :     case wasm::kExprI32UConvertF32:
    1495             :     case wasm::kExprI32UConvertF64:
    1496             :     case wasm::kExprI32UConvertSatF32:
    1497             :     case wasm::kExprI32UConvertSatF64:
    1498             :       return MachineType::Uint32();
    1499             :     case wasm::kExprI64SConvertF32:
    1500             :     case wasm::kExprI64SConvertF64:
    1501             :     case wasm::kExprI64SConvertSatF32:
    1502             :     case wasm::kExprI64SConvertSatF64:
    1503             :       return MachineType::Int64();
    1504             :     case wasm::kExprI64UConvertF32:
    1505             :     case wasm::kExprI64UConvertF64:
    1506             :     case wasm::kExprI64UConvertSatF32:
    1507             :     case wasm::kExprI64UConvertSatF64:
    1508             :       return MachineType::Uint64();
    1509             :     default:
    1510           0 :       UNREACHABLE();
    1511             :   }
    1512             : }
    1513             : 
    1514        1165 : MachineType FloatConvertType(wasm::WasmOpcode opcode) {
    1515        1165 :   switch (opcode) {
    1516             :     case wasm::kExprI32SConvertF32:
    1517             :     case wasm::kExprI32UConvertF32:
    1518             :     case wasm::kExprI32SConvertSatF32:
    1519             :     case wasm::kExprI64SConvertF32:
    1520             :     case wasm::kExprI64UConvertF32:
    1521             :     case wasm::kExprI32UConvertSatF32:
    1522             :     case wasm::kExprI64SConvertSatF32:
    1523             :     case wasm::kExprI64UConvertSatF32:
    1524             :       return MachineType::Float32();
    1525             :     case wasm::kExprI32SConvertF64:
    1526             :     case wasm::kExprI32UConvertF64:
    1527             :     case wasm::kExprI64SConvertF64:
    1528             :     case wasm::kExprI64UConvertF64:
    1529             :     case wasm::kExprI32SConvertSatF64:
    1530             :     case wasm::kExprI32UConvertSatF64:
    1531             :     case wasm::kExprI64SConvertSatF64:
    1532             :     case wasm::kExprI64UConvertSatF64:
    1533             :       return MachineType::Float64();
    1534             :     default:
    1535           0 :       UNREACHABLE();
    1536             :   }
    1537             : }
    1538             : 
    1539        2330 : const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
    1540        1165 :   switch (opcode) {
    1541             :     case wasm::kExprI32SConvertF32:
    1542             :     case wasm::kExprI32SConvertSatF32:
    1543         124 :       return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
    1544             :     case wasm::kExprI32UConvertF32:
    1545             :     case wasm::kExprI32UConvertSatF32:
    1546          56 :       return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
    1547             :     case wasm::kExprI32SConvertF64:
    1548             :     case wasm::kExprI32SConvertSatF64:
    1549         124 :       return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
    1550             :     case wasm::kExprI32UConvertF64:
    1551             :     case wasm::kExprI32UConvertSatF64:
    1552          48 :       return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
    1553             :     case wasm::kExprI64SConvertF32:
    1554             :     case wasm::kExprI64SConvertSatF32:
    1555          48 :       return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
    1556             :     case wasm::kExprI64UConvertF32:
    1557             :     case wasm::kExprI64UConvertSatF32:
    1558          48 :       return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
    1559             :     case wasm::kExprI64SConvertF64:
    1560             :     case wasm::kExprI64SConvertSatF64:
    1561         661 :       return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
    1562             :     case wasm::kExprI64UConvertF64:
    1563             :     case wasm::kExprI64UConvertSatF64:
    1564          56 :       return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
    1565             :     default:
    1566           0 :       UNREACHABLE();
    1567             :   }
    1568             : }
    1569             : 
    1570         352 : wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
    1571         352 :   switch (opcode) {
    1572             :     case wasm::kExprI32SConvertF32:
    1573             :     case wasm::kExprI32SConvertSatF32:
    1574             :       return wasm::kExprF32SConvertI32;
    1575             :     case wasm::kExprI32UConvertF32:
    1576             :     case wasm::kExprI32UConvertSatF32:
    1577          56 :       return wasm::kExprF32UConvertI32;
    1578             :     case wasm::kExprI32SConvertF64:
    1579             :     case wasm::kExprI32SConvertSatF64:
    1580         124 :       return wasm::kExprF64SConvertI32;
    1581             :     case wasm::kExprI32UConvertF64:
    1582             :     case wasm::kExprI32UConvertSatF64:
    1583          48 :       return wasm::kExprF64UConvertI32;
    1584             :     default:
    1585           0 :       UNREACHABLE();
    1586             :   }
    1587             : }
    1588             : 
    1589        1164 : bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
    1590        1164 :   switch (opcode) {
    1591             :     case wasm::kExprI32SConvertF32:
    1592             :     case wasm::kExprI32UConvertF32:
    1593             :     case wasm::kExprI32SConvertF64:
    1594             :     case wasm::kExprI32UConvertF64:
    1595             :     case wasm::kExprI64SConvertF32:
    1596             :     case wasm::kExprI64UConvertF32:
    1597             :     case wasm::kExprI64SConvertF64:
    1598             :     case wasm::kExprI64UConvertF64:
    1599             :       return true;
    1600             :     case wasm::kExprI32SConvertSatF64:
    1601             :     case wasm::kExprI32UConvertSatF64:
    1602             :     case wasm::kExprI32SConvertSatF32:
    1603             :     case wasm::kExprI32UConvertSatF32:
    1604             :     case wasm::kExprI64SConvertSatF32:
    1605             :     case wasm::kExprI64UConvertSatF32:
    1606             :     case wasm::kExprI64SConvertSatF64:
    1607             :     case wasm::kExprI64UConvertSatF64:
    1608          64 :       return false;
    1609             :     default:
    1610           0 :       UNREACHABLE();
    1611             :   }
    1612             : }
    1613             : 
    1614         128 : Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
    1615         128 :   switch (ty.representation()) {
    1616             :     case MachineRepresentation::kWord32:
    1617          32 :       return builder->Int32Constant(0);
    1618             :     case MachineRepresentation::kWord64:
    1619          32 :       return builder->Int64Constant(0);
    1620             :     case MachineRepresentation::kFloat32:
    1621          32 :       return builder->Float32Constant(0.0);
    1622             :     case MachineRepresentation::kFloat64:
    1623          32 :       return builder->Float64Constant(0.0);
    1624             :     default:
    1625           0 :       UNREACHABLE();
    1626             :   }
    1627             : }
    1628             : 
    1629          64 : Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
    1630          64 :   switch (ty.semantic()) {
    1631             :     case MachineSemantic::kInt32:
    1632          16 :       return builder->Int32Constant(std::numeric_limits<int32_t>::min());
    1633             :     case MachineSemantic::kUint32:
    1634          16 :       return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
    1635             :     case MachineSemantic::kInt64:
    1636          16 :       return builder->Int64Constant(std::numeric_limits<int64_t>::min());
    1637             :     case MachineSemantic::kUint64:
    1638          16 :       return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
    1639             :     default:
    1640           0 :       UNREACHABLE();
    1641             :   }
    1642             : }
    1643             : 
    1644          64 : Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
    1645          64 :   switch (ty.semantic()) {
    1646             :     case MachineSemantic::kInt32:
    1647          16 :       return builder->Int32Constant(std::numeric_limits<int32_t>::max());
    1648             :     case MachineSemantic::kUint32:
    1649          16 :       return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
    1650             :     case MachineSemantic::kInt64:
    1651          16 :       return builder->Int64Constant(std::numeric_limits<int64_t>::max());
    1652             :     case MachineSemantic::kUint64:
    1653          16 :       return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
    1654             :     default:
    1655           0 :       UNREACHABLE();
    1656             :   }
    1657             : }
    1658             : 
    1659         352 : wasm::WasmOpcode TruncOp(const MachineType& ty) {
    1660         352 :   switch (ty.representation()) {
    1661             :     case MachineRepresentation::kFloat32:
    1662             :       return wasm::kExprF32Trunc;
    1663             :     case MachineRepresentation::kFloat64:
    1664         172 :       return wasm::kExprF64Trunc;
    1665             :     default:
    1666           0 :       UNREACHABLE();
    1667             :   }
    1668             : }
    1669             : 
    1670         415 : wasm::WasmOpcode NeOp(const MachineType& ty) {
    1671         415 :   switch (ty.representation()) {
    1672             :     case MachineRepresentation::kFloat32:
    1673             :       return wasm::kExprF32Ne;
    1674             :     case MachineRepresentation::kFloat64:
    1675         203 :       return wasm::kExprF64Ne;
    1676             :     default:
    1677           0 :       UNREACHABLE();
    1678             :   }
    1679             : }
    1680             : 
    1681          64 : wasm::WasmOpcode LtOp(const MachineType& ty) {
    1682          64 :   switch (ty.representation()) {
    1683             :     case MachineRepresentation::kFloat32:
    1684             :       return wasm::kExprF32Lt;
    1685             :     case MachineRepresentation::kFloat64:
    1686          32 :       return wasm::kExprF64Lt;
    1687             :     default:
    1688           0 :       UNREACHABLE();
    1689             :   }
    1690             : }
    1691             : 
    1692        1164 : Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
    1693        1164 :                       const MachineType& int_ty, const MachineType& float_ty,
    1694             :                       Node* trunc, Node* converted_value) {
    1695        1164 :   if (int_ty.representation() == MachineRepresentation::kWord32) {
    1696         352 :     Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
    1697         351 :     return builder->Binop(NeOp(float_ty), trunc, check);
    1698             :   }
    1699             :   return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
    1700        1625 :                                    trunc, builder->graph()->start());
    1701             : }
    1702             : 
    1703          64 : Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
    1704          64 :                           const MachineType& int_ty,
    1705             :                           const MachineType& float_ty, Node* trunc,
    1706             :                           Node* converted_value) {
    1707             :   Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
    1708          64 :                                converted_value);
    1709          64 :   if (int_ty.representation() == MachineRepresentation::kWord64) {
    1710          32 :     test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
    1711             :   }
    1712          64 :   return test;
    1713             : }
    1714             : 
    1715             : }  // namespace
    1716             : 
    1717        1161 : Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
    1718             :                                              wasm::WasmCodePosition position,
    1719         256 :                                              wasm::WasmOpcode opcode) {
    1720        1161 :   const MachineType int_ty = IntConvertType(opcode);
    1721        1164 :   const MachineType float_ty = FloatConvertType(opcode);
    1722        1167 :   const Operator* conv_op = ConvertOp(this, opcode);
    1723             :   Node* trunc = nullptr;
    1724             :   Node* converted_value = nullptr;
    1725             :   const bool is_int32 =
    1726        1164 :       int_ty.representation() == MachineRepresentation::kWord32;
    1727        1164 :   if (is_int32) {
    1728         352 :     trunc = Unop(TruncOp(float_ty), input);
    1729             :     converted_value = graph()->NewNode(conv_op, trunc);
    1730             :   } else {
    1731             :     trunc = graph()->NewNode(conv_op, input);
    1732             :     converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
    1733         811 :                                        trunc, graph()->start());
    1734             :   }
    1735        1167 :   if (IsTrappingConvertOp(opcode)) {
    1736             :     Node* test =
    1737        1104 :         ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
    1738        1102 :     if (is_int32) {
    1739         319 :       TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
    1740             :     } else {
    1741             :       ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
    1742             :     }
    1743        1097 :     return converted_value;
    1744             :   }
    1745             :   Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
    1746          64 :                                    converted_value);
    1747         128 :   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
    1748          64 :   tl_d.Chain(Control());
    1749          64 :   Node* nan_test = Binop(NeOp(float_ty), input, input);
    1750         128 :   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
    1751          64 :   nan_d.Nest(tl_d, true);
    1752          64 :   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
    1753         128 :   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
    1754          64 :   sat_d.Nest(nan_d, false);
    1755             :   Node* sat_val =
    1756          64 :       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
    1757             :   Node* nan_val =
    1758          64 :       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
    1759          64 :   return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
    1760             : }
    1761             : 
    1762           8 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
    1763           8 :   MachineOperatorBuilder* m = mcgraph()->machine();
    1764             :   // asm.js must use the wacky JS semantics.
    1765           8 :   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
    1766          16 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1767             : }
    1768             : 
    1769         108 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
    1770         108 :   MachineOperatorBuilder* m = mcgraph()->machine();
    1771             :   // asm.js must use the wacky JS semantics.
    1772         216 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1773             : }
    1774             : 
    1775           8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
    1776           8 :   MachineOperatorBuilder* m = mcgraph()->machine();
    1777             :   // asm.js must use the wacky JS semantics.
    1778           8 :   input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
    1779          16 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1780             : }
    1781             : 
    1782           8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
    1783           8 :   MachineOperatorBuilder* m = mcgraph()->machine();
    1784             :   // asm.js must use the wacky JS semantics.
    1785          16 :   return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
    1786             : }
    1787             : 
    1788           0 : Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
    1789           0 :                                              MachineRepresentation input_type) {
    1790             :   Node* stack_slot_param =
    1791           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
    1792             : 
    1793             :   const Operator* store_op = mcgraph()->machine()->Store(
    1794           0 :       StoreRepresentation(input_type, kNoWriteBarrier));
    1795             :   SetEffect(graph()->NewNode(store_op, stack_slot_param,
    1796             :                              mcgraph()->Int32Constant(0), input, Effect(),
    1797           0 :                              Control()));
    1798             : 
    1799           0 :   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    1800             :   MachineSignature sig(1, 1, sig_types);
    1801             : 
    1802           0 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    1803             : 
    1804           0 :   return BuildCCall(&sig, function, stack_slot_param);
    1805             : }
    1806             : 
    1807           0 : Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
    1808             :   return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
    1809           0 :                               MachineRepresentation::kWord32);
    1810             : }
    1811             : 
    1812           0 : Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
    1813             :   return Unop(wasm::kExprI64UConvertI32,
    1814             :               BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
    1815           0 :                                    MachineRepresentation::kWord64));
    1816             : }
    1817             : 
    1818           0 : Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
    1819             :   return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
    1820           0 :                               MachineRepresentation::kWord32);
    1821             : }
    1822             : 
    1823           0 : Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
    1824             :   return Unop(
    1825             :       wasm::kExprI64UConvertI32,
    1826             :       BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
    1827           0 :                            MachineRepresentation::kWord64));
    1828             : }
    1829             : 
    1830           0 : Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
    1831           0 :   MachineType type = MachineType::Float32();
    1832           0 :   ExternalReference ref = ExternalReference::wasm_f32_trunc();
    1833             : 
    1834           0 :   return BuildCFuncInstruction(ref, type, input);
    1835             : }
    1836             : 
    1837           0 : Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
    1838           0 :   MachineType type = MachineType::Float32();
    1839           0 :   ExternalReference ref = ExternalReference::wasm_f32_floor();
    1840           0 :   return BuildCFuncInstruction(ref, type, input);
    1841             : }
    1842             : 
    1843           0 : Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
    1844           0 :   MachineType type = MachineType::Float32();
    1845           0 :   ExternalReference ref = ExternalReference::wasm_f32_ceil();
    1846           0 :   return BuildCFuncInstruction(ref, type, input);
    1847             : }
    1848             : 
    1849           0 : Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
    1850           0 :   MachineType type = MachineType::Float32();
    1851           0 :   ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
    1852           0 :   return BuildCFuncInstruction(ref, type, input);
    1853             : }
    1854             : 
    1855           0 : Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
    1856           0 :   MachineType type = MachineType::Float64();
    1857           0 :   ExternalReference ref = ExternalReference::wasm_f64_trunc();
    1858           0 :   return BuildCFuncInstruction(ref, type, input);
    1859             : }
    1860             : 
    1861           0 : Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
    1862           0 :   MachineType type = MachineType::Float64();
    1863           0 :   ExternalReference ref = ExternalReference::wasm_f64_floor();
    1864           0 :   return BuildCFuncInstruction(ref, type, input);
    1865             : }
    1866             : 
    1867           0 : Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
    1868           0 :   MachineType type = MachineType::Float64();
    1869           0 :   ExternalReference ref = ExternalReference::wasm_f64_ceil();
    1870           0 :   return BuildCFuncInstruction(ref, type, input);
    1871             : }
    1872             : 
    1873           0 : Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
    1874           0 :   MachineType type = MachineType::Float64();
    1875           0 :   ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
    1876           0 :   return BuildCFuncInstruction(ref, type, input);
    1877             : }
    1878             : 
    1879          21 : Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
    1880          21 :   MachineType type = MachineType::Float64();
    1881          21 :   ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
    1882          21 :   return BuildCFuncInstruction(ref, type, input);
    1883             : }
    1884             : 
    1885          21 : Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
    1886          21 :   MachineType type = MachineType::Float64();
    1887          21 :   ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
    1888          21 :   return BuildCFuncInstruction(ref, type, input);
    1889             : }
    1890             : 
    1891           9 : Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
    1892           9 :   MachineType type = MachineType::Float64();
    1893           9 :   ExternalReference ref = ExternalReference::wasm_float64_pow();
    1894           9 :   return BuildCFuncInstruction(ref, type, left, right);
    1895             : }
    1896             : 
    1897          20 : Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
    1898          20 :   MachineType type = MachineType::Float64();
    1899          20 :   ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
    1900          20 :   return BuildCFuncInstruction(ref, type, left, right);
    1901             : }
    1902             : 
    1903          71 : Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
    1904             :                                               MachineType type, Node* input0,
    1905         968 :                                               Node* input1) {
    1906             :   // We do truncation by calling a C function which calculates the result.
    1907             :   // The input is passed to the C function as a byte buffer holding the two
    1908             :   // input doubles. We reserve this byte buffer as a stack slot, store the
    1909             :   // parameters in this buffer slots, pass a pointer to the buffer to the C
    1910             :   // function, and after calling the C function we collect the return value from
    1911             :   // the buffer.
    1912             : 
    1913             :   const int type_size = ElementSizeInBytes(type.representation());
    1914          71 :   const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
    1915             :   Node* stack_slot =
    1916          71 :       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
    1917             : 
    1918             :   const Operator* store_op = mcgraph()->machine()->Store(
    1919          71 :       StoreRepresentation(type.representation(), kNoWriteBarrier));
    1920             :   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
    1921          71 :                              input0, Effect(), Control()));
    1922             : 
    1923          71 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    1924             : 
    1925          71 :   if (input1 != nullptr) {
    1926             :     SetEffect(graph()->NewNode(store_op, stack_slot,
    1927             :                                mcgraph()->Int32Constant(type_size), input1,
    1928          29 :                                Effect(), Control()));
    1929             :   }
    1930             : 
    1931          71 :   MachineType sig_types[] = {MachineType::Pointer()};
    1932             :   MachineSignature sig(0, 1, sig_types);
    1933          71 :   BuildCCall(&sig, function, stack_slot);
    1934             : 
    1935             :   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
    1936             :                                     stack_slot, mcgraph()->Int32Constant(0),
    1937         213 :                                     Effect(), Control()));
    1938             : }
    1939             : 
    1940           0 : Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
    1941             :   // TODO(titzer/bradnelson): Check handlng of asm.js case.
    1942             :   return BuildIntToFloatConversionInstruction(
    1943             :       input, ExternalReference::wasm_int64_to_float32(),
    1944           0 :       MachineRepresentation::kWord64, MachineType::Float32());
    1945             : }
    1946           0 : Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
    1947             :   // TODO(titzer/bradnelson): Check handlng of asm.js case.
    1948             :   return BuildIntToFloatConversionInstruction(
    1949             :       input, ExternalReference::wasm_uint64_to_float32(),
    1950           0 :       MachineRepresentation::kWord64, MachineType::Float32());
    1951             : }
    1952           0 : Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
    1953             :   return BuildIntToFloatConversionInstruction(
    1954             :       input, ExternalReference::wasm_int64_to_float64(),
    1955           0 :       MachineRepresentation::kWord64, MachineType::Float64());
    1956             : }
    1957           0 : Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
    1958             :   return BuildIntToFloatConversionInstruction(
    1959             :       input, ExternalReference::wasm_uint64_to_float64(),
    1960           0 :       MachineRepresentation::kWord64, MachineType::Float64());
    1961             : }
    1962             : 
    1963           0 : Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
    1964             :     Node* input, ExternalReference ref,
    1965             :     MachineRepresentation parameter_representation,
    1966           0 :     const MachineType result_type) {
    1967             :   int stack_slot_size =
    1968             :       std::max(ElementSizeInBytes(parameter_representation),
    1969           0 :                ElementSizeInBytes(result_type.representation()));
    1970             :   Node* stack_slot =
    1971           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
    1972             :   const Operator* store_op = mcgraph()->machine()->Store(
    1973           0 :       StoreRepresentation(parameter_representation, kNoWriteBarrier));
    1974             :   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
    1975           0 :                              input, Effect(), Control()));
    1976           0 :   MachineType sig_types[] = {MachineType::Pointer()};
    1977             :   MachineSignature sig(0, 1, sig_types);
    1978           0 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    1979           0 :   BuildCCall(&sig, function, stack_slot);
    1980             :   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
    1981             :                                     stack_slot, mcgraph()->Int32Constant(0),
    1982           0 :                                     Effect(), Control()));
    1983             : }
    1984             : 
    1985             : namespace {
    1986             : 
    1987           0 : ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
    1988             :                                     wasm::WasmOpcode opcode) {
    1989           0 :   switch (opcode) {
    1990             :     case wasm::kExprI64SConvertF32:
    1991             :     case wasm::kExprI64SConvertSatF32:
    1992           0 :       return ExternalReference::wasm_float32_to_int64();
    1993             :     case wasm::kExprI64UConvertF32:
    1994             :     case wasm::kExprI64UConvertSatF32:
    1995           0 :       return ExternalReference::wasm_float32_to_uint64();
    1996             :     case wasm::kExprI64SConvertF64:
    1997             :     case wasm::kExprI64SConvertSatF64:
    1998           0 :       return ExternalReference::wasm_float64_to_int64();
    1999             :     case wasm::kExprI64UConvertF64:
    2000             :     case wasm::kExprI64UConvertSatF64:
    2001           0 :       return ExternalReference::wasm_float64_to_uint64();
    2002             :     default:
    2003           0 :       UNREACHABLE();
    2004             :   }
    2005             : }
    2006             : 
    2007             : }  // namespace
    2008             : 
    2009           0 : Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
    2010             :                                                wasm::WasmCodePosition position,
    2011           0 :                                                wasm::WasmOpcode opcode) {
    2012           0 :   const MachineType int_ty = IntConvertType(opcode);
    2013           0 :   const MachineType float_ty = FloatConvertType(opcode);
    2014           0 :   ExternalReference call_ref = convert_ccall_ref(this, opcode);
    2015           0 :   int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
    2016           0 :                                  ElementSizeInBytes(float_ty.representation()));
    2017             :   Node* stack_slot =
    2018           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
    2019             :   const Operator* store_op = mcgraph()->machine()->Store(
    2020           0 :       StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
    2021             :   SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
    2022             :                              Effect(), Control()));
    2023           0 :   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    2024             :   MachineSignature sig(1, 1, sig_types);
    2025             :   Node* function =
    2026           0 :       graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
    2027           0 :   Node* overflow = BuildCCall(&sig, function, stack_slot);
    2028           0 :   if (IsTrappingConvertOp(opcode)) {
    2029             :     ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
    2030             :     return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
    2031             :                                       stack_slot, Int32Constant(0), Effect(),
    2032           0 :                                       Control()));
    2033             :   }
    2034           0 :   Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
    2035           0 :   Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
    2036           0 :   tl_d.Chain(Control());
    2037           0 :   Node* nan_test = Binop(NeOp(float_ty), input, input);
    2038           0 :   Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
    2039           0 :   nan_d.Nest(tl_d, true);
    2040           0 :   Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
    2041           0 :   Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
    2042           0 :   sat_d.Nest(nan_d, false);
    2043             :   Node* sat_val =
    2044           0 :       sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
    2045             :   Node* load =
    2046             :       SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
    2047           0 :                                  Int32Constant(0), Effect(), Control()));
    2048             :   Node* nan_val =
    2049           0 :       nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
    2050           0 :   return tl_d.Phi(int_ty.representation(), nan_val, load);
    2051             : }
    2052             : 
    2053        8960 : Node* WasmGraphBuilder::MemoryGrow(Node* input) {
    2054        1120 :   needs_stack_check_ = true;
    2055             : 
    2056             :   WasmMemoryGrowDescriptor interface_descriptor;
    2057             :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    2058             :       mcgraph()->zone(),                              // zone
    2059             :       interface_descriptor,                           // descriptor
    2060             :       interface_descriptor.GetStackParameterCount(),  // stack parameter count
    2061             :       CallDescriptor::kNoFlags,                       // flags
    2062             :       Operator::kNoProperties,                        // properties
    2063        1120 :       StubCallMode::kCallWasmRuntimeStub);            // stub call mode
    2064             :   // A direct call to a wasm runtime stub defined in this module.
    2065             :   // Just encode the stub index. This will be patched at relocation.
    2066             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    2067        1120 :       wasm::WasmCode::kWasmMemoryGrow, RelocInfo::WASM_STUB_CALL);
    2068             :   return SetEffect(
    2069             :       SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    2070        2240 :                                   call_target, input, Effect(), Control())));
    2071             : }
    2072             : 
    2073         503 : Node* WasmGraphBuilder::Throw(uint32_t exception_index,
    2074             :                               const wasm::WasmException* exception,
    2075        4190 :                               const Vector<Node*> values) {
    2076         503 :   needs_stack_check_ = true;
    2077         503 :   uint32_t encoded_size = WasmExceptionPackage::GetEncodedSize(exception);
    2078             :   Node* create_parameters[] = {
    2079         503 :       LoadExceptionTagFromTable(exception_index),
    2080        1008 :       BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
    2081             :   Node* except_obj =
    2082             :       BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
    2083         504 :                          arraysize(create_parameters));
    2084             :   Node* values_array =
    2085         504 :       BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
    2086         503 :   uint32_t index = 0;
    2087        1534 :   const wasm::WasmExceptionSig* sig = exception->sig;
    2088         503 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2089        1534 :   for (size_t i = 0; i < sig->parameter_count(); ++i) {
    2090         528 :     Node* value = values[i];
    2091         264 :     switch (sig->GetParam(i)) {
    2092             :       case wasm::kWasmF32:
    2093          16 :         value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
    2094             :         V8_FALLTHROUGH;
    2095             :       case wasm::kWasmI32:
    2096         184 :         BuildEncodeException32BitValue(values_array, &index, value);
    2097         184 :         break;
    2098             :       case wasm::kWasmF64:
    2099          16 :         value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
    2100             :         V8_FALLTHROUGH;
    2101             :       case wasm::kWasmI64: {
    2102             :         Node* upper32 = graph()->NewNode(
    2103             :             m->TruncateInt64ToInt32(),
    2104          32 :             Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
    2105          32 :         BuildEncodeException32BitValue(values_array, &index, upper32);
    2106          32 :         Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
    2107          32 :         BuildEncodeException32BitValue(values_array, &index, lower32);
    2108          32 :         break;
    2109             :       }
    2110             :       case wasm::kWasmS128:
    2111             :         BuildEncodeException32BitValue(
    2112             :             values_array, &index,
    2113          32 :             graph()->NewNode(m->I32x4ExtractLane(0), value));
    2114             :         BuildEncodeException32BitValue(
    2115             :             values_array, &index,
    2116          32 :             graph()->NewNode(m->I32x4ExtractLane(1), value));
    2117             :         BuildEncodeException32BitValue(
    2118             :             values_array, &index,
    2119          32 :             graph()->NewNode(m->I32x4ExtractLane(2), value));
    2120             :         BuildEncodeException32BitValue(
    2121             :             values_array, &index,
    2122          32 :             graph()->NewNode(m->I32x4ExtractLane(3), value));
    2123          16 :         break;
    2124             :       case wasm::kWasmAnyRef:
    2125         128 :         STORE_FIXED_ARRAY_SLOT_ANY(values_array, index, value);
    2126          32 :         ++index;
    2127          32 :         break;
    2128             :       default:
    2129           0 :         UNREACHABLE();
    2130             :     }
    2131             :   }
    2132             :   DCHECK_EQ(encoded_size, index);
    2133             :   WasmThrowDescriptor interface_descriptor;
    2134             :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    2135             :       mcgraph()->zone(), interface_descriptor,
    2136             :       interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
    2137         503 :       Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
    2138             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    2139         504 :       wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
    2140             :   return SetEffect(SetControl(
    2141             :       graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
    2142        1512 :                        except_obj, Effect(), Control())));
    2143             : }
    2144             : 
    2145         312 : void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
    2146             :                                                       uint32_t* index,
    2147        3432 :                                                       Node* value) {
    2148         312 :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2149             :   Node* upper_halfword_as_smi = BuildChangeUint31ToSmi(
    2150         624 :       graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16)));
    2151        1248 :   STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, upper_halfword_as_smi);
    2152         312 :   ++(*index);
    2153             :   Node* lower_halfword_as_smi = BuildChangeUint31ToSmi(
    2154         624 :       graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu)));
    2155        1248 :   STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, lower_halfword_as_smi);
    2156         312 :   ++(*index);
    2157         312 : }
    2158             : 
    2159         272 : Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
    2160        2720 :                                                        uint32_t* index) {
    2161         272 :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2162             :   Node* upper =
    2163        1088 :       BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
    2164         272 :   (*index)++;
    2165         272 :   upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
    2166             :   Node* lower =
    2167        1088 :       BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
    2168         272 :   (*index)++;
    2169         272 :   Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
    2170         272 :   return value;
    2171             : }
    2172             : 
    2173          16 : Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
    2174             :                                                        uint32_t* index) {
    2175             :   Node* upper = Binop(wasm::kExprI64Shl,
    2176             :                       Unop(wasm::kExprI64UConvertI32,
    2177             :                            BuildDecodeException32BitValue(values_array, index)),
    2178          16 :                       Int64Constant(32));
    2179             :   Node* lower = Unop(wasm::kExprI64UConvertI32,
    2180          16 :                      BuildDecodeException32BitValue(values_array, index));
    2181          16 :   return Binop(wasm::kExprI64Ior, upper, lower);
    2182             : }
    2183             : 
    2184        3385 : Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
    2185         423 :   needs_stack_check_ = true;
    2186             :   WasmThrowDescriptor interface_descriptor;
    2187             :   auto call_descriptor = Linkage::GetStubCallDescriptor(
    2188             :       mcgraph()->zone(), interface_descriptor,
    2189             :       interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
    2190         423 :       Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
    2191             :   Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    2192         422 :       wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
    2193             :   return SetEffect(SetControl(
    2194             :       graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
    2195         847 :                        except_obj, Effect(), Control())));
    2196             : }
    2197             : 
    2198         376 : Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
    2199         376 :                                           Node* expected_tag) {
    2200         376 :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2201         752 :   return graph()->NewNode(machine->WordEqual(), caught_tag, expected_tag);
    2202             : }
    2203             : 
    2204        8796 : Node* WasmGraphBuilder::LoadExceptionTagFromTable(uint32_t exception_index) {
    2205             :   Node* exceptions_table =
    2206        1758 :       LOAD_INSTANCE_FIELD(ExceptionsTable, MachineType::TaggedPointer());
    2207        2640 :   Node* tag = LOAD_FIXED_ARRAY_SLOT_PTR(exceptions_table, exception_index);
    2208         880 :   return tag;
    2209             : }
    2210             : 
    2211         376 : Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
    2212         376 :   needs_stack_check_ = true;
    2213         376 :   return BuildCallToRuntime(Runtime::kWasmExceptionGetTag, &except_obj, 1);
    2214             : }
    2215             : 
    2216         376 : Node** WasmGraphBuilder::GetExceptionValues(
    2217         112 :     Node* except_obj, const wasm::WasmException* exception) {
    2218             :   Node* values_array =
    2219         376 :       BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
    2220         375 :   uint32_t index = 0;
    2221        1246 :   const wasm::WasmExceptionSig* sig = exception->sig;
    2222         375 :   Node** values = Buffer(sig->parameter_count());
    2223        1246 :   for (size_t i = 0; i < sig->parameter_count(); ++i) {
    2224             :     Node* value;
    2225         248 :     switch (sig->GetParam(i)) {
    2226             :       case wasm::kWasmI32:
    2227         200 :         value = BuildDecodeException32BitValue(values_array, &index);
    2228         200 :         break;
    2229             :       case wasm::kWasmI64:
    2230           8 :         value = BuildDecodeException64BitValue(values_array, &index);
    2231           8 :         break;
    2232             :       case wasm::kWasmF32: {
    2233             :         value = Unop(wasm::kExprF32ReinterpretI32,
    2234           8 :                      BuildDecodeException32BitValue(values_array, &index));
    2235           8 :         break;
    2236             :       }
    2237             :       case wasm::kWasmF64: {
    2238             :         value = Unop(wasm::kExprF64ReinterpretI64,
    2239           8 :                      BuildDecodeException64BitValue(values_array, &index));
    2240           8 :         break;
    2241             :       }
    2242             :       case wasm::kWasmS128:
    2243             :         value = graph()->NewNode(
    2244             :             mcgraph()->machine()->I32x4Splat(),
    2245          16 :             BuildDecodeException32BitValue(values_array, &index));
    2246             :         value = graph()->NewNode(
    2247             :             mcgraph()->machine()->I32x4ReplaceLane(1), value,
    2248          16 :             BuildDecodeException32BitValue(values_array, &index));
    2249             :         value = graph()->NewNode(
    2250             :             mcgraph()->machine()->I32x4ReplaceLane(2), value,
    2251          16 :             BuildDecodeException32BitValue(values_array, &index));
    2252             :         value = graph()->NewNode(
    2253             :             mcgraph()->machine()->I32x4ReplaceLane(3), value,
    2254          16 :             BuildDecodeException32BitValue(values_array, &index));
    2255           8 :         break;
    2256             :       case wasm::kWasmAnyRef:
    2257          48 :         value = LOAD_FIXED_ARRAY_SLOT_ANY(values_array, index);
    2258          16 :         ++index;
    2259          16 :         break;
    2260             :       default:
    2261           0 :         UNREACHABLE();
    2262             :     }
    2263         248 :     values[i] = value;
    2264             :   }
    2265             :   DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(exception));
    2266         375 :   return values;
    2267             : }
    2268             : 
    2269       14180 : Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
    2270       86992 :                                      wasm::WasmCodePosition position) {
    2271       14180 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2272             :   ZeroCheck32(wasm::kTrapDivByZero, right, position);
    2273             :   Node* before = Control();
    2274             :   Node* denom_is_m1;
    2275             :   Node* denom_is_not_m1;
    2276             :   BranchExpectFalse(
    2277             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
    2278       14180 :       &denom_is_m1, &denom_is_not_m1);
    2279       14180 :   SetControl(denom_is_m1);
    2280       14180 :   TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
    2281       14180 :   if (Control() != denom_is_m1) {
    2282             :     SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
    2283        1912 :                                 Control()));
    2284             :   } else {
    2285             :     SetControl(before);
    2286             :   }
    2287       28360 :   return graph()->NewNode(m->Int32Div(), left, right, Control());
    2288             : }
    2289             : 
    2290       14104 : Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
    2291       70520 :                                      wasm::WasmCodePosition position) {
    2292       14104 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2293             : 
    2294             :   ZeroCheck32(wasm::kTrapRemByZero, right, position);
    2295             : 
    2296             :   Diamond d(
    2297             :       graph(), mcgraph()->common(),
    2298             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
    2299       42312 :       BranchHint::kFalse);
    2300       14104 :   d.Chain(Control());
    2301             : 
    2302             :   return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
    2303       28208 :                graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
    2304             : }
    2305             : 
    2306       14124 : Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
    2307       14124 :                                      wasm::WasmCodePosition position) {
    2308       14124 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2309             :   return graph()->NewNode(m->Uint32Div(), left, right,
    2310       28248 :                           ZeroCheck32(wasm::kTrapDivByZero, right, position));
    2311             : }
    2312             : 
    2313       14084 : Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
    2314       14084 :                                      wasm::WasmCodePosition position) {
    2315       14084 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2316             :   return graph()->NewNode(m->Uint32Mod(), left, right,
    2317       28168 :                           ZeroCheck32(wasm::kTrapRemByZero, right, position));
    2318             : }
    2319             : 
    2320        1016 : Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
    2321         385 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2322             : 
    2323             :   Int32Matcher mr(right);
    2324         385 :   if (mr.HasValue()) {
    2325         317 :     if (mr.Value() == 0) {
    2326          13 :       return mcgraph()->Int32Constant(0);
    2327         304 :     } else if (mr.Value() == -1) {
    2328             :       // The result is the negation of the left input.
    2329          26 :       return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
    2330             :     }
    2331         582 :     return graph()->NewNode(m->Int32Div(), left, right, Control());
    2332             :   }
    2333             : 
    2334             :   // asm.js semantics return 0 on divide or mod by zero.
    2335          68 :   if (m->Int32DivIsSafe()) {
    2336             :     // The hardware instruction does the right thing (e.g. arm).
    2337           0 :     return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
    2338             :   }
    2339             : 
    2340             :   // Check denominator for zero.
    2341             :   Diamond z(
    2342             :       graph(), mcgraph()->common(),
    2343             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
    2344         204 :       BranchHint::kFalse);
    2345             : 
    2346             :   // Check numerator for -1. (avoid minint / -1 case).
    2347             :   Diamond n(
    2348             :       graph(), mcgraph()->common(),
    2349             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
    2350         204 :       BranchHint::kFalse);
    2351             : 
    2352          68 :   Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
    2353             :   Node* neg =
    2354          68 :       graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
    2355             : 
    2356             :   return n.Phi(
    2357             :       MachineRepresentation::kWord32, neg,
    2358          68 :       z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
    2359             : }
    2360             : 
    2361         816 : Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
    2362         421 :   CommonOperatorBuilder* c = mcgraph()->common();
    2363         421 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2364         421 :   Node* const zero = mcgraph()->Int32Constant(0);
    2365             : 
    2366             :   Int32Matcher mr(right);
    2367         421 :   if (mr.HasValue()) {
    2368         360 :     if (mr.Value() == 0 || mr.Value() == -1) {
    2369             :       return zero;
    2370             :     }
    2371         668 :     return graph()->NewNode(m->Int32Mod(), left, right, Control());
    2372             :   }
    2373             : 
    2374             :   // General case for signed integer modulus, with optimization for (unknown)
    2375             :   // power of 2 right hand side.
    2376             :   //
    2377             :   //   if 0 < right then
    2378             :   //     msk = right - 1
    2379             :   //     if right & msk != 0 then
    2380             :   //       left % right
    2381             :   //     else
    2382             :   //       if left < 0 then
    2383             :   //         -(-left & msk)
    2384             :   //       else
    2385             :   //         left & msk
    2386             :   //   else
    2387             :   //     if right < -1 then
    2388             :   //       left % right
    2389             :   //     else
    2390             :   //       zero
    2391             :   //
    2392             :   // Note: We do not use the Diamond helper class here, because it really hurts
    2393             :   // readability with nested diamonds.
    2394          61 :   Node* const minus_one = mcgraph()->Int32Constant(-1);
    2395             : 
    2396          61 :   const Operator* const merge_op = c->Merge(2);
    2397          61 :   const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
    2398             : 
    2399          61 :   Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
    2400             :   Node* branch0 =
    2401          61 :       graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
    2402             : 
    2403          61 :   Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
    2404             :   Node* true0;
    2405             :   {
    2406          61 :     Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
    2407             : 
    2408          61 :     Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
    2409          61 :     Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
    2410             : 
    2411          61 :     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
    2412          61 :     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
    2413             : 
    2414          61 :     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
    2415             :     Node* false1;
    2416             :     {
    2417          61 :       Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
    2418             :       Node* branch2 =
    2419          61 :           graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
    2420             : 
    2421          61 :       Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
    2422             :       Node* true2 = graph()->NewNode(
    2423             :           m->Int32Sub(), zero,
    2424             :           graph()->NewNode(m->Word32And(),
    2425         183 :                            graph()->NewNode(m->Int32Sub(), zero, left), msk));
    2426             : 
    2427          61 :       Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
    2428          61 :       Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
    2429             : 
    2430             :       if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
    2431             :       false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
    2432             :     }
    2433             : 
    2434             :     if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
    2435             :     true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
    2436             :   }
    2437             : 
    2438          61 :   Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
    2439             :   Node* false0;
    2440             :   {
    2441          61 :     Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
    2442             :     Node* branch1 =
    2443          61 :         graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
    2444             : 
    2445          61 :     Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
    2446          61 :     Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
    2447             : 
    2448          61 :     Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
    2449             :     Node* false1 = zero;
    2450             : 
    2451             :     if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
    2452             :     false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
    2453             :   }
    2454             : 
    2455             :   Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
    2456          61 :   return graph()->NewNode(phi_op, true0, false0, merge0);
    2457             : }
    2458             : 
    2459         548 : Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
    2460         137 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2461             :   // asm.js semantics return 0 on divide or mod by zero.
    2462         137 :   if (m->Uint32DivIsSafe()) {
    2463             :     // The hardware instruction does the right thing (e.g. arm).
    2464           0 :     return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
    2465             :   }
    2466             : 
    2467             :   // Explicit check for x % 0.
    2468             :   Diamond z(
    2469             :       graph(), mcgraph()->common(),
    2470             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
    2471         411 :       BranchHint::kFalse);
    2472             : 
    2473             :   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
    2474             :                graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
    2475         411 :                                 z.if_false));
    2476             : }
    2477             : 
    2478         744 : Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
    2479         186 :   MachineOperatorBuilder* m = mcgraph()->machine();
    2480             :   // asm.js semantics return 0 on divide or mod by zero.
    2481             :   // Explicit check for x % 0.
    2482             :   Diamond z(
    2483             :       graph(), mcgraph()->common(),
    2484             :       graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
    2485         558 :       BranchHint::kFalse);
    2486             : 
    2487             :   Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
    2488         372 :                                z.if_false);
    2489             :   return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
    2490         186 :                rem);
    2491             : }
    2492             : 
    2493         720 : Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
    2494        7192 :                                      wasm::WasmCodePosition position) {
    2495        1440 :   if (mcgraph()->machine()->Is32()) {
    2496             :     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
    2497           0 :                           MachineType::Int64(), wasm::kTrapDivByZero, position);
    2498             :   }
    2499             :   ZeroCheck64(wasm::kTrapDivByZero, right, position);
    2500             :   Node* before = Control();
    2501             :   Node* denom_is_m1;
    2502             :   Node* denom_is_not_m1;
    2503             :   BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
    2504             :                                      mcgraph()->Int64Constant(-1)),
    2505        1440 :                     &denom_is_m1, &denom_is_not_m1);
    2506         720 :   SetControl(denom_is_m1);
    2507             :   TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
    2508         720 :              std::numeric_limits<int64_t>::min(), position);
    2509         720 :   if (Control() != denom_is_m1) {
    2510             :     SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
    2511        1432 :                                 Control()));
    2512             :   } else {
    2513             :     SetControl(before);
    2514             :   }
    2515             :   return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
    2516        1440 :                           Control());
    2517             : }
    2518             : 
    2519         644 : Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
    2520        4508 :                                      wasm::WasmCodePosition position) {
    2521        1288 :   if (mcgraph()->machine()->Is32()) {
    2522             :     return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
    2523           0 :                           MachineType::Int64(), wasm::kTrapRemByZero, position);
    2524             :   }
    2525             :   ZeroCheck64(wasm::kTrapRemByZero, right, position);
    2526             :   Diamond d(mcgraph()->graph(), mcgraph()->common(),
    2527             :             graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
    2528        1932 :                              mcgraph()->Int64Constant(-1)));
    2529             : 
    2530         644 :   d.Chain(Control());
    2531             : 
    2532             :   Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
    2533        1288 :                                d.if_false);
    2534             : 
    2535             :   return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
    2536         644 :                rem);
    2537             : }
    2538             : 
    2539         660 : Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
    2540        1320 :                                      wasm::WasmCodePosition position) {
    2541        1320 :   if (mcgraph()->machine()->Is32()) {
    2542             :     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
    2543           0 :                           MachineType::Int64(), wasm::kTrapDivByZero, position);
    2544             :   }
    2545             :   return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
    2546        1320 :                           ZeroCheck64(wasm::kTrapDivByZero, right, position));
    2547             : }
    2548         644 : Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
    2549        1288 :                                      wasm::WasmCodePosition position) {
    2550        1288 :   if (mcgraph()->machine()->Is32()) {
    2551             :     return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
    2552           0 :                           MachineType::Int64(), wasm::kTrapRemByZero, position);
    2553             :   }
    2554             :   return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
    2555        1288 :                           ZeroCheck64(wasm::kTrapRemByZero, right, position));
    2556             : }
    2557             : 
    2558           0 : Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
    2559             :                                        ExternalReference ref,
    2560             :                                        MachineType result_type,
    2561             :                                        wasm::TrapReason trap_zero,
    2562           0 :                                        wasm::WasmCodePosition position) {
    2563             :   Node* stack_slot =
    2564           0 :       graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
    2565             : 
    2566             :   const Operator* store_op = mcgraph()->machine()->Store(
    2567           0 :       StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
    2568             :   SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
    2569           0 :                              left, Effect(), Control()));
    2570             :   SetEffect(graph()->NewNode(store_op, stack_slot,
    2571             :                              mcgraph()->Int32Constant(sizeof(double)), right,
    2572           0 :                              Effect(), Control()));
    2573             : 
    2574           0 :   MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
    2575             :   MachineSignature sig(1, 1, sig_types);
    2576             : 
    2577           0 :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
    2578           0 :   Node* call = BuildCCall(&sig, function, stack_slot);
    2579             : 
    2580             :   ZeroCheck32(trap_zero, call, position);
    2581           0 :   TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
    2582             :   return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
    2583             :                                     stack_slot, mcgraph()->Int32Constant(0),
    2584           0 :                                     Effect(), Control()));
    2585             : }
    2586             : 
    2587             : template <typename... Args>
    2588         159 : Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
    2589         440 :                                    Args... args) {
    2590             :   DCHECK_LE(sig->return_count(), 1);
    2591             :   DCHECK_EQ(sizeof...(args), sig->parameter_count());
    2592         477 :   Node* const call_args[] = {function, args..., Effect(), Control()};
    2593             : 
    2594             :   auto call_descriptor =
    2595         159 :       Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
    2596             : 
    2597         159 :   const Operator* op = mcgraph()->common()->Call(call_descriptor);
    2598         318 :   return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
    2599             : }
    2600             : 
    2601      512350 : Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
    2602             :                                       Node*** rets,
    2603             :                                       wasm::WasmCodePosition position,
    2604             :                                       Node* instance_node,
    2605      853922 :                                       UseRetpoline use_retpoline) {
    2606      170783 :   if (instance_node == nullptr) {
    2607             :     DCHECK_NOT_NULL(instance_node_);
    2608             :     instance_node = instance_node_.get();
    2609             :   }
    2610      170783 :   needs_stack_check_ = true;
    2611             :   const size_t params = sig->parameter_count();
    2612             :   const size_t extra = 3;  // instance_node, effect, and control.
    2613      170783 :   const size_t count = 1 + params + extra;
    2614             : 
    2615             :   // Reallocate the buffer to make space for extra inputs.
    2616      170783 :   args = Realloc(args, 1 + params, count);
    2617             : 
    2618             :   // Make room for the instance_node parameter at index 1, just after code.
    2619      170783 :   memmove(&args[2], &args[1], params * sizeof(Node*));
    2620      170783 :   args[1] = instance_node;
    2621             : 
    2622             :   // Add effect and control inputs.
    2623      341566 :   args[params + 2] = Effect();
    2624      341566 :   args[params + 3] = Control();
    2625             : 
    2626             :   auto call_descriptor =
    2627      341566 :       GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
    2628      170786 :   const Operator* op = mcgraph()->common()->Call(call_descriptor);
    2629      341570 :   Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
    2630             :   DCHECK(position == wasm::kNoCodePosition || position > 0);
    2631      170787 :   if (position > 0) SetSourcePosition(call, position);
    2632             : 
    2633             :   size_t ret_count = sig->return_count();
    2634      170784 :   if (ret_count == 0) return call;  // No return value.
    2635             : 
    2636       37761 :   *rets = Buffer(ret_count);
    2637       37762 :   if (ret_count == 1) {
    2638             :     // Only a single return value.
    2639       36626 :     (*rets)[0] = call;
    2640             :   } else {
    2641             :     // Create projections for all return values.
    2642        2288 :     for (size_t i = 0; i < ret_count; i++) {
    2643        2288 :       (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
    2644        4576 :                                     graph()->start());
    2645             :     }
    2646             :   }
    2647             :   return call;
    2648             : }
    2649             : 
    2650        6942 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
    2651             :                                         Node*** rets,
    2652             :                                         wasm::WasmCodePosition position,
    2653      138872 :                                         int func_index) {
    2654             :   // Load the imported function refs array from the instance.
    2655             :   Node* imported_function_refs =
    2656       13885 :       LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
    2657             :   Node* ref_node =
    2658       13886 :       LOAD_FIXED_ARRAY_SLOT_PTR(imported_function_refs, func_index);
    2659             : 
    2660             :   // Load the target from the imported_targets array at a known offset.
    2661             :   Node* imported_targets =
    2662       13887 :       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
    2663             :   Node* target_node = SetEffect(graph()->NewNode(
    2664             :       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
    2665             :       mcgraph()->Int32Constant(func_index * kSystemPointerSize), Effect(),
    2666       20831 :       Control()));
    2667        6944 :   args[0] = target_node;
    2668             :   return BuildWasmCall(sig, args, rets, position, ref_node,
    2669        6944 :                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
    2670             : }
    2671             : 
    2672         464 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
    2673             :                                         Node*** rets,
    2674             :                                         wasm::WasmCodePosition position,
    2675       10208 :                                         Node* func_index) {
    2676             :   // Load the imported function refs array from the instance.
    2677             :   Node* imported_function_refs =
    2678         928 :       LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
    2679             :   // Access fixed array at {header_size - tag + func_index * kTaggedSize}.
    2680             :   Node* imported_instances_data = graph()->NewNode(
    2681             :       mcgraph()->machine()->IntAdd(), imported_function_refs,
    2682             :       mcgraph()->IntPtrConstant(
    2683         928 :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
    2684             :   Node* func_index_times_tagged_size = graph()->NewNode(
    2685             :       mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
    2686         928 :       mcgraph()->Int32Constant(kTaggedSize));
    2687             :   Node* ref_node = SetEffect(
    2688             :       graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    2689             :                        imported_instances_data, func_index_times_tagged_size,
    2690         464 :                        Effect(), Control()));
    2691             : 
    2692             :   // Load the target from the imported_targets array at the offset of
    2693             :   // {func_index}.
    2694             :   STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
    2695             :   Node* func_index_times_pointersize = func_index_times_tagged_size;
    2696             :   Node* imported_targets =
    2697         928 :       LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
    2698             :   Node* target_node = SetEffect(graph()->NewNode(
    2699             :       mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
    2700         464 :       func_index_times_pointersize, Effect(), Control()));
    2701         464 :   args[0] = target_node;
    2702             :   return BuildWasmCall(sig, args, rets, position, ref_node,
    2703         464 :                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
    2704             : }
    2705             : 
    2706       29123 : Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
    2707       22179 :                                    wasm::WasmCodePosition position) {
    2708             :   DCHECK_NULL(args[0]);
    2709       58246 :   wasm::FunctionSig* sig = env_->module->functions[index].sig;
    2710             : 
    2711       29123 :   if (env_ && index < env_->module->num_imported_functions) {
    2712             :     // Call to an imported function.
    2713        6944 :     return BuildImportCall(sig, args, rets, position, index);
    2714             :   }
    2715             : 
    2716             :   // A direct call to a wasm function defined in this module.
    2717             :   // Just encode the function index. This will be patched at instantiation.
    2718             :   Address code = static_cast<Address>(index);
    2719       44358 :   args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
    2720             : 
    2721       22183 :   return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
    2722             : }
    2723             : 
    2724        3285 : Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
    2725             :                                      Node*** rets,
    2726       98546 :                                      wasm::WasmCodePosition position) {
    2727             :   DCHECK_NOT_NULL(args[0]);
    2728             :   DCHECK_NOT_NULL(env_);
    2729             : 
    2730             :   // Assume only one table for now.
    2731        6570 :   wasm::FunctionSig* sig = env_->module->signatures[sig_index];
    2732             : 
    2733             :   Node* ift_size =
    2734        6570 :       LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
    2735             : 
    2736        3285 :   MachineOperatorBuilder* machine = mcgraph()->machine();
    2737        3285 :   Node* key = args[0];
    2738             : 
    2739             :   // Bounds check against the table size.
    2740        3285 :   Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
    2741        3285 :   TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
    2742             : 
    2743             :   // Mask the key to prevent SSCA.
    2744        3285 :   if (untrusted_code_mitigations_) {
    2745             :     // mask = ((key - size) & ~key) >> 31
    2746             :     Node* neg_key =
    2747           0 :         graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
    2748             :     Node* masked_diff = graph()->NewNode(
    2749             :         machine->Word32And(),
    2750           0 :         graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
    2751             :     Node* mask =
    2752           0 :         graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
    2753           0 :     key = graph()->NewNode(machine->Word32And(), key, mask);
    2754             :   }
    2755             : 
    2756             :   // Load signature from the table and check.
    2757             :   Node* ift_sig_ids =
    2758        6569 :       LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
    2759             : 
    2760        6570 :   int32_t expected_sig_id = env_->module->signature_ids[sig_index];
    2761             :   Node* int32_scaled_key = Uint32ToUintptr(
    2762        6570 :       graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
    2763             : 
    2764             :   Node* loaded_sig = SetEffect(
    2765             :       graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
    2766        3285 :                        int32_scaled_key, Effect(), Control()));
    2767             :   Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
    2768        3285 :                                      Int32Constant(expected_sig_id));
    2769             : 
    2770        3285 :   TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
    2771             : 
    2772             :   Node* ift_targets =
    2773        6569 :       LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
    2774        6570 :   Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableRefs,
    2775             :                                             MachineType::TaggedPointer());
    2776             : 
    2777             :   Node* intptr_scaled_key = graph()->NewNode(
    2778        3285 :       machine->Word32Shl(), key, Int32Constant(kSystemPointerSizeLog2));
    2779             : 
    2780             :   Node* target = SetEffect(
    2781             :       graph()->NewNode(machine->Load(MachineType::Pointer()), ift_targets,
    2782        3285 :                        intptr_scaled_key, Effect(), Control()));
    2783             : 
    2784             :   STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
    2785             :   Node* tagged_scaled_key = intptr_scaled_key;
    2786             : 
    2787             :   Node* target_instance = SetEffect(graph()->NewNode(
    2788             :       machine->Load(MachineType::TaggedPointer()),
    2789             :       graph()->NewNode(machine->IntAdd(), ift_instances, tagged_scaled_key),
    2790             :       Int32Constant(wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)),
    2791        6570 :       Effect(), Control()));
    2792             : 
    2793        3285 :   args[0] = target;
    2794             : 
    2795             :   return BuildWasmCall(sig, args, rets, position, target_instance,
    2796        3285 :                        untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
    2797             : }
    2798             : 
    2799       26944 : Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
    2800             :   // Implement Rol by Ror since TurboFan does not have Rol opcode.
    2801             :   // TODO(weiliang): support Word32Rol opcode in TurboFan.
    2802             :   Int32Matcher m(right);
    2803       13472 :   if (m.HasValue()) {
    2804             :     return Binop(wasm::kExprI32Ror, left,
    2805       26912 :                  mcgraph()->Int32Constant(32 - (m.Value() & 0x1F)));
    2806             :   } else {
    2807             :     return Binop(wasm::kExprI32Ror, left,
    2808          16 :                  Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
    2809             :   }
    2810             : }
    2811             : 
    2812         120 : Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
    2813             :   // Implement Rol by Ror since TurboFan does not have Rol opcode.
    2814             :   // TODO(weiliang): support Word64Rol opcode in TurboFan.
    2815             :   Int64Matcher m(right);
    2816          60 :   if (m.HasValue()) {
    2817             :     return Binop(wasm::kExprI64Ror, left,
    2818          32 :                  mcgraph()->Int64Constant(64 - (m.Value() & 0x3F)));
    2819             :   } else {
    2820             :     return Binop(wasm::kExprI64Ror, left,
    2821          44 :                  Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
    2822             :   }
    2823             : }
    2824             : 
    2825       39394 : Node* WasmGraphBuilder::Invert(Node* node) {
    2826       39394 :   return Unop(wasm::kExprI32Eqz, node);
    2827             : }
    2828             : 
    2829           0 : bool CanCover(Node* value, IrOpcode::Value opcode) {
    2830           0 :   if (value->opcode() != opcode) return false;
    2831             :   bool first = true;
    2832           0 :   for (Edge const edge : value->use_edges()) {
    2833           0 :     if (NodeProperties::IsControlEdge(edge)) continue;
    2834           0 :     if (NodeProperties::IsEffectEdge(edge)) continue;
    2835             :     DCHECK(NodeProperties::IsValueEdge(edge));
    2836           0 :     if (!first) return false;
    2837             :     first = false;
    2838             :   }
    2839           0 :   return true;
    2840             : }
    2841             : 
    2842       63398 : Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
    2843      126796 :   if (mcgraph()->machine()->Is64()) {
    2844       63398 :     value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
    2845             :   }
    2846       63398 :   return value;
    2847             : }
    2848             : 
    2849      126797 : Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
    2850       63398 :   value = BuildChangeInt32ToIntPtr(value);
    2851             :   return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
    2852      126797 :                           BuildSmiShiftBitsConstant());
    2853             : }
    2854             : 
    2855        2832 : Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
    2856             :   return graph()->NewNode(mcgraph()->machine()->WordShl(),
    2857        4248 :                           Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
    2858             : }
    2859             : 
    2860      285187 : Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
    2861      285187 :   return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
    2862             : }
    2863             : 
    2864      661121 : Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
    2865             :   value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
    2866      220374 :                            BuildSmiShiftBitsConstant());
    2867      440748 :   if (mcgraph()->machine()->Is64()) {
    2868             :     value =
    2869      220374 :         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
    2870             :   }
    2871      220373 :   return value;
    2872             : }
    2873             : 
    2874         288 : Node* WasmGraphBuilder::BuildConvertUint32ToSmiWithSaturation(Node* value,
    2875        1152 :                                                               uint32_t maxval) {
    2876             :   DCHECK(Smi::IsValid(maxval));
    2877             :   Node* max = Uint32Constant(maxval);
    2878             :   Node* check = graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(),
    2879         288 :                                  value, max);
    2880         288 :   Node* valsmi = BuildChangeUint31ToSmi(value);
    2881         576 :   Node* maxsmi = graph()->NewNode(mcgraph()->common()->NumberConstant(maxval));
    2882         576 :   Diamond d(graph(), mcgraph()->common(), check, BranchHint::kTrue);
    2883         288 :   d.Chain(Control());
    2884         288 :   return d.Phi(MachineRepresentation::kTagged, valsmi, maxsmi);
    2885             : }
    2886             : 
    2887      537147 : void WasmGraphBuilder::InitInstanceCache(
    2888     5372007 :     WasmInstanceCacheNodes* instance_cache) {
    2889             :   DCHECK_NOT_NULL(instance_node_);
    2890             : 
    2891             :   // Load the memory start.
    2892             :   instance_cache->mem_start = SetEffect(graph()->NewNode(
    2893             :       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
    2894             :       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
    2895     1611587 :       Effect(), Control()));
    2896             : 
    2897             :   // Load the memory size.
    2898             :   instance_cache->mem_size = SetEffect(graph()->NewNode(
    2899             :       mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
    2900             :       mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
    2901     1611644 :       Effect(), Control()));
    2902             : 
    2903      537246 :   if (untrusted_code_mitigations_) {
    2904             :     // Load the memory mask.
    2905             :     instance_cache->mem_mask =
    2906           0 :         LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr());
    2907             :   } else {
    2908             :     // Explicitly set to nullptr to ensure a SEGV when we try to use it.
    2909      537246 :     instance_cache->mem_mask = nullptr;
    2910             :   }
    2911      537246 : }
    2912             : 
    2913        4259 : void WasmGraphBuilder::PrepareInstanceCacheForLoop(
    2914        8518 :     WasmInstanceCacheNodes* instance_cache, Node* control) {
    2915             : #define INTRODUCE_PHI(field, rep)                                            \
    2916             :   instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
    2917             :                                            instance_cache->field, control);
    2918             : 
    2919       12777 :   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
    2920       12777 :   INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
    2921        4259 :   if (untrusted_code_mitigations_) {
    2922           0 :     INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation());
    2923             :   }
    2924             : 
    2925             : #undef INTRODUCE_PHI
    2926        4259 : }
    2927             : 
    2928       34467 : void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
    2929             :                                              WasmInstanceCacheNodes* from,
    2930       25841 :                                              Node* merge) {
    2931             : #define INTRODUCE_PHI(field, rep)                                            \
    2932             :   if (to->field != from->field) {                                            \
    2933             :     Node* vals[] = {to->field, from->field, merge};                          \
    2934             :     to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
    2935             :   }
    2936             : 
    2937       60307 :   INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
    2938       60310 :   INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
    2939       34468 :   if (untrusted_code_mitigations_) {
    2940           0 :     INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
    2941             :   }
    2942             : 
    2943             : #undef INTRODUCE_PHI
    2944       34468 : }
    2945             : 
    2946      213528 : void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
    2947             :                                               WasmInstanceCacheNodes* from,
    2948             :                                               Node* merge) {
    2949             :   to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
    2950      213528 :                                       merge, to->mem_size, from->mem_size);
    2951             :   to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
    2952      213529 :                                        merge, to->mem_start, from->mem_start);
    2953      213530 :   if (untrusted_code_mitigations_) {
    2954             :     to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
    2955           0 :                                         merge, to->mem_mask, from->mem_mask);
    2956             :   }
    2957      213530 : }
    2958             : 
    2959      992944 : Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
    2960             :                                              Node* merge, Node* tnode,
    2961       13302 :                                              Node* fnode) {
    2962      992944 :   if (IsPhiWithMerge(tnode, merge)) {
    2963       94402 :     AppendToPhi(tnode, fnode);
    2964      898544 :   } else if (tnode != fnode) {
    2965       13302 :     uint32_t count = merge->InputCount();
    2966             :     // + 1 for the merge node.
    2967       13302 :     Node** vals = Buffer(count + 1);
    2968       13302 :     for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
    2969       13302 :     vals[count - 1] = fnode;
    2970       13302 :     vals[count] = merge;
    2971             :     return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
    2972       39906 :                             vals);
    2973             :   }
    2974      979645 :   return tnode;
    2975             : }
    2976             : 
    2977      213529 : Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
    2978             :                                                    Node* fnode) {
    2979      213529 :   if (IsPhiWithMerge(tnode, merge)) {
    2980       13448 :     AppendToPhi(tnode, fnode);
    2981      200082 :   } else if (tnode != fnode) {
    2982         128 :     uint32_t count = merge->InputCount();
    2983         128 :     Node** effects = Buffer(count);
    2984         493 :     for (uint32_t j = 0; j < count - 1; j++) {
    2985         365 :       effects[j] = tnode;
    2986             :     }
    2987         128 :     effects[count - 1] = fnode;
    2988         128 :     tnode = EffectPhi(count, effects, merge);
    2989             :   }
    2990      213530 :   return tnode;
    2991             : }
    2992             : 
    2993         479 : Node* WasmGraphBuilder::GetImportedMutableGlobals() {
    2994         248 :   if (imported_mutable_globals_ == nullptr) {
    2995             :     // Load imported_mutable_globals_ from the instance object at runtime.
    2996         463 :     imported_mutable_globals_ = graph()->NewNode(
    2997             :         mcgraph()->machine()->Load(MachineType::UintPtr()),
    2998             :         instance_node_.get(),
    2999             :         mcgraph()->Int32Constant(
    3000             :             WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
    3001             :         graph()->start(), graph()->start());
    3002             :   }
    3003         248 :   return imported_mutable_globals_.get();
    3004             : }
    3005             : 
    3006       51368 : void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
    3007             :                                               const wasm::WasmGlobal& global,
    3008             :                                               Node** base_node,
    3009       82689 :                                               Node** offset_node) {
    3010             :   DCHECK_NOT_NULL(instance_node_);
    3011       51368 :   if (global.mutability && global.imported) {
    3012             :     *base_node = SetEffect(graph()->NewNode(
    3013             :         mcgraph()->machine()->Load(MachineType::UintPtr()),
    3014             :         GetImportedMutableGlobals(),
    3015             :         mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
    3016         544 :         Control()));
    3017         136 :     *offset_node = mcgraph()->Int32Constant(0);
    3018             :   } else {
    3019       51232 :     if (globals_start_ == nullptr) {
    3020             :       // Load globals_start from the instance object at runtime.
    3021             :       // TODO(wasm): we currently generate only one load of the {globals_start}
    3022             :       // start per graph, which means it can be placed anywhere by the
    3023             :       // scheduler. This is legal because the globals_start should never change.
    3024             :       // However, in some cases (e.g. if the instance object is already in a
    3025             :       // register), it is slightly more efficient to reload this value from the
    3026             :       // instance object. Since this depends on register allocation, it is not
    3027             :       // possible to express in the graph, and would essentially constitute a
    3028             :       // "mem2reg" optimization in TurboFan.
    3029       19834 :       globals_start_ = graph()->NewNode(
    3030             :           mcgraph()->machine()->Load(MachineType::UintPtr()),
    3031             :           instance_node_.get(),
    3032             :           mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
    3033             :           graph()->start(), graph()->start());
    3034             :     }
    3035       51235 :     *base_node = globals_start_.get();
    3036      102470 :     *offset_node = mcgraph()->Int32Constant(global.offset);
    3037             : 
    3038       51231 :     if (mem_type == MachineType::Simd128() && global.offset != 0) {
    3039             :       // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
    3040             :       *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
    3041       31080 :                                     *offset_node);
    3042       10360 :       *offset_node = mcgraph()->Int32Constant(0);
    3043             :     }
    3044             :   }
    3045       51367 : }
    3046             : 
    3047         112 : void WasmGraphBuilder::GetBaseAndOffsetForImportedMutableAnyRefGlobal(
    3048        2126 :     const wasm::WasmGlobal& global, Node** base, Node** offset) {
    3049             :   // Load the base from the ImportedMutableGlobalsBuffer of the instance.
    3050         224 :   Node* buffers = LOAD_INSTANCE_FIELD(ImportedMutableGlobalsBuffers,
    3051             :                                       MachineType::TaggedPointer());
    3052         448 :   *base = LOAD_FIXED_ARRAY_SLOT_ANY(buffers, global.index);
    3053             : 
    3054             :   // For the offset we need the index of the global in the buffer, and then
    3055             :   // calculate the actual offset from the index. Load the index from the
    3056             :   // ImportedMutableGlobals array of the instance.
    3057             :   Node* index = SetEffect(
    3058             :       graph()->NewNode(mcgraph()->machine()->Load(MachineType::UintPtr()),
    3059             :                        GetImportedMutableGlobals(),
    3060             :                        mcgraph()->Int32Constant(global.index * sizeof(Address)),
    3061         336 :                        Effect(), Control()));
    3062             : 
    3063             :   // From the index, calculate the actual offset in the FixeArray. This
    3064             :   // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
    3065             :   // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
    3066             :   Node* index_times_tagged_size =
    3067             :       graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
    3068         223 :                        mcgraph()->Int32Constant(kTaggedSize));
    3069             :   *offset = graph()->NewNode(
    3070             :       mcgraph()->machine()->IntAdd(), index_times_tagged_size,
    3071             :       mcgraph()->IntPtrConstant(
    3072         336 :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
    3073         112 : }
    3074             : 
    3075      365431 : Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
    3076             :   DCHECK_NOT_NULL(instance_cache_);
    3077      275601 :   Node* mem_start = instance_cache_->mem_start;
    3078             :   DCHECK_NOT_NULL(mem_start);
    3079      275601 :   if (offset == 0) return mem_start;
    3080             :   return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
    3081      179658 :                           mcgraph()->IntPtrConstant(offset));
    3082             : }
    3083             : 
    3084        1472 : Node* WasmGraphBuilder::CurrentMemoryPages() {
    3085             :   // CurrentMemoryPages can not be called from asm.js.
    3086             :   DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
    3087             :   DCHECK_NOT_NULL(instance_cache_);
    3088         368 :   Node* mem_size = instance_cache_->mem_size;
    3089             :   DCHECK_NOT_NULL(mem_size);
    3090             :   Node* result =
    3091             :       graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
    3092         736 :                        mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
    3093         736 :   if (mcgraph()->machine()->Is64()) {
    3094             :     result =
    3095         368 :         graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
    3096             :   }
    3097         368 :   return result;
    3098             : }
    3099             : 
    3100        6220 : Node* WasmGraphBuilder::BuildLoadBuiltinFromInstance(int builtin_index) {
    3101             :   DCHECK(Builtins::IsBuiltinId(builtin_index));
    3102        1244 :   Node* isolate_root = LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
    3103        1866 :   return LOAD_TAGGED_POINTER(isolate_root,
    3104             :                              IsolateData::builtin_slot_offset(builtin_index));
    3105             : }
    3106             : 
    3107             : // Only call this function for code which is not reused across instantiations,
    3108             : // as we do not patch the embedded js_context.
    3109        5261 : Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(
    3110             :     Runtime::FunctionId f, Node* js_context, Node** parameters,
    3111       52609 :     int parameter_count, Node** effect, Node* control) {
    3112        5261 :   const Runtime::Function* fun = Runtime::FunctionForId(f);
    3113             :   auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
    3114             :       mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
    3115       10522 :       CallDescriptor::kNoFlags);
    3116             :   // The CEntryStub is loaded from the instance_node so that generated code is
    3117             :   // Isolate independent. At the moment this is only done for CEntryStub(1).
    3118             :   DCHECK_EQ(1, fun->result_size);
    3119             :   Node* centry_stub =
    3120       10522 :       LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
    3121             :   // TODO(titzer): allow arbitrary number of runtime arguments
    3122             :   // At the moment we only allow 5 parameters. If more parameters are needed,
    3123             :   // increase this constant accordingly.
    3124             :   static const int kMaxParams = 5;
    3125             :   DCHECK_GE(kMaxParams, parameter_count);
    3126             :   Node* inputs[kMaxParams + 6];
    3127             :   int count = 0;
    3128        5259 :   inputs[count++] = centry_stub;
    3129       12990 :   for (int i = 0; i < parameter_count; i++) {
    3130        7731 :     inputs[count++] = parameters[i];
    3131             :   }
    3132        5259 :   inputs[count++] =
    3133       10518 :       mcgraph()->ExternalConstant(ExternalReference::Create(f));  // ref
    3134       10520 :   inputs[count++] = mcgraph()->Int32Constant(fun->nargs);         // arity
    3135        5262 :   inputs[count++] = js_context;                                   // js_context
    3136        5262 :   inputs[count++] = *effect;
    3137        5262 :   inputs[count++] = control;
    3138             : 
    3139             :   Node* call = mcgraph()->graph()->NewNode(
    3140       10522 :       mcgraph()->common()->Call(call_descriptor), count, inputs);
    3141        5263 :   *effect = call;
    3142        5263 :   return call;
    3143             : }
    3144             : 
    3145        4340 : Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
    3146             :                                            Node** parameters,
    3147        4340 :                                            int parameter_count) {
    3148             :   return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
    3149        8678 :                                        parameter_count, effect_, Control());
    3150             : }
    3151             : 
    3152      182017 : Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
    3153       72292 :   const wasm::WasmGlobal& global = env_->module->globals[index];
    3154       36250 :   if (global.type == wasm::ValueType::kWasmAnyRef) {
    3155         206 :     if (global.mutability && global.imported) {
    3156          80 :       Node* base = nullptr;
    3157          80 :       Node* offset = nullptr;
    3158          80 :       GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
    3159             :       return SetEffect(
    3160             :           graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
    3161         160 :                            base, offset, Effect(), Control()));
    3162             :     }
    3163             :     Node* globals_buffer =
    3164         254 :         LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
    3165         510 :     return LOAD_FIXED_ARRAY_SLOT_ANY(globals_buffer, global.offset);
    3166             :   }
    3167             : 
    3168             :   MachineType mem_type =
    3169       36044 :       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
    3170       36042 :   Node* base = nullptr;
    3171       36042 :   Node* offset = nullptr;
    3172       36042 :   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
    3173       36042 :                          &offset);
    3174             :   Node* result = SetEffect(graph()->NewNode(
    3175       72088 :       mcgraph()->machine()->Load(mem_type), base, offset, Effect(), Control()));
    3176             : #if defined(V8_TARGET_BIG_ENDIAN)
    3177             :   result = BuildChangeEndiannessLoad(result, mem_type,
    3178             :                                      env_->module->globals[index].type);
    3179             : #endif
    3180       36043 :   return result;
    3181             : }
    3182             : 
    3183       77758 : Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
    3184       30860 :   const wasm::WasmGlobal& global = env_->module->globals[index];
    3185       15446 :   if (global.type == wasm::ValueType::kWasmAnyRef) {
    3186         120 :     if (global.mutability && global.imported) {
    3187          32 :       Node* base = nullptr;
    3188          32 :       Node* offset = nullptr;
    3189          32 :       GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
    3190             : 
    3191             :       return SetEffect(graph()->NewNode(
    3192             :           mcgraph()->machine()->Store(StoreRepresentation(
    3193             :               MachineRepresentation::kTagged, kFullWriteBarrier)),
    3194          96 :           base, offset, val, Effect(), Control()));
    3195             :     }
    3196             :     Node* globals_buffer =
    3197         176 :         LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
    3198         440 :     return STORE_FIXED_ARRAY_SLOT_ANY(globals_buffer,
    3199             :                                       env_->module->globals[index].offset, val);
    3200             :   }
    3201             : 
    3202             :   MachineType mem_type =
    3203       15326 :       wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
    3204       15326 :   Node* base = nullptr;
    3205       15326 :   Node* offset = nullptr;
    3206       15326 :   GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
    3207       15326 :                          &offset);
    3208             :   const Operator* op = mcgraph()->machine()->Store(
    3209       30652 :       StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
    3210             : #if defined(V8_TARGET_BIG_ENDIAN)
    3211             :   val = BuildChangeEndiannessStore(val, mem_type.representation(),
    3212             :                                    env_->module->globals[index].type);
    3213             : #endif
    3214             :   return SetEffect(
    3215       30652 :       graph()->NewNode(op, base, offset, val, Effect(), Control()));
    3216             : }
    3217             : 
    3218           0 : void WasmGraphBuilder::GetTableBaseAndOffset(uint32_t table_index, Node* index,
    3219             :                                              wasm::WasmCodePosition position,
    3220             :                                              Node** base_node,
    3221           0 :                                              Node** offset_node) {
    3222           0 :   Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
    3223           0 :   Node* table = LOAD_FIXED_ARRAY_SLOT_ANY(tables, table_index);
    3224             : 
    3225             :   int storage_field_size = WasmTableObject::kElementsOffsetEnd -
    3226             :                            WasmTableObject::kElementsOffset + 1;
    3227           0 :   Node* storage = LOAD_RAW(
    3228             :       table, wasm::ObjectAccess::ToTagged(WasmTableObject::kElementsOffset),
    3229             :       assert_size(storage_field_size, MachineType::TaggedPointer()));
    3230             : 
    3231             :   int length_field_size =
    3232             :       FixedArray::kLengthOffsetEnd - FixedArray::kLengthOffset + 1;
    3233             :   Node* storage_size =
    3234           0 :       LOAD_RAW(storage, wasm::ObjectAccess::ToTagged(FixedArray::kLengthOffset),
    3235             :                assert_size(length_field_size, MachineType::TaggedSigned()));
    3236             : 
    3237           0 :   storage_size = BuildChangeSmiToInt32(storage_size);
    3238             :   // Bounds check against the table size.
    3239             :   Node* in_bounds = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
    3240           0 :                                      index, storage_size);
    3241           0 :   TrapIfFalse(wasm::kTrapTableOutOfBounds, in_bounds, position);
    3242             : 
    3243             :   // From the index, calculate the actual offset in the FixeArray. This
    3244             :   // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
    3245             :   // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
    3246             :   Node* index_times_tagged_size =
    3247             :       graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
    3248           0 :                        mcgraph()->Int32Constant(kTaggedSize));
    3249             : 
    3250             :   *offset_node = graph()->NewNode(
    3251             :       mcgraph()->machine()->IntAdd(), index_times_tagged_size,
    3252             :       mcgraph()->IntPtrConstant(
    3253           0 :           wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
    3254             : 
    3255           0 :   *base_node = storage;
    3256           0 : }
    3257             : 
    3258           0 : Node* WasmGraphBuilder::GetTable(uint32_t table_index, Node* index,
    3259           0 :                                  wasm::WasmCodePosition position) {
    3260           0 :   Node* base = nullptr;
    3261           0 :   Node* offset = nullptr;
    3262           0 :   GetTableBaseAndOffset(table_index, index, position, &base, &offset);
    3263             :   return SetEffect(
    3264             :       graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
    3265           0 :                        base, offset, Effect(), Control()));
    3266             : }
    3267             : 
    3268           0 : Node* WasmGraphBuilder::SetTable(uint32_t table_index, Node* index, Node* val,
    3269           0 :                                  wasm::WasmCodePosition position) {
    3270           0 :   Node* base = nullptr;
    3271           0 :   Node* offset = nullptr;
    3272           0 :   GetTableBaseAndOffset(table_index, index, position, &base, &offset);
    3273             : 
    3274             :   const Operator* op = mcgraph()->machine()->Store(
    3275           0 :       StoreRepresentation(MachineRepresentation::kTagged, kFullWriteBarrier));
    3276             : 
    3277           0 :   Node* store = graph()->NewNode(op, base, offset, val, Effect(), Control());
    3278           0 :   return SetEffect(store);
    3279             : }
    3280             : 
    3281       32641 : Node* WasmGraphBuilder::CheckBoundsAndAlignment(
    3282             :     uint8_t access_size, Node* index, uint32_t offset,
    3283       20663 :     wasm::WasmCodePosition position) {
    3284             :   // Atomic operations need bounds checks until the backend can emit protected
    3285             :   // loads.
    3286             :   index =
    3287       32641 :       BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
    3288             : 
    3289       32643 :   const uintptr_t align_mask = access_size - 1;
    3290             : 
    3291             :   // Don't emit an alignment check if the index is a constant.
    3292             :   // TODO(wasm): a constant match is also done above in {BoundsCheckMem}.
    3293             :   UintPtrMatcher match(index);
    3294       32643 :   if (match.HasValue()) {
    3295       27479 :     uintptr_t effective_offset = match.Value() + offset;
    3296       27479 :     if ((effective_offset & align_mask) != 0) {
    3297             :       // statically known to be unaligned; trap.
    3298           0 :       TrapIfEq32(wasm::kTrapUnalignedAccess, Int32Constant(0), 0, position);
    3299             :     }
    3300             :     return index;
    3301             :   }
    3302             : 
    3303             :   // Unlike regular memory accesses, atomic memory accesses should trap if
    3304             :   // the effective offset is misaligned.
    3305             :   // TODO(wasm): this addition is redundant with one inserted by {MemBuffer}.
    3306             :   Node* effective_offset = graph()->NewNode(mcgraph()->machine()->IntAdd(),
    3307       10327 :                                             MemBuffer(offset), index);
    3308             : 
    3309             :   Node* cond = graph()->NewNode(mcgraph()->machine()->WordAnd(),
    3310       10333 :                                 effective_offset, IntPtrConstant(align_mask));
    3311             :   TrapIfFalse(wasm::kTrapUnalignedAccess,
    3312             :               graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
    3313             :                                mcgraph()->Int32Constant(0)),
    3314       15500 :               position);
    3315        5167 :   return index;
    3316             : }
    3317             : 
    3318             : // Insert code to bounds check a memory access if necessary. Return the
    3319             : // bounds-checked index, which is guaranteed to have (the equivalent of)
    3320             : // {uintptr_t} representation.
    3321      270971 : Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
    3322             :                                        uint32_t offset,
    3323             :                                        wasm::WasmCodePosition position,
    3324      303696 :                                        EnforceBoundsCheck enforce_check) {
    3325             :   DCHECK_LE(1, access_size);
    3326      270971 :   index = Uint32ToUintptr(index);
    3327      271042 :   if (FLAG_wasm_no_bounds_checks) return index;
    3328             : 
    3329      271049 :   if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
    3330             :     return index;
    3331             :   }
    3332             : 
    3333       65334 :   if (!IsInBounds(offset, access_size, env_->max_memory_size)) {
    3334             :     // The access will be out of bounds, even for the largest memory.
    3335          16 :     TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
    3336          16 :     return mcgraph()->IntPtrConstant(0);
    3337             :   }
    3338       32651 :   uint64_t end_offset = uint64_t{offset} + access_size - 1u;
    3339       32651 :   Node* end_offset_node = IntPtrConstant(end_offset);
    3340             : 
    3341             :   // The accessed memory is [index + offset, index + end_offset].
    3342             :   // Check that the last read byte (at {index + end_offset}) is in bounds.
    3343             :   // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
    3344             :   //    compute {effective_size} as {mem_size - end_offset)}.
    3345             :   //    {effective_size} is >= 1 if condition 1) holds.
    3346             :   // 2) Check that {index + end_offset < mem_size} by
    3347             :   //    - computing {effective_size} as {mem_size - end_offset} and
    3348             :   //    - checking that {index < effective_size}.
    3349             : 
    3350       32631 :   auto m = mcgraph()->machine();
    3351       32631 :   Node* mem_size = instance_cache_->mem_size;
    3352       32631 :   if (end_offset >= env_->min_memory_size) {
    3353             :     // The end offset is larger than the smallest memory.
    3354             :     // Dynamically check the end offset against the dynamic memory size.
    3355        5903 :     Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
    3356        5903 :     TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
    3357             :   } else {
    3358             :     // The end offset is smaller than the smallest memory, so only one check is
    3359             :     // required. Check to see if the index is also a constant.
    3360             :     UintPtrMatcher match(index);
    3361       26728 :     if (match.HasValue()) {
    3362             :       uintptr_t index_val = match.Value();
    3363       26732 :       if (index_val < env_->min_memory_size - end_offset) {
    3364             :         // The input index is a constant and everything is statically within
    3365             :         // bounds of the smallest possible memory.
    3366             :         return index;
    3367             :       }
    3368             :     }
    3369             :   }
    3370             : 
    3371             :   // This produces a positive number, since {end_offset < min_size <= mem_size}.
    3372             :   Node* effective_size =
    3373        5920 :       graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
    3374             : 
    3375             :   // Introduce the actual bounds check.
    3376        5919 :   Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
    3377        5920 :   TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
    3378             : 
    3379        5918 :   if (untrusted_code_mitigations_) {
    3380             :     // In the fallthrough case, condition the index with the memory mask.
    3381           0 :     Node* mem_mask = instance_cache_->mem_mask;
    3382             :     DCHECK_NOT_NULL(mem_mask);
    3383           0 :     index = graph()->NewNode(m->WordAnd(), index, mem_mask);
    3384             :   }
    3385        5918 :   return index;
    3386             : }
    3387             : 
    3388             : // Check that the range [start, start + size) is in the range [0, max).
    3389         152 : void WasmGraphBuilder::BoundsCheckRange(Node* start, Node* size, Node* max,
    3390         152 :                                         wasm::WasmCodePosition position) {
    3391             :   // The accessed memory is [start, end), where {end} is {start + size}. We
    3392             :   // want to check that {start + size <= max}, making sure that {start + size}
    3393             :   // doesn't overflow. This can be expressed as {start <= max - size} as long
    3394             :   // as {max - size} isn't negative, which is true if {size <= max}.
    3395         152 :   auto m = mcgraph()->machine();
    3396         152 :   Node* cond = graph()->NewNode(m->Uint32LessThanOrEqual(), size, max);
    3397         152 :   TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
    3398             : 
    3399             :   // This produces a positive number, since {size <= max}.
    3400         152 :   Node* effective_size = graph()->NewNode(m->Int32Sub(), max, size);
    3401             : 
    3402             :   // Introduce the actual bounds check.
    3403             :   Node* check =
    3404         152 :       graph()->NewNode(m->Uint32LessThanOrEqual(), start, effective_size);
    3405         152 :   TrapIfFalse(wasm::kTrapMemOutOfBounds, check, position);
    3406             : 
    3407             :   // TODO(binji): Does this need addtional untrusted_code_mitigations_ mask
    3408             :   // like BoundsCheckMem above?
    3409         152 : }
    3410             : 
    3411         112 : Node* WasmGraphBuilder::BoundsCheckMemRange(Node* start, Node* size,
    3412         112 :                                             wasm::WasmCodePosition position) {
    3413             :   // TODO(binji): Support trap handler.
    3414         112 :   if (!FLAG_wasm_no_bounds_checks) {
    3415         112 :     BoundsCheckRange(start, size, instance_cache_->mem_size, position);
    3416             :   }
    3417             :   return graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(0),
    3418         336 :                           Uint32ToUintptr(start));
    3419             : }
    3420             : 
    3421        4676 : const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
    3422        4680 :                                                       wasm::ValueType type) {
    3423        4676 :   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
    3424        4676 :   MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
    3425        4680 :   if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
    3426           8 :                             wasm::ValueTypes::MachineRepresentationFor(type))) {
    3427        4676 :     return mcgraph()->machine()->Load(mach_type);
    3428             :   }
    3429           0 :   return mcgraph()->machine()->UnalignedLoad(mach_type);
    3430             : }
    3431             : 
    3432        9064 : const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
    3433        9076 :                                                        wasm::ValueType type) {
    3434        9064 :   int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
    3435        9064 :   MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
    3436        9088 :   if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
    3437             :     StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
    3438        9064 :     return mcgraph()->machine()->Store(store_rep);
    3439             :   }
    3440             :   UnalignedStoreRepresentation store_rep(rep);
    3441           0 :   return mcgraph()->machine()->UnalignedStore(store_rep);
    3442             : }
    3443             : 
    3444          40 : Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
    3445             :                                              MachineRepresentation rep,
    3446             :                                              Node* index, uint32_t offset,
    3447         160 :                                              wasm::WasmCodePosition position) {
    3448             :   int kAlign = 4;  // Ensure that the LSB is 0, such that this looks like a Smi.
    3449             :   Node* info = graph()->NewNode(
    3450          80 :       mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
    3451             : 
    3452             :   Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    3453          80 :                                    Int32Constant(offset), index);
    3454         120 :   auto store = [&](int offset, MachineRepresentation rep, Node* data) {
    3455             :     SetEffect(graph()->NewNode(
    3456             :         mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
    3457         720 :         info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
    3458         160 :   };
    3459             :   // Store address, is_store, and mem_rep.
    3460             :   store(offsetof(wasm::MemoryTracingInfo, address),
    3461          40 :         MachineRepresentation::kWord32, address);
    3462             :   store(offsetof(wasm::MemoryTracingInfo, is_store),
    3463             :         MachineRepresentation::kWord8,
    3464          80 :         mcgraph()->Int32Constant(is_store ? 1 : 0));
    3465             :   store(offsetof(wasm::MemoryTracingInfo, mem_rep),
    3466             :         MachineRepresentation::kWord8,
    3467          80 :         mcgraph()->Int32Constant(static_cast<int>(rep)));
    3468             : 
    3469          40 :   Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
    3470          40 :   SetSourcePosition(call, position);
    3471          40 :   return call;
    3472             : }
    3473             : 
    3474       94247 : Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
    3475             :                                 Node* index, uint32_t offset,
    3476             :                                 uint32_t alignment,
    3477      565076 :                                 wasm::WasmCodePosition position) {
    3478             :   Node* load;
    3479             : 
    3480             :   // Wasm semantics throw on OOB. Introduce explicit bounds check and
    3481             :   // conditioning when not using the trap handler.
    3482             :   index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
    3483       94250 :                          position, kCanOmitBoundsCheck);
    3484             : 
    3485      187339 :   if (memtype.representation() == MachineRepresentation::kWord8 ||
    3486       93079 :       mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
    3487       94260 :     if (use_trap_handler()) {
    3488             :       load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
    3489      188524 :                               MemBuffer(offset), index, Effect(), Control());
    3490       94269 :       SetSourcePosition(load, position);
    3491             :     } else {
    3492             :       load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
    3493           2 :                               MemBuffer(offset), index, Effect(), Control());
    3494             :     }
    3495             :   } else {
    3496             :     // TODO(eholk): Support unaligned loads with trap handlers.
    3497             :     DCHECK(!use_trap_handler());
    3498             :     load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
    3499           0 :                             MemBuffer(offset), index, Effect(), Control());
    3500             :   }
    3501             : 
    3502             :   SetEffect(load);
    3503             : 
    3504             : #if defined(V8_TARGET_BIG_ENDIAN)
    3505             :   load = BuildChangeEndiannessLoad(load, memtype, type);
    3506             : #endif
    3507             : 
    3508      135514 :   if (type == wasm::kWasmI64 &&
    3509             :       ElementSizeInBytes(memtype.representation()) < 8) {
    3510             :     // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
    3511         689 :     if (memtype.IsSigned()) {
    3512             :       // sign extend
    3513         352 :       load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
    3514             :     } else {
    3515             :       // zero extend
    3516             :       load =
    3517         337 :           graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
    3518             :     }
    3519             :   }
    3520             : 
    3521       94262 :   if (FLAG_trace_wasm_memory) {
    3522             :     TraceMemoryOperation(false, memtype.representation(), index, offset,
    3523          24 :                          position);
    3524             :   }
    3525             : 
    3526       94262 :   return load;
    3527             : }
    3528             : 
    3529      144131 : Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
    3530             :                                  uint32_t offset, uint32_t alignment, Node* val,
    3531             :                                  wasm::WasmCodePosition position,
    3532      863913 :                                  wasm::ValueType type) {
    3533             :   Node* store;
    3534             : 
    3535             :   index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
    3536      144140 :                          position, kCanOmitBoundsCheck);
    3537             : 
    3538             : #if defined(V8_TARGET_BIG_ENDIAN)
    3539             :   val = BuildChangeEndiannessStore(val, mem_rep, type);
    3540             : #endif
    3541             : 
    3542      287281 :   if (mem_rep == MachineRepresentation::kWord8 ||
    3543      143141 :       mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
    3544      144140 :     if (use_trap_handler()) {
    3545             :       store =
    3546             :           graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
    3547      288284 :                            MemBuffer(offset), index, val, Effect(), Control());
    3548      144207 :       SetSourcePosition(store, position);
    3549             :     } else {
    3550             :       StoreRepresentation rep(mem_rep, kNoWriteBarrier);
    3551             :       store =
    3552             :           graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
    3553           0 :                            index, val, Effect(), Control());
    3554             :     }
    3555             :   } else {
    3556             :     // TODO(eholk): Support unaligned stores with trap handlers.
    3557             :     DCHECK(!use_trap_handler());
    3558             :     UnalignedStoreRepresentation rep(mem_rep);
    3559             :     store =
    3560             :         graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
    3561           0 :                          MemBuffer(offset), index, val, Effect(), Control());
    3562             :   }
    3563             : 
    3564             :   SetEffect(store);
    3565             : 
    3566      144208 :   if (FLAG_trace_wasm_memory) {
    3567          16 :     TraceMemoryOperation(true, mem_rep, index, offset, position);
    3568             :   }
    3569             : 
    3570      144208 :   return store;
    3571             : }
    3572             : 
    3573             : namespace {
    3574       56390 : Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
    3575       56390 :   switch (rep) {
    3576             :     case MachineRepresentation::kWord8:
    3577             :     case MachineRepresentation::kWord16:
    3578             :     case MachineRepresentation::kWord32:
    3579       50149 :       return mcgraph->Int32Constant(0);
    3580             :     case MachineRepresentation::kWord64:
    3581           0 :       return mcgraph->Int64Constant(0);
    3582             :     case MachineRepresentation::kFloat32:
    3583        5396 :       return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
    3584             :     case MachineRepresentation::kFloat64:
    3585         845 :       return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
    3586             :     default:
    3587           0 :       UNREACHABLE();
    3588             :   }
    3589             : }
    3590             : }  // namespace
    3591             : 
    3592      563900 : Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
    3593             :   DCHECK_NOT_NULL(instance_cache_);
    3594       56390 :   Node* mem_start = instance_cache_->mem_start;
    3595       56390 :   Node* mem_size = instance_cache_->mem_size;
    3596             :   DCHECK_NOT_NULL(mem_start);
    3597             :   DCHECK_NOT_NULL(mem_size);
    3598             : 
    3599             :   // Asm.js semantics are defined in terms of typed arrays, hence OOB
    3600             :   // reads return {undefined} coerced to the result type (0 for integers, NaN
    3601             :   // for float and double).
    3602             :   // Note that we check against the memory size ignoring the size of the
    3603             :   // stored value, which is conservative if misaligned. Technically, asm.js
    3604             :   // should never have misaligned accesses.
    3605       56390 :   index = Uint32ToUintptr(index);
    3606             :   Diamond bounds_check(
    3607             :       graph(), mcgraph()->common(),
    3608             :       graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
    3609      169170 :       BranchHint::kTrue);
    3610       56390 :   bounds_check.Chain(Control());
    3611             : 
    3612       56390 :   if (untrusted_code_mitigations_) {
    3613             :     // Condition the index with the memory mask.
    3614           0 :     Node* mem_mask = instance_cache_->mem_mask;
    3615             :     DCHECK_NOT_NULL(mem_mask);
    3616           0 :     index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
    3617             :   }
    3618             : 
    3619             :   Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
    3620      112780 :                                 index, Effect(), bounds_check.if_true);
    3621       56390 :   SetEffect(bounds_check.EffectPhi(load, Effect()));
    3622       56390 :   SetControl(bounds_check.merge);
    3623             :   return bounds_check.Phi(type.representation(), load,
    3624       56390 :                           GetAsmJsOOBValue(type.representation(), mcgraph()));
    3625             : }
    3626             : 
    3627      372887 : Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
    3628      745774 :   if (mcgraph()->machine()->Is32()) return node;
    3629             :   // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
    3630             :   Uint32Matcher matcher(node);
    3631      372920 :   if (matcher.HasValue()) {
    3632      258281 :     uintptr_t value = matcher.Value();
    3633      258281 :     return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
    3634             :   }
    3635      229279 :   return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
    3636             : }
    3637             : 
    3638       40014 : Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
    3639      320112 :                                            Node* val) {
    3640             :   DCHECK_NOT_NULL(instance_cache_);
    3641       40014 :   Node* mem_start = instance_cache_->mem_start;
    3642       40014 :   Node* mem_size = instance_cache_->mem_size;
    3643             :   DCHECK_NOT_NULL(mem_start);
    3644             :   DCHECK_NOT_NULL(mem_size);
    3645             : 
    3646             :   // Asm.js semantics are to ignore OOB writes.
    3647             :   // Note that we check against the memory size ignoring the size of the
    3648             :   // stored value, which is conservative if misaligned. Technically, asm.js
    3649             :   // should never have misaligned accesses.
    3650             :   Diamond bounds_check(
    3651             :       graph(), mcgraph()->common(),
    3652             :       graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
    3653      120042 :       BranchHint::kTrue);
    3654       40014 :   bounds_check.Chain(Control());
    3655             : 
    3656       40014 :   if (untrusted_code_mitigations_) {
    3657             :     // Condition the index with the memory mask.
    3658           0 :     Node* mem_mask = instance_cache_->mem_mask;
    3659             :     DCHECK_NOT_NULL(mem_mask);
    3660             :     index =
    3661           0 :         graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
    3662             :   }
    3663             : 
    3664       40014 :   index = Uint32ToUintptr(index);
    3665             :   const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
    3666       80028 :       type.representation(), WriteBarrierKind::kNoWriteBarrier));
    3667             :   Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
    3668       40014 :                                  bounds_check.if_true);
    3669       40014 :   SetEffect(bounds_check.EffectPhi(store, Effect()));
    3670       40014 :   SetControl(bounds_check.merge);
    3671       40014 :   return val;
    3672             : }
    3673             : 
    3674           0 : void WasmGraphBuilder::PrintDebugName(Node* node) {
    3675           0 :   PrintF("#%d:%s", node->id(), node->op()->mnemonic());
    3676           0 : }
    3677             : 
    3678    11923125 : Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
    3679             : 
    3680             : namespace {
    3681        4068 : Signature<MachineRepresentation>* CreateMachineSignature(
    3682        8136 :     Zone* zone, wasm::FunctionSig* sig) {
    3683             :   Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
    3684             :                                                     sig->parameter_count());
    3685        8136 :   for (auto ret : sig->returns()) {
    3686        4068 :     builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
    3687             :   }
    3688             : 
    3689        6464 :   for (auto param : sig->parameters()) {
    3690        2396 :     builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
    3691             :   }
    3692        4068 :   return builder.Build();
    3693             : }
    3694             : }  // namespace
    3695             : 
    3696      496993 : void WasmGraphBuilder::LowerInt64() {
    3697     1490979 :   if (mcgraph()->machine()->Is64()) return;
    3698             :   Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
    3699             :                   mcgraph()->zone(),
    3700           0 :                   CreateMachineSignature(mcgraph()->zone(), sig_));
    3701           0 :   r.LowerGraph();
    3702             : }
    3703             : 
    3704           0 : void WasmGraphBuilder::SimdScalarLoweringForTesting() {
    3705             :   SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
    3706           0 :       .LowerGraph();
    3707           0 : }
    3708             : 
    3709      412095 : void WasmGraphBuilder::SetSourcePosition(Node* node,
    3710             :                                          wasm::WasmCodePosition position) {
    3711             :   DCHECK_NE(position, wasm::kNoCodePosition);
    3712      412095 :   if (source_position_table_)
    3713      325101 :     source_position_table_->SetSourcePosition(node, SourcePosition(position));
    3714      412120 : }
    3715             : 
    3716        4312 : Node* WasmGraphBuilder::S128Zero() {
    3717        2156 :   has_simd_ = true;
    3718        4312 :   return graph()->NewNode(mcgraph()->machine()->S128Zero());
    3719             : }
    3720             : 
    3721        8328 : Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
    3722        4164 :   has_simd_ = true;
    3723        4164 :   switch (opcode) {
    3724             :     case wasm::kExprF32x4Splat:
    3725         840 :       return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
    3726             :     case wasm::kExprF32x4SConvertI32x4:
    3727             :       return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
    3728          24 :                               inputs[0]);
    3729             :     case wasm::kExprF32x4UConvertI32x4:
    3730             :       return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
    3731          24 :                               inputs[0]);
    3732             :     case wasm::kExprF32x4Abs:
    3733          24 :       return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
    3734             :     case wasm::kExprF32x4Neg:
    3735          24 :       return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
    3736             :     case wasm::kExprF32x4RecipApprox:
    3737             :       return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
    3738          24 :                               inputs[0]);
    3739             :     case wasm::kExprF32x4RecipSqrtApprox:
    3740             :       return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
    3741          24 :                               inputs[0]);
    3742             :     case wasm::kExprF32x4Add:
    3743             :       return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
    3744          72 :                               inputs[1]);
    3745             :     case wasm::kExprF32x4AddHoriz:
    3746             :       return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
    3747          24 :                               inputs[1]);
    3748             :     case wasm::kExprF32x4Sub:
    3749             :       return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
    3750          24 :                               inputs[1]);
    3751             :     case wasm::kExprF32x4Mul:
    3752             :       return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
    3753          24 :                               inputs[1]);
    3754             :     case wasm::kExprF32x4Min:
    3755             :       return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
    3756          24 :                               inputs[1]);
    3757             :     case wasm::kExprF32x4Max:
    3758             :       return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
    3759          24 :                               inputs[1]);
    3760             :     case wasm::kExprF32x4Eq:
    3761             :       return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
    3762          24 :                               inputs[1]);
    3763             :     case wasm::kExprF32x4Ne:
    3764             :       return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
    3765          24 :                               inputs[1]);
    3766             :     case wasm::kExprF32x4Lt:
    3767             :       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
    3768          24 :                               inputs[1]);
    3769             :     case wasm::kExprF32x4Le:
    3770             :       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
    3771          24 :                               inputs[1]);
    3772             :     case wasm::kExprF32x4Gt:
    3773             :       return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
    3774          24 :                               inputs[0]);
    3775             :     case wasm::kExprF32x4Ge:
    3776             :       return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
    3777          24 :                               inputs[0]);
    3778             :     case wasm::kExprI32x4Splat:
    3779        3732 :       return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
    3780             :     case wasm::kExprI32x4SConvertF32x4:
    3781             :       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
    3782          24 :                               inputs[0]);
    3783             :     case wasm::kExprI32x4UConvertF32x4:
    3784             :       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
    3785          24 :                               inputs[0]);
    3786             :     case wasm::kExprI32x4SConvertI16x8Low:
    3787             :       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
    3788          24 :                               inputs[0]);
    3789             :     case wasm::kExprI32x4SConvertI16x8High:
    3790             :       return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
    3791          24 :                               inputs[0]);
    3792             :     case wasm::kExprI32x4Neg:
    3793          24 :       return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
    3794             :     case wasm::kExprI32x4Add:
    3795             :       return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
    3796          72 :                               inputs[1]);
    3797             :     case wasm::kExprI32x4AddHoriz:
    3798             :       return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
    3799          24 :                               inputs[1]);
    3800             :     case wasm::kExprI32x4Sub:
    3801             :       return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
    3802          24 :                               inputs[1]);
    3803             :     case wasm::kExprI32x4Mul:
    3804             :       return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
    3805          24 :                               inputs[1]);
    3806             :     case wasm::kExprI32x4MinS:
    3807             :       return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
    3808          24 :                               inputs[1]);
    3809             :     case wasm::kExprI32x4MaxS:
    3810             :       return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
    3811          24 :                               inputs[1]);
    3812             :     case wasm::kExprI32x4Eq:
    3813             :       return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
    3814         120 :                               inputs[1]);
    3815             :     case wasm::kExprI32x4Ne:
    3816             :       return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
    3817         132 :                               inputs[1]);
    3818             :     case wasm::kExprI32x4LtS:
    3819             :       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
    3820          24 :                               inputs[0]);
    3821             :     case wasm::kExprI32x4LeS:
    3822             :       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
    3823          24 :                               inputs[0]);
    3824             :     case wasm::kExprI32x4GtS:
    3825             :       return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
    3826          24 :                               inputs[1]);
    3827             :     case wasm::kExprI32x4GeS:
    3828             :       return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
    3829          24 :                               inputs[1]);
    3830             :     case wasm::kExprI32x4UConvertI16x8Low:
    3831             :       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
    3832          24 :                               inputs[0]);
    3833             :     case wasm::kExprI32x4UConvertI16x8High:
    3834             :       return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
    3835          24 :                               inputs[0]);
    3836             :     case wasm::kExprI32x4MinU:
    3837             :       return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
    3838          24 :                               inputs[1]);
    3839             :     case wasm::kExprI32x4MaxU:
    3840             :       return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
    3841          24 :                               inputs[1]);
    3842             :     case wasm::kExprI32x4LtU:
    3843             :       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
    3844          24 :                               inputs[0]);
    3845             :     case wasm::kExprI32x4LeU:
    3846             :       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
    3847          24 :                               inputs[0]);
    3848             :     case wasm::kExprI32x4GtU:
    3849             :       return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
    3850          24 :                               inputs[1]);
    3851             :     case wasm::kExprI32x4GeU:
    3852             :       return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
    3853          24 :                               inputs[1]);
    3854             :     case wasm::kExprI16x8Splat:
    3855        2352 :       return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
    3856             :     case wasm::kExprI16x8SConvertI8x16Low:
    3857             :       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
    3858          24 :                               inputs[0]);
    3859             :     case wasm::kExprI16x8SConvertI8x16High:
    3860             :       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
    3861          24 :                               inputs[0]);
    3862             :     case wasm::kExprI16x8Neg:
    3863          24 :       return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
    3864             :     case wasm::kExprI16x8SConvertI32x4:
    3865             :       return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
    3866          24 :                               inputs[0], inputs[1]);
    3867             :     case wasm::kExprI16x8Add:
    3868             :       return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
    3869          24 :                               inputs[1]);
    3870             :     case wasm::kExprI16x8AddSaturateS:
    3871             :       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
    3872          24 :                               inputs[0], inputs[1]);
    3873             :     case wasm::kExprI16x8AddHoriz:
    3874             :       return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
    3875          24 :                               inputs[1]);
    3876             :     case wasm::kExprI16x8Sub:
    3877             :       return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
    3878          24 :                               inputs[1]);
    3879             :     case wasm::kExprI16x8SubSaturateS:
    3880             :       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
    3881          24 :                               inputs[0], inputs[1]);
    3882             :     case wasm::kExprI16x8Mul:
    3883             :       return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
    3884          24 :                               inputs[1]);
    3885             :     case wasm::kExprI16x8MinS:
    3886             :       return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
    3887          24 :                               inputs[1]);
    3888             :     case wasm::kExprI16x8MaxS:
    3889             :       return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
    3890          24 :                               inputs[1]);
    3891             :     case wasm::kExprI16x8Eq:
    3892             :       return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
    3893         120 :                               inputs[1]);
    3894             :     case wasm::kExprI16x8Ne:
    3895             :       return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
    3896         132 :                               inputs[1]);
    3897             :     case wasm::kExprI16x8LtS:
    3898             :       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
    3899          24 :                               inputs[0]);
    3900             :     case wasm::kExprI16x8LeS:
    3901             :       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
    3902          24 :                               inputs[0]);
    3903             :     case wasm::kExprI16x8GtS:
    3904             :       return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
    3905          24 :                               inputs[1]);
    3906             :     case wasm::kExprI16x8GeS:
    3907             :       return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
    3908          24 :                               inputs[1]);
    3909             :     case wasm::kExprI16x8UConvertI8x16Low:
    3910             :       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
    3911          24 :                               inputs[0]);
    3912             :     case wasm::kExprI16x8UConvertI8x16High:
    3913             :       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
    3914          24 :                               inputs[0]);
    3915             :     case wasm::kExprI16x8UConvertI32x4:
    3916             :       return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
    3917          24 :                               inputs[0], inputs[1]);
    3918             :     case wasm::kExprI16x8AddSaturateU:
    3919             :       return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
    3920          24 :                               inputs[0], inputs[1]);
    3921             :     case wasm::kExprI16x8SubSaturateU:
    3922             :       return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
    3923          24 :                               inputs[0], inputs[1]);
    3924             :     case wasm::kExprI16x8MinU:
    3925             :       return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
    3926          24 :                               inputs[1]);
    3927             :     case wasm::kExprI16x8MaxU:
    3928             :       return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
    3929          24 :                               inputs[1]);
    3930             :     case wasm::kExprI16x8LtU:
    3931             :       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
    3932          24 :                               inputs[0]);
    3933             :     case wasm::kExprI16x8LeU:
    3934             :       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
    3935          24 :                               inputs[0]);
    3936             :     case wasm::kExprI16x8GtU:
    3937             :       return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
    3938          24 :                               inputs[1]);
    3939             :     case wasm::kExprI16x8GeU:
    3940             :       return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
    3941          24 :                               inputs[1]);
    3942             :     case wasm::kExprI8x16Splat:
    3943        1728 :       return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
    3944             :     case wasm::kExprI8x16Neg:
    3945          24 :       return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
    3946             :     case wasm::kExprI8x16SConvertI16x8:
    3947             :       return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
    3948          24 :                               inputs[0], inputs[1]);
    3949             :     case wasm::kExprI8x16Add:
    3950             :       return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
    3951          24 :                               inputs[1]);
    3952             :     case wasm::kExprI8x16AddSaturateS:
    3953             :       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
    3954          24 :                               inputs[0], inputs[1]);
    3955             :     case wasm::kExprI8x16Sub:
    3956             :       return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
    3957          24 :                               inputs[1]);
    3958             :     case wasm::kExprI8x16SubSaturateS:
    3959             :       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
    3960          24 :                               inputs[0], inputs[1]);
    3961             :     case wasm::kExprI8x16Mul:
    3962             :       return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
    3963          24 :                               inputs[1]);
    3964             :     case wasm::kExprI8x16MinS:
    3965             :       return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
    3966          24 :                               inputs[1]);
    3967             :     case wasm::kExprI8x16MaxS:
    3968             :       return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
    3969          24 :                               inputs[1]);
    3970             :     case wasm::kExprI8x16Eq:
    3971             :       return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
    3972         120 :                               inputs[1]);
    3973             :     case wasm::kExprI8x16Ne:
    3974             :       return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
    3975         132 :                               inputs[1]);
    3976             :     case wasm::kExprI8x16LtS:
    3977             :       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
    3978          24 :                               inputs[0]);
    3979             :     case wasm::kExprI8x16LeS:
    3980             :       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
    3981          24 :                               inputs[0]);
    3982             :     case wasm::kExprI8x16GtS:
    3983             :       return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
    3984          24 :                               inputs[1]);
    3985             :     case wasm::kExprI8x16GeS:
    3986             :       return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
    3987          24 :                               inputs[1]);
    3988             :     case wasm::kExprI8x16UConvertI16x8:
    3989             :       return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
    3990          24 :                               inputs[0], inputs[1]);
    3991             :     case wasm::kExprI8x16AddSaturateU:
    3992             :       return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
    3993          24 :                               inputs[0], inputs[1]);
    3994             :     case wasm::kExprI8x16SubSaturateU:
    3995             :       return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
    3996          24 :                               inputs[0], inputs[1]);
    3997             :     case wasm::kExprI8x16MinU:
    3998             :       return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
    3999          24 :                               inputs[1]);
    4000             :     case wasm::kExprI8x16MaxU:
    4001             :       return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
    4002          24 :                               inputs[1]);
    4003             :     case wasm::kExprI8x16LtU:
    4004             :       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
    4005          24 :                               inputs[0]);
    4006             :     case wasm::kExprI8x16LeU:
    4007             :       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
    4008          24 :                               inputs[0]);
    4009             :     case wasm::kExprI8x16GtU:
    4010             :       return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
    4011          24 :                               inputs[1]);
    4012             :     case wasm::kExprI8x16GeU:
    4013             :       return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
    4014          24 :                               inputs[1]);
    4015             :     case wasm::kExprS128And:
    4016             :       return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
    4017          24 :                               inputs[1]);
    4018             :     case wasm::kExprS128Or:
    4019             :       return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
    4020          24 :                               inputs[1]);
    4021             :     case wasm::kExprS128Xor:
    4022             :       return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
    4023          24 :                               inputs[1]);
    4024             :     case wasm::kExprS128Not:
    4025          24 :       return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
    4026             :     case wasm::kExprS128Select:
    4027             :       return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[2],
    4028          84 :                               inputs[0], inputs[1]);
    4029             :     case wasm::kExprS1x4AnyTrue:
    4030         108 :       return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
    4031             :     case wasm::kExprS1x4AllTrue:
    4032         108 :       return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
    4033             :     case wasm::kExprS1x8AnyTrue:
    4034         108 :       return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
    4035             :     case wasm::kExprS1x8AllTrue:
    4036         108 :       return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
    4037             :     case wasm::kExprS1x16AnyTrue:
    4038         108 :       return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
    4039             :     case wasm::kExprS1x16AllTrue:
    4040         108 :       return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
    4041             :     default:
    4042           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4043             :   }
    4044             : }
    4045             : 
    4046       18676 : Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
    4047       18676 :                                    Node* const* inputs) {
    4048       18676 :   has_simd_ = true;
    4049       18676 :   switch (opcode) {
    4050             :     case wasm::kExprF32x4ExtractLane:
    4051             :       return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
    4052        2112 :                               inputs[0]);
    4053             :     case wasm::kExprF32x4ReplaceLane:
    4054             :       return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
    4055         192 :                               inputs[0], inputs[1]);
    4056             :     case wasm::kExprI32x4ExtractLane:
    4057             :       return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
    4058       13116 :                               inputs[0]);
    4059             :     case wasm::kExprI32x4ReplaceLane:
    4060             :       return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
    4061         288 :                               inputs[0], inputs[1]);
    4062             :     case wasm::kExprI16x8ExtractLane:
    4063             :       return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
    4064       15840 :                               inputs[0]);
    4065             :     case wasm::kExprI16x8ReplaceLane:
    4066             :       return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
    4067         288 :                               inputs[0], inputs[1]);
    4068             :     case wasm::kExprI8x16ExtractLane:
    4069             :       return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
    4070       23712 :                               inputs[0]);
    4071             :     case wasm::kExprI8x16ReplaceLane:
    4072             :       return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
    4073         480 :                               inputs[0], inputs[1]);
    4074             :     default:
    4075           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4076             :   }
    4077             : }
    4078             : 
    4079        1272 : Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
    4080        1272 :                                     Node* const* inputs) {
    4081        1272 :   has_simd_ = true;
    4082        1272 :   switch (opcode) {
    4083             :     case wasm::kExprI32x4Shl:
    4084         744 :       return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
    4085             :     case wasm::kExprI32x4ShrS:
    4086             :       return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
    4087         744 :                               inputs[0]);
    4088             :     case wasm::kExprI32x4ShrU:
    4089             :       return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
    4090         744 :                               inputs[0]);
    4091             :     case wasm::kExprI16x8Shl:
    4092         360 :       return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
    4093             :     case wasm::kExprI16x8ShrS:
    4094             :       return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
    4095         360 :                               inputs[0]);
    4096             :     case wasm::kExprI16x8ShrU:
    4097             :       return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
    4098         360 :                               inputs[0]);
    4099             :     case wasm::kExprI8x16Shl:
    4100         168 :       return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
    4101             :     case wasm::kExprI8x16ShrS:
    4102             :       return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
    4103         168 :                               inputs[0]);
    4104             :     case wasm::kExprI8x16ShrU:
    4105             :       return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
    4106         168 :                               inputs[0]);
    4107             :     default:
    4108           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4109             :   }
    4110             : }
    4111             : 
    4112       14544 : Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
    4113       14544 :                                           Node* const* inputs) {
    4114       14544 :   has_simd_ = true;
    4115             :   return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
    4116       43632 :                           inputs[0], inputs[1]);
    4117             : }
    4118             : 
    4119             : #define ATOMIC_BINOP_LIST(V)                        \
    4120             :   V(I32AtomicAdd, Add, Uint32, Word32)              \
    4121             :   V(I64AtomicAdd, Add, Uint64, Word64)              \
    4122             :   V(I32AtomicAdd8U, Add, Uint8, Word32)             \
    4123             :   V(I32AtomicAdd16U, Add, Uint16, Word32)           \
    4124             :   V(I64AtomicAdd8U, Add, Uint8, Word64)             \
    4125             :   V(I64AtomicAdd16U, Add, Uint16, Word64)           \
    4126             :   V(I64AtomicAdd32U, Add, Uint32, Word64)           \
    4127             :   V(I32AtomicSub, Sub, Uint32, Word32)              \
    4128             :   V(I64AtomicSub, Sub, Uint64, Word64)              \
    4129             :   V(I32AtomicSub8U, Sub, Uint8, Word32)             \
    4130             :   V(I32AtomicSub16U, Sub, Uint16, Word32)           \
    4131             :   V(I64AtomicSub8U, Sub, Uint8, Word64)             \
    4132             :   V(I64AtomicSub16U, Sub, Uint16, Word64)           \
    4133             :   V(I64AtomicSub32U, Sub, Uint32, Word64)           \
    4134             :   V(I32AtomicAnd, And, Uint32, Word32)              \
    4135             :   V(I64AtomicAnd, And, Uint64, Word64)              \
    4136             :   V(I32AtomicAnd8U, And, Uint8, Word32)             \
    4137             :   V(I64AtomicAnd16U, And, Uint16, Word64)           \
    4138             :   V(I32AtomicAnd16U, And, Uint16, Word32)           \
    4139             :   V(I64AtomicAnd8U, And, Uint8, Word64)             \
    4140             :   V(I64AtomicAnd32U, And, Uint32, Word64)           \
    4141             :   V(I32AtomicOr, Or, Uint32, Word32)                \
    4142             :   V(I64AtomicOr, Or, Uint64, Word64)                \
    4143             :   V(I32AtomicOr8U, Or, Uint8, Word32)               \
    4144             :   V(I32AtomicOr16U, Or, Uint16, Word32)             \
    4145             :   V(I64AtomicOr8U, Or, Uint8, Word64)               \
    4146             :   V(I64AtomicOr16U, Or, Uint16, Word64)             \
    4147             :   V(I64AtomicOr32U, Or, Uint32, Word64)             \
    4148             :   V(I32AtomicXor, Xor, Uint32, Word32)              \
    4149             :   V(I64AtomicXor, Xor, Uint64, Word64)              \
    4150             :   V(I32AtomicXor8U, Xor, Uint8, Word32)             \
    4151             :   V(I32AtomicXor16U, Xor, Uint16, Word32)           \
    4152             :   V(I64AtomicXor8U, Xor, Uint8, Word64)             \
    4153             :   V(I64AtomicXor16U, Xor, Uint16, Word64)           \
    4154             :   V(I64AtomicXor32U, Xor, Uint32, Word64)           \
    4155             :   V(I32AtomicExchange, Exchange, Uint32, Word32)    \
    4156             :   V(I64AtomicExchange, Exchange, Uint64, Word64)    \
    4157             :   V(I32AtomicExchange8U, Exchange, Uint8, Word32)   \
    4158             :   V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
    4159             :   V(I64AtomicExchange8U, Exchange, Uint8, Word64)   \
    4160             :   V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
    4161             :   V(I64AtomicExchange32U, Exchange, Uint32, Word64)
    4162             : 
    4163             : #define ATOMIC_CMP_EXCHG_LIST(V)                 \
    4164             :   V(I32AtomicCompareExchange, Uint32, Word32)    \
    4165             :   V(I64AtomicCompareExchange, Uint64, Word64)    \
    4166             :   V(I32AtomicCompareExchange8U, Uint8, Word32)   \
    4167             :   V(I32AtomicCompareExchange16U, Uint16, Word32) \
    4168             :   V(I64AtomicCompareExchange8U, Uint8, Word64)   \
    4169             :   V(I64AtomicCompareExchange16U, Uint16, Word64) \
    4170             :   V(I64AtomicCompareExchange32U, Uint32, Word64)
    4171             : 
    4172             : #define ATOMIC_LOAD_LIST(V)           \
    4173             :   V(I32AtomicLoad, Uint32, Word32)    \
    4174             :   V(I64AtomicLoad, Uint64, Word64)    \
    4175             :   V(I32AtomicLoad8U, Uint8, Word32)   \
    4176             :   V(I32AtomicLoad16U, Uint16, Word32) \
    4177             :   V(I64AtomicLoad8U, Uint8, Word64)   \
    4178             :   V(I64AtomicLoad16U, Uint16, Word64) \
    4179             :   V(I64AtomicLoad32U, Uint32, Word64)
    4180             : 
    4181             : #define ATOMIC_STORE_LIST(V)                    \
    4182             :   V(I32AtomicStore, Uint32, kWord32, Word32)    \
    4183             :   V(I64AtomicStore, Uint64, kWord64, Word64)    \
    4184             :   V(I32AtomicStore8U, Uint8, kWord8, Word32)    \
    4185             :   V(I32AtomicStore16U, Uint16, kWord16, Word32) \
    4186             :   V(I64AtomicStore8U, Uint8, kWord8, Word64)    \
    4187             :   V(I64AtomicStore16U, Uint16, kWord16, Word64) \
    4188             :   V(I64AtomicStore32U, Uint32, kWord32, Word64)
    4189             : 
    4190       32678 : Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
    4191             :                                  uint32_t alignment, uint32_t offset,
    4192      134036 :                                  wasm::WasmCodePosition position) {
    4193             :   Node* node;
    4194       32678 :   switch (opcode) {
    4195             : #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix)                     \
    4196             :   case wasm::kExpr##Name: {                                                   \
    4197             :     Node* index = CheckBoundsAndAlignment(                                    \
    4198             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,    \
    4199             :         position);                                                            \
    4200             :     node = graph()->NewNode(                                                  \
    4201             :         mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
    4202             :         MemBuffer(offset), index, inputs[1], Effect(), Control());            \
    4203             :     break;                                                                    \
    4204             :   }
    4205       96688 :     ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
    4206             : #undef BUILD_ATOMIC_BINOP
    4207             : 
    4208             : #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix)                            \
    4209             :   case wasm::kExpr##Name: {                                                   \
    4210             :     Node* index = CheckBoundsAndAlignment(                                    \
    4211             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,    \
    4212             :         position);                                                            \
    4213             :     node = graph()->NewNode(                                                  \
    4214             :         mcgraph()->machine()->Prefix##AtomicCompareExchange(                  \
    4215             :             MachineType::Type()),                                             \
    4216             :         MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
    4217             :     break;                                                                    \
    4218             :   }
    4219         733 :     ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
    4220             : #undef BUILD_ATOMIC_CMP_EXCHG
    4221             : 
    4222             : #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix)                           \
    4223             :   case wasm::kExpr##Name: {                                                \
    4224             :     Node* index = CheckBoundsAndAlignment(                                 \
    4225             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
    4226             :         position);                                                         \
    4227             :     node = graph()->NewNode(                                               \
    4228             :         mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()),     \
    4229             :         MemBuffer(offset), index, Effect(), Control());                    \
    4230             :     break;                                                                 \
    4231             :   }
    4232       16392 :     ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
    4233             : #undef BUILD_ATOMIC_LOAD_OP
    4234             : 
    4235             : #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix)                         \
    4236             :   case wasm::kExpr##Name: {                                                    \
    4237             :     Node* index = CheckBoundsAndAlignment(                                     \
    4238             :         wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset,     \
    4239             :         position);                                                             \
    4240             :     node = graph()->NewNode(                                                   \
    4241             :         mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
    4242             :         MemBuffer(offset), index, inputs[1], Effect(), Control());             \
    4243             :     break;                                                                     \
    4244             :   }
    4245       15808 :     ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
    4246             : #undef BUILD_ATOMIC_STORE_OP
    4247             :     case wasm::kExprAtomicWake: {
    4248             :       Node* index = CheckBoundsAndAlignment(
    4249             :           wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
    4250         320 :           position);
    4251             :       // Now that we've bounds-checked, compute the effective address.
    4252             :       Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    4253         160 :                                        Uint32Constant(offset), index);
    4254             :       WasmAtomicWakeDescriptor interface_descriptor;
    4255             :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4256             :           mcgraph()->zone(), interface_descriptor,
    4257             :           interface_descriptor.GetStackParameterCount(),
    4258             :           CallDescriptor::kNoFlags, Operator::kNoProperties,
    4259         160 :           StubCallMode::kCallWasmRuntimeStub);
    4260             :       Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    4261         160 :           wasm::WasmCode::kWasmAtomicWake, RelocInfo::WASM_STUB_CALL);
    4262             :       node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4263             :                               call_target, address, inputs[1], Effect(),
    4264         320 :                               Control());
    4265             :       break;
    4266             :     }
    4267             : 
    4268             :     case wasm::kExprI32AtomicWait: {
    4269             :       Node* index = CheckBoundsAndAlignment(
    4270             :           wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
    4271         508 :           position);
    4272             :       // Now that we've bounds-checked, compute the effective address.
    4273             :       Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    4274         256 :                                        Uint32Constant(offset), index);
    4275             :       Node* timeout;
    4276         512 :       if (mcgraph()->machine()->Is32()) {
    4277           0 :         timeout = BuildF64SConvertI64(inputs[2]);
    4278             :       } else {
    4279             :         timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
    4280         256 :                                    inputs[2]);
    4281             :       }
    4282             :       WasmI32AtomicWaitDescriptor interface_descriptor;
    4283             :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4284             :           mcgraph()->zone(), interface_descriptor,
    4285             :           interface_descriptor.GetStackParameterCount(),
    4286             :           CallDescriptor::kNoFlags, Operator::kNoProperties,
    4287         256 :           StubCallMode::kCallWasmRuntimeStub);
    4288             :       Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    4289         254 :           wasm::WasmCode::kWasmI32AtomicWait, RelocInfo::WASM_STUB_CALL);
    4290             :       node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4291             :                               call_target, address, inputs[1], timeout,
    4292         512 :                               Effect(), Control());
    4293             :       break;
    4294             :     }
    4295             : 
    4296             :     case wasm::kExprI64AtomicWait: {
    4297             :       Node* index = CheckBoundsAndAlignment(
    4298             :           wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
    4299         506 :           position);
    4300             :       // Now that we've bounds-checked, compute the effective address.
    4301             :       Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
    4302         256 :                                        Uint32Constant(offset), index);
    4303             :       Node* timeout;
    4304         510 :       if (mcgraph()->machine()->Is32()) {
    4305           0 :         timeout = BuildF64SConvertI64(inputs[2]);
    4306             :       } else {
    4307             :         timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
    4308         255 :                                    inputs[2]);
    4309             :       }
    4310             :       Node* expected_value_low = graph()->NewNode(
    4311         510 :           mcgraph()->machine()->TruncateInt64ToInt32(), inputs[1]);
    4312             :       Node* tmp = graph()->NewNode(mcgraph()->machine()->Word64Shr(), inputs[1],
    4313         512 :                                    Int64Constant(32));
    4314             :       Node* expected_value_high =
    4315         255 :           graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), tmp);
    4316             :       WasmI64AtomicWaitDescriptor interface_descriptor;
    4317             :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4318             :           mcgraph()->zone(), interface_descriptor,
    4319             :           interface_descriptor.GetStackParameterCount(),
    4320             :           CallDescriptor::kNoFlags, Operator::kNoProperties,
    4321         256 :           StubCallMode::kCallWasmRuntimeStub);
    4322             :       Node* call_target = mcgraph()->RelocatableIntPtrConstant(
    4323         254 :           wasm::WasmCode::kWasmI64AtomicWait, RelocInfo::WASM_STUB_CALL);
    4324             :       node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4325             :                               call_target, address, expected_value_high,
    4326         256 :                               expected_value_low, timeout, Effect(), Control());
    4327             :       break;
    4328             :     }
    4329             : 
    4330             :     default:
    4331           0 :       FATAL_UNSUPPORTED_OPCODE(opcode);
    4332             :   }
    4333       32686 :   return SetEffect(node);
    4334             : }
    4335             : 
    4336             : #undef ATOMIC_BINOP_LIST
    4337             : #undef ATOMIC_CMP_EXCHG_LIST
    4338             : #undef ATOMIC_LOAD_LIST
    4339             : #undef ATOMIC_STORE_LIST
    4340             : 
    4341          64 : Node* WasmGraphBuilder::CheckDataSegmentIsPassiveAndNotDropped(
    4342         640 :     uint32_t data_segment_index, wasm::WasmCodePosition position) {
    4343             :   // The data segment index must be in bounds since it is required by
    4344             :   // validation.
    4345             :   DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments);
    4346             : 
    4347             :   Node* dropped_data_segments =
    4348         128 :       LOAD_INSTANCE_FIELD(DroppedDataSegments, MachineType::Pointer());
    4349             :   Node* is_segment_dropped = SetEffect(graph()->NewNode(
    4350             :       mcgraph()->machine()->Load(MachineType::Uint8()), dropped_data_segments,
    4351         192 :       mcgraph()->IntPtrConstant(data_segment_index), Effect(), Control()));
    4352          64 :   TrapIfTrue(wasm::kTrapDataSegmentDropped, is_segment_dropped, position);
    4353          64 :   return dropped_data_segments;
    4354             : }
    4355             : 
    4356          40 : Node* WasmGraphBuilder::MemoryInit(uint32_t data_segment_index, Node* dst,
    4357             :                                    Node* src, Node* size,
    4358         720 :                                    wasm::WasmCodePosition position) {
    4359          40 :   CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
    4360          40 :   dst = BoundsCheckMemRange(dst, size, position);
    4361          40 :   MachineOperatorBuilder* m = mcgraph()->machine();
    4362             : 
    4363             :   Node* seg_index = Uint32Constant(data_segment_index);
    4364             : 
    4365             :   {
    4366             :     // Load segment size from WasmInstanceObject::data_segment_sizes.
    4367             :     Node* seg_size_array =
    4368          80 :         LOAD_INSTANCE_FIELD(DataSegmentSizes, MachineType::Pointer());
    4369             :     STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2);
    4370             :     Node* scaled_index = Uint32ToUintptr(
    4371          80 :         graph()->NewNode(m->Word32Shl(), seg_index, Int32Constant(2)));
    4372             :     Node* seg_size = SetEffect(graph()->NewNode(m->Load(MachineType::Uint32()),
    4373             :                                                 seg_size_array, scaled_index,
    4374          40 :                                                 Effect(), Control()));
    4375             : 
    4376             :     // Bounds check the src index against the segment size.
    4377          40 :     BoundsCheckRange(src, size, seg_size, position);
    4378             :   }
    4379             : 
    4380             :   {
    4381             :     // Load segment's base pointer from WasmInstanceObject::data_segment_starts.
    4382             :     Node* seg_start_array =
    4383          80 :         LOAD_INSTANCE_FIELD(DataSegmentStarts, MachineType::Pointer());
    4384             :     STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <=
    4385             :                   kMaxUInt32 / kSystemPointerSize);
    4386             :     Node* scaled_index = Uint32ToUintptr(graph()->NewNode(
    4387          80 :         m->Word32Shl(), seg_index, Int32Constant(kSystemPointerSizeLog2)));
    4388             :     Node* seg_start = SetEffect(
    4389             :         graph()->NewNode(m->Load(MachineType::Pointer()), seg_start_array,
    4390          40 :                          scaled_index, Effect(), Control()));
    4391             : 
    4392             :     // Convert src index to pointer.
    4393          40 :     src = graph()->NewNode(m->IntAdd(), seg_start, Uint32ToUintptr(src));
    4394             :   }
    4395             : 
    4396             :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
    4397         120 :       ExternalReference::wasm_memory_copy()));
    4398             :   MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
    4399          40 :                              MachineType::Uint32()};
    4400             :   MachineSignature sig(0, 3, sig_types);
    4401          40 :   return BuildCCall(&sig, function, dst, src, size);
    4402             : }
    4403             : 
    4404          24 : Node* WasmGraphBuilder::DataDrop(uint32_t data_segment_index,
    4405         144 :                                  wasm::WasmCodePosition position) {
    4406             :   Node* dropped_data_segments =
    4407          24 :       CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
    4408             :   const Operator* store_op = mcgraph()->machine()->Store(
    4409          24 :       StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
    4410             :   return SetEffect(
    4411             :       graph()->NewNode(store_op, dropped_data_segments,
    4412             :                        mcgraph()->IntPtrConstant(data_segment_index),
    4413          72 :                        mcgraph()->Int32Constant(1), Effect(), Control()));
    4414             : }
    4415             : 
    4416          24 : Node* WasmGraphBuilder::MemoryCopy(Node* dst, Node* src, Node* size,
    4417          24 :                                    wasm::WasmCodePosition position) {
    4418          24 :   dst = BoundsCheckMemRange(dst, size, position);
    4419          24 :   src = BoundsCheckMemRange(src, size, position);
    4420             :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
    4421          72 :       ExternalReference::wasm_memory_copy()));
    4422             :   MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
    4423          24 :                              MachineType::Uint32()};
    4424             :   MachineSignature sig(0, 3, sig_types);
    4425          24 :   return BuildCCall(&sig, function, dst, src, size);
    4426             : }
    4427             : 
    4428          24 : Node* WasmGraphBuilder::MemoryFill(Node* dst, Node* value, Node* size,
    4429          24 :                                    wasm::WasmCodePosition position) {
    4430          24 :   dst = BoundsCheckMemRange(dst, size, position);
    4431             :   Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
    4432          72 :       ExternalReference::wasm_memory_fill()));
    4433             :   MachineType sig_types[] = {MachineType::Pointer(), MachineType::Uint32(),
    4434          24 :                              MachineType::Uint32()};
    4435             :   MachineSignature sig(0, 3, sig_types);
    4436          24 :   return BuildCCall(&sig, function, dst, value, size);
    4437             : }
    4438             : 
    4439          48 : Node* WasmGraphBuilder::CheckElemSegmentIsPassiveAndNotDropped(
    4440         480 :     uint32_t elem_segment_index, wasm::WasmCodePosition position) {
    4441             :   // The elem segment index must be in bounds since it is required by
    4442             :   // validation.
    4443             :   DCHECK_LT(elem_segment_index, env_->module->elem_segments.size());
    4444             : 
    4445             :   Node* dropped_elem_segments =
    4446          96 :       LOAD_INSTANCE_FIELD(DroppedElemSegments, MachineType::Pointer());
    4447             :   Node* is_segment_dropped = SetEffect(graph()->NewNode(
    4448             :       mcgraph()->machine()->Load(MachineType::Uint8()), dropped_elem_segments,
    4449         144 :       mcgraph()->IntPtrConstant(elem_segment_index), Effect(), Control()));
    4450          48 :   TrapIfTrue(wasm::kTrapElemSegmentDropped, is_segment_dropped, position);
    4451          48 :   return dropped_elem_segments;
    4452             : }
    4453             : 
    4454          31 : Node* WasmGraphBuilder::TableInit(uint32_t table_index,
    4455             :                                   uint32_t elem_segment_index, Node* dst,
    4456             :                                   Node* src, Node* size,
    4457          64 :                                   wasm::WasmCodePosition position) {
    4458          31 :   CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
    4459             :   Node* args[] = {
    4460          64 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
    4461          64 :       graph()->NewNode(mcgraph()->common()->NumberConstant(elem_segment_index)),
    4462          32 :       BuildConvertUint32ToSmiWithSaturation(dst, wasm::kV8MaxWasmTableSize),
    4463          32 :       BuildConvertUint32ToSmiWithSaturation(src, wasm::kV8MaxWasmTableSize),
    4464         128 :       BuildConvertUint32ToSmiWithSaturation(size, wasm::kV8MaxWasmTableSize)};
    4465             :   Node* result =
    4466          32 :       BuildCallToRuntime(Runtime::kWasmTableInit, args, arraysize(args));
    4467             : 
    4468          32 :   return result;
    4469             : }
    4470             : 
    4471          16 : Node* WasmGraphBuilder::ElemDrop(uint32_t elem_segment_index,
    4472          96 :                                  wasm::WasmCodePosition position) {
    4473             :   Node* dropped_elem_segments =
    4474          16 :       CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
    4475             :   const Operator* store_op = mcgraph()->machine()->Store(
    4476          16 :       StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
    4477             :   return SetEffect(
    4478             :       graph()->NewNode(store_op, dropped_elem_segments,
    4479             :                        mcgraph()->IntPtrConstant(elem_segment_index),
    4480          48 :                        mcgraph()->Int32Constant(1), Effect(), Control()));
    4481             : }
    4482             : 
    4483          64 : Node* WasmGraphBuilder::TableCopy(uint32_t table_src_index,
    4484             :                                   uint32_t table_dst_index, Node* dst,
    4485             :                                   Node* src, Node* size,
    4486         128 :                                   wasm::WasmCodePosition position) {
    4487             :   Node* args[] = {
    4488         128 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_src_index)),
    4489         128 :       graph()->NewNode(mcgraph()->common()->NumberConstant(table_dst_index)),
    4490          64 :       BuildConvertUint32ToSmiWithSaturation(dst, wasm::kV8MaxWasmTableSize),
    4491          64 :       BuildConvertUint32ToSmiWithSaturation(src, wasm::kV8MaxWasmTableSize),
    4492         256 :       BuildConvertUint32ToSmiWithSaturation(size, wasm::kV8MaxWasmTableSize)};
    4493             :   Node* result =
    4494          64 :       BuildCallToRuntime(Runtime::kWasmTableCopy, args, arraysize(args));
    4495             : 
    4496          64 :   return result;
    4497             : }
    4498             : 
    4499           0 : class WasmDecorator final : public GraphDecorator {
    4500             :  public:
    4501             :   explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
    4502           0 :       : origins_(origins), decoder_(decoder) {}
    4503             : 
    4504           0 :   void Decorate(Node* node) final {
    4505             :     origins_->SetNodeOrigin(
    4506             :         node, NodeOrigin("wasm graph creation", "n/a",
    4507           0 :                          NodeOrigin::kWasmBytecode, decoder_->position()));
    4508           0 :   }
    4509             : 
    4510             :  private:
    4511             :   compiler::NodeOriginTable* origins_;
    4512             :   wasm::Decoder* decoder_;
    4513             : };
    4514             : 
    4515           0 : void WasmGraphBuilder::AddBytecodePositionDecorator(
    4516             :     NodeOriginTable* node_origins, wasm::Decoder* decoder) {
    4517             :   DCHECK_NULL(decorator_);
    4518           0 :   decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
    4519           0 :   graph()->AddDecorator(decorator_);
    4520           0 : }
    4521             : 
    4522           0 : void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
    4523             :   DCHECK_NOT_NULL(decorator_);
    4524           0 :   graph()->RemoveDecorator(decorator_);
    4525           0 :   decorator_ = nullptr;
    4526           0 : }
    4527             : 
    4528             : namespace {
    4529      278213 : bool must_record_function_compilation(Isolate* isolate) {
    4530      278213 :   return isolate->logger()->is_listening_to_code_events() ||
    4531             :          isolate->is_profiling();
    4532             : }
    4533             : 
    4534             : PRINTF_FORMAT(4, 5)
    4535           0 : void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
    4536             :                                Isolate* isolate, Handle<Code> code,
    4537             :                                const char* format, ...) {
    4538             :   DCHECK(must_record_function_compilation(isolate));
    4539             : 
    4540             :   ScopedVector<char> buffer(128);
    4541             :   va_list arguments;
    4542           0 :   va_start(arguments, format);
    4543           0 :   int len = VSNPrintF(buffer, format, arguments);
    4544           0 :   CHECK_LT(0, len);
    4545           0 :   va_end(arguments);
    4546             :   Handle<String> name_str =
    4547           0 :       isolate->factory()->NewStringFromAsciiChecked(buffer.start());
    4548           0 :   PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
    4549           0 : }
    4550             : 
    4551             : class WasmWrapperGraphBuilder : public WasmGraphBuilder {
    4552             :  public:
    4553      149250 :   WasmWrapperGraphBuilder(Zone* zone, JSGraph* jsgraph, wasm::FunctionSig* sig,
    4554             :                           compiler::SourcePositionTable* spt,
    4555             :                           StubCallMode stub_mode, wasm::WasmFeatures features)
    4556             :       : WasmGraphBuilder(nullptr, zone, jsgraph, sig, spt),
    4557             :         isolate_(jsgraph->isolate()),
    4558             :         jsgraph_(jsgraph),
    4559             :         stub_mode_(stub_mode),
    4560      298500 :         enabled_features_(features) {}
    4561             : 
    4562       46970 :   Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
    4563      129366 :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4564       39447 :     CommonOperatorBuilder* common = mcgraph()->common();
    4565       39447 :     Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    4566             :                        ? mcgraph()->RelocatableIntPtrConstant(
    4567             :                              wasm::WasmCode::kWasmAllocateHeapNumber,
    4568             :                              RelocInfo::WASM_STUB_CALL)
    4569             :                        : jsgraph()->HeapConstant(
    4570       46970 :                              BUILTIN_CODE(isolate_, AllocateHeapNumber));
    4571       39447 :     if (!allocate_heap_number_operator_.is_set()) {
    4572             :       auto call_descriptor = Linkage::GetStubCallDescriptor(
    4573             :           mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
    4574       33075 :           CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
    4575       11025 :       allocate_heap_number_operator_.set(common->Call(call_descriptor));
    4576             :     }
    4577             :     Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
    4578             :                                          target, Effect(), control);
    4579             :     SetEffect(
    4580             :         graph()->NewNode(machine->Store(StoreRepresentation(
    4581             :                              MachineRepresentation::kFloat64, kNoWriteBarrier)),
    4582             :                          heap_number, BuildHeapNumberValueIndexConstant(),
    4583       78894 :                          value, heap_number, control));
    4584       39447 :     return heap_number;
    4585             :   }
    4586             : 
    4587       81410 :   Node* BuildChangeSmiToFloat64(Node* value) {
    4588             :     return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
    4589      162820 :                             BuildChangeSmiToInt32(value));
    4590             :   }
    4591             : 
    4592       81409 :   Node* BuildTestHeapObject(Node* value) {
    4593             :     return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
    4594      162819 :                             mcgraph()->IntPtrConstant(kHeapObjectTag));
    4595             :   }
    4596             : 
    4597       81409 :   Node* BuildLoadHeapNumberValue(Node* value) {
    4598             :     return SetEffect(graph()->NewNode(
    4599             :         mcgraph()->machine()->Load(MachineType::Float64()), value,
    4600      407048 :         BuildHeapNumberValueIndexConstant(), Effect(), Control()));
    4601             :   }
    4602             : 
    4603             :   Node* BuildHeapNumberValueIndexConstant() {
    4604      120856 :     return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
    4605             :   }
    4606             : 
    4607             :   Node* BuildChangeInt32ToTagged(Node* value) {
    4608             :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4609             :     CommonOperatorBuilder* common = mcgraph()->common();
    4610             : 
    4611             :     if (SmiValuesAre32Bits()) {
    4612       23952 :       return BuildChangeInt32ToSmi(value);
    4613             :     }
    4614             :     DCHECK(SmiValuesAre31Bits());
    4615             : 
    4616             :     Node* effect = Effect();
    4617             :     Node* control = Control();
    4618             :     Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
    4619             :                                  graph()->start());
    4620             : 
    4621             :     Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
    4622             :     Node* branch =
    4623             :         graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
    4624             : 
    4625             :     Node* if_true = graph()->NewNode(common->IfTrue(), branch);
    4626             :     Node* vtrue = BuildAllocateHeapNumberWithValue(
    4627             :         graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
    4628             :     Node* etrue = Effect();
    4629             : 
    4630             :     Node* if_false = graph()->NewNode(common->IfFalse(), branch);
    4631             :     Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
    4632             :     vfalse = BuildChangeInt32ToIntPtr(vfalse);
    4633             : 
    4634             :     Node* merge =
    4635             :         SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
    4636             :     SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
    4637             :     return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
    4638             :                             vtrue, vfalse, merge);
    4639             :   }
    4640             : 
    4641       39447 :   Node* BuildChangeFloat64ToTagged(Node* value) {
    4642      315576 :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4643       39447 :     CommonOperatorBuilder* common = mcgraph()->common();
    4644             : 
    4645             :     // Check several conditions:
    4646             :     //  i32?
    4647             :     //  ├─ true: zero?
    4648             :     //  │        ├─ true: negative?
    4649             :     //  │        │        ├─ true: box
    4650             :     //  │        │        └─ false: potentially Smi
    4651             :     //  │        └─ false: potentially Smi
    4652             :     //  └─ false: box
    4653             :     // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
    4654             :     // still need to check whether the value fits in a Smi.
    4655             : 
    4656             :     Node* effect = Effect();
    4657             :     Node* control = Control();
    4658       39447 :     Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
    4659             :     Node* check_i32 = graph()->NewNode(
    4660             :         machine->Float64Equal(), value,
    4661       78894 :         graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
    4662       39447 :     Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
    4663             : 
    4664       39447 :     Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
    4665       39447 :     Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
    4666             : 
    4667             :     // We only need to check for -0 if the {value} can potentially contain -0.
    4668             :     Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
    4669       39447 :                                         mcgraph()->Int32Constant(0));
    4670             :     Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
    4671       39447 :                                          check_zero, if_i32);
    4672             : 
    4673       39447 :     Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
    4674       39447 :     Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
    4675             : 
    4676             :     // In case of 0, we need to check the high bits for the IEEE -0 pattern.
    4677             :     Node* check_negative = graph()->NewNode(
    4678             :         machine->Int32LessThan(),
    4679             :         graph()->NewNode(machine->Float64ExtractHighWord32(), value),
    4680       78894 :         mcgraph()->Int32Constant(0));
    4681             :     Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
    4682       39447 :                                              check_negative, if_zero);
    4683             : 
    4684       39447 :     Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
    4685             :     Node* if_not_negative =
    4686       39447 :         graph()->NewNode(common->IfFalse(), branch_negative);
    4687             : 
    4688             :     // We need to create a box for negative 0.
    4689             :     Node* if_smi =
    4690       39447 :         graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
    4691       39447 :     Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
    4692             : 
    4693             :     // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
    4694             :     // 32-bit machines we need to deal with potential overflow and fallback to
    4695             :     // boxing.
    4696             :     Node* vsmi;
    4697             :     if (SmiValuesAre32Bits()) {
    4698       39447 :       vsmi = BuildChangeInt32ToSmi(value32);
    4699             :     } else {
    4700             :       DCHECK(SmiValuesAre31Bits());
    4701             :       Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
    4702             :                                        value32, if_smi);
    4703             : 
    4704             :       Node* check_ovf =
    4705             :           graph()->NewNode(common->Projection(1), smi_tag, if_smi);
    4706             :       Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
    4707             :                                           check_ovf, if_smi);
    4708             : 
    4709             :       Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
    4710             :       if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
    4711             : 
    4712             :       if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
    4713             :       vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
    4714             :       vsmi = BuildChangeInt32ToIntPtr(vsmi);
    4715             :     }
    4716             : 
    4717             :     // Allocate the box for the {value}.
    4718       39447 :     Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
    4719             :     Node* ebox = Effect();
    4720             : 
    4721             :     Node* merge =
    4722       39447 :         SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
    4723       39447 :     SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
    4724             :     return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
    4725       78894 :                             vsmi, vbox, merge);
    4726             :   }
    4727             : 
    4728        6266 :   int AddArgumentNodes(Node** args, int pos, int param_count,
    4729       43007 :                        wasm::FunctionSig* sig) {
    4730             :     // Convert wasm numbers to JS values.
    4731       55539 :     for (int i = 0; i < param_count; ++i) {
    4732             :       Node* param =
    4733       43007 :           Param(i + 1);  // Start from index 1 to drop the instance_node.
    4734       86014 :       args[pos++] = ToJS(param, sig->GetParam(i));
    4735             :     }
    4736        6266 :     return pos;
    4737             :   }
    4738             : 
    4739      159459 :   Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
    4740             :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    4741             :         mcgraph()->zone(), TypeConversionDescriptor{}, 0,
    4742      654637 :         CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
    4743             :     Node* stub_code =
    4744       81410 :         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    4745             :             ? mcgraph()->RelocatableIntPtrConstant(
    4746             :                   wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
    4747      162820 :             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
    4748             : 
    4749             :     Node* result = SetEffect(
    4750             :         graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
    4751       81409 :                          node, js_context, Effect(), Control()));
    4752             : 
    4753       81409 :     SetSourcePosition(result, 1);
    4754             : 
    4755       81410 :     return result;
    4756             :   }
    4757             : 
    4758       81410 :   Node* BuildChangeTaggedToFloat64(Node* value) {
    4759     1221141 :     MachineOperatorBuilder* machine = mcgraph()->machine();
    4760       81410 :     CommonOperatorBuilder* common = mcgraph()->common();
    4761             : 
    4762             :     // Implement the following decision tree:
    4763             :     //  heap object?
    4764             :     //  ├─ true: undefined?
    4765             :     //  │        ├─ true: f64 const
    4766             :     //  │        └─ false: load heap number value
    4767             :     //  └─ false: smi to float64
    4768             : 
    4769       81410 :     Node* check_heap_object = BuildTestHeapObject(value);
    4770             :     Diamond is_heap_object(graph(), common, check_heap_object,
    4771       81410 :                            BranchHint::kFalse);
    4772       81409 :     is_heap_object.Chain(Control());
    4773             : 
    4774       81409 :     SetControl(is_heap_object.if_true);
    4775             :     Node* orig_effect = Effect();
    4776             : 
    4777             :     Node* undefined_node =
    4778      162819 :         LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
    4779             :     Node* check_undefined =
    4780       81409 :         graph()->NewNode(machine->WordEqual(), value, undefined_node);
    4781             :     Node* effect_tagged = Effect();
    4782             : 
    4783       81409 :     Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
    4784       81410 :     is_undefined.Nest(is_heap_object, true);
    4785             : 
    4786       81409 :     SetControl(is_undefined.if_false);
    4787       81409 :     Node* vheap_number = BuildLoadHeapNumberValue(value);
    4788             :     Node* effect_undefined = Effect();
    4789             : 
    4790       81410 :     SetControl(is_undefined.merge);
    4791             :     Node* vundefined =
    4792       81410 :         mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
    4793             :     Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
    4794       81410 :                                      vundefined, vheap_number);
    4795             : 
    4796       81410 :     effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
    4797             : 
    4798             :     // If input is Smi: just convert to float64.
    4799       81410 :     Node* vfrom_smi = BuildChangeSmiToFloat64(value);
    4800             : 
    4801       81410 :     SetControl(is_heap_object.merge);
    4802       81410 :     SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
    4803             :     return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
    4804       81409 :                               vfrom_smi);
    4805             :   }
    4806             : 
    4807       63887 :   Node* ToJS(Node* node, wasm::ValueType type) {
    4808       63887 :     switch (type) {
    4809             :       case wasm::kWasmI32:
    4810       23951 :         return BuildChangeInt32ToTagged(node);
    4811             :       case wasm::kWasmS128:
    4812           0 :         UNREACHABLE();
    4813             :       case wasm::kWasmI64: {
    4814             :         DCHECK(enabled_features_.bigint);
    4815          32 :         return BuildChangeInt64ToBigInt(node);
    4816             :       }
    4817             :       case wasm::kWasmF32:
    4818             :         node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
    4819       11021 :                                 node);
    4820       11021 :         return BuildChangeFloat64ToTagged(node);
    4821             :       case wasm::kWasmF64:
    4822       28426 :         return BuildChangeFloat64ToTagged(node);
    4823             :       case wasm::kWasmAnyRef:
    4824             :       case wasm::kWasmAnyFunc:
    4825             :         return node;
    4826             :       default:
    4827           0 :         UNREACHABLE();
    4828             :     }
    4829             :   }
    4830             : 
    4831          56 :   Node* BuildChangeInt64ToBigInt(Node* input) {
    4832             :     BigIntToWasmI64Descriptor interface_descriptor;
    4833             : 
    4834             :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    4835             :         mcgraph()->zone(),                              // zone
    4836             :         interface_descriptor,                           // descriptor
    4837             :         interface_descriptor.GetStackParameterCount(),  // stack parameter count
    4838             :         CallDescriptor::kNoFlags,                       // flags
    4839             :         Operator::kNoProperties,                        // properties
    4840         264 :         stub_mode_);                                    // stub call mode
    4841             : 
    4842             :     Node* target =
    4843          32 :         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    4844             :             ? mcgraph()->RelocatableIntPtrConstant(
    4845             :                   wasm::WasmCode::kBigIntToWasmI64, RelocInfo::WASM_STUB_CALL)
    4846          64 :             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, I64ToBigInt));
    4847             : 
    4848             :     return SetEffect(
    4849             :         SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
    4850          64 :                                     target, input, Effect(), Control())));
    4851             :   }
    4852             : 
    4853          24 :   Node* BuildChangeBigIntToInt64(Node* input, Node* context) {
    4854             :     BigIntToI64Descriptor interface_descriptor;
    4855             : 
    4856             :     auto call_descriptor = Linkage::GetStubCallDescriptor(
    4857             :         mcgraph()->zone(),                              // zone
    4858             :         interface_descriptor,                           // descriptor
    4859             :         interface_descriptor.GetStackParameterCount(),  // stack parameter count
    4860             :         CallDescriptor::kNoFlags,                       // flags
    4861             :         Operator::kNoProperties,                        // properties
    4862         136 :         stub_mode_);                                    // stub call mode
    4863             : 
    4864             :     Node* target =
    4865          16 :         (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
    4866             :             ? mcgraph()->RelocatableIntPtrConstant(
    4867             :                   wasm::WasmCode::kWasmBigIntToI64, RelocInfo::WASM_STUB_CALL)
    4868          32 :             : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, BigIntToI64));
    4869             : 
    4870             :     return SetEffect(SetControl(
    4871             :         graph()->NewNode(mcgraph()->common()->Call(call_descriptor), target,
    4872          32 :                          input, context, Effect(), Control())));
    4873             :   }
    4874             : 
    4875       83434 :   Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
    4876             :     DCHECK_NE(wasm::kWasmStmt, type);
    4877             : 
    4878             :     // The parameter is of type AnyRef, we take it as is.
    4879       83434 :     if (type == wasm::kWasmAnyRef) {
    4880        1960 :       return node;
    4881             :     }
    4882             : 
    4883       81474 :     if (type == wasm::kWasmAnyFunc) {
    4884             :       Node* check =
    4885             :           BuildChangeSmiToInt32(SetEffect(BuildCallToRuntimeWithContext(
    4886             :               Runtime::kWasmIsValidAnyFuncValue, js_context, &node, 1, effect_,
    4887       41056 :               Control())));
    4888             : 
    4889             :       Diamond type_check(graph(), mcgraph()->common(), check,
    4890          96 :                          BranchHint::kTrue);
    4891          48 :       type_check.Chain(Control());
    4892             : 
    4893          48 :       Node* effect = Effect();
    4894             :       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
    4895          48 :                                     nullptr, 0, &effect, type_check.if_false);
    4896             : 
    4897          96 :       SetEffect(type_check.EffectPhi(Effect(), effect));
    4898             : 
    4899          48 :       SetControl(type_check.merge);
    4900             : 
    4901          48 :       return node;
    4902             :     }
    4903             :     Node* num = nullptr;
    4904             : 
    4905       81426 :     if (type != wasm::kWasmI64) {
    4906             :       // Do a JavaScript ToNumber.
    4907       81410 :       num = BuildJavaScriptToNumber(node, js_context);
    4908             : 
    4909             :       // Change representation.
    4910       81410 :       num = BuildChangeTaggedToFloat64(num);
    4911             :     }
    4912             : 
    4913       81426 :     switch (type) {
    4914             :       case wasm::kWasmI32: {
    4915             :         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
    4916       28634 :                                num);
    4917       28634 :         break;
    4918             :       }
    4919             :       case wasm::kWasmI64: {
    4920             :         DCHECK(enabled_features_.bigint);
    4921          16 :         num = BuildChangeBigIntToInt64(node, js_context);
    4922          16 :         break;
    4923             :       }
    4924             :       case wasm::kWasmF32:
    4925             :         num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
    4926       12038 :                                num);
    4927       12038 :         break;
    4928             :       case wasm::kWasmF64:
    4929             :         break;
    4930             :       case wasm::kWasmS128:
    4931           0 :         UNREACHABLE();
    4932             :       default:
    4933           0 :         UNREACHABLE();
    4934             :     }
    4935             :     DCHECK_NOT_NULL(num);
    4936             : 
    4937       81426 :     return num;
    4938             :   }
    4939             : 
    4940      289292 :   void BuildModifyThreadInWasmFlag(bool new_value) {
    4941      578552 :     if (!trap_handler::IsTrapHandlerEnabled()) return;
    4942             :     Node* isolate_root =
    4943     4624944 :         LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
    4944             : 
    4945             :     Node* thread_in_wasm_flag_address =
    4946      867138 :         LOAD_RAW(isolate_root, Isolate::thread_in_wasm_flag_address_offset(),
    4947             :                  MachineType::Pointer());
    4948             : 
    4949      289046 :     if (FLAG_debug_code) {
    4950             :       Node* flag_value = SetEffect(
    4951             :           graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
    4952             :                            thread_in_wasm_flag_address,
    4953          32 :                            mcgraph()->Int32Constant(0), Effect(), Control()));
    4954             :       Node* check =
    4955             :           graph()->NewNode(mcgraph()->machine()->Word32Equal(), flag_value,
    4956          48 :                            mcgraph()->Int32Constant(new_value ? 0 : 1));
    4957             : 
    4958             :       Diamond flag_check(graph(), mcgraph()->common(), check,
    4959          32 :                          BranchHint::kTrue);
    4960          16 :       flag_check.Chain(Control());
    4961             :       Node* message_id = jsgraph()->SmiConstant(static_cast<int32_t>(
    4962             :           new_value ? AbortReason::kUnexpectedThreadInWasmSet
    4963          32 :                     : AbortReason::kUnexpectedThreadInWasmUnset));
    4964             : 
    4965          16 :       Node* effect = Effect();
    4966             :       BuildCallToRuntimeWithContext(Runtime::kAbort, NoContextConstant(),
    4967             :                                     &message_id, 1, &effect,
    4968          32 :                                     flag_check.if_false);
    4969             : 
    4970          32 :       SetEffect(flag_check.EffectPhi(Effect(), effect));
    4971             : 
    4972          16 :       SetControl(flag_check.merge);
    4973             :     }
    4974             : 
    4975             :     SetEffect(graph()->NewNode(
    4976             :         mcgraph()->machine()->Store(StoreRepresentation(
    4977             :             MachineRepresentation::kWord32, kNoWriteBarrier)),
    4978             :         thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
    4979     1445230 :         mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
    4980             :   }
    4981             : 
    4982      695540 :   Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
    4983             :     Node* shared = SetEffect(graph()->NewNode(
    4984             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
    4985             :         jsgraph()->Int32Constant(
    4986             :             wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()),
    4987     1112864 :         Effect(), Control()));
    4988             :     return SetEffect(graph()->NewNode(
    4989             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
    4990             :         jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
    4991             :                                  kHeapObjectTag),
    4992      417324 :         Effect(), Control()));
    4993             :   }
    4994             : 
    4995      417324 :   Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
    4996             :     return SetEffect(graph()->NewNode(
    4997             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
    4998             :         jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
    4999             :                                  kHeapObjectTag),
    5000      834648 :         Effect(), Control()));
    5001             :   }
    5002             : 
    5003        1392 :   Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
    5004             :     Node* function_index_smi = SetEffect(graph()->NewNode(
    5005             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
    5006             :         jsgraph()->Int32Constant(
    5007             :             WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
    5008        2320 :         Effect(), Control()));
    5009         464 :     Node* function_index = BuildChangeSmiToInt32(function_index_smi);
    5010         464 :     return function_index;
    5011             :   }
    5012             : 
    5013      413722 :   Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
    5014             :     Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
    5015             :         jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
    5016             :         jsgraph()->Int32Constant(
    5017             :             WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
    5018      689537 :         Effect(), Control()));
    5019      137908 :     Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
    5020      137908 :     return jump_table_offset;
    5021             :   }
    5022             : 
    5023      396444 :   void BuildJSToWasmWrapper(bool is_import) {
    5024      378425 :     const int wasm_count = static_cast<int>(sig_->parameter_count());
    5025             : 
    5026             :     // Build the start and the JS parameter nodes.
    5027     1106397 :     SetEffect(SetControl(Start(wasm_count + 5)));
    5028             : 
    5029             :     // Create the js_closure and js_context parameters.
    5030             :     Node* js_closure =
    5031             :         graph()->NewNode(jsgraph()->common()->Parameter(
    5032             :                              Linkage::kJSCallClosureParamIndex, "%closure"),
    5033      278216 :                          graph()->start());
    5034             :     Node* js_context = graph()->NewNode(
    5035             :         mcgraph()->common()->Parameter(
    5036             :             Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
    5037      278216 :         graph()->start());
    5038             : 
    5039             :     // Create the instance_node node to pass as parameter. It is loaded from
    5040             :     // an actual reference to an instance or a placeholder reference,
    5041             :     // called {WasmExportedFunction} via the {WasmExportedFunctionData}
    5042             :     // structure.
    5043      139108 :     Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
    5044             :     instance_node_.set(
    5045      139108 :         BuildLoadInstanceFromExportedFunctionData(function_data));
    5046             : 
    5047      139108 :     if (!wasm::IsJSCompatibleSignature(sig_, enabled_features_.bigint)) {
    5048             :       // Throw a TypeError. Use the js_context of the calling javascript
    5049             :       // function (passed as a parameter), such that the generated code is
    5050             :       // js_context independent.
    5051             :       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
    5052         736 :                                     nullptr, 0, effect_, Control());
    5053             :       Return(jsgraph()->SmiConstant(0));
    5054      139843 :       return;
    5055             :     }
    5056             : 
    5057      138372 :     const int args_count = wasm_count + 1;  // +1 for wasm_code.
    5058      138372 :     Node** args = Buffer(args_count);
    5059             :     Node** rets;
    5060             : 
    5061             :     // Convert JS parameters to wasm numbers.
    5062      356809 :     for (int i = 0; i < wasm_count; ++i) {
    5063       80065 :       Node* param = Param(i + 1);
    5064      160130 :       Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
    5065       80065 :       args[i + 1] = wasm_param;
    5066             :     }
    5067             : 
    5068             :     // Set the ThreadInWasm flag before we do the actual call.
    5069      138372 :     BuildModifyThreadInWasmFlag(true);
    5070             : 
    5071      138371 :     if (is_import) {
    5072             :       // Call to an imported function.
    5073             :       // Load function index from {WasmExportedFunctionData}.
    5074             :       Node* function_index =
    5075         464 :           BuildLoadFunctionIndexFromExportedFunctionData(function_data);
    5076         464 :       BuildImportCall(sig_, args, &rets, wasm::kNoCodePosition, function_index);
    5077             :     } else {
    5078             :       // Call to a wasm function defined in this module.
    5079             :       // The call target is the jump table slot for that function.
    5080             :       Node* jump_table_start =
    5081      275815 :           LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
    5082             :       Node* jump_table_offset =
    5083      137908 :           BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
    5084             :       Node* jump_table_slot = graph()->NewNode(
    5085      137908 :           mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
    5086      137908 :       args[0] = jump_table_slot;
    5087             : 
    5088             :       BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
    5089      137908 :                     kNoRetpoline);
    5090             :     }
    5091             : 
    5092             :     // Clear the ThreadInWasm flag.
    5093      138372 :     BuildModifyThreadInWasmFlag(false);
    5094             : 
    5095      138372 :     Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
    5096      276744 :                                             : ToJS(rets[0], sig_->GetReturn());
    5097             :     Return(jsval);
    5098             :   }
    5099             : 
    5100        6342 :   bool BuildWasmImportCallWrapper(WasmImportCallKind kind) {
    5101       15977 :     int wasm_count = static_cast<int>(sig_->parameter_count());
    5102             : 
    5103             :     // Build the start and the parameter nodes.
    5104      163871 :     SetEffect(SetControl(Start(wasm_count + 4)));
    5105             : 
    5106        6342 :     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
    5107             : 
    5108             :     Node* native_context =
    5109       12684 :         LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
    5110             : 
    5111        6342 :     if (kind == WasmImportCallKind::kRuntimeTypeError) {
    5112             :       // =======================================================================
    5113             :       // === Runtime TypeError =================================================
    5114             :       // =======================================================================
    5115             :       BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
    5116             :                                     native_context, nullptr, 0, effect_,
    5117          76 :                                     Control());
    5118             :       // We don't need to return a value here, as the runtime call will not
    5119             :       // return anyway (the c entry stub will trigger stack unwinding).
    5120             :       ReturnVoid();
    5121          76 :       return false;
    5122             :     }
    5123             : 
    5124             :     // The callable is passed as the last parameter, after WASM arguments.
    5125        6266 :     Node* callable_node = Param(wasm_count + 1);
    5126             : 
    5127             :     Node* undefined_node =
    5128       12532 :         LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
    5129             : 
    5130             :     Node* call = nullptr;
    5131             :     bool sloppy_receiver = true;
    5132             : 
    5133             :     // Clear the ThreadInWasm flag.
    5134        6266 :     BuildModifyThreadInWasmFlag(false);
    5135             : 
    5136        6266 :     switch (kind) {
    5137             :       // =======================================================================
    5138             :       // === JS Functions with matching arity ==================================
    5139             :       // =======================================================================
    5140             :       case WasmImportCallKind::kJSFunctionArityMatch:
    5141             :         sloppy_receiver = false;
    5142             :         V8_FALLTHROUGH;  // fallthru
    5143             :       case WasmImportCallKind::kJSFunctionArityMatchSloppy: {
    5144        3972 :         Node** args = Buffer(wasm_count + 9);
    5145             :         int pos = 0;
    5146             :         Node* function_context = SetEffect(graph()->NewNode(
    5147             :             mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    5148             :             callable_node,
    5149             :             mcgraph()->Int32Constant(
    5150             :                 wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
    5151        7944 :             Effect(), Control()));
    5152        3972 :         args[pos++] = callable_node;  // target callable.
    5153             :         // Receiver.
    5154        3972 :         if (sloppy_receiver) {
    5155        7800 :           Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
    5156             :               native_context, Context::GLOBAL_PROXY_INDEX);
    5157        3900 :           args[pos++] = global_proxy;
    5158             :         } else {
    5159          72 :           args[pos++] = undefined_node;
    5160             :         }
    5161             : 
    5162             :         auto call_descriptor = Linkage::GetJSCallDescriptor(
    5163        3972 :             graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
    5164             : 
    5165             :         // Convert wasm numbers to JS values.
    5166        3972 :         pos = AddArgumentNodes(args, pos, wasm_count, sig_);
    5167             : 
    5168        3972 :         args[pos++] = undefined_node;                        // new target
    5169        7944 :         args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
    5170        3972 :         args[pos++] = function_context;
    5171        7944 :         args[pos++] = Effect();
    5172        7944 :         args[pos++] = Control();
    5173             : 
    5174             :         call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
    5175        7944 :                                 args);
    5176        3972 :         break;
    5177             :       }
    5178             :       // =======================================================================
    5179             :       // === JS Functions with arguments adapter ===============================
    5180             :       // =======================================================================
    5181             :       case WasmImportCallKind::kJSFunctionArityMismatch:
    5182             :         sloppy_receiver = false;
    5183             :         V8_FALLTHROUGH;  // fallthru
    5184             :       case WasmImportCallKind::kJSFunctionArityMismatchSloppy: {
    5185         622 :         Node** args = Buffer(wasm_count + 9);
    5186             :         int pos = 0;
    5187             :         Node* function_context = SetEffect(graph()->NewNode(
    5188             :             mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    5189             :             callable_node,
    5190             :             mcgraph()->Int32Constant(
    5191             :                 wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
    5192        1244 :             Effect(), Control()));
    5193             :         args[pos++] =
    5194         622 :             BuildLoadBuiltinFromInstance(Builtins::kArgumentsAdaptorTrampoline);
    5195         622 :         args[pos++] = callable_node;                         // target callable
    5196         622 :         args[pos++] = undefined_node;                        // new target
    5197         622 :         args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
    5198             : 
    5199             :         // Load shared function info, and then the formal parameter count.
    5200             :         Node* shared_function_info = SetEffect(graph()->NewNode(
    5201             :             mcgraph()->machine()->Load(MachineType::TaggedPointer()),
    5202             :             callable_node,
    5203             :             mcgraph()->Int32Constant(
    5204             :                 wasm::ObjectAccess::
    5205             :                     SharedFunctionInfoOffsetInTaggedJSFunction()),
    5206        1244 :             Effect(), Control()));
    5207             :         Node* formal_param_count = SetEffect(graph()->NewNode(
    5208             :             mcgraph()->machine()->Load(MachineType::Uint16()),
    5209             :             shared_function_info,
    5210             :             mcgraph()->Int32Constant(
    5211             :                 wasm::ObjectAccess::
    5212             :                     FormalParameterCountOffsetInSharedFunctionInfo()),
    5213        1244 :             Effect(), Control()));
    5214         622 :         args[pos++] = formal_param_count;
    5215             : 
    5216             :         // Receiver.
    5217         622 :         if (sloppy_receiver) {
    5218        1144 :           Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
    5219             :               native_context, Context::GLOBAL_PROXY_INDEX);
    5220         572 :           args[pos++] = global_proxy;
    5221             :         } else {
    5222          50 :           args[pos++] = undefined_node;
    5223             :         }
    5224             : 
    5225             : #ifdef V8_TARGET_ARCH_IA32
    5226             :         // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative
    5227             :         // call instead once builtins are embedded in every build configuration.
    5228             :         CallDescriptor::Flags flags = CallDescriptor::kAllowCallThroughSlot;
    5229             : #else
    5230             :         CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
    5231             : #endif
    5232             :         auto call_descriptor = Linkage::GetStubCallDescriptor(
    5233             :             mcgraph()->zone(), ArgumentsAdaptorDescriptor{}, 1 + wasm_count,
    5234        1244 :             flags, Operator::kNoProperties);
    5235             : 
    5236             :         // Convert wasm numbers to JS values.
    5237         622 :         pos = AddArgumentNodes(args, pos, wasm_count, sig_);
    5238         622 :         args[pos++] = function_context;
    5239        1244 :         args[pos++] = Effect();
    5240        1244 :         args[pos++] = Control();
    5241             :         call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
    5242        1244 :                                 args);
    5243             :         break;
    5244             :       }
    5245             :       // =======================================================================
    5246             :       // === General case of unknown callable ==================================
    5247             :       // =======================================================================
    5248             :       case WasmImportCallKind::kUseCallBuiltin: {
    5249        1672 :         Node** args = Buffer(wasm_count + 9);
    5250             :         int pos = 0;
    5251             :         args[pos++] = mcgraph()->RelocatableIntPtrConstant(
    5252        1672 :             wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
    5253        1672 :         args[pos++] = callable_node;
    5254        1672 :         args[pos++] = mcgraph()->Int32Constant(wasm_count);  // argument count
    5255        1672 :         args[pos++] = undefined_node;                        // receiver
    5256             : 
    5257             :         auto call_descriptor = Linkage::GetStubCallDescriptor(
    5258             :             graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
    5259             :             CallDescriptor::kNoFlags, Operator::kNoProperties,
    5260        3344 :             StubCallMode::kCallWasmRuntimeStub);
    5261             : 
    5262             :         // Convert wasm numbers to JS values.
    5263        1672 :         pos = AddArgumentNodes(args, pos, wasm_count, sig_);
    5264             : 
    5265             :         // The native_context is sufficient here, because all kind of callables
    5266             :         // which depend on the context provide their own context. The context
    5267             :         // here is only needed if the target is a constructor to throw a
    5268             :         // TypeError, if the target is a native function, or if the target is a
    5269             :         // callable JSObject, which can only be constructed by the runtime.
    5270        1672 :         args[pos++] = native_context;
    5271        3344 :         args[pos++] = Effect();
    5272        3344 :         args[pos++] = Control();
    5273             : 
    5274             :         call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
    5275        3344 :                                 args);
    5276        1672 :         break;
    5277             :       }
    5278             :       default:
    5279           0 :         UNREACHABLE();
    5280             :     }
    5281             :     DCHECK_NOT_NULL(call);
    5282             : 
    5283             :     SetEffect(call);
    5284        6266 :     SetSourcePosition(call, 0);
    5285             : 
    5286             :     // Convert the return value back.
    5287        6266 :     Node* val = sig_->return_count() == 0
    5288             :                     ? mcgraph()->Int32Constant(0)
    5289       12532 :                     : FromJS(call, native_context, sig_->GetReturn());
    5290             : 
    5291             :     // Set the ThreadInWasm flag again.
    5292        6266 :     BuildModifyThreadInWasmFlag(true);
    5293             : 
    5294             :     Return(val);
    5295        6266 :     return true;
    5296             :   }
    5297             : 
    5298        4888 :   void BuildWasmInterpreterEntry(uint32_t func_index) {
    5299       16180 :     int param_count = static_cast<int>(sig_->parameter_count());
    5300             : 
    5301             :     // Build the start and the parameter nodes.
    5302       43436 :     SetEffect(SetControl(Start(param_count + 3)));
    5303             : 
    5304             :     // Create the instance_node from the passed parameter.
    5305        2444 :     instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
    5306             : 
    5307             :     // Compute size for the argument buffer.
    5308        2444 :     int args_size_bytes = 0;
    5309       13952 :     for (wasm::ValueType type : sig_->parameters()) {
    5310        9064 :       args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
    5311             :     }
    5312             : 
    5313             :     // The return value is also passed via this buffer:
    5314             :     DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
    5315             :     // TODO(wasm): Handle multi-value returns.
    5316             :     DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
    5317             :     int return_size_bytes =
    5318             :         sig_->return_count() == 0
    5319             :             ? 0
    5320        4672 :             : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
    5321             : 
    5322             :     // Get a stack slot for the arguments.
    5323             :     Node* arg_buffer =
    5324        2444 :         args_size_bytes == 0 && return_size_bytes == 0
    5325             :             ? mcgraph()->IntPtrConstant(0)
    5326             :             : graph()->NewNode(mcgraph()->machine()->StackSlot(
    5327        7220 :                   std::max(args_size_bytes, return_size_bytes), 8));
    5328             : 
    5329             :     // Now store all our arguments to the buffer.
    5330             :     int offset = 0;
    5331             : 
    5332       13952 :     for (int i = 0; i < param_count; ++i) {
    5333        9064 :       wasm::ValueType type = sig_->GetParam(i);
    5334             :       // Start from the parameter with index 1 to drop the instance_node.
    5335             :       SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
    5336        9064 :                                  Int32Constant(offset), Param(i + 1), Effect(),
    5337       18128 :                                  Control()));
    5338        9064 :       offset += wasm::ValueTypes::ElementSizeInBytes(type);
    5339             :     }
    5340             :     DCHECK_EQ(args_size_bytes, offset);
    5341             : 
    5342             :     // We are passing the raw arg_buffer here. To the GC and other parts, it
    5343             :     // looks like a Smi (lowest bit not set). In the runtime function however,
    5344             :     // don't call Smi::value on it, but just cast it to a byte pointer.
    5345        4888 :     Node* parameters[] = {jsgraph()->SmiConstant(func_index), arg_buffer};
    5346             :     BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
    5347        2444 :                        arraysize(parameters));
    5348             : 
    5349             :     // Read back the return value.
    5350        4888 :     if (sig_->return_count() == 0) {
    5351             :       Return(Int32Constant(0));
    5352             :     } else {
    5353             :       // TODO(wasm): Implement multi-return.
    5354             :       DCHECK_EQ(1, sig_->return_count());
    5355             :       MachineType load_rep =
    5356        2228 :           wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
    5357             :       Node* val = SetEffect(
    5358             :           graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
    5359        2228 :                            Int32Constant(0), Effect(), Control()));
    5360             :       Return(val);
    5361             :     }
    5362             : 
    5363        4888 :     if (ContainsInt64(sig_)) LowerInt64();
    5364        2444 :   }
    5365             : 
    5366        2712 :   void BuildCWasmEntry() {
    5367             :     // Build the start and the JS parameter nodes.
    5368       30092 :     SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
    5369             : 
    5370             :     // Create parameter nodes (offset by 1 for the receiver parameter).
    5371        1356 :     Node* code_entry = Param(CWasmEntryParameters::kCodeEntry + 1);
    5372        1356 :     Node* object_ref_node = Param(CWasmEntryParameters::kObjectRef + 1);
    5373        1356 :     Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
    5374             : 
    5375        4016 :     int wasm_arg_count = static_cast<int>(sig_->parameter_count());
    5376             :     int arg_count =
    5377        1356 :         wasm_arg_count + 4;  // code, object_ref_node, control, effect
    5378        1356 :     Node** args = Buffer(arg_count);
    5379             : 
    5380             :     int pos = 0;
    5381        1356 :     args[pos++] = code_entry;
    5382        1356 :     args[pos++] = object_ref_node;
    5383             : 
    5384             :     int offset = 0;
    5385        7388 :     for (wasm::ValueType type : sig_->parameters()) {
    5386             :       Node* arg_load = SetEffect(
    5387             :           graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
    5388        4676 :                            Int32Constant(offset), Effect(), Control()));
    5389        4676 :       args[pos++] = arg_load;
    5390        4676 :       offset += wasm::ValueTypes::ElementSizeInBytes(type);
    5391             :     }
    5392             : 
    5393        2712 :     args[pos++] = Effect();
    5394        2712 :     args[pos++] = Control();
    5395             :     DCHECK_EQ(arg_count, pos);
    5396             : 
    5397             :     // Call the wasm code.
    5398        2712 :     auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
    5399             : 
    5400             :     Node* call = SetEffect(graph()->NewNode(
    5401        2712 :         mcgraph()->common()->Call(call_descriptor), arg_count, args));
    5402             : 
    5403             :     // Store the return value.
    5404             :     DCHECK_GE(1, sig_->return_count());
    5405        2712 :     if (sig_->return_count() == 1) {
    5406             :       StoreRepresentation store_rep(
    5407             :           wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
    5408        1304 :           kNoWriteBarrier);
    5409             :       SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
    5410             :                                  arg_buffer, Int32Constant(0), call, Effect(),
    5411        1304 :                                  Control()));
    5412             :     }
    5413             :     Return(jsgraph()->SmiConstant(0));
    5414             : 
    5415        2712 :     if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
    5416             :       MachineRepresentation sig_reps[] = {
    5417             :           MachineRepresentation::kWord32,  // return value
    5418             :           MachineRepresentation::kTagged,  // receiver
    5419             :           MachineRepresentation::kTagged,  // arg0 (code)
    5420             :           MachineRepresentation::kTagged   // arg1 (buffer)
    5421           0 :       };
    5422             :       Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
    5423             :       Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
    5424           0 :                       mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
    5425           0 :       r.LowerGraph();
    5426             :     }
    5427        1356 :   }
    5428             : 
    5429             :   JSGraph* jsgraph() { return jsgraph_; }
    5430             : 
    5431             :  private:
    5432             :   Isolate* const isolate_;
    5433             :   JSGraph* jsgraph_;
    5434             :   StubCallMode stub_mode_;
    5435             :   SetOncePointer<const Operator> allocate_heap_number_operator_;
    5436             :   wasm::WasmFeatures enabled_features_;
    5437             : };
    5438             : 
    5439      140464 : void AppendSignature(char* buffer, size_t max_name_len,
    5440      140464 :                      wasm::FunctionSig* sig) {
    5441      140464 :   size_t name_len = strlen(buffer);
    5442             :   auto append_name_char = [&](char c) {
    5443      249009 :     if (name_len + 1 < max_name_len) buffer[name_len++] = c;
    5444             :   };
    5445      226169 :   for (wasm::ValueType t : sig->parameters()) {
    5446             :     append_name_char(wasm::ValueTypes::ShortNameOf(t));
    5447             :   }
    5448             :   append_name_char(':');
    5449      163304 :   for (wasm::ValueType t : sig->returns()) {
    5450             :     append_name_char(wasm::ValueTypes::ShortNameOf(t));
    5451             :   }
    5452      140464 :   buffer[name_len] = '\0';
    5453      140464 : }
    5454             : 
    5455             : }  // namespace
    5456             : 
    5457      278216 : MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
    5458      139108 :                                          wasm::FunctionSig* sig,
    5459             :                                          bool is_import) {
    5460      278216 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5461             :                "CompileJSToWasmWrapper");
    5462             :   //----------------------------------------------------------------------------
    5463             :   // Create the Graph.
    5464             :   //----------------------------------------------------------------------------
    5465      278215 :   Zone zone(isolate->allocator(), ZONE_NAME);
    5466      139108 :   Graph graph(&zone);
    5467      139108 :   CommonOperatorBuilder common(&zone);
    5468             :   MachineOperatorBuilder machine(
    5469             :       &zone, MachineType::PointerRepresentation(),
    5470             :       InstructionSelector::SupportedMachineOperatorFlags(),
    5471      139108 :       InstructionSelector::AlignmentRequirements());
    5472      139108 :   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
    5473             : 
    5474      139108 :   Node* control = nullptr;
    5475      139108 :   Node* effect = nullptr;
    5476             : 
    5477             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
    5478             :                                   StubCallMode::kCallCodeObject,
    5479      139108 :                                   wasm::WasmFeaturesFromIsolate(isolate));
    5480             :   builder.set_control_ptr(&control);
    5481             :   builder.set_effect_ptr(&effect);
    5482      139108 :   builder.BuildJSToWasmWrapper(is_import);
    5483             : 
    5484             :   //----------------------------------------------------------------------------
    5485             :   // Run the compilation pipeline.
    5486             :   //----------------------------------------------------------------------------
    5487             :   static constexpr size_t kMaxNameLen = 128;
    5488      139107 :   char debug_name[kMaxNameLen] = "js_to_wasm:";
    5489      139107 :   AppendSignature(debug_name, kMaxNameLen, sig);
    5490             : 
    5491             :   // Schedule and compile to machine code.
    5492      139108 :   int params = static_cast<int>(sig->parameter_count());
    5493             :   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
    5494      139108 :       &zone, false, params + 1, CallDescriptor::kNoFlags);
    5495             : 
    5496             :   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
    5497             :       isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
    5498      139108 :       WasmAssemblerOptions());
    5499             :   Handle<Code> code;
    5500      139106 :   if (!maybe_code.ToHandle(&code)) {
    5501           0 :     return maybe_code;
    5502             :   }
    5503             : #ifdef ENABLE_DISASSEMBLER
    5504             :   if (FLAG_print_opt_code) {
    5505             :     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
    5506             :     OFStream os(tracing_scope.file());
    5507             :     code->Disassemble(debug_name, os);
    5508             :   }
    5509             : #endif
    5510             : 
    5511      139108 :   if (must_record_function_compilation(isolate)) {
    5512             :     RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
    5513           0 :                               debug_name);
    5514             :   }
    5515             : 
    5516      278214 :   return code;
    5517             : }
    5518             : 
    5519      119577 : WasmImportCallKind GetWasmImportCallKind(Handle<JSReceiver> target,
    5520       10229 :                                          wasm::FunctionSig* expected_sig,
    5521             :                                          bool has_bigint_feature) {
    5522      119577 :   if (WasmExportedFunction::IsWasmExportedFunction(*target)) {
    5523      107288 :     auto imported_function = WasmExportedFunction::cast(*target);
    5524             :     wasm::FunctionSig* imported_sig =
    5525             :         imported_function->instance()
    5526      214576 :             ->module()
    5527      214576 :             ->functions[imported_function->function_index()]
    5528      107288 :             .sig;
    5529      107288 :     if (*imported_sig != *expected_sig) {
    5530             :       return WasmImportCallKind::kLinkError;
    5531             :     }
    5532      107128 :     return WasmImportCallKind::kWasmToWasm;
    5533             :   }
    5534             :   // Assuming we are calling to JS, check whether this would be a runtime error.
    5535       12289 :   if (!wasm::IsJSCompatibleSignature(expected_sig, has_bigint_feature)) {
    5536             :     return WasmImportCallKind::kRuntimeTypeError;
    5537             :   }
    5538             :   // For JavaScript calls, determine whether the target has an arity match
    5539             :   // and whether it has a sloppy receiver.
    5540       24426 :   if (target->IsJSFunction()) {
    5541       10469 :     Handle<JSFunction> function = Handle<JSFunction>::cast(target);
    5542       10469 :     SharedFunctionInfo shared = function->shared();
    5543             : 
    5544             : // Check for math intrinsics.
    5545             : #define COMPARE_SIG_FOR_BUILTIN(name)                                         \
    5546             :   {                                                                           \
    5547             :     wasm::FunctionSig* sig = wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
    5548             :     if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name);     \
    5549             :     DCHECK_NOT_NULL(sig);                                                     \
    5550             :     if (*expected_sig == *sig) return WasmImportCallKind::k##name;            \
    5551             :   }
    5552             : #define COMPARE_SIG_FOR_BUILTIN_F64(name) \
    5553             :   case Builtins::kMath##name:             \
    5554             :     COMPARE_SIG_FOR_BUILTIN(F64##name);   \
    5555             :     break;
    5556             : #define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
    5557             :   case Builtins::kMath##name:                 \
    5558             :     COMPARE_SIG_FOR_BUILTIN(F64##name);       \
    5559             :     COMPARE_SIG_FOR_BUILTIN(F32##name);       \
    5560             :     break;
    5561             : 
    5562       10469 :     if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
    5563         504 :       switch (shared->builtin_id()) {
    5564          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Acos);
    5565          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Asin);
    5566          17 :         COMPARE_SIG_FOR_BUILTIN_F64(Atan);
    5567          27 :         COMPARE_SIG_FOR_BUILTIN_F64(Cos);
    5568          27 :         COMPARE_SIG_FOR_BUILTIN_F64(Sin);
    5569          17 :         COMPARE_SIG_FOR_BUILTIN_F64(Tan);
    5570          17 :         COMPARE_SIG_FOR_BUILTIN_F64(Exp);
    5571          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Log);
    5572          18 :         COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
    5573             :         //===========================================================
    5574             :         // TODO(8505): Math.pow for wasm does not match JS.
    5575             :         //        COMPARE_SIG_FOR_BUILTIN_F64(Pow);
    5576             :         //===========================================================
    5577         106 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
    5578          24 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
    5579          34 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
    5580          34 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
    5581          43 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
    5582          52 :         COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
    5583             :         case Builtins::kMathFround:
    5584           8 :           COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
    5585             :           break;
    5586             :         default:
    5587             :           break;
    5588             :       }
    5589             :     }
    5590             : 
    5591             : #undef COMPARE_SIG_FOR_BUILTIN
    5592             : #undef COMPARE_SIG_FOR_BUILTIN_F64
    5593             : #undef COMPARE_SIG_FOR_BUILTIN_F32_F64
    5594             : 
    5595       10245 :     if (IsClassConstructor(shared->kind())) {
    5596             :       // Class constructor will throw anyway.
    5597             :       return WasmImportCallKind::kUseCallBuiltin;
    5598             :     }
    5599       20146 :     bool sloppy = is_sloppy(shared->language_mode()) && !shared->native();
    5600       20458 :     if (shared->internal_formal_parameter_count() ==
    5601             :         expected_sig->parameter_count()) {
    5602             :       return sloppy ? WasmImportCallKind::kJSFunctionArityMatchSloppy
    5603        7651 :                     : WasmImportCallKind::kJSFunctionArityMatch;
    5604             :     }
    5605             :     return sloppy ? WasmImportCallKind::kJSFunctionArityMismatchSloppy
    5606        2578 :                   : WasmImportCallKind::kJSFunctionArityMismatch;
    5607             :   }
    5608             :   // Unknown case. Use the call builtin.
    5609             :   return WasmImportCallKind::kUseCallBuiltin;
    5610             : }
    5611             : 
    5612         224 : wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
    5613             :                                         const char** name_ptr) {
    5614             : #define CASE(name)                          \
    5615             :   case WasmImportCallKind::k##name:         \
    5616             :     *name_ptr = "WasmMathIntrinsic:" #name; \
    5617             :     return wasm::kExpr##name
    5618         224 :   switch (kind) {
    5619           8 :     CASE(F64Acos);
    5620           8 :     CASE(F64Asin);
    5621           8 :     CASE(F64Atan);
    5622           8 :     CASE(F64Cos);
    5623           8 :     CASE(F64Sin);
    5624           8 :     CASE(F64Tan);
    5625           8 :     CASE(F64Exp);
    5626           8 :     CASE(F64Log);
    5627           8 :     CASE(F64Atan2);
    5628           0 :     CASE(F64Pow);
    5629          16 :     CASE(F64Ceil);
    5630          16 :     CASE(F64Floor);
    5631          16 :     CASE(F64Sqrt);
    5632          16 :     CASE(F64Min);
    5633          16 :     CASE(F64Max);
    5634          16 :     CASE(F64Abs);
    5635           8 :     CASE(F32Min);
    5636           8 :     CASE(F32Max);
    5637           8 :     CASE(F32Abs);
    5638           8 :     CASE(F32Ceil);
    5639           8 :     CASE(F32Floor);
    5640           8 :     CASE(F32Sqrt);
    5641           8 :     CASE(F32ConvertF64);
    5642             :     default:
    5643           0 :       UNREACHABLE();
    5644             :       return wasm::kExprUnreachable;
    5645             :   }
    5646             : #undef CASE
    5647             : }
    5648             : 
    5649         224 : wasm::WasmCode* CompileWasmMathIntrinsic(wasm::WasmEngine* wasm_engine,
    5650             :                                          wasm::NativeModule* native_module,
    5651             :                                          WasmImportCallKind kind,
    5652         448 :                                          wasm::FunctionSig* sig) {
    5653             :   DCHECK_EQ(1, sig->return_count());
    5654             : 
    5655         448 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5656             :                "CompileWasmMathIntrinsic");
    5657             : 
    5658         448 :   Zone zone(wasm_engine->allocator(), ZONE_NAME);
    5659             : 
    5660             :   // Compile a WASM function with a single bytecode and let TurboFan
    5661             :   // generate either inlined machine code or a call to a helper.
    5662             :   SourcePositionTable* source_positions = nullptr;
    5663         224 :   MachineGraph* mcgraph = new (&zone) MachineGraph(
    5664         448 :       new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
    5665             :       new (&zone) MachineOperatorBuilder(
    5666             :           &zone, MachineType::PointerRepresentation(),
    5667             :           InstructionSelector::SupportedMachineOperatorFlags(),
    5668         672 :           InstructionSelector::AlignmentRequirements()));
    5669             : 
    5670             :   wasm::CompilationEnv env(
    5671             :       native_module->module(), wasm::UseTrapHandler::kNoTrapHandler,
    5672             :       wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
    5673             :       wasm::kAllWasmFeatures, wasm::LowerSimd::kNoLowerSimd);
    5674             : 
    5675             :   WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
    5676         224 :                            source_positions);
    5677             : 
    5678             :   // Set up the graph start.
    5679         224 :   Node* start = builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
    5680         224 :   Node* effect = start;
    5681         224 :   Node* control = start;
    5682             :   builder.set_effect_ptr(&effect);
    5683             :   builder.set_control_ptr(&control);
    5684         224 :   builder.set_instance_node(builder.Param(wasm::kWasmInstanceParameterIndex));
    5685             : 
    5686             :   // Generate either a unop or a binop.
    5687             :   Node* result = nullptr;
    5688         224 :   const char* debug_name = "WasmMathIntrinsic";
    5689         224 :   auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
    5690         224 :   switch (sig->parameter_count()) {
    5691             :     case 1:
    5692         168 :       result = builder.Unop(opcode, builder.Param(1));
    5693         168 :       break;
    5694             :     case 2:
    5695          56 :       result = builder.Binop(opcode, builder.Param(1), builder.Param(2));
    5696          56 :       break;
    5697             :     default:
    5698           0 :       UNREACHABLE();
    5699             :       break;
    5700             :   }
    5701             : 
    5702             :   builder.Return(result);
    5703             : 
    5704             :   // Run the compiler pipeline to generate machine code.
    5705         224 :   auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
    5706         224 :   if (mcgraph->machine()->Is32()) {
    5707             :     call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
    5708             :   }
    5709             : 
    5710             :   wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
    5711             :       wasm_engine, call_descriptor, mcgraph, Code::WASM_FUNCTION,
    5712             :       wasm::WasmCode::kFunction, debug_name, WasmStubAssemblerOptions(),
    5713         224 :       native_module, source_positions);
    5714         224 :   CHECK_NOT_NULL(wasm_code);
    5715             :   // TODO(titzer): add counters for math intrinsic code size / allocation
    5716             : 
    5717         224 :   return wasm_code;
    5718             : }
    5719             : 
    5720        6566 : wasm::WasmCode* CompileWasmImportCallWrapper(wasm::WasmEngine* wasm_engine,
    5721             :                                              wasm::NativeModule* native_module,
    5722             :                                              WasmImportCallKind kind,
    5723             :                                              wasm::FunctionSig* sig,
    5724             :                                              bool source_positions) {
    5725             :   DCHECK_NE(WasmImportCallKind::kLinkError, kind);
    5726             :   DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind);
    5727             : 
    5728             :   // Check for math intrinsics first.
    5729        6566 :   if (FLAG_wasm_math_intrinsics &&
    5730        6566 :       kind >= WasmImportCallKind::kFirstMathIntrinsic &&
    5731             :       kind <= WasmImportCallKind::kLastMathIntrinsic) {
    5732         224 :     return CompileWasmMathIntrinsic(wasm_engine, native_module, kind, sig);
    5733             :   }
    5734             : 
    5735       12684 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5736             :                "CompileWasmImportCallWrapper");
    5737             :   //----------------------------------------------------------------------------
    5738             :   // Create the Graph
    5739             :   //----------------------------------------------------------------------------
    5740       12684 :   Zone zone(wasm_engine->allocator(), ZONE_NAME);
    5741        6342 :   Graph graph(&zone);
    5742        6342 :   CommonOperatorBuilder common(&zone);
    5743             :   MachineOperatorBuilder machine(
    5744             :       &zone, MachineType::PointerRepresentation(),
    5745             :       InstructionSelector::SupportedMachineOperatorFlags(),
    5746        6342 :       InstructionSelector::AlignmentRequirements());
    5747        6342 :   JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
    5748             : 
    5749        6342 :   Node* control = nullptr;
    5750        6342 :   Node* effect = nullptr;
    5751             : 
    5752             :   SourcePositionTable* source_position_table =
    5753        6832 :       source_positions ? new (&zone) SourcePositionTable(&graph) : nullptr;
    5754             : 
    5755             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, source_position_table,
    5756             :                                   StubCallMode::kCallWasmRuntimeStub,
    5757             :                                   native_module->enabled_features());
    5758             :   builder.set_control_ptr(&control);
    5759             :   builder.set_effect_ptr(&effect);
    5760        6342 :   builder.BuildWasmImportCallWrapper(kind);
    5761             : 
    5762             :   const char* func_name = "wasm-to-js";
    5763             : 
    5764             :   // Schedule and compile to machine code.
    5765             :   CallDescriptor* incoming =
    5766             :       GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline,
    5767        6342 :                             WasmGraphBuilder::kExtraCallableParam);
    5768        6342 :   if (machine.Is32()) {
    5769             :     incoming = GetI32WasmCallDescriptor(&zone, incoming);
    5770             :   }
    5771             :   wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
    5772             :       wasm_engine, incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
    5773             :       wasm::WasmCode::kWasmToJsWrapper, func_name, WasmStubAssemblerOptions(),
    5774        6342 :       native_module, source_position_table);
    5775        6342 :   CHECK_NOT_NULL(wasm_code);
    5776             : 
    5777        6342 :   return wasm_code;
    5778             : }
    5779             : 
    5780        2444 : wasm::WasmCode* CompileWasmInterpreterEntry(wasm::WasmEngine* wasm_engine,
    5781             :                                             wasm::NativeModule* native_module,
    5782             :                                             uint32_t func_index,
    5783             :                                             wasm::FunctionSig* sig) {
    5784             :   //----------------------------------------------------------------------------
    5785             :   // Create the Graph
    5786             :   //----------------------------------------------------------------------------
    5787        2444 :   Zone zone(wasm_engine->allocator(), ZONE_NAME);
    5788        2444 :   Graph graph(&zone);
    5789        2444 :   CommonOperatorBuilder common(&zone);
    5790             :   MachineOperatorBuilder machine(
    5791             :       &zone, MachineType::PointerRepresentation(),
    5792             :       InstructionSelector::SupportedMachineOperatorFlags(),
    5793        2444 :       InstructionSelector::AlignmentRequirements());
    5794        2444 :   JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
    5795             : 
    5796        2444 :   Node* control = nullptr;
    5797        2444 :   Node* effect = nullptr;
    5798             : 
    5799             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
    5800             :                                   StubCallMode::kCallWasmRuntimeStub,
    5801             :                                   native_module->enabled_features());
    5802             :   builder.set_control_ptr(&control);
    5803             :   builder.set_effect_ptr(&effect);
    5804        2444 :   builder.BuildWasmInterpreterEntry(func_index);
    5805             : 
    5806             :   // Schedule and compile to machine code.
    5807        2444 :   CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
    5808        2444 :   if (machine.Is32()) {
    5809             :     incoming = GetI32WasmCallDescriptor(&zone, incoming);
    5810             :   }
    5811             : 
    5812             :   EmbeddedVector<char, 32> func_name;
    5813             :   func_name.Truncate(
    5814        2444 :       SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
    5815             : 
    5816             :   wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
    5817             :       wasm_engine, incoming, &jsgraph, Code::WASM_INTERPRETER_ENTRY,
    5818        2444 :       wasm::WasmCode::kInterpreterEntry, func_name.start(),
    5819        2444 :       WasmStubAssemblerOptions(), native_module);
    5820        2444 :   CHECK_NOT_NULL(wasm_code);
    5821             : 
    5822        2444 :   return wasm_code;
    5823             : }
    5824             : 
    5825        1356 : MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
    5826        1356 :   Zone zone(isolate->allocator(), ZONE_NAME);
    5827        1356 :   Graph graph(&zone);
    5828        1356 :   CommonOperatorBuilder common(&zone);
    5829             :   MachineOperatorBuilder machine(
    5830             :       &zone, MachineType::PointerRepresentation(),
    5831             :       InstructionSelector::SupportedMachineOperatorFlags(),
    5832        1356 :       InstructionSelector::AlignmentRequirements());
    5833        1356 :   JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
    5834             : 
    5835        1356 :   Node* control = nullptr;
    5836        1356 :   Node* effect = nullptr;
    5837             : 
    5838             :   WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
    5839             :                                   StubCallMode::kCallCodeObject,
    5840        1356 :                                   wasm::WasmFeaturesFromIsolate(isolate));
    5841             :   builder.set_control_ptr(&control);
    5842             :   builder.set_effect_ptr(&effect);
    5843        1356 :   builder.BuildCWasmEntry();
    5844             : 
    5845             :   // Schedule and compile to machine code.
    5846             :   CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
    5847             :       &zone, false, CWasmEntryParameters::kNumParameters + 1,
    5848        1356 :       CallDescriptor::kNoFlags);
    5849             : 
    5850             :   // Build a name in the form "c-wasm-entry:<params>:<returns>".
    5851             :   static constexpr size_t kMaxNameLen = 128;
    5852        1356 :   char debug_name[kMaxNameLen] = "c-wasm-entry:";
    5853        1356 :   AppendSignature(debug_name, kMaxNameLen, sig);
    5854             : 
    5855             :   MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
    5856             :       isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
    5857        1356 :       AssemblerOptions::Default(isolate));
    5858             :   Handle<Code> code;
    5859        1356 :   if (!maybe_code.ToHandle(&code)) {
    5860           0 :     return maybe_code;
    5861             :   }
    5862             : #ifdef ENABLE_DISASSEMBLER
    5863             :   if (FLAG_print_opt_code) {
    5864             :     CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
    5865             :     OFStream os(tracing_scope.file());
    5866             :     code->Disassemble(debug_name, os);
    5867             :   }
    5868             : #endif
    5869             : 
    5870        1356 :   return code;
    5871             : }
    5872             : 
    5873      504196 : TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
    5874             :     wasm::WasmCompilationUnit* wasm_unit)
    5875      504196 :     : wasm_unit_(wasm_unit) {}
    5876             : 
    5877             : // Clears unique_ptrs, but (part of) the type is forward declared in the header.
    5878             : TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
    5879             : 
    5880      503096 : bool TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
    5881             :     wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
    5882             :     wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph,
    5883             :     NodeOriginTable* node_origins, SourcePositionTable* source_positions,
    5884             :     wasm::WasmError* error_out) {
    5885             :   base::ElapsedTimer decode_timer;
    5886             :   if (FLAG_trace_wasm_decode_time) {
    5887             :     decode_timer.Start();
    5888             :   }
    5889             : 
    5890             :   // Create a TF graph during decoding.
    5891             :   WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
    5892     1006192 :                            source_positions);
    5893             :   wasm::VoidResult graph_construction_result = wasm::BuildTFGraph(
    5894             :       wasm_unit_->wasm_engine_->allocator(), env->enabled_features, env->module,
    5895     1006370 :       &builder, detected, func_body, node_origins);
    5896      503220 :   if (graph_construction_result.failed()) {
    5897             :     if (FLAG_trace_wasm_compiler) {
    5898             :       StdoutStream{} << "Compilation failed: "
    5899             :                      << graph_construction_result.error().message()
    5900             :                      << std::endl;
    5901             :     }
    5902             :     *error_out = graph_construction_result.error();
    5903        6688 :     return false;
    5904             :   }
    5905             : 
    5906      496532 :   builder.LowerInt64();
    5907             : 
    5908     1001160 :   if (builder.has_simd() &&
    5909        8204 :       (!CpuFeatures::SupportsWasmSimd128() || env->lower_simd)) {
    5910             :     SimdScalarLowering(mcgraph,
    5911             :                        CreateMachineSignature(mcgraph->zone(), func_body.sig))
    5912       12204 :         .LowerGraph();
    5913             :   }
    5914             : 
    5915      992970 :   if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
    5916      496492 :       wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
    5917             :     PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(), func_body,
    5918           0 :                      env->module, wasm::kPrintLocals);
    5919             :   }
    5920             :   if (FLAG_trace_wasm_decode_time) {
    5921             :     *decode_ms = decode_timer.Elapsed().InMillisecondsF();
    5922             :   }
    5923             :   return true;
    5924             : }
    5925             : 
    5926             : namespace {
    5927      503145 : Vector<const char> GetDebugName(Zone* zone, int index) {
    5928             :   // TODO(herhut): Use name from module if available.
    5929             :   constexpr int kBufferLength = 24;
    5930             : 
    5931             :   EmbeddedVector<char, kBufferLength> name_vector;
    5932      503145 :   int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
    5933             :   DCHECK(name_len > 0 && name_len < name_vector.length());
    5934             : 
    5935      503185 :   char* index_name = zone->NewArray<char>(name_len);
    5936      503198 :   memcpy(index_name, name_vector.start(), name_len);
    5937      503198 :   return Vector<const char>(index_name, name_len);
    5938             : }
    5939             : }  // namespace
    5940             : 
    5941      503211 : wasm::WasmCompilationResult TurbofanWasmCompilationUnit::ExecuteCompilation(
    5942             :     wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
    5943             :     Counters* counters, wasm::WasmFeatures* detected) {
    5944     1006423 :   TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
    5945             :                "ExecuteTurbofanCompilation");
    5946      503212 :   double decode_ms = 0;
    5947             :   size_t node_count = 0;
    5948             : 
    5949     1006178 :   Zone zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
    5950     1496025 :   MachineGraph* mcgraph = new (&zone) MachineGraph(
    5951     1006452 :       new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
    5952             :       new (&zone) MachineOperatorBuilder(
    5953             :           &zone, MachineType::PointerRepresentation(),
    5954             :           InstructionSelector::SupportedMachineOperatorFlags(),
    5955     1509599 :           InstructionSelector::AlignmentRequirements()));
    5956             : 
    5957             :   OptimizedCompilationInfo info(GetDebugName(&zone, wasm_unit_->func_index_),
    5958     1006397 :                                 &zone, Code::WASM_FUNCTION);
    5959      503167 :   if (env->runtime_exception_support) {
    5960             :     info.SetWasmRuntimeExceptionSupport();
    5961             :   }
    5962             : 
    5963      503167 :   if (info.trace_turbo_json_enabled()) {
    5964           0 :     TurboCfgFile tcf;
    5965           0 :     tcf << AsC1VCompilation(&info);
    5966             :   }
    5967             : 
    5968             :   NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
    5969             :                                       ? new (&zone)
    5970           0 :                                             NodeOriginTable(mcgraph->graph())
    5971      503167 :                                       : nullptr;
    5972             :   SourcePositionTable* source_positions =
    5973      503163 :       new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
    5974             :   wasm::WasmError error;
    5975      503196 :   if (!BuildGraphForWasmFunction(env, func_body, detected, &decode_ms, mcgraph,
    5976      503095 :                                  node_origins, source_positions, &error)) {
    5977             :     DCHECK(!error.empty());
    5978       13376 :     return wasm::WasmCompilationResult{std::move(error)};
    5979             :   }
    5980             : 
    5981      496508 :   if (node_origins) {
    5982           0 :     node_origins->AddDecorator();
    5983             :   }
    5984             : 
    5985             :   base::ElapsedTimer pipeline_timer;
    5986             :   if (FLAG_trace_wasm_decode_time) {
    5987             :     node_count = mcgraph->graph()->NodeCount();
    5988             :     pipeline_timer.Start();
    5989             :   }
    5990             : 
    5991             :   // Run the compiler pipeline to generate machine code.
    5992      496508 :   auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
    5993      496441 :   if (mcgraph->machine()->Is32()) {
    5994             :     call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
    5995             :   }
    5996             : 
    5997             :   Pipeline::GenerateCodeForWasmFunction(
    5998             :       &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
    5999             :       source_positions, node_origins, func_body, env->module,
    6000      496441 :       wasm_unit_->func_index_);
    6001             : 
    6002             :   if (FLAG_trace_wasm_decode_time) {
    6003             :     double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
    6004             :     PrintF(
    6005             :         "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
    6006             :         "%0.3f ms pipeline\n",
    6007             :         static_cast<unsigned>(func_body.end - func_body.start), decode_ms,
    6008             :         node_count, pipeline_ms);
    6009             :   }
    6010             :   // TODO(bradnelson): Improve histogram handling of size_t.
    6011             :   counters->wasm_compile_function_peak_memory_bytes()->AddSample(
    6012      992834 :       static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
    6013     1992691 :   return std::move(*info.ReleaseWasmCompilationResult());
    6014             : }
    6015             : 
    6016             : namespace {
    6017             : // Helper for allocating either an GP or FP reg, or the next stack slot.
    6018             : class LinkageLocationAllocator {
    6019             :  public:
    6020             :   template <size_t kNumGpRegs, size_t kNumFpRegs>
    6021             :   constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
    6022             :                                      const DoubleRegister (&fp)[kNumFpRegs])
    6023             :       : allocator_(wasm::LinkageAllocator(gp, fp)) {}
    6024             : 
    6025     4828551 :   LinkageLocation Next(MachineRepresentation rep) {
    6026     4828551 :     MachineType type = MachineType::TypeForRepresentation(rep);
    6027     4828601 :     if (IsFloatingPoint(rep)) {
    6028     9081147 :       if (allocator_.CanAllocateFP(rep)) {
    6029             :         int reg_code = allocator_.NextFpReg(rep);
    6030             :         return LinkageLocation::ForRegister(reg_code, type);
    6031             :       }
    6032     4317350 :     } else if (allocator_.CanAllocateGP()) {
    6033             :       int reg_code = allocator_.NextGpReg();
    6034             :       return LinkageLocation::ForRegister(reg_code, type);
    6035             :     }
    6036             :     // Cannot use register; use stack slot.
    6037      154699 :     int index = -1 - allocator_.NextStackSlot(rep);
    6038             :     return LinkageLocation::ForCallerFrameSlot(index, type);
    6039             :   }
    6040             : 
    6041             :   void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
    6042     4251874 :   int NumStackSlots() const { return allocator_.NumStackSlots(); }
    6043             : 
    6044             :  private:
    6045             :   wasm::LinkageAllocator allocator_;
    6046             : };
    6047             : }  // namespace
    6048             : 
    6049             : // General code uses the above configuration data.
    6050     2125885 : CallDescriptor* GetWasmCallDescriptor(
    6051     9938397 :     Zone* zone, wasm::FunctionSig* fsig,
    6052             :     WasmGraphBuilder::UseRetpoline use_retpoline,
    6053             :     WasmGraphBuilder::ExtraCallableParam extra_callable_param) {
    6054             :   // The extra here is to accomodate the instance object as first parameter
    6055             :   // and, in the case of an import wrapper, the additional callable.
    6056     2125885 :   int extra_params = extra_callable_param ? 2 : 1;
    6057             :   LocationSignature::Builder locations(zone, fsig->return_count(),
    6058     2125885 :                                        fsig->parameter_count() + extra_params);
    6059             : 
    6060             :   // Add register and/or stack parameter(s).
    6061             :   LinkageLocationAllocator params(wasm::kGpParamRegisters,
    6062     2125956 :                                   wasm::kFpParamRegisters);
    6063             : 
    6064             :   // The instance object.
    6065     2125956 :   locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
    6066             :   const size_t param_offset = 1;  // Actual params start here.
    6067             : 
    6068             :   // Parameters are separated into two groups (first all untagged, then all
    6069             :   // tagged parameters). This allows for easy iteration of tagged parameters
    6070             :   // during frame iteration.
    6071             :   const size_t parameter_count = fsig->parameter_count();
    6072     2983888 :   for (size_t i = 0; i < parameter_count; i++) {
    6073             :     MachineRepresentation param =
    6074      857972 :         wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
    6075             :     // Skip tagged parameters (e.g. any-ref).
    6076      864562 :     if (IsAnyTagged(param)) continue;
    6077      851314 :     auto l = params.Next(param);
    6078      851363 :     locations.AddParamAt(i + param_offset, l);
    6079             :   }
    6080      858012 :   for (size_t i = 0; i < parameter_count; i++) {
    6081             :     MachineRepresentation param =
    6082      857980 :         wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
    6083             :     // Skip untagged parameters.
    6084     1709376 :     if (!IsAnyTagged(param)) continue;
    6085        6624 :     auto l = params.Next(param);
    6086        6636 :     locations.AddParamAt(i + param_offset, l);
    6087             :   }
    6088             : 
    6089             :   // Import call wrappers have an additional (implicit) parameter, the callable.
    6090             :   // For consistency with JS, we use the JSFunction register.
    6091     2125948 :   if (extra_callable_param) {
    6092             :     locations.AddParam(LinkageLocation::ForRegister(
    6093             :         kJSFunctionRegister.code(), MachineType::TaggedPointer()));
    6094             :   }
    6095             : 
    6096             :   // Add return location(s).
    6097             :   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
    6098     2125948 :                                 wasm::kFpReturnRegisters);
    6099             : 
    6100             :   int parameter_slots = params.NumStackSlots();
    6101             :   if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
    6102             : 
    6103             :   rets.SetStackOffset(parameter_slots);
    6104             : 
    6105     2125948 :   const int return_count = static_cast<int>(locations.return_count_);
    6106     3970727 :   for (int i = 0; i < return_count; i++) {
    6107             :     MachineRepresentation ret =
    6108     3689548 :         wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
    6109     1844774 :     auto l = rets.Next(ret);
    6110     1844779 :     locations.AddReturn(l);
    6111             :   }
    6112             : 
    6113             :   const RegList kCalleeSaveRegisters = 0;
    6114             :   const RegList kCalleeSaveFPRegisters = 0;
    6115             : 
    6116             :   // The target for wasm calls is always a code object.
    6117             :   MachineType target_type = MachineType::Pointer();
    6118             :   LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
    6119             : 
    6120             :   CallDescriptor::Kind kind = extra_callable_param
    6121             :                                   ? CallDescriptor::kCallWasmImportWrapper
    6122     2125953 :                                   : CallDescriptor::kCallWasmFunction;
    6123             : 
    6124             :   CallDescriptor::Flags flags =
    6125     2125953 :       use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
    6126             :   return new (zone) CallDescriptor(             // --
    6127             :       kind,                                     // kind
    6128             :       target_type,                              // target MachineType
    6129             :       target_loc,                               // target location
    6130             :       locations.Build(),                        // location_sig
    6131             :       parameter_slots,                          // stack_parameter_count
    6132             :       compiler::Operator::kNoProperties,        // properties
    6133             :       kCalleeSaveRegisters,                     // callee-saved registers
    6134             :       kCalleeSaveFPRegisters,                   // callee-saved fp regs
    6135             :       flags,                                    // flags
    6136             :       "wasm-call",                              // debug name
    6137             :       0,                                        // allocatable registers
    6138     8503731 :       rets.NumStackSlots() - parameter_slots);  // stack_return_count
    6139             : }
    6140             : 
    6141             : namespace {
    6142           0 : CallDescriptor* ReplaceTypeInCallDescriptorWith(
    6143           0 :     Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
    6144             :     MachineType input_type, MachineRepresentation output_type) {
    6145             :   size_t parameter_count = call_descriptor->ParameterCount();
    6146             :   size_t return_count = call_descriptor->ReturnCount();
    6147           0 :   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
    6148           0 :     if (call_descriptor->GetParameterType(i) == input_type) {
    6149           0 :       parameter_count += num_replacements - 1;
    6150             :     }
    6151             :   }
    6152           0 :   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
    6153           0 :     if (call_descriptor->GetReturnType(i) == input_type) {
    6154           0 :       return_count += num_replacements - 1;
    6155             :     }
    6156             :   }
    6157           0 :   if (parameter_count == call_descriptor->ParameterCount() &&
    6158             :       return_count == call_descriptor->ReturnCount()) {
    6159             :     return call_descriptor;
    6160             :   }
    6161             : 
    6162             :   LocationSignature::Builder locations(zone, return_count, parameter_count);
    6163             : 
    6164             :   LinkageLocationAllocator params(wasm::kGpParamRegisters,
    6165           0 :                                   wasm::kFpParamRegisters);
    6166           0 :   for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
    6167           0 :     if (call_descriptor->GetParameterType(i) == input_type) {
    6168           0 :       for (size_t j = 0; j < num_replacements; j++) {
    6169           0 :         locations.AddParam(params.Next(output_type));
    6170             :       }
    6171             :     } else {
    6172             :       locations.AddParam(
    6173           0 :           params.Next(call_descriptor->GetParameterType(i).representation()));
    6174             :     }
    6175             :   }
    6176             : 
    6177             :   LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
    6178           0 :                                 wasm::kFpReturnRegisters);
    6179             :   rets.SetStackOffset(params.NumStackSlots());
    6180           0 :   for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
    6181           0 :     if (call_descriptor->GetReturnType(i) == input_type) {
    6182           0 :       for (size_t j = 0; j < num_replacements; j++) {
    6183           0 :         locations.AddReturn(rets.Next(output_type));
    6184             :       }
    6185             :     } else {
    6186             :       locations.AddReturn(
    6187           0 :           rets.Next(call_descriptor->GetReturnType(i).representation()));
    6188             :     }
    6189             :   }
    6190             : 
    6191             :   return new (zone) CallDescriptor(                    // --
    6192             :       call_descriptor->kind(),                         // kind
    6193             :       call_descriptor->GetInputType(0),                // target MachineType
    6194             :       call_descriptor->GetInputLocation(0),            // target location
    6195             :       locations.Build(),                               // location_sig
    6196             :       params.NumStackSlots(),                          // stack_parameter_count
    6197             :       call_descriptor->properties(),                   // properties
    6198             :       call_descriptor->CalleeSavedRegisters(),         // callee-saved registers
    6199             :       call_descriptor->CalleeSavedFPRegisters(),       // callee-saved fp regs
    6200             :       call_descriptor->flags(),                        // flags
    6201             :       call_descriptor->debug_name(),                   // debug name
    6202             :       call_descriptor->AllocatableRegisters(),         // allocatable registers
    6203           0 :       rets.NumStackSlots() - params.NumStackSlots());  // stack_return_count
    6204             : }
    6205             : }  // namespace
    6206             : 
    6207           0 : CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
    6208             :                                          CallDescriptor* call_descriptor) {
    6209             :   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
    6210             :                                          MachineType::Int64(),
    6211           0 :                                          MachineRepresentation::kWord32);
    6212             : }
    6213             : 
    6214           0 : CallDescriptor* GetI32WasmCallDescriptorForSimd(
    6215             :     Zone* zone, CallDescriptor* call_descriptor) {
    6216             :   return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
    6217             :                                          MachineType::Simd128(),
    6218           0 :                                          MachineRepresentation::kWord32);
    6219             : }
    6220             : 
    6221      496467 : AssemblerOptions WasmAssemblerOptions() {
    6222      635575 :   AssemblerOptions options;
    6223             :   // Relocation info required to serialize {WasmCode} for proper functions.
    6224             :   options.record_reloc_info_for_serialization = true;
    6225             :   options.enable_root_array_delta_access = false;
    6226      496467 :   return options;
    6227             : }
    6228             : 
    6229           0 : AssemblerOptions WasmStubAssemblerOptions() {
    6230        9010 :   AssemblerOptions options;
    6231             :   // Relocation info not necessary because stubs are not serialized.
    6232        9010 :   options.record_reloc_info_for_serialization = false;
    6233             :   options.enable_root_array_delta_access = false;
    6234           0 :   return options;
    6235             : }
    6236             : 
    6237             : #undef WASM_64
    6238             : #undef FATAL_UNSUPPORTED_OPCODE
    6239             : #undef WASM_INSTANCE_OBJECT_SIZE
    6240             : #undef WASM_INSTANCE_OBJECT_OFFSET
    6241             : #undef LOAD_RAW
    6242             : #undef LOAD_INSTANCE_FIELD
    6243             : #undef LOAD_TAGGED_POINTER
    6244             : #undef LOAD_TAGGED_ANY
    6245             : #undef LOAD_FIXED_ARRAY_SLOT
    6246             : #undef LOAD_FIXED_ARRAY_SLOT_SMI
    6247             : #undef LOAD_FIXED_ARRAY_SLOT_PTR
    6248             : #undef LOAD_FIXED_ARRAY_SLOT_ANY
    6249             : #undef STORE_FIXED_ARRAY_SLOT_SMI
    6250             : #undef STORE_FIXED_ARRAY_SLOT_ANY
    6251             : 
    6252             : }  // namespace compiler
    6253             : }  // namespace internal
    6254      178779 : }  // namespace v8

Generated by: LCOV version 1.10