LCOV - code coverage report
Current view: top level - src/compiler/x64 - code-generator-x64.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 1097 1244 88.2 %
Date: 2017-04-26 Functions: 53 105 50.5 %

          Line data    Source code
       1             : // Copyright 2013 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #include "src/compiler/code-generator.h"
       6             : 
       7             : #include <limits>
       8             : 
       9             : #include "src/compilation-info.h"
      10             : #include "src/compiler/code-generator-impl.h"
      11             : #include "src/compiler/gap-resolver.h"
      12             : #include "src/compiler/node-matchers.h"
      13             : #include "src/compiler/osr.h"
      14             : #include "src/heap/heap-inl.h"
      15             : #include "src/wasm/wasm-module.h"
      16             : #include "src/x64/assembler-x64.h"
      17             : #include "src/x64/macro-assembler-x64.h"
      18             : 
      19             : namespace v8 {
      20             : namespace internal {
      21             : namespace compiler {
      22             : 
      23             : #define __ masm()->
      24             : 
      25             : // Adds X64 specific methods for decoding operands.
      26             : class X64OperandConverter : public InstructionOperandConverter {
      27             :  public:
      28             :   X64OperandConverter(CodeGenerator* gen, Instruction* instr)
      29             :       : InstructionOperandConverter(gen, instr) {}
      30             : 
      31             :   Immediate InputImmediate(size_t index) {
      32     2577061 :     return ToImmediate(instr_->InputAt(index));
      33             :   }
      34             : 
      35      525602 :   Operand InputOperand(size_t index, int extra = 0) {
      36     1576807 :     return ToOperand(instr_->InputAt(index), extra);
      37             :   }
      38             : 
      39           0 :   Operand OutputOperand() { return ToOperand(instr_->Output()); }
      40             : 
      41     2139456 :   Immediate ToImmediate(InstructionOperand* operand) {
      42     2139456 :     Constant constant = ToConstant(operand);
      43     2139456 :     if (constant.type() == Constant::kFloat64) {
      44             :       DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64()));
      45      105799 :       return Immediate(0);
      46             :     }
      47     2033657 :     if (RelocInfo::IsWasmReference(constant.rmode())) {
      48       12563 :       return Immediate(constant.ToInt32(), constant.rmode());
      49             :     }
      50     2021094 :     return Immediate(constant.ToInt32());
      51             :   }
      52             : 
      53             :   Operand ToOperand(InstructionOperand* op, int extra = 0) {
      54             :     DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
      55     9260195 :     return SlotToOperand(AllocatedOperand::cast(op)->index(), extra);
      56             :   }
      57             : 
      58     9269799 :   Operand SlotToOperand(int slot_index, int extra = 0) {
      59    18539598 :     FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index);
      60             :     return Operand(offset.from_stack_pointer() ? rsp : rbp,
      61    18539598 :                    offset.offset() + extra);
      62             :   }
      63             : 
      64             :   static size_t NextOffset(size_t* offset) {
      65     7513281 :     size_t i = *offset;
      66    13563676 :     (*offset)++;
      67             :     return i;
      68             :   }
      69             : 
      70             :   static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
      71             :     STATIC_ASSERT(0 == static_cast<int>(times_1));
      72             :     STATIC_ASSERT(1 == static_cast<int>(times_2));
      73             :     STATIC_ASSERT(2 == static_cast<int>(times_4));
      74             :     STATIC_ASSERT(3 == static_cast<int>(times_8));
      75      606275 :     int scale = static_cast<int>(mode - one);
      76             :     DCHECK(scale >= 0 && scale < 4);
      77      606275 :     return static_cast<ScaleFactor>(scale);
      78             :   }
      79             : 
      80     7513281 :   Operand MemoryOperand(size_t* offset) {
      81     7513281 :     AddressingMode mode = AddressingModeField::decode(instr_->opcode());
      82     7513281 :     switch (mode) {
      83             :       case kMode_MR: {
      84      351444 :         Register base = InputRegister(NextOffset(offset));
      85             :         int32_t disp = 0;
      86      351444 :         return Operand(base, disp);
      87             :       }
      88             :       case kMode_MRI: {
      89     5131650 :         Register base = InputRegister(NextOffset(offset));
      90     5131650 :         int32_t disp = InputInt32(NextOffset(offset));
      91     5131650 :         return Operand(base, disp);
      92             :       }
      93             :       case kMode_MR1:
      94             :       case kMode_MR2:
      95             :       case kMode_MR4:
      96             :       case kMode_MR8: {
      97      241508 :         Register base = InputRegister(NextOffset(offset));
      98      241508 :         Register index = InputRegister(NextOffset(offset));
      99             :         ScaleFactor scale = ScaleFor(kMode_MR1, mode);
     100             :         int32_t disp = 0;
     101      241508 :         return Operand(base, index, scale, disp);
     102             :       }
     103             :       case kMode_MR1I:
     104             :       case kMode_MR2I:
     105             :       case kMode_MR4I:
     106             :       case kMode_MR8I: {
     107      314106 :         Register base = InputRegister(NextOffset(offset));
     108      314106 :         Register index = InputRegister(NextOffset(offset));
     109             :         ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
     110      314106 :         int32_t disp = InputInt32(NextOffset(offset));
     111      314106 :         return Operand(base, index, scale, disp);
     112             :       }
     113             :       case kMode_M1: {
     114           0 :         Register base = InputRegister(NextOffset(offset));
     115             :         int32_t disp = 0;
     116           0 :         return Operand(base, disp);
     117             :       }
     118             :       case kMode_M2:
     119           0 :         UNREACHABLE();  // Should use kModeMR with more compact encoding instead
     120             :         return Operand(no_reg, 0);
     121             :       case kMode_M4:
     122             :       case kMode_M8: {
     123        1636 :         Register index = InputRegister(NextOffset(offset));
     124             :         ScaleFactor scale = ScaleFor(kMode_M1, mode);
     125             :         int32_t disp = 0;
     126        1636 :         return Operand(index, scale, disp);
     127             :       }
     128             :       case kMode_M1I:
     129             :       case kMode_M2I:
     130             :       case kMode_M4I:
     131             :       case kMode_M8I: {
     132       49025 :         Register index = InputRegister(NextOffset(offset));
     133             :         ScaleFactor scale = ScaleFor(kMode_M1I, mode);
     134       49025 :         int32_t disp = InputInt32(NextOffset(offset));
     135       49025 :         return Operand(index, scale, disp);
     136             :       }
     137             :       case kMode_Root: {
     138     1423912 :         Register base = kRootRegister;
     139     1423912 :         int32_t disp = InputInt32(NextOffset(offset));
     140     1423912 :         return Operand(base, disp);
     141             :       }
     142             :       case kMode_None:
     143           0 :         UNREACHABLE();
     144             :         return Operand(no_reg, 0);
     145             :     }
     146           0 :     UNREACHABLE();
     147             :     return Operand(no_reg, 0);
     148             :   }
     149             : 
     150             :   Operand MemoryOperand(size_t first_input = 0) {
     151     3522244 :     return MemoryOperand(&first_input);
     152             :   }
     153             : };
     154             : 
     155             : 
     156             : namespace {
     157             : 
     158             : bool HasImmediateInput(Instruction* instr, size_t index) {
     159    12873963 :   return instr->InputAt(index)->IsImmediate();
     160             : }
     161             : 
     162             : 
     163           0 : class OutOfLineLoadZero final : public OutOfLineCode {
     164             :  public:
     165             :   OutOfLineLoadZero(CodeGenerator* gen, Register result)
     166         133 :       : OutOfLineCode(gen), result_(result) {}
     167             : 
     168         266 :   void Generate() final { __ xorl(result_, result_); }
     169             : 
     170             :  private:
     171             :   Register const result_;
     172             : };
     173             : 
     174           0 : class OutOfLineLoadFloat32NaN final : public OutOfLineCode {
     175             :  public:
     176             :   OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result)
     177         121 :       : OutOfLineCode(gen), result_(result) {}
     178             : 
     179         121 :   void Generate() final {
     180         121 :     __ Xorps(result_, result_);
     181         121 :     __ Divss(result_, result_);
     182         121 :   }
     183             : 
     184             :  private:
     185             :   XMMRegister const result_;
     186             : };
     187             : 
     188           0 : class OutOfLineLoadFloat64NaN final : public OutOfLineCode {
     189             :  public:
     190             :   OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result)
     191         313 :       : OutOfLineCode(gen), result_(result) {}
     192             : 
     193         313 :   void Generate() final {
     194         313 :     __ Xorpd(result_, result_);
     195         313 :     __ Divsd(result_, result_);
     196         313 :   }
     197             : 
     198             :  private:
     199             :   XMMRegister const result_;
     200             : };
     201             : 
     202           0 : class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
     203             :  public:
     204             :   OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
     205             :                              XMMRegister input,
     206             :                              UnwindingInfoWriter* unwinding_info_writer)
     207             :       : OutOfLineCode(gen),
     208             :         result_(result),
     209             :         input_(input),
     210       70260 :         unwinding_info_writer_(unwinding_info_writer) {}
     211             : 
     212       70260 :   void Generate() final {
     213      351300 :     __ subp(rsp, Immediate(kDoubleSize));
     214             :     unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
     215      140520 :                                                       kDoubleSize);
     216      140520 :     __ Movsd(MemOperand(rsp, 0), input_);
     217       70260 :     __ SlowTruncateToI(result_, rsp, 0);
     218       70260 :     __ addp(rsp, Immediate(kDoubleSize));
     219             :     unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
     220      140520 :                                                       -kDoubleSize);
     221       70260 :   }
     222             : 
     223             :  private:
     224             :   Register const result_;
     225             :   XMMRegister const input_;
     226             :   UnwindingInfoWriter* const unwinding_info_writer_;
     227             : };
     228             : 
     229             : 
     230           0 : class OutOfLineRecordWrite final : public OutOfLineCode {
     231             :  public:
     232             :   OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand,
     233             :                        Register value, Register scratch0, Register scratch1,
     234             :                        RecordWriteMode mode)
     235             :       : OutOfLineCode(gen),
     236             :         object_(object),
     237             :         operand_(operand),
     238             :         value_(value),
     239             :         scratch0_(scratch0),
     240             :         scratch1_(scratch1),
     241      345680 :         mode_(mode) {}
     242             : 
     243      345680 :   void Generate() final {
     244      345680 :     if (mode_ > RecordWriteMode::kValueIsPointer) {
     245     1670844 :       __ JumpIfSmi(value_, exit());
     246             :     }
     247             :     __ CheckPageFlag(value_, scratch0_,
     248             :                      MemoryChunk::kPointersToHereAreInterestingMask, zero,
     249      691360 :                      exit());
     250             :     RememberedSetAction const remembered_set_action =
     251             :         mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
     252      345680 :                                              : OMIT_REMEMBERED_SET;
     253             :     SaveFPRegsMode const save_fp_mode =
     254      691360 :         frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
     255             :     RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
     256      345680 :                          remembered_set_action, save_fp_mode);
     257      345680 :     __ leap(scratch1_, operand_);
     258      345680 :     __ CallStub(&stub);
     259      345680 :   }
     260             : 
     261             :  private:
     262             :   Register const object_;
     263             :   Operand const operand_;
     264             :   Register const value_;
     265             :   Register const scratch0_;
     266             :   Register const scratch1_;
     267             :   RecordWriteMode const mode_;
     268             : };
     269             : 
     270           0 : class WasmOutOfLineTrap final : public OutOfLineCode {
     271             :  public:
     272             :   WasmOutOfLineTrap(CodeGenerator* gen, int pc, bool frame_elided,
     273             :                     int32_t position)
     274             :       : OutOfLineCode(gen),
     275             :         gen_(gen),
     276             :         pc_(pc),
     277             :         frame_elided_(frame_elided),
     278        1650 :         position_(position) {}
     279             : 
     280             :   // TODO(eholk): Refactor this method to take the code generator as a
     281             :   // parameter.
     282        1650 :   void Generate() final {
     283        8798 :     __ RecordProtectedInstructionLanding(pc_);
     284             : 
     285        1650 :     if (frame_elided_) {
     286         548 :       __ EnterFrame(StackFrame::WASM_COMPILED);
     287             :     }
     288             : 
     289             :     wasm::TrapReason trap_id = wasm::kTrapMemOutOfBounds;
     290        1650 :     int trap_reason = wasm::WasmOpcodes::TrapReasonToMessageId(trap_id);
     291        1650 :     __ Push(Smi::FromInt(trap_reason));
     292             :     // TODO(eholk): use AssembleSourcePosition instead of passing in position_
     293             :     // as a parameter. See AssembleArchTrap as an example. Consider sharing code
     294             :     // with AssembleArchTrap.
     295        3300 :     __ Push(Smi::FromInt(position_));
     296             :     __ Move(rsi, Smi::kZero);
     297        1650 :     __ CallRuntime(Runtime::kThrowWasmError);
     298             : 
     299             :     ReferenceMap* reference_map =
     300        1650 :         new (gen_->code()->zone()) ReferenceMap(gen_->code()->zone());
     301             :     gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
     302        1650 :                           Safepoint::kNoLazyDeopt);
     303        1650 :   }
     304             : 
     305             :  private:
     306             :   CodeGenerator* gen_;
     307             :   int pc_;
     308             :   bool frame_elided_;
     309             :   int32_t position_;
     310             : };
     311             : 
     312     4924485 : void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
     313             :                          InstructionCode opcode, size_t input_count,
     314             :                          X64OperandConverter& i, int pc) {
     315             :   const X64MemoryProtection protection =
     316     4922835 :       static_cast<X64MemoryProtection>(MiscField::decode(opcode));
     317     4922835 :   if (protection == X64MemoryProtection::kProtected) {
     318        1650 :     const bool frame_elided = !codegen->frame_access_state()->has_frame();
     319        1650 :     const int32_t position = i.InputInt32(input_count - 1);
     320             :     new (zone) WasmOutOfLineTrap(codegen, pc, frame_elided, position);
     321             :   }
     322     4922835 : }
     323             : }  // namespace
     324             : 
     325             : 
     326             : #define ASSEMBLE_UNOP(asm_instr)         \
     327             :   do {                                   \
     328             :     if (instr->Output()->IsRegister()) { \
     329             :       __ asm_instr(i.OutputRegister());  \
     330             :     } else {                             \
     331             :       __ asm_instr(i.OutputOperand());   \
     332             :     }                                    \
     333             :   } while (0)
     334             : 
     335             : #define ASSEMBLE_BINOP(asm_instr)                                     \
     336             :   do {                                                                \
     337             :     if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
     338             :       size_t index = 1;                                               \
     339             :       Operand right = i.MemoryOperand(&index);                        \
     340             :       __ asm_instr(i.InputRegister(0), right);                        \
     341             :     } else {                                                          \
     342             :       if (HasImmediateInput(instr, 1)) {                              \
     343             :         if (instr->InputAt(0)->IsRegister()) {                        \
     344             :           __ asm_instr(i.InputRegister(0), i.InputImmediate(1));      \
     345             :         } else {                                                      \
     346             :           __ asm_instr(i.InputOperand(0), i.InputImmediate(1));       \
     347             :         }                                                             \
     348             :       } else {                                                        \
     349             :         if (instr->InputAt(1)->IsRegister()) {                        \
     350             :           __ asm_instr(i.InputRegister(0), i.InputRegister(1));       \
     351             :         } else {                                                      \
     352             :           __ asm_instr(i.InputRegister(0), i.InputOperand(1));        \
     353             :         }                                                             \
     354             :       }                                                               \
     355             :     }                                                                 \
     356             :   } while (0)
     357             : 
     358             : #define ASSEMBLE_COMPARE(asm_instr)                                   \
     359             :   do {                                                                \
     360             :     if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
     361             :       size_t index = 0;                                               \
     362             :       Operand left = i.MemoryOperand(&index);                         \
     363             :       if (HasImmediateInput(instr, index)) {                          \
     364             :         __ asm_instr(left, i.InputImmediate(index));                  \
     365             :       } else {                                                        \
     366             :         __ asm_instr(left, i.InputRegister(index));                   \
     367             :       }                                                               \
     368             :     } else {                                                          \
     369             :       if (HasImmediateInput(instr, 1)) {                              \
     370             :         if (instr->InputAt(0)->IsRegister()) {                        \
     371             :           __ asm_instr(i.InputRegister(0), i.InputImmediate(1));      \
     372             :         } else {                                                      \
     373             :           __ asm_instr(i.InputOperand(0), i.InputImmediate(1));       \
     374             :         }                                                             \
     375             :       } else {                                                        \
     376             :         if (instr->InputAt(1)->IsRegister()) {                        \
     377             :           __ asm_instr(i.InputRegister(0), i.InputRegister(1));       \
     378             :         } else {                                                      \
     379             :           __ asm_instr(i.InputRegister(0), i.InputOperand(1));        \
     380             :         }                                                             \
     381             :       }                                                               \
     382             :     }                                                                 \
     383             :   } while (0)
     384             : 
     385             : #define ASSEMBLE_MULT(asm_instr)                              \
     386             :   do {                                                        \
     387             :     if (HasImmediateInput(instr, 1)) {                        \
     388             :       if (instr->InputAt(0)->IsRegister()) {                  \
     389             :         __ asm_instr(i.OutputRegister(), i.InputRegister(0),  \
     390             :                      i.InputImmediate(1));                    \
     391             :       } else {                                                \
     392             :         __ asm_instr(i.OutputRegister(), i.InputOperand(0),   \
     393             :                      i.InputImmediate(1));                    \
     394             :       }                                                       \
     395             :     } else {                                                  \
     396             :       if (instr->InputAt(1)->IsRegister()) {                  \
     397             :         __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
     398             :       } else {                                                \
     399             :         __ asm_instr(i.OutputRegister(), i.InputOperand(1));  \
     400             :       }                                                       \
     401             :     }                                                         \
     402             :   } while (0)
     403             : 
     404             : 
     405             : #define ASSEMBLE_SHIFT(asm_instr, width)                                   \
     406             :   do {                                                                     \
     407             :     if (HasImmediateInput(instr, 1)) {                                     \
     408             :       if (instr->Output()->IsRegister()) {                                 \
     409             :         __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
     410             :       } else {                                                             \
     411             :         __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1)));  \
     412             :       }                                                                    \
     413             :     } else {                                                               \
     414             :       if (instr->Output()->IsRegister()) {                                 \
     415             :         __ asm_instr##_cl(i.OutputRegister());                             \
     416             :       } else {                                                             \
     417             :         __ asm_instr##_cl(i.OutputOperand());                              \
     418             :       }                                                                    \
     419             :     }                                                                      \
     420             :   } while (0)
     421             : 
     422             : 
     423             : #define ASSEMBLE_MOVX(asm_instr)                            \
     424             :   do {                                                      \
     425             :     if (instr->addressing_mode() != kMode_None) {           \
     426             :       __ asm_instr(i.OutputRegister(), i.MemoryOperand());  \
     427             :     } else if (instr->InputAt(0)->IsRegister()) {           \
     428             :       __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
     429             :     } else {                                                \
     430             :       __ asm_instr(i.OutputRegister(), i.InputOperand(0));  \
     431             :     }                                                       \
     432             :   } while (0)
     433             : 
     434             : #define ASSEMBLE_SSE_BINOP(asm_instr)                                   \
     435             :   do {                                                                  \
     436             :     if (instr->InputAt(1)->IsFPRegister()) {                            \
     437             :       __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
     438             :     } else {                                                            \
     439             :       __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1));        \
     440             :     }                                                                   \
     441             :   } while (0)
     442             : 
     443             : #define ASSEMBLE_SSE_UNOP(asm_instr)                                    \
     444             :   do {                                                                  \
     445             :     if (instr->InputAt(0)->IsFPRegister()) {                            \
     446             :       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
     447             :     } else {                                                            \
     448             :       __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0));        \
     449             :     }                                                                   \
     450             :   } while (0)
     451             : 
     452             : #define ASSEMBLE_AVX_BINOP(asm_instr)                                  \
     453             :   do {                                                                 \
     454             :     CpuFeatureScope avx_scope(masm(), AVX);                            \
     455             :     if (instr->InputAt(1)->IsFPRegister()) {                           \
     456             :       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
     457             :                    i.InputDoubleRegister(1));                          \
     458             :     } else {                                                           \
     459             :       __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
     460             :                    i.InputOperand(1));                                 \
     461             :     }                                                                  \
     462             :   } while (0)
     463             : 
     464             : #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN)             \
     465             :   do {                                                                       \
     466             :     auto result = i.OutputDoubleRegister();                                  \
     467             :     auto buffer = i.InputRegister(0);                                        \
     468             :     auto index1 = i.InputRegister(1);                                        \
     469             :     auto index2 = i.InputUint32(2);                                          \
     470             :     OutOfLineCode* ool;                                                      \
     471             :     if (instr->InputAt(3)->IsRegister()) {                                   \
     472             :       auto length = i.InputRegister(3);                                      \
     473             :       DCHECK_EQ(0u, index2);                                                 \
     474             :       __ cmpl(index1, length);                                               \
     475             :       ool = new (zone()) OutOfLineLoadNaN(this, result);                     \
     476             :     } else {                                                                 \
     477             :       auto length = i.InputUint32(3);                                        \
     478             :       RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode();       \
     479             :       DCHECK_LE(index2, length);                                             \
     480             :       __ cmpl(index1, Immediate(length - index2, rmode));                    \
     481             :       class OutOfLineLoadFloat final : public OutOfLineCode {                \
     482             :        public:                                                               \
     483             :         OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result,           \
     484             :                            Register buffer, Register index1, int32_t index2, \
     485             :                            int32_t length, RelocInfo::Mode rmode)            \
     486             :             : OutOfLineCode(gen),                                            \
     487             :               result_(result),                                               \
     488             :               buffer_(buffer),                                               \
     489             :               index1_(index1),                                               \
     490             :               index2_(index2),                                               \
     491             :               length_(length),                                               \
     492             :               rmode_(rmode) {}                                               \
     493             :                                                                              \
     494             :         void Generate() final {                                              \
     495             :           __ leal(kScratchRegister, Operand(index1_, index2_));              \
     496             :           __ Pcmpeqd(result_, result_);                                      \
     497             :           __ cmpl(kScratchRegister, Immediate(length_, rmode_));             \
     498             :           __ j(above_equal, exit());                                         \
     499             :           __ asm_instr(result_,                                              \
     500             :                        Operand(buffer_, kScratchRegister, times_1, 0));      \
     501             :         }                                                                    \
     502             :                                                                              \
     503             :        private:                                                              \
     504             :         XMMRegister const result_;                                           \
     505             :         Register const buffer_;                                              \
     506             :         Register const index1_;                                              \
     507             :         int32_t const index2_;                                               \
     508             :         int32_t const length_;                                               \
     509             :         RelocInfo::Mode rmode_;                                              \
     510             :       };                                                                     \
     511             :       ool = new (zone()) OutOfLineLoadFloat(this, result, buffer, index1,    \
     512             :                                             index2, length, rmode);          \
     513             :     }                                                                        \
     514             :     __ j(above_equal, ool->entry());                                         \
     515             :     __ asm_instr(result, Operand(buffer, index1, times_1, index2));          \
     516             :     __ bind(ool->exit());                                                    \
     517             :   } while (false)
     518             : 
     519             : #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr)                               \
     520             :   do {                                                                         \
     521             :     auto result = i.OutputRegister();                                          \
     522             :     auto buffer = i.InputRegister(0);                                          \
     523             :     auto index1 = i.InputRegister(1);                                          \
     524             :     auto index2 = i.InputUint32(2);                                            \
     525             :     OutOfLineCode* ool;                                                        \
     526             :     if (instr->InputAt(3)->IsRegister()) {                                     \
     527             :       auto length = i.InputRegister(3);                                        \
     528             :       DCHECK_EQ(0u, index2);                                                   \
     529             :       __ cmpl(index1, length);                                                 \
     530             :       ool = new (zone()) OutOfLineLoadZero(this, result);                      \
     531             :     } else {                                                                   \
     532             :       auto length = i.InputUint32(3);                                          \
     533             :       RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode();         \
     534             :       DCHECK_LE(index2, length);                                               \
     535             :       __ cmpl(index1, Immediate(length - index2, rmode));                      \
     536             :       class OutOfLineLoadInteger final : public OutOfLineCode {                \
     537             :        public:                                                                 \
     538             :         OutOfLineLoadInteger(CodeGenerator* gen, Register result,              \
     539             :                              Register buffer, Register index1, int32_t index2, \
     540             :                              int32_t length, RelocInfo::Mode rmode)            \
     541             :             : OutOfLineCode(gen),                                              \
     542             :               result_(result),                                                 \
     543             :               buffer_(buffer),                                                 \
     544             :               index1_(index1),                                                 \
     545             :               index2_(index2),                                                 \
     546             :               length_(length),                                                 \
     547             :               rmode_(rmode) {}                                                 \
     548             :                                                                                \
     549             :         void Generate() final {                                                \
     550             :           Label oob;                                                           \
     551             :           __ leal(kScratchRegister, Operand(index1_, index2_));                \
     552             :           __ cmpl(kScratchRegister, Immediate(length_, rmode_));               \
     553             :           __ j(above_equal, &oob, Label::kNear);                               \
     554             :           __ asm_instr(result_,                                                \
     555             :                        Operand(buffer_, kScratchRegister, times_1, 0));        \
     556             :           __ jmp(exit());                                                      \
     557             :           __ bind(&oob);                                                       \
     558             :           __ xorl(result_, result_);                                           \
     559             :         }                                                                      \
     560             :                                                                                \
     561             :        private:                                                                \
     562             :         Register const result_;                                                \
     563             :         Register const buffer_;                                                \
     564             :         Register const index1_;                                                \
     565             :         int32_t const index2_;                                                 \
     566             :         int32_t const length_;                                                 \
     567             :         RelocInfo::Mode const rmode_;                                          \
     568             :       };                                                                       \
     569             :       ool = new (zone()) OutOfLineLoadInteger(this, result, buffer, index1,    \
     570             :                                               index2, length, rmode);          \
     571             :     }                                                                          \
     572             :     __ j(above_equal, ool->entry());                                           \
     573             :     __ asm_instr(result, Operand(buffer, index1, times_1, index2));            \
     574             :     __ bind(ool->exit());                                                      \
     575             :   } while (false)
     576             : 
     577             : #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr)                              \
     578             :   do {                                                                       \
     579             :     auto buffer = i.InputRegister(0);                                        \
     580             :     auto index1 = i.InputRegister(1);                                        \
     581             :     auto index2 = i.InputUint32(2);                                          \
     582             :     auto value = i.InputDoubleRegister(4);                                   \
     583             :     if (instr->InputAt(3)->IsRegister()) {                                   \
     584             :       auto length = i.InputRegister(3);                                      \
     585             :       DCHECK_EQ(0u, index2);                                                 \
     586             :       Label done;                                                            \
     587             :       __ cmpl(index1, length);                                               \
     588             :       __ j(above_equal, &done, Label::kNear);                                \
     589             :       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
     590             :       __ bind(&done);                                                        \
     591             :     } else {                                                                 \
     592             :       auto length = i.InputUint32(3);                                        \
     593             :       RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode();       \
     594             :       DCHECK_LE(index2, length);                                             \
     595             :       __ cmpl(index1, Immediate(length - index2, rmode));                    \
     596             :       class OutOfLineStoreFloat final : public OutOfLineCode {               \
     597             :        public:                                                               \
     598             :         OutOfLineStoreFloat(CodeGenerator* gen, Register buffer,             \
     599             :                             Register index1, int32_t index2, int32_t length, \
     600             :                             XMMRegister value, RelocInfo::Mode rmode)        \
     601             :             : OutOfLineCode(gen),                                            \
     602             :               buffer_(buffer),                                               \
     603             :               index1_(index1),                                               \
     604             :               index2_(index2),                                               \
     605             :               length_(length),                                               \
     606             :               value_(value),                                                 \
     607             :               rmode_(rmode) {}                                               \
     608             :                                                                              \
     609             :         void Generate() final {                                              \
     610             :           __ leal(kScratchRegister, Operand(index1_, index2_));              \
     611             :           __ cmpl(kScratchRegister, Immediate(length_, rmode_));             \
     612             :           __ j(above_equal, exit());                                         \
     613             :           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),       \
     614             :                        value_);                                              \
     615             :         }                                                                    \
     616             :                                                                              \
     617             :        private:                                                              \
     618             :         Register const buffer_;                                              \
     619             :         Register const index1_;                                              \
     620             :         int32_t const index2_;                                               \
     621             :         int32_t const length_;                                               \
     622             :         XMMRegister const value_;                                            \
     623             :         RelocInfo::Mode rmode_;                                              \
     624             :       };                                                                     \
     625             :       auto ool = new (zone()) OutOfLineStoreFloat(                           \
     626             :           this, buffer, index1, index2, length, value, rmode);               \
     627             :       __ j(above_equal, ool->entry());                                       \
     628             :       __ asm_instr(Operand(buffer, index1, times_1, index2), value);         \
     629             :       __ bind(ool->exit());                                                  \
     630             :     }                                                                        \
     631             :   } while (false)
     632             : 
     633             : #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value)                  \
     634             :   do {                                                                         \
     635             :     auto buffer = i.InputRegister(0);                                          \
     636             :     auto index1 = i.InputRegister(1);                                          \
     637             :     auto index2 = i.InputUint32(2);                                            \
     638             :     if (instr->InputAt(3)->IsRegister()) {                                     \
     639             :       auto length = i.InputRegister(3);                                        \
     640             :       DCHECK_EQ(0u, index2);                                                   \
     641             :       Label done;                                                              \
     642             :       __ cmpl(index1, length);                                                 \
     643             :       __ j(above_equal, &done, Label::kNear);                                  \
     644             :       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
     645             :       __ bind(&done);                                                          \
     646             :     } else {                                                                   \
     647             :       auto length = i.InputUint32(3);                                          \
     648             :       RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode();         \
     649             :       DCHECK_LE(index2, length);                                               \
     650             :       __ cmpl(index1, Immediate(length - index2, rmode));                      \
     651             :       class OutOfLineStoreInteger final : public OutOfLineCode {               \
     652             :        public:                                                                 \
     653             :         OutOfLineStoreInteger(CodeGenerator* gen, Register buffer,             \
     654             :                               Register index1, int32_t index2, int32_t length, \
     655             :                               Value value, RelocInfo::Mode rmode)              \
     656             :             : OutOfLineCode(gen),                                              \
     657             :               buffer_(buffer),                                                 \
     658             :               index1_(index1),                                                 \
     659             :               index2_(index2),                                                 \
     660             :               length_(length),                                                 \
     661             :               value_(value),                                                   \
     662             :               rmode_(rmode) {}                                                 \
     663             :                                                                                \
     664             :         void Generate() final {                                                \
     665             :           __ leal(kScratchRegister, Operand(index1_, index2_));                \
     666             :           __ cmpl(kScratchRegister, Immediate(length_, rmode_));               \
     667             :           __ j(above_equal, exit());                                           \
     668             :           __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0),         \
     669             :                        value_);                                                \
     670             :         }                                                                      \
     671             :                                                                                \
     672             :        private:                                                                \
     673             :         Register const buffer_;                                                \
     674             :         Register const index1_;                                                \
     675             :         int32_t const index2_;                                                 \
     676             :         int32_t const length_;                                                 \
     677             :         Value const value_;                                                    \
     678             :         RelocInfo::Mode rmode_;                                                \
     679             :       };                                                                       \
     680             :       auto ool = new (zone()) OutOfLineStoreInteger(                           \
     681             :           this, buffer, index1, index2, length, value, rmode);                 \
     682             :       __ j(above_equal, ool->entry());                                         \
     683             :       __ asm_instr(Operand(buffer, index1, times_1, index2), value);           \
     684             :       __ bind(ool->exit());                                                    \
     685             :     }                                                                          \
     686             :   } while (false)
     687             : 
     688             : #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr)                \
     689             :   do {                                                           \
     690             :     if (instr->InputAt(4)->IsRegister()) {                       \
     691             :       Register value = i.InputRegister(4);                       \
     692             :       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register);  \
     693             :     } else {                                                     \
     694             :       Immediate value = i.InputImmediate(4);                     \
     695             :       ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
     696             :     }                                                            \
     697             :   } while (false)
     698             : 
     699             : #define ASSEMBLE_IEEE754_BINOP(name)                                          \
     700             :   do {                                                                        \
     701             :     __ PrepareCallCFunction(2);                                               \
     702             :     __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
     703             :                      2);                                                      \
     704             :   } while (false)
     705             : 
     706             : #define ASSEMBLE_IEEE754_UNOP(name)                                           \
     707             :   do {                                                                        \
     708             :     __ PrepareCallCFunction(1);                                               \
     709             :     __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
     710             :                      1);                                                      \
     711             :   } while (false)
     712             : 
     713             : #define ASSEMBLE_ATOMIC_BINOP(bin_inst, mov_inst, cmpxchg_inst) \
     714             :   do {                                                          \
     715             :     Label binop;                                                \
     716             :     __ bind(&binop);                                            \
     717             :     __ mov_inst(rax, i.MemoryOperand(1));                       \
     718             :     __ movl(i.TempRegister(0), rax);                            \
     719             :     __ bin_inst(i.TempRegister(0), i.InputRegister(0));         \
     720             :     __ lock();                                                  \
     721             :     __ cmpxchg_inst(i.MemoryOperand(1), i.TempRegister(0));     \
     722             :     __ j(not_equal, &binop);                                    \
     723             :   } while (false)
     724             : 
     725      869523 : void CodeGenerator::AssembleDeconstructFrame() {
     726     1739046 :   unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
     727      869523 :   __ movq(rsp, rbp);
     728      869523 :   __ popq(rbp);
     729      869523 : }
     730             : 
     731      298452 : void CodeGenerator::AssemblePrepareTailCall() {
     732      149226 :   if (frame_access_state()->has_frame()) {
     733       31366 :     __ movq(rbp, MemOperand(rbp, 0));
     734             :   }
     735             :   frame_access_state()->SetFrameAccessToSP();
     736      149226 : }
     737             : 
     738        1041 : void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
     739             :                                                      Register scratch1,
     740             :                                                      Register scratch2,
     741             :                                                      Register scratch3) {
     742             :   DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
     743             :   Label done;
     744             : 
     745             :   // Check if current frame is an arguments adaptor frame.
     746             :   __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset),
     747        2082 :           Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
     748        1041 :   __ j(not_equal, &done, Label::kNear);
     749             : 
     750             :   // Load arguments count from current arguments adaptor frame (note, it
     751             :   // does not include receiver).
     752        1041 :   Register caller_args_count_reg = scratch1;
     753             :   __ SmiToInteger32(
     754             :       caller_args_count_reg,
     755        1041 :       Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset));
     756             : 
     757             :   ParameterCount callee_args_count(args_reg);
     758             :   __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
     759        1041 :                         scratch3, ReturnAddressState::kOnStack);
     760        1041 :   __ bind(&done);
     761        1041 : }
     762             : 
     763             : namespace {
     764             : 
     765      340283 : void AdjustStackPointerForTailCall(Assembler* assembler,
     766             :                                    FrameAccessState* state,
     767             :                                    int new_slot_above_sp,
     768             :                                    bool allow_shrinkage = true) {
     769             :   int current_sp_offset = state->GetSPToFPSlotCount() +
     770      340283 :                           StandardFrameConstants::kFixedSlotCountAboveFp;
     771      340283 :   int stack_slot_delta = new_slot_above_sp - current_sp_offset;
     772      340283 :   if (stack_slot_delta > 0) {
     773         100 :     assembler->subq(rsp, Immediate(stack_slot_delta * kPointerSize));
     774             :     state->IncreaseSPDelta(stack_slot_delta);
     775      340233 :   } else if (allow_shrinkage && stack_slot_delta < 0) {
     776       30936 :     assembler->addq(rsp, Immediate(-stack_slot_delta * kPointerSize));
     777             :     state->IncreaseSPDelta(stack_slot_delta);
     778             :   }
     779      340283 : }
     780             : 
     781             : }  // namespace
     782             : 
     783      149226 : void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
     784      232888 :                                               int first_unused_stack_slot) {
     785             :   CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush);
     786             :   ZoneVector<MoveOperands*> pushes(zone());
     787      149226 :   GetPushCompatibleMoves(instr, flags, &pushes);
     788             : 
     789      158837 :   if (!pushes.empty() &&
     790       19222 :       (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
     791             :        first_unused_stack_slot)) {
     792             :     X64OperandConverter g(this, instr);
     793       61053 :     for (auto move : pushes) {
     794             :       LocationOperand destination_location(
     795             :           LocationOperand::cast(move->destination()));
     796       41831 :       InstructionOperand source(move->source());
     797       41831 :       AdjustStackPointerForTailCall(masm(), frame_access_state(),
     798       41831 :                                     destination_location.index());
     799       41831 :       if (source.IsStackSlot()) {
     800             :         LocationOperand source_location(LocationOperand::cast(source));
     801        9611 :         __ Push(g.SlotToOperand(source_location.index()));
     802       32220 :       } else if (source.IsRegister()) {
     803             :         LocationOperand source_location(LocationOperand::cast(source));
     804       32220 :         __ Push(source_location.GetRegister());
     805           0 :       } else if (source.IsImmediate()) {
     806           0 :         __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
     807             :       } else {
     808             :         // Pushes of non-scalar data types is not supported.
     809           0 :         UNIMPLEMENTED();
     810             :       }
     811             :       frame_access_state()->IncreaseSPDelta(1);
     812             :       move->Eliminate();
     813             :     }
     814             :   }
     815      149226 :   AdjustStackPointerForTailCall(masm(), frame_access_state(),
     816      149226 :                                 first_unused_stack_slot, false);
     817      149226 : }
     818             : 
     819      149226 : void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
     820      149226 :                                              int first_unused_stack_slot) {
     821      149226 :   AdjustStackPointerForTailCall(masm(), frame_access_state(),
     822      149226 :                                 first_unused_stack_slot);
     823      149226 : }
     824             : 
     825             : // Assembles an instruction after register allocation, producing machine code.
     826    38922581 : CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     827    51256074 :     Instruction* instr) {
     828             :   X64OperandConverter i(this, instr);
     829             :   InstructionCode opcode = instr->opcode();
     830    38922581 :   ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
     831    38922581 :   switch (arch_opcode) {
     832             :     case kArchCallCodeObject: {
     833     3228606 :       EnsureSpaceForLazyDeopt();
     834     3228605 :       if (HasImmediateInput(instr, 0)) {
     835     3225591 :         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
     836     3225591 :         __ Call(code, RelocInfo::CODE_TARGET);
     837             :       } else {
     838        6030 :         Register reg = i.InputRegister(0);
     839        3015 :         __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
     840        3015 :         __ call(reg);
     841             :       }
     842     3228606 :       RecordCallPosition(instr);
     843             :       frame_access_state()->ClearSPDelta();
     844             :       break;
     845             :     }
     846             :     case kArchTailCallCodeObjectFromJSFunction:
     847             :     case kArchTailCallCodeObject: {
     848      125313 :       if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
     849             :         AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
     850             :                                          i.TempRegister(0), i.TempRegister(1),
     851         864 :                                          i.TempRegister(2));
     852             :       }
     853      125313 :       if (HasImmediateInput(instr, 0)) {
     854      124625 :         Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
     855      124625 :         __ jmp(code, RelocInfo::CODE_TARGET);
     856             :       } else {
     857        1376 :         Register reg = i.InputRegister(0);
     858         688 :         __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
     859         688 :         __ jmp(reg);
     860             :       }
     861             :       unwinding_info_writer_.MarkBlockWillExit();
     862             :       frame_access_state()->ClearSPDelta();
     863      125313 :       frame_access_state()->SetFrameAccessToDefault();
     864      125313 :       break;
     865             :     }
     866             :     case kArchTailCallAddress: {
     867       23736 :       CHECK(!HasImmediateInput(instr, 0));
     868       23736 :       Register reg = i.InputRegister(0);
     869       23736 :       __ jmp(reg);
     870             :       unwinding_info_writer_.MarkBlockWillExit();
     871             :       frame_access_state()->ClearSPDelta();
     872       23736 :       frame_access_state()->SetFrameAccessToDefault();
     873             :       break;
     874             :     }
     875             :     case kArchCallJSFunction: {
     876      150311 :       EnsureSpaceForLazyDeopt();
     877      150311 :       Register func = i.InputRegister(0);
     878      150311 :       if (FLAG_debug_code) {
     879             :         // Check the function's context matches the context argument.
     880          16 :         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
     881          16 :         __ Assert(equal, kWrongFunctionContext);
     882             :       }
     883      150311 :       __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
     884             :       frame_access_state()->ClearSPDelta();
     885      150311 :       RecordCallPosition(instr);
     886             :       break;
     887             :     }
     888             :     case kArchTailCallJSFunctionFromJSFunction: {
     889             :       Register func = i.InputRegister(0);
     890         177 :       if (FLAG_debug_code) {
     891             :         // Check the function's context matches the context argument.
     892           0 :         __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
     893           0 :         __ Assert(equal, kWrongFunctionContext);
     894             :       }
     895             :       AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
     896             :                                        i.TempRegister(0), i.TempRegister(1),
     897         354 :                                        i.TempRegister(2));
     898         177 :       __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
     899             :       frame_access_state()->ClearSPDelta();
     900         177 :       frame_access_state()->SetFrameAccessToDefault();
     901             :       break;
     902             :     }
     903             :     case kArchPrepareCallCFunction: {
     904             :       // Frame alignment requires using FP-relative frame addressing.
     905             :       frame_access_state()->SetFrameAccessToFP();
     906             :       int const num_parameters = MiscField::decode(instr->opcode());
     907        2084 :       __ PrepareCallCFunction(num_parameters);
     908        2084 :       break;
     909             :     }
     910             :     case kArchPrepareTailCall:
     911      149226 :       AssemblePrepareTailCall();
     912      149226 :       break;
     913             :     case kArchCallCFunction: {
     914             :       int const num_parameters = MiscField::decode(instr->opcode());
     915        2084 :       if (HasImmediateInput(instr, 0)) {
     916        1624 :         ExternalReference ref = i.InputExternalReference(0);
     917        1624 :         __ CallCFunction(ref, num_parameters);
     918             :       } else {
     919         460 :         Register func = i.InputRegister(0);
     920         460 :         __ CallCFunction(func, num_parameters);
     921             :       }
     922        2084 :       frame_access_state()->SetFrameAccessToDefault();
     923             :       frame_access_state()->ClearSPDelta();
     924             :       break;
     925             :     }
     926             :     case kArchJmp:
     927     3267552 :       AssembleArchJump(i.InputRpo(0));
     928     3267553 :       break;
     929             :     case kArchLookupSwitch:
     930       16755 :       AssembleArchLookupSwitch(instr);
     931       16755 :       break;
     932             :     case kArchTableSwitch:
     933        6072 :       AssembleArchTableSwitch(instr);
     934        6072 :       break;
     935             :     case kArchComment: {
     936             :       Address comment_string = i.InputExternalReference(0).address();
     937           0 :       __ RecordComment(reinterpret_cast<const char*>(comment_string));
     938           0 :       break;
     939             :     }
     940             :     case kArchDebugBreak:
     941        9087 :       __ int3();
     942        9087 :       break;
     943             :     case kArchNop:
     944             :     case kArchThrowTerminator:
     945             :       // don't emit code for nops.
     946             :       break;
     947             :     case kArchDeoptimize: {
     948             :       int deopt_state_id =
     949       15564 :           BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
     950             :       CodeGenResult result =
     951       15564 :           AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
     952       15564 :       if (result != kSuccess) return result;
     953             :       break;
     954             :     }
     955             :     case kArchRet:
     956     1087724 :       AssembleReturn(instr->InputAt(0));
     957     1087723 :       break;
     958             :     case kArchStackPointer:
     959          43 :       __ movq(i.OutputRegister(), rsp);
     960             :       break;
     961             :     case kArchFramePointer:
     962        4760 :       __ movq(i.OutputRegister(), rbp);
     963             :       break;
     964             :     case kArchParentFramePointer:
     965       53041 :       if (frame_access_state()->has_frame()) {
     966       38847 :         __ movq(i.OutputRegister(), Operand(rbp, 0));
     967             :       } else {
     968       40092 :         __ movq(i.OutputRegister(), rbp);
     969             :       }
     970             :       break;
     971             :     case kArchTruncateDoubleToI: {
     972             :       auto result = i.OutputRegister();
     973             :       auto input = i.InputDoubleRegister(0);
     974             :       auto ool = new (zone()) OutOfLineTruncateDoubleToI(
     975       70260 :           this, result, input, &unwinding_info_writer_);
     976             :       // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The
     977             :       // use of Cvttsd2siq requires the movl below to avoid sign extension.
     978       70260 :       __ Cvttsd2siq(result, input);
     979       70260 :       __ cmpq(result, Immediate(1));
     980       70260 :       __ j(overflow, ool->entry());
     981       70260 :       __ bind(ool->exit());
     982             :       __ movl(result, result);
     983             :       break;
     984             :     }
     985             :     case kArchStoreWithWriteBarrier: {
     986             :       RecordWriteMode mode =
     987             :           static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
     988             :       Register object = i.InputRegister(0);
     989      345680 :       size_t index = 0;
     990      345680 :       Operand operand = i.MemoryOperand(&index);
     991      345680 :       Register value = i.InputRegister(index);
     992             :       Register scratch0 = i.TempRegister(0);
     993             :       Register scratch1 = i.TempRegister(1);
     994             :       auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value,
     995      691360 :                                                    scratch0, scratch1, mode);
     996      345680 :       __ movp(operand, value);
     997             :       __ CheckPageFlag(object, scratch0,
     998             :                        MemoryChunk::kPointersFromHereAreInterestingMask,
     999      345680 :                        not_zero, ool->entry());
    1000      345680 :       __ bind(ool->exit());
    1001             :       break;
    1002             :     }
    1003             :     case kArchStackSlot: {
    1004             :       FrameOffset offset =
    1005        2836 :           frame_access_state()->GetFrameOffset(i.InputInt32(0));
    1006             :       Register base;
    1007        1418 :       if (offset.from_stack_pointer()) {
    1008           0 :         base = rsp;
    1009             :       } else {
    1010        1418 :         base = rbp;
    1011             :       }
    1012        4254 :       __ leaq(i.OutputRegister(), Operand(base, offset.offset()));
    1013             :       break;
    1014             :     }
    1015             :     case kIeee754Float64Acos:
    1016          50 :       ASSEMBLE_IEEE754_UNOP(acos);
    1017          50 :       break;
    1018             :     case kIeee754Float64Acosh:
    1019          49 :       ASSEMBLE_IEEE754_UNOP(acosh);
    1020          49 :       break;
    1021             :     case kIeee754Float64Asin:
    1022          50 :       ASSEMBLE_IEEE754_UNOP(asin);
    1023          50 :       break;
    1024             :     case kIeee754Float64Asinh:
    1025          49 :       ASSEMBLE_IEEE754_UNOP(asinh);
    1026          49 :       break;
    1027             :     case kIeee754Float64Atan:
    1028          66 :       ASSEMBLE_IEEE754_UNOP(atan);
    1029          66 :       break;
    1030             :     case kIeee754Float64Atanh:
    1031          49 :       ASSEMBLE_IEEE754_UNOP(atanh);
    1032          49 :       break;
    1033             :     case kIeee754Float64Atan2:
    1034          73 :       ASSEMBLE_IEEE754_BINOP(atan2);
    1035          73 :       break;
    1036             :     case kIeee754Float64Cbrt:
    1037          49 :       ASSEMBLE_IEEE754_UNOP(cbrt);
    1038          49 :       break;
    1039             :     case kIeee754Float64Cos:
    1040         293 :       ASSEMBLE_IEEE754_UNOP(cos);
    1041         293 :       break;
    1042             :     case kIeee754Float64Cosh:
    1043          58 :       ASSEMBLE_IEEE754_UNOP(cosh);
    1044          58 :       break;
    1045             :     case kIeee754Float64Exp:
    1046          66 :       ASSEMBLE_IEEE754_UNOP(exp);
    1047          66 :       break;
    1048             :     case kIeee754Float64Expm1:
    1049          49 :       ASSEMBLE_IEEE754_UNOP(expm1);
    1050          49 :       break;
    1051             :     case kIeee754Float64Log:
    1052         168 :       ASSEMBLE_IEEE754_UNOP(log);
    1053         168 :       break;
    1054             :     case kIeee754Float64Log1p:
    1055          49 :       ASSEMBLE_IEEE754_UNOP(log1p);
    1056          49 :       break;
    1057             :     case kIeee754Float64Log2:
    1058          49 :       ASSEMBLE_IEEE754_UNOP(log2);
    1059          49 :       break;
    1060             :     case kIeee754Float64Log10:
    1061          49 :       ASSEMBLE_IEEE754_UNOP(log10);
    1062          49 :       break;
    1063             :     case kIeee754Float64Pow: {
    1064             :       // TODO(bmeurer): Improve integration of the stub.
    1065         105 :       __ Movsd(xmm2, xmm0);
    1066         105 :       MathPowStub stub(isolate(), MathPowStub::DOUBLE);
    1067         105 :       __ CallStub(&stub);
    1068         105 :       __ Movsd(xmm0, xmm3);
    1069             :       break;
    1070             :     }
    1071             :     case kIeee754Float64Sin:
    1072         293 :       ASSEMBLE_IEEE754_UNOP(sin);
    1073         293 :       break;
    1074             :     case kIeee754Float64Sinh:
    1075          58 :       ASSEMBLE_IEEE754_UNOP(sinh);
    1076          58 :       break;
    1077             :     case kIeee754Float64Tan:
    1078          66 :       ASSEMBLE_IEEE754_UNOP(tan);
    1079          66 :       break;
    1080             :     case kIeee754Float64Tanh:
    1081          58 :       ASSEMBLE_IEEE754_UNOP(tanh);
    1082          58 :       break;
    1083             :     case kX64Add32:
    1084      296210 :       ASSEMBLE_BINOP(addl);
    1085             :       break;
    1086             :     case kX64Add:
    1087      240376 :       ASSEMBLE_BINOP(addq);
    1088             :       break;
    1089             :     case kX64Sub32:
    1090      268198 :       ASSEMBLE_BINOP(subl);
    1091             :       break;
    1092             :     case kX64Sub:
    1093      196402 :       ASSEMBLE_BINOP(subq);
    1094             :       break;
    1095             :     case kX64And32:
    1096      388266 :       ASSEMBLE_BINOP(andl);
    1097             :       break;
    1098             :     case kX64And:
    1099      658037 :       ASSEMBLE_BINOP(andq);
    1100             :       break;
    1101             :     case kX64Cmp8:
    1102      444944 :       ASSEMBLE_COMPARE(cmpb);
    1103             :       break;
    1104             :     case kX64Cmp16:
    1105          87 :       ASSEMBLE_COMPARE(cmpw);
    1106             :       break;
    1107             :     case kX64Cmp32:
    1108     2744441 :       ASSEMBLE_COMPARE(cmpl);
    1109             :       break;
    1110             :     case kX64Cmp:
    1111     6517852 :       ASSEMBLE_COMPARE(cmpq);
    1112             :       break;
    1113             :     case kX64Test8:
    1114      351317 :       ASSEMBLE_COMPARE(testb);
    1115             :       break;
    1116             :     case kX64Test16:
    1117        8344 :       ASSEMBLE_COMPARE(testw);
    1118             :       break;
    1119             :     case kX64Test32:
    1120      122255 :       ASSEMBLE_COMPARE(testl);
    1121             :       break;
    1122             :     case kX64Test:
    1123     2073968 :       ASSEMBLE_COMPARE(testq);
    1124             :       break;
    1125             :     case kX64Imul32:
    1126      189986 :       ASSEMBLE_MULT(imull);
    1127             :       break;
    1128             :     case kX64Imul:
    1129       16387 :       ASSEMBLE_MULT(imulq);
    1130             :       break;
    1131             :     case kX64ImulHigh32:
    1132       10242 :       if (instr->InputAt(1)->IsRegister()) {
    1133        5121 :         __ imull(i.InputRegister(1));
    1134             :       } else {
    1135           0 :         __ imull(i.InputOperand(1));
    1136             :       }
    1137             :       break;
    1138             :     case kX64UmulHigh32:
    1139        1708 :       if (instr->InputAt(1)->IsRegister()) {
    1140         854 :         __ mull(i.InputRegister(1));
    1141             :       } else {
    1142           0 :         __ mull(i.InputOperand(1));
    1143             :       }
    1144             :       break;
    1145             :     case kX64Idiv32:
    1146        2755 :       __ cdq();
    1147        2755 :       __ idivl(i.InputRegister(1));
    1148             :       break;
    1149             :     case kX64Idiv:
    1150         894 :       __ cqo();
    1151         894 :       __ idivq(i.InputRegister(1));
    1152             :       break;
    1153             :     case kX64Udiv32:
    1154        1084 :       __ xorl(rdx, rdx);
    1155        1084 :       __ divl(i.InputRegister(1));
    1156             :       break;
    1157             :     case kX64Udiv:
    1158         824 :       __ xorq(rdx, rdx);
    1159         824 :       __ divq(i.InputRegister(1));
    1160             :       break;
    1161             :     case kX64Not:
    1162         132 :       ASSEMBLE_UNOP(notq);
    1163             :       break;
    1164             :     case kX64Not32:
    1165        7262 :       ASSEMBLE_UNOP(notl);
    1166             :       break;
    1167             :     case kX64Neg:
    1168         470 :       ASSEMBLE_UNOP(negq);
    1169             :       break;
    1170             :     case kX64Neg32:
    1171       10314 :       ASSEMBLE_UNOP(negl);
    1172             :       break;
    1173             :     case kX64Or32:
    1174       61038 :       ASSEMBLE_BINOP(orl);
    1175             :       break;
    1176             :     case kX64Or:
    1177       58234 :       ASSEMBLE_BINOP(orq);
    1178             :       break;
    1179             :     case kX64Xor32:
    1180       41210 :       ASSEMBLE_BINOP(xorl);
    1181             :       break;
    1182             :     case kX64Xor:
    1183        1662 :       ASSEMBLE_BINOP(xorq);
    1184             :       break;
    1185             :     case kX64Shl32:
    1186       35394 :       ASSEMBLE_SHIFT(shll, 5);
    1187             :       break;
    1188             :     case kX64Shl:
    1189     1101651 :       ASSEMBLE_SHIFT(shlq, 6);
    1190             :       break;
    1191             :     case kX64Shr32:
    1192      139505 :       ASSEMBLE_SHIFT(shrl, 5);
    1193             :       break;
    1194             :     case kX64Shr:
    1195     1296795 :       ASSEMBLE_SHIFT(shrq, 6);
    1196             :       break;
    1197             :     case kX64Sar32:
    1198       48719 :       ASSEMBLE_SHIFT(sarl, 5);
    1199             :       break;
    1200             :     case kX64Sar:
    1201       89735 :       ASSEMBLE_SHIFT(sarq, 6);
    1202             :       break;
    1203             :     case kX64Ror32:
    1204        3664 :       ASSEMBLE_SHIFT(rorl, 5);
    1205             :       break;
    1206             :     case kX64Ror:
    1207         238 :       ASSEMBLE_SHIFT(rorq, 6);
    1208             :       break;
    1209             :     case kX64Lzcnt:
    1210          56 :       if (instr->InputAt(0)->IsRegister()) {
    1211          28 :         __ Lzcntq(i.OutputRegister(), i.InputRegister(0));
    1212             :       } else {
    1213           0 :         __ Lzcntq(i.OutputRegister(), i.InputOperand(0));
    1214             :       }
    1215             :       break;
    1216             :     case kX64Lzcnt32:
    1217        1694 :       if (instr->InputAt(0)->IsRegister()) {
    1218         802 :         __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
    1219             :       } else {
    1220          90 :         __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
    1221             :       }
    1222             :       break;
    1223             :     case kX64Tzcnt:
    1224          26 :       if (instr->InputAt(0)->IsRegister()) {
    1225          13 :         __ Tzcntq(i.OutputRegister(), i.InputRegister(0));
    1226             :       } else {
    1227           0 :         __ Tzcntq(i.OutputRegister(), i.InputOperand(0));
    1228             :       }
    1229             :       break;
    1230             :     case kX64Tzcnt32:
    1231         936 :       if (instr->InputAt(0)->IsRegister()) {
    1232         468 :         __ Tzcntl(i.OutputRegister(), i.InputRegister(0));
    1233             :       } else {
    1234           0 :         __ Tzcntl(i.OutputRegister(), i.InputOperand(0));
    1235             :       }
    1236             :       break;
    1237             :     case kX64Popcnt:
    1238          70 :       if (instr->InputAt(0)->IsRegister()) {
    1239          35 :         __ Popcntq(i.OutputRegister(), i.InputRegister(0));
    1240             :       } else {
    1241           0 :         __ Popcntq(i.OutputRegister(), i.InputOperand(0));
    1242             :       }
    1243             :       break;
    1244             :     case kX64Popcnt32:
    1245         196 :       if (instr->InputAt(0)->IsRegister()) {
    1246          98 :         __ Popcntl(i.OutputRegister(), i.InputRegister(0));
    1247             :       } else {
    1248           0 :         __ Popcntl(i.OutputRegister(), i.InputOperand(0));
    1249             :       }
    1250             :       break;
    1251             :     case kSSEFloat32Cmp:
    1252           0 :       ASSEMBLE_SSE_BINOP(Ucomiss);
    1253             :       break;
    1254             :     case kSSEFloat32Add:
    1255           0 :       ASSEMBLE_SSE_BINOP(addss);
    1256             :       break;
    1257             :     case kSSEFloat32Sub:
    1258           0 :       ASSEMBLE_SSE_BINOP(subss);
    1259             :       break;
    1260             :     case kSSEFloat32Mul:
    1261           0 :       ASSEMBLE_SSE_BINOP(mulss);
    1262             :       break;
    1263             :     case kSSEFloat32Div:
    1264           0 :       ASSEMBLE_SSE_BINOP(divss);
    1265             :       // Don't delete this mov. It may improve performance on some CPUs,
    1266             :       // when there is a (v)mulss depending on the result.
    1267           0 :       __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
    1268           0 :       break;
    1269             :     case kSSEFloat32Abs: {
    1270             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1271           0 :       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
    1272           0 :       __ psrlq(kScratchDoubleReg, 33);
    1273           0 :       __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
    1274           0 :       break;
    1275             :     }
    1276             :     case kSSEFloat32Neg: {
    1277             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1278           0 :       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
    1279           0 :       __ psllq(kScratchDoubleReg, 31);
    1280           0 :       __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
    1281           0 :       break;
    1282             :     }
    1283             :     case kSSEFloat32Sqrt:
    1284          90 :       ASSEMBLE_SSE_UNOP(sqrtss);
    1285             :       break;
    1286             :     case kSSEFloat32ToFloat64:
    1287       33264 :       ASSEMBLE_SSE_UNOP(Cvtss2sd);
    1288             :       break;
    1289             :     case kSSEFloat32Round: {
    1290             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    1291             :       RoundingMode const mode =
    1292             :           static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
    1293         239 :       __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
    1294             :       break;
    1295             :     }
    1296             :     case kSSEFloat32ToInt32:
    1297         264 :       if (instr->InputAt(0)->IsFPRegister()) {
    1298         132 :         __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0));
    1299             :       } else {
    1300           0 :         __ Cvttss2si(i.OutputRegister(), i.InputOperand(0));
    1301             :       }
    1302             :       break;
    1303             :     case kSSEFloat32ToUint32: {
    1304          26 :       if (instr->InputAt(0)->IsFPRegister()) {
    1305          13 :         __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
    1306             :       } else {
    1307           0 :         __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
    1308             :       }
    1309             :       break;
    1310             :     }
    1311             :     case kSSEFloat64Cmp:
    1312        1932 :       ASSEMBLE_SSE_BINOP(Ucomisd);
    1313             :       break;
    1314             :     case kSSEFloat64Add:
    1315         333 :       ASSEMBLE_SSE_BINOP(addsd);
    1316             :       break;
    1317             :     case kSSEFloat64Sub:
    1318         498 :       ASSEMBLE_SSE_BINOP(subsd);
    1319             :       break;
    1320             :     case kSSEFloat64Mul:
    1321          51 :       ASSEMBLE_SSE_BINOP(mulsd);
    1322             :       break;
    1323             :     case kSSEFloat64Div:
    1324          15 :       ASSEMBLE_SSE_BINOP(divsd);
    1325             :       // Don't delete this mov. It may improve performance on some CPUs,
    1326             :       // when there is a (v)mulsd depending on the result.
    1327          10 :       __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
    1328           5 :       break;
    1329             :     case kSSEFloat64Mod: {
    1330        1014 :       __ subq(rsp, Immediate(kDoubleSize));
    1331             :       unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    1332        2028 :                                                        kDoubleSize);
    1333             :       // Move values to st(0) and st(1).
    1334        2028 :       __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
    1335        1014 :       __ fld_d(Operand(rsp, 0));
    1336        2028 :       __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
    1337        1014 :       __ fld_d(Operand(rsp, 0));
    1338             :       // Loop while fprem isn't done.
    1339             :       Label mod_loop;
    1340        1014 :       __ bind(&mod_loop);
    1341             :       // This instructions traps on all kinds inputs, but we are assuming the
    1342             :       // floating point control word is set to ignore them all.
    1343        1014 :       __ fprem();
    1344             :       // The following 2 instruction implicitly use rax.
    1345        1014 :       __ fnstsw_ax();
    1346        1014 :       if (CpuFeatures::IsSupported(SAHF)) {
    1347             :         CpuFeatureScope sahf_scope(masm(), SAHF);
    1348        1005 :         __ sahf();
    1349             :       } else {
    1350             :         __ shrl(rax, Immediate(8));
    1351           9 :         __ andl(rax, Immediate(0xFF));
    1352           9 :         __ pushq(rax);
    1353             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    1354          18 :                                                          kPointerSize);
    1355           9 :         __ popfq();
    1356             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    1357          18 :                                                          -kPointerSize);
    1358             :       }
    1359        1014 :       __ j(parity_even, &mod_loop);
    1360             :       // Move output to stack and clean up.
    1361        1014 :       __ fstp(1);
    1362        1014 :       __ fstp_d(Operand(rsp, 0));
    1363        2028 :       __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
    1364        1014 :       __ addq(rsp, Immediate(kDoubleSize));
    1365             :       unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    1366        2028 :                                                        -kDoubleSize);
    1367             :       break;
    1368             :     }
    1369             :     case kSSEFloat32Max: {
    1370             :       Label compare_nan, compare_swap, done_compare;
    1371          58 :       if (instr->InputAt(1)->IsFPRegister()) {
    1372          29 :         __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1373             :       } else {
    1374           0 :         __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
    1375             :       }
    1376             :       auto ool =
    1377          29 :           new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
    1378          29 :       __ j(parity_even, ool->entry());
    1379          29 :       __ j(above, &done_compare, Label::kNear);
    1380          29 :       __ j(below, &compare_swap, Label::kNear);
    1381          58 :       __ Movmskps(kScratchRegister, i.InputDoubleRegister(0));
    1382             :       __ testl(kScratchRegister, Immediate(1));
    1383          29 :       __ j(zero, &done_compare, Label::kNear);
    1384          29 :       __ bind(&compare_swap);
    1385          58 :       if (instr->InputAt(1)->IsFPRegister()) {
    1386          58 :         __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1387             :       } else {
    1388           0 :         __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
    1389             :       }
    1390          29 :       __ bind(&done_compare);
    1391          29 :       __ bind(ool->exit());
    1392             :       break;
    1393             :     }
    1394             :     case kSSEFloat32Min: {
    1395             :       Label compare_swap, done_compare;
    1396          58 :       if (instr->InputAt(1)->IsFPRegister()) {
    1397          29 :         __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1398             :       } else {
    1399           0 :         __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
    1400             :       }
    1401             :       auto ool =
    1402          29 :           new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
    1403          29 :       __ j(parity_even, ool->entry());
    1404          29 :       __ j(below, &done_compare, Label::kNear);
    1405          29 :       __ j(above, &compare_swap, Label::kNear);
    1406          58 :       if (instr->InputAt(1)->IsFPRegister()) {
    1407          58 :         __ Movmskps(kScratchRegister, i.InputDoubleRegister(1));
    1408             :       } else {
    1409           0 :         __ Movss(kScratchDoubleReg, i.InputOperand(1));
    1410           0 :         __ Movmskps(kScratchRegister, kScratchDoubleReg);
    1411             :       }
    1412             :       __ testl(kScratchRegister, Immediate(1));
    1413          29 :       __ j(zero, &done_compare, Label::kNear);
    1414          29 :       __ bind(&compare_swap);
    1415          58 :       if (instr->InputAt(1)->IsFPRegister()) {
    1416          58 :         __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1417             :       } else {
    1418           0 :         __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
    1419             :       }
    1420          29 :       __ bind(&done_compare);
    1421          29 :       __ bind(ool->exit());
    1422             :       break;
    1423             :     }
    1424             :     case kSSEFloat64Max: {
    1425             :       Label compare_nan, compare_swap, done_compare;
    1426         194 :       if (instr->InputAt(1)->IsFPRegister()) {
    1427          97 :         __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1428             :       } else {
    1429           0 :         __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
    1430             :       }
    1431             :       auto ool =
    1432          97 :           new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
    1433          97 :       __ j(parity_even, ool->entry());
    1434          97 :       __ j(above, &done_compare, Label::kNear);
    1435          97 :       __ j(below, &compare_swap, Label::kNear);
    1436         194 :       __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0));
    1437             :       __ testl(kScratchRegister, Immediate(1));
    1438          97 :       __ j(zero, &done_compare, Label::kNear);
    1439          97 :       __ bind(&compare_swap);
    1440         194 :       if (instr->InputAt(1)->IsFPRegister()) {
    1441         194 :         __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1442             :       } else {
    1443           0 :         __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
    1444             :       }
    1445          97 :       __ bind(&done_compare);
    1446          97 :       __ bind(ool->exit());
    1447             :       break;
    1448             :     }
    1449             :     case kSSEFloat64Min: {
    1450             :       Label compare_swap, done_compare;
    1451         306 :       if (instr->InputAt(1)->IsFPRegister()) {
    1452         153 :         __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1453             :       } else {
    1454           0 :         __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
    1455             :       }
    1456             :       auto ool =
    1457         153 :           new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
    1458         153 :       __ j(parity_even, ool->entry());
    1459         153 :       __ j(below, &done_compare, Label::kNear);
    1460         153 :       __ j(above, &compare_swap, Label::kNear);
    1461         306 :       if (instr->InputAt(1)->IsFPRegister()) {
    1462         306 :         __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1));
    1463             :       } else {
    1464           0 :         __ Movsd(kScratchDoubleReg, i.InputOperand(1));
    1465           0 :         __ Movmskpd(kScratchRegister, kScratchDoubleReg);
    1466             :       }
    1467             :       __ testl(kScratchRegister, Immediate(1));
    1468         153 :       __ j(zero, &done_compare, Label::kNear);
    1469         153 :       __ bind(&compare_swap);
    1470         306 :       if (instr->InputAt(1)->IsFPRegister()) {
    1471         306 :         __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1472             :       } else {
    1473           0 :         __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
    1474             :       }
    1475         153 :       __ bind(&done_compare);
    1476         153 :       __ bind(ool->exit());
    1477             :       break;
    1478             :     }
    1479             :     case kSSEFloat64Abs: {
    1480             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1481           3 :       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
    1482           3 :       __ psrlq(kScratchDoubleReg, 1);
    1483           6 :       __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
    1484           3 :       break;
    1485             :     }
    1486             :     case kSSEFloat64Neg: {
    1487             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1488          78 :       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
    1489          78 :       __ psllq(kScratchDoubleReg, 63);
    1490         156 :       __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
    1491          78 :       break;
    1492             :     }
    1493             :     case kSSEFloat64Sqrt:
    1494         774 :       ASSEMBLE_SSE_UNOP(Sqrtsd);
    1495             :       break;
    1496             :     case kSSEFloat64Round: {
    1497             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    1498             :       RoundingMode const mode =
    1499             :           static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
    1500       18490 :       __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
    1501             :       break;
    1502             :     }
    1503             :     case kSSEFloat64ToFloat32:
    1504       23808 :       ASSEMBLE_SSE_UNOP(Cvtsd2ss);
    1505             :       break;
    1506             :     case kSSEFloat64ToInt32:
    1507      164542 :       if (instr->InputAt(0)->IsFPRegister()) {
    1508       73935 :         __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
    1509             :       } else {
    1510       16672 :         __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0));
    1511             :       }
    1512             :       break;
    1513             :     case kSSEFloat64ToUint32: {
    1514         774 :       if (instr->InputAt(0)->IsFPRegister()) {
    1515         387 :         __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
    1516             :       } else {
    1517           0 :         __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
    1518             :       }
    1519         774 :       if (MiscField::decode(instr->opcode())) {
    1520         760 :         __ AssertZeroExtended(i.OutputRegister());
    1521             :       }
    1522             :       break;
    1523             :     }
    1524             :     case kSSEFloat32ToInt64:
    1525          52 :       if (instr->InputAt(0)->IsFPRegister()) {
    1526          26 :         __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
    1527             :       } else {
    1528           0 :         __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
    1529             :       }
    1530          26 :       if (instr->OutputCount() > 1) {
    1531          40 :         __ Set(i.OutputRegister(1), 1);
    1532             :         Label done;
    1533             :         Label fail;
    1534             :         __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN));
    1535          40 :         if (instr->InputAt(0)->IsFPRegister()) {
    1536          40 :           __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0));
    1537             :         } else {
    1538           0 :           __ Ucomiss(kScratchDoubleReg, i.InputOperand(0));
    1539             :         }
    1540             :         // If the input is NaN, then the conversion fails.
    1541          20 :         __ j(parity_even, &fail);
    1542             :         // If the input is INT64_MIN, then the conversion succeeds.
    1543          20 :         __ j(equal, &done);
    1544          40 :         __ cmpq(i.OutputRegister(0), Immediate(1));
    1545             :         // If the conversion results in INT64_MIN, but the input was not
    1546             :         // INT64_MIN, then the conversion fails.
    1547          20 :         __ j(no_overflow, &done);
    1548          20 :         __ bind(&fail);
    1549          40 :         __ Set(i.OutputRegister(1), 0);
    1550          20 :         __ bind(&done);
    1551             :       }
    1552             :       break;
    1553             :     case kSSEFloat64ToInt64:
    1554          98 :       if (instr->InputAt(0)->IsFPRegister()) {
    1555          49 :         __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0));
    1556             :       } else {
    1557           0 :         __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0));
    1558             :       }
    1559          49 :       if (instr->OutputCount() > 1) {
    1560          86 :         __ Set(i.OutputRegister(1), 1);
    1561             :         Label done;
    1562             :         Label fail;
    1563             :         __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN));
    1564          86 :         if (instr->InputAt(0)->IsFPRegister()) {
    1565          86 :           __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0));
    1566             :         } else {
    1567           0 :           __ Ucomisd(kScratchDoubleReg, i.InputOperand(0));
    1568             :         }
    1569             :         // If the input is NaN, then the conversion fails.
    1570          43 :         __ j(parity_even, &fail);
    1571             :         // If the input is INT64_MIN, then the conversion succeeds.
    1572          43 :         __ j(equal, &done);
    1573          86 :         __ cmpq(i.OutputRegister(0), Immediate(1));
    1574             :         // If the conversion results in INT64_MIN, but the input was not
    1575             :         // INT64_MIN, then the conversion fails.
    1576          43 :         __ j(no_overflow, &done);
    1577          43 :         __ bind(&fail);
    1578          86 :         __ Set(i.OutputRegister(1), 0);
    1579          43 :         __ bind(&done);
    1580             :       }
    1581             :       break;
    1582             :     case kSSEFloat32ToUint64: {
    1583             :       Label done;
    1584             :       Label success;
    1585          26 :       if (instr->OutputCount() > 1) {
    1586          20 :         __ Set(i.OutputRegister(1), 0);
    1587             :       }
    1588             :       // There does not exist a Float32ToUint64 instruction, so we have to use
    1589             :       // the Float32ToInt64 instruction.
    1590          52 :       if (instr->InputAt(0)->IsFPRegister()) {
    1591          52 :         __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
    1592             :       } else {
    1593           0 :         __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
    1594             :       }
    1595             :       // Check if the result of the Float32ToInt64 conversion is positive, we
    1596             :       // are already done.
    1597          52 :       __ testq(i.OutputRegister(), i.OutputRegister());
    1598          26 :       __ j(positive, &success);
    1599             :       // The result of the first conversion was negative, which means that the
    1600             :       // input value was not within the positive int64 range. We subtract 2^64
    1601             :       // and convert it again to see if it is within the uint64 range.
    1602          26 :       __ Move(kScratchDoubleReg, -9223372036854775808.0f);
    1603          52 :       if (instr->InputAt(0)->IsFPRegister()) {
    1604          52 :         __ addss(kScratchDoubleReg, i.InputDoubleRegister(0));
    1605             :       } else {
    1606           0 :         __ addss(kScratchDoubleReg, i.InputOperand(0));
    1607             :       }
    1608          52 :       __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg);
    1609          26 :       __ testq(i.OutputRegister(), i.OutputRegister());
    1610             :       // The only possible negative value here is 0x80000000000000000, which is
    1611             :       // used on x64 to indicate an integer overflow.
    1612          26 :       __ j(negative, &done);
    1613             :       // The input value is within uint64 range and the second conversion worked
    1614             :       // successfully, but we still have to undo the subtraction we did
    1615             :       // earlier.
    1616          26 :       __ Set(kScratchRegister, 0x8000000000000000);
    1617          52 :       __ orq(i.OutputRegister(), kScratchRegister);
    1618          26 :       __ bind(&success);
    1619          26 :       if (instr->OutputCount() > 1) {
    1620          40 :         __ Set(i.OutputRegister(1), 1);
    1621             :       }
    1622          26 :       __ bind(&done);
    1623             :       break;
    1624             :     }
    1625             :     case kSSEFloat64ToUint64: {
    1626             :       Label done;
    1627             :       Label success;
    1628         155 :       if (instr->OutputCount() > 1) {
    1629          20 :         __ Set(i.OutputRegister(1), 0);
    1630             :       }
    1631             :       // There does not exist a Float64ToUint64 instruction, so we have to use
    1632             :       // the Float64ToInt64 instruction.
    1633         310 :       if (instr->InputAt(0)->IsFPRegister()) {
    1634         310 :         __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
    1635             :       } else {
    1636           0 :         __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
    1637             :       }
    1638             :       // Check if the result of the Float64ToInt64 conversion is positive, we
    1639             :       // are already done.
    1640         310 :       __ testq(i.OutputRegister(), i.OutputRegister());
    1641         155 :       __ j(positive, &success);
    1642             :       // The result of the first conversion was negative, which means that the
    1643             :       // input value was not within the positive int64 range. We subtract 2^64
    1644             :       // and convert it again to see if it is within the uint64 range.
    1645         155 :       __ Move(kScratchDoubleReg, -9223372036854775808.0);
    1646         310 :       if (instr->InputAt(0)->IsFPRegister()) {
    1647         310 :         __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0));
    1648             :       } else {
    1649           0 :         __ addsd(kScratchDoubleReg, i.InputOperand(0));
    1650             :       }
    1651         310 :       __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg);
    1652         155 :       __ testq(i.OutputRegister(), i.OutputRegister());
    1653             :       // The only possible negative value here is 0x80000000000000000, which is
    1654             :       // used on x64 to indicate an integer overflow.
    1655         155 :       __ j(negative, &done);
    1656             :       // The input value is within uint64 range and the second conversion worked
    1657             :       // successfully, but we still have to undo the subtraction we did
    1658             :       // earlier.
    1659         155 :       __ Set(kScratchRegister, 0x8000000000000000);
    1660         310 :       __ orq(i.OutputRegister(), kScratchRegister);
    1661         155 :       __ bind(&success);
    1662         155 :       if (instr->OutputCount() > 1) {
    1663          40 :         __ Set(i.OutputRegister(1), 1);
    1664             :       }
    1665         155 :       __ bind(&done);
    1666             :       break;
    1667             :     }
    1668             :     case kSSEInt32ToFloat64:
    1669      498344 :       if (instr->InputAt(0)->IsRegister()) {
    1670      245122 :         __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
    1671             :       } else {
    1672        8100 :         __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
    1673             :       }
    1674             :       break;
    1675             :     case kSSEInt32ToFloat32:
    1676         618 :       if (instr->InputAt(0)->IsRegister()) {
    1677         309 :         __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
    1678             :       } else {
    1679           0 :         __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
    1680             :       }
    1681             :       break;
    1682             :     case kSSEInt64ToFloat32:
    1683          26 :       if (instr->InputAt(0)->IsRegister()) {
    1684          13 :         __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
    1685             :       } else {
    1686           0 :         __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
    1687             :       }
    1688             :       break;
    1689             :     case kSSEInt64ToFloat64:
    1690        1518 :       if (instr->InputAt(0)->IsRegister()) {
    1691         759 :         __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
    1692             :       } else {
    1693           0 :         __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
    1694             :       }
    1695             :       break;
    1696             :     case kSSEUint64ToFloat32:
    1697          26 :       if (instr->InputAt(0)->IsRegister()) {
    1698          13 :         __ movq(kScratchRegister, i.InputRegister(0));
    1699             :       } else {
    1700           0 :         __ movq(kScratchRegister, i.InputOperand(0));
    1701             :       }
    1702             :       __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister,
    1703          26 :                    i.TempRegister(0));
    1704          13 :       break;
    1705             :     case kSSEUint64ToFloat64:
    1706          26 :       if (instr->InputAt(0)->IsRegister()) {
    1707          13 :         __ movq(kScratchRegister, i.InputRegister(0));
    1708             :       } else {
    1709           0 :         __ movq(kScratchRegister, i.InputOperand(0));
    1710             :       }
    1711             :       __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister,
    1712          26 :                    i.TempRegister(0));
    1713          13 :       break;
    1714             :     case kSSEUint32ToFloat64:
    1715        4002 :       if (instr->InputAt(0)->IsRegister()) {
    1716        1368 :         __ movl(kScratchRegister, i.InputRegister(0));
    1717             :       } else {
    1718         633 :         __ movl(kScratchRegister, i.InputOperand(0));
    1719             :       }
    1720        4002 :       __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
    1721        2001 :       break;
    1722             :     case kSSEUint32ToFloat32:
    1723          74 :       if (instr->InputAt(0)->IsRegister()) {
    1724          37 :         __ movl(kScratchRegister, i.InputRegister(0));
    1725             :       } else {
    1726           0 :         __ movl(kScratchRegister, i.InputOperand(0));
    1727             :       }
    1728          74 :       __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister);
    1729          37 :       break;
    1730             :     case kSSEFloat64ExtractLowWord32:
    1731          12 :       if (instr->InputAt(0)->IsFPStackSlot()) {
    1732           0 :         __ movl(i.OutputRegister(), i.InputOperand(0));
    1733             :       } else {
    1734           6 :         __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
    1735             :       }
    1736             :       break;
    1737             :     case kSSEFloat64ExtractHighWord32:
    1738       87714 :       if (instr->InputAt(0)->IsFPStackSlot()) {
    1739       63786 :         __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
    1740             :       } else {
    1741       11964 :         __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
    1742             :       }
    1743             :       break;
    1744             :     case kSSEFloat64InsertLowWord32:
    1745          12 :       if (instr->InputAt(1)->IsRegister()) {
    1746           6 :         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
    1747             :       } else {
    1748           0 :         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
    1749             :       }
    1750             :       break;
    1751             :     case kSSEFloat64InsertHighWord32:
    1752          12 :       if (instr->InputAt(1)->IsRegister()) {
    1753           6 :         __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
    1754             :       } else {
    1755           0 :         __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
    1756             :       }
    1757             :       break;
    1758             :     case kSSEFloat64LoadLowWord32:
    1759           0 :       if (instr->InputAt(0)->IsRegister()) {
    1760           0 :         __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
    1761             :       } else {
    1762           0 :         __ Movd(i.OutputDoubleRegister(), i.InputOperand(0));
    1763             :       }
    1764             :       break;
    1765             :     case kAVXFloat32Cmp: {
    1766             :       CpuFeatureScope avx_scope(masm(), AVX);
    1767        2380 :       if (instr->InputAt(1)->IsFPRegister()) {
    1768        1172 :         __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1769             :       } else {
    1770          36 :         __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
    1771             :       }
    1772             :       break;
    1773             :     }
    1774             :     case kAVXFloat32Add:
    1775         354 :       ASSEMBLE_AVX_BINOP(vaddss);
    1776             :       break;
    1777             :     case kAVXFloat32Sub:
    1778        4368 :       ASSEMBLE_AVX_BINOP(vsubss);
    1779             :       break;
    1780             :     case kAVXFloat32Mul:
    1781         141 :       ASSEMBLE_AVX_BINOP(vmulss);
    1782             :       break;
    1783             :     case kAVXFloat32Div:
    1784         132 :       ASSEMBLE_AVX_BINOP(vdivss);
    1785             :       // Don't delete this mov. It may improve performance on some CPUs,
    1786             :       // when there is a (v)mulss depending on the result.
    1787          88 :       __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
    1788          44 :       break;
    1789             :     case kAVXFloat64Cmp: {
    1790             :       CpuFeatureScope avx_scope(masm(), AVX);
    1791      401940 :       if (instr->InputAt(1)->IsFPRegister()) {
    1792      191629 :         __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
    1793             :       } else {
    1794       28023 :         __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
    1795             :       }
    1796             :       break;
    1797             :     }
    1798             :     case kAVXFloat64Add:
    1799      163488 :       ASSEMBLE_AVX_BINOP(vaddsd);
    1800             :       break;
    1801             :     case kAVXFloat64Sub:
    1802       75417 :       ASSEMBLE_AVX_BINOP(vsubsd);
    1803             :       break;
    1804             :     case kAVXFloat64Mul:
    1805       46617 :       ASSEMBLE_AVX_BINOP(vmulsd);
    1806             :       break;
    1807             :     case kAVXFloat64Div:
    1808       36705 :       ASSEMBLE_AVX_BINOP(vdivsd);
    1809             :       // Don't delete this mov. It may improve performance on some CPUs,
    1810             :       // when there is a (v)mulsd depending on the result.
    1811       24470 :       __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
    1812       12235 :       break;
    1813             :     case kAVXFloat32Abs: {
    1814             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1815             :       CpuFeatureScope avx_scope(masm(), AVX);
    1816          50 :       __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
    1817             :       __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33);
    1818         100 :       if (instr->InputAt(0)->IsFPRegister()) {
    1819             :         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
    1820          50 :                   i.InputDoubleRegister(0));
    1821             :       } else {
    1822             :         __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
    1823           0 :                   i.InputOperand(0));
    1824             :       }
    1825             :       break;
    1826             :     }
    1827             :     case kAVXFloat32Neg: {
    1828             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1829             :       CpuFeatureScope avx_scope(masm(), AVX);
    1830          28 :       __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
    1831             :       __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31);
    1832          56 :       if (instr->InputAt(0)->IsFPRegister()) {
    1833             :         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
    1834          28 :                   i.InputDoubleRegister(0));
    1835             :       } else {
    1836             :         __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
    1837           0 :                   i.InputOperand(0));
    1838             :       }
    1839             :       break;
    1840             :     }
    1841             :     case kAVXFloat64Abs: {
    1842             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1843             :       CpuFeatureScope avx_scope(masm(), AVX);
    1844         240 :       __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
    1845             :       __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1);
    1846         480 :       if (instr->InputAt(0)->IsFPRegister()) {
    1847             :         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
    1848         240 :                   i.InputDoubleRegister(0));
    1849             :       } else {
    1850             :         __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
    1851           0 :                   i.InputOperand(0));
    1852             :       }
    1853             :       break;
    1854             :     }
    1855             :     case kAVXFloat64Neg: {
    1856             :       // TODO(bmeurer): Use RIP relative 128-bit constants.
    1857             :       CpuFeatureScope avx_scope(masm(), AVX);
    1858        9734 :       __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
    1859             :       __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63);
    1860       19468 :       if (instr->InputAt(0)->IsFPRegister()) {
    1861             :         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
    1862        9657 :                   i.InputDoubleRegister(0));
    1863             :       } else {
    1864             :         __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
    1865         154 :                   i.InputOperand(0));
    1866             :       }
    1867             :       break;
    1868             :     }
    1869             :     case kSSEFloat64SilenceNaN:
    1870        1757 :       __ Xorpd(kScratchDoubleReg, kScratchDoubleReg);
    1871        3514 :       __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg);
    1872        1757 :       break;
    1873             :     case kX64Movsxbl:
    1874             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1875       14672 :                           __ pc_offset());
    1876       22438 :       ASSEMBLE_MOVX(movsxbl);
    1877       14672 :       __ AssertZeroExtended(i.OutputRegister());
    1878        7336 :       break;
    1879             :     case kX64Movzxbl:
    1880             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1881      183234 :                           __ pc_offset());
    1882      276393 :       ASSEMBLE_MOVX(movzxbl);
    1883      183234 :       __ AssertZeroExtended(i.OutputRegister());
    1884       91617 :       break;
    1885             :     case kX64Movsxbq:
    1886             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1887       10860 :                           __ pc_offset());
    1888       16290 :       ASSEMBLE_MOVX(movsxbq);
    1889             :       break;
    1890             :     case kX64Movzxbq:
    1891             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1892         172 :                           __ pc_offset());
    1893         258 :       ASSEMBLE_MOVX(movzxbq);
    1894         172 :       __ AssertZeroExtended(i.OutputRegister());
    1895          86 :       break;
    1896             :     case kX64Movb: {
    1897             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1898       15330 :                           __ pc_offset());
    1899        7665 :       size_t index = 0;
    1900        7665 :       Operand operand = i.MemoryOperand(&index);
    1901       15330 :       if (HasImmediateInput(instr, index)) {
    1902        2770 :         __ movb(operand, Immediate(i.InputInt8(index)));
    1903             :       } else {
    1904       12560 :         __ movb(operand, i.InputRegister(index));
    1905             :       }
    1906             :       break;
    1907             :     }
    1908             :     case kX64Movsxwl:
    1909             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1910        4086 :                           __ pc_offset());
    1911        6427 :       ASSEMBLE_MOVX(movsxwl);
    1912        4086 :       __ AssertZeroExtended(i.OutputRegister());
    1913        2043 :       break;
    1914             :     case kX64Movzxwl:
    1915             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1916       31364 :                           __ pc_offset());
    1917       51606 :       ASSEMBLE_MOVX(movzxwl);
    1918       31364 :       __ AssertZeroExtended(i.OutputRegister());
    1919       15682 :       break;
    1920             :     case kX64Movsxwq:
    1921             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1922        8194 :                           __ pc_offset());
    1923       12291 :       ASSEMBLE_MOVX(movsxwq);
    1924             :       break;
    1925             :     case kX64Movzxwq:
    1926             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1927           0 :                           __ pc_offset());
    1928           0 :       ASSEMBLE_MOVX(movzxwq);
    1929           0 :       __ AssertZeroExtended(i.OutputRegister());
    1930           0 :       break;
    1931             :     case kX64Movw: {
    1932             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1933       10858 :                           __ pc_offset());
    1934        5429 :       size_t index = 0;
    1935        5429 :       Operand operand = i.MemoryOperand(&index);
    1936       10858 :       if (HasImmediateInput(instr, index)) {
    1937         478 :         __ movw(operand, Immediate(i.InputInt16(index)));
    1938             :       } else {
    1939       10380 :         __ movw(operand, i.InputRegister(index));
    1940             :       }
    1941             :       break;
    1942             :     }
    1943             :     case kX64Movl:
    1944             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1945     1040092 :                           __ pc_offset());
    1946      520046 :       if (instr->HasOutput()) {
    1947      318410 :         if (instr->addressing_mode() == kMode_None) {
    1948       76424 :           if (instr->InputAt(0)->IsRegister()) {
    1949       74310 :             __ movl(i.OutputRegister(), i.InputRegister(0));
    1950             :           } else {
    1951        2114 :             __ movl(i.OutputRegister(), i.InputOperand(0));
    1952             :           }
    1953             :         } else {
    1954      560396 :           __ movl(i.OutputRegister(), i.MemoryOperand());
    1955             :         }
    1956      636820 :         __ AssertZeroExtended(i.OutputRegister());
    1957             :       } else {
    1958      201636 :         size_t index = 0;
    1959      201636 :         Operand operand = i.MemoryOperand(&index);
    1960      403272 :         if (HasImmediateInput(instr, index)) {
    1961       43076 :           __ movl(operand, i.InputImmediate(index));
    1962             :         } else {
    1963      317120 :           __ movl(operand, i.InputRegister(index));
    1964             :         }
    1965             :       }
    1966             :       break;
    1967             :     case kX64Movsxlq:
    1968             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1969      162884 :                           __ pc_offset());
    1970      288199 :       ASSEMBLE_MOVX(movsxlq);
    1971             :       break;
    1972             :     case kX64Movq:
    1973             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1974     7571340 :                           __ pc_offset());
    1975     3785670 :       if (instr->HasOutput()) {
    1976     4244484 :         __ movq(i.OutputRegister(), i.MemoryOperand());
    1977             :       } else {
    1978     1663429 :         size_t index = 0;
    1979     1663429 :         Operand operand = i.MemoryOperand(&index);
    1980     3326858 :         if (HasImmediateInput(instr, index)) {
    1981      144988 :           __ movq(operand, i.InputImmediate(index));
    1982             :         } else {
    1983     3036882 :           __ movq(operand, i.InputRegister(index));
    1984             :         }
    1985             :       }
    1986             :       break;
    1987             :     case kX64Movss:
    1988             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    1989       36760 :                           __ pc_offset());
    1990       18380 :       if (instr->HasOutput()) {
    1991       21014 :         __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
    1992             :       } else {
    1993        7873 :         size_t index = 0;
    1994        7873 :         Operand operand = i.MemoryOperand(&index);
    1995       15746 :         __ movss(operand, i.InputDoubleRegister(index));
    1996             :       }
    1997             :       break;
    1998             :     case kX64Movsd:
    1999             :       EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
    2000      755828 :                           __ pc_offset());
    2001      377914 :       if (instr->HasOutput()) {
    2002      575356 :         __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand());
    2003             :       } else {
    2004       90236 :         size_t index = 0;
    2005       90236 :         Operand operand = i.MemoryOperand(&index);
    2006      180472 :         __ Movsd(operand, i.InputDoubleRegister(index));
    2007             :       }
    2008             :       break;
    2009             :     case kX64BitcastFI:
    2010         228 :       if (instr->InputAt(0)->IsFPStackSlot()) {
    2011           0 :         __ movl(i.OutputRegister(), i.InputOperand(0));
    2012             :       } else {
    2013         114 :         __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
    2014             :       }
    2015             :       break;
    2016             :     case kX64BitcastDL:
    2017         514 :       if (instr->InputAt(0)->IsFPStackSlot()) {
    2018           0 :         __ movq(i.OutputRegister(), i.InputOperand(0));
    2019             :       } else {
    2020         257 :         __ Movq(i.OutputRegister(), i.InputDoubleRegister(0));
    2021             :       }
    2022             :       break;
    2023             :     case kX64BitcastIF:
    2024         104 :       if (instr->InputAt(0)->IsRegister()) {
    2025          52 :         __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
    2026             :       } else {
    2027           0 :         __ movss(i.OutputDoubleRegister(), i.InputOperand(0));
    2028             :       }
    2029             :       break;
    2030             :     case kX64BitcastLD:
    2031         262 :       if (instr->InputAt(0)->IsRegister()) {
    2032         131 :         __ Movq(i.OutputDoubleRegister(), i.InputRegister(0));
    2033             :       } else {
    2034           0 :         __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0));
    2035             :       }
    2036             :       break;
    2037             :     case kX64Lea32: {
    2038             :       AddressingMode mode = AddressingModeField::decode(instr->opcode());
    2039             :       // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
    2040             :       // and addressing mode just happens to work out. The "addl"/"subl" forms
    2041             :       // in these cases are faster based on measurements.
    2042      208214 :       if (i.InputRegister(0).is(i.OutputRegister())) {
    2043       90290 :         if (mode == kMode_MRI) {
    2044       67971 :           int32_t constant_summand = i.InputInt32(1);
    2045       67971 :           if (constant_summand > 0) {
    2046       82768 :             __ addl(i.OutputRegister(), Immediate(constant_summand));
    2047       26587 :           } else if (constant_summand < 0) {
    2048       79635 :             __ subl(i.OutputRegister(), Immediate(-constant_summand));
    2049             :           }
    2050       22319 :         } else if (mode == kMode_MR1) {
    2051       10971 :           if (i.InputRegister(1).is(i.OutputRegister())) {
    2052         521 :             __ shll(i.OutputRegister(), Immediate(1));
    2053             :           } else {
    2054       10450 :             __ addl(i.OutputRegister(), i.InputRegister(1));
    2055             :           }
    2056       11348 :         } else if (mode == kMode_M2) {
    2057           0 :           __ shll(i.OutputRegister(), Immediate(1));
    2058       11348 :         } else if (mode == kMode_M4) {
    2059         247 :           __ shll(i.OutputRegister(), Immediate(2));
    2060       11101 :         } else if (mode == kMode_M8) {
    2061         177 :           __ shll(i.OutputRegister(), Immediate(3));
    2062             :         } else {
    2063       21848 :           __ leal(i.OutputRegister(), i.MemoryOperand());
    2064             :         }
    2065      139365 :       } else if (mode == kMode_MR1 &&
    2066             :                  i.InputRegister(1).is(i.OutputRegister())) {
    2067       14373 :         __ addl(i.OutputRegister(), i.InputRegister(0));
    2068             :       } else {
    2069      207102 :         __ leal(i.OutputRegister(), i.MemoryOperand());
    2070             :       }
    2071      416428 :       __ AssertZeroExtended(i.OutputRegister());
    2072      208214 :       break;
    2073             :     }
    2074             :     case kX64Lea: {
    2075             :       AddressingMode mode = AddressingModeField::decode(instr->opcode());
    2076             :       // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation
    2077             :       // and addressing mode just happens to work out. The "addq"/"subq" forms
    2078             :       // in these cases are faster based on measurements.
    2079     1010125 :       if (i.InputRegister(0).is(i.OutputRegister())) {
    2080      464070 :         if (mode == kMode_MRI) {
    2081      433909 :           int32_t constant_summand = i.InputInt32(1);
    2082      433909 :           if (constant_summand > 0) {
    2083      630516 :             __ addq(i.OutputRegister(), Immediate(constant_summand));
    2084      118651 :           } else if (constant_summand < 0) {
    2085      355863 :             __ subq(i.OutputRegister(), Immediate(-constant_summand));
    2086             :           }
    2087       30161 :         } else if (mode == kMode_MR1) {
    2088       15194 :           if (i.InputRegister(1).is(i.OutputRegister())) {
    2089        1591 :             __ shlq(i.OutputRegister(), Immediate(1));
    2090             :           } else {
    2091       13603 :             __ addq(i.OutputRegister(), i.InputRegister(1));
    2092             :           }
    2093       14967 :         } else if (mode == kMode_M2) {
    2094           0 :           __ shlq(i.OutputRegister(), Immediate(1));
    2095       14967 :         } else if (mode == kMode_M4) {
    2096         129 :           __ shlq(i.OutputRegister(), Immediate(2));
    2097       14838 :         } else if (mode == kMode_M8) {
    2098        1118 :           __ shlq(i.OutputRegister(), Immediate(3));
    2099             :         } else {
    2100       27440 :           __ leaq(i.OutputRegister(), i.MemoryOperand());
    2101             :         }
    2102      583867 :       } else if (mode == kMode_MR1 &&
    2103             :                  i.InputRegister(1).is(i.OutputRegister())) {
    2104       13119 :         __ addq(i.OutputRegister(), i.InputRegister(0));
    2105             :       } else {
    2106     1065872 :         __ leaq(i.OutputRegister(), i.MemoryOperand());
    2107             :       }
    2108             :       break;
    2109             :     }
    2110             :     case kX64Dec32:
    2111           0 :       __ decl(i.OutputRegister());
    2112             :       break;
    2113             :     case kX64Inc32:
    2114           0 :       __ incl(i.OutputRegister());
    2115             :       break;
    2116             :     case kX64Push:
    2117     3671754 :       if (AddressingModeField::decode(instr->opcode()) != kMode_None) {
    2118      111410 :         size_t index = 0;
    2119      111410 :         Operand operand = i.MemoryOperand(&index);
    2120      111410 :         __ pushq(operand);
    2121             :         frame_access_state()->IncreaseSPDelta(1);
    2122             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    2123      222820 :                                                          kPointerSize);
    2124     3560344 :       } else if (HasImmediateInput(instr, 0)) {
    2125      238406 :         __ pushq(i.InputImmediate(0));
    2126             :         frame_access_state()->IncreaseSPDelta(1);
    2127             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    2128      476812 :                                                          kPointerSize);
    2129     3321938 :       } else if (instr->InputAt(0)->IsRegister()) {
    2130     2913864 :         __ pushq(i.InputRegister(0));
    2131             :         frame_access_state()->IncreaseSPDelta(1);
    2132             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    2133     5827728 :                                                          kPointerSize);
    2134      408074 :       } else if (instr->InputAt(0)->IsFPRegister()) {
    2135             :         // TODO(titzer): use another machine instruction?
    2136       12622 :         __ subq(rsp, Immediate(kDoubleSize));
    2137             :         frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
    2138             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    2139       25244 :                                                          kDoubleSize);
    2140       25244 :         __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
    2141             :       } else {
    2142      395452 :         __ pushq(i.InputOperand(0));
    2143             :         frame_access_state()->IncreaseSPDelta(1);
    2144             :         unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    2145      790904 :                                                          kPointerSize);
    2146             :       }
    2147             :       break;
    2148             :     case kX64Poke: {
    2149             :       int const slot = MiscField::decode(instr->opcode());
    2150        1348 :       if (HasImmediateInput(instr, 0)) {
    2151         902 :         __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
    2152             :       } else {
    2153        1794 :         __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
    2154             :       }
    2155             :       break;
    2156             :     }
    2157             :     case kX64I32x4Splat: {
    2158         210 :       XMMRegister dst = i.OutputSimd128Register();
    2159         210 :       __ movd(dst, i.InputRegister(0));
    2160         210 :       __ pshufd(dst, dst, 0x0);
    2161             :       break;
    2162             :     }
    2163             :     case kX64I32x4ExtractLane: {
    2164             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2165        1008 :       __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1));
    2166             :       break;
    2167             :     }
    2168             :     case kX64I32x4ReplaceLane: {
    2169             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2170          84 :       if (instr->InputAt(2)->IsRegister()) {
    2171             :         __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2),
    2172          84 :                   i.InputInt8(1));
    2173             :       } else {
    2174           0 :         __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
    2175             :       }
    2176             :       break;
    2177             :     }
    2178             :     case kX64I32x4Shl: {
    2179          14 :       __ pslld(i.OutputSimd128Register(), i.InputInt8(1));
    2180           7 :       break;
    2181             :     }
    2182             :     case kX64I32x4ShrS: {
    2183          14 :       __ psrad(i.OutputSimd128Register(), i.InputInt8(1));
    2184           7 :       break;
    2185             :     }
    2186             :     case kX64I32x4Add: {
    2187           7 :       __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2188             :       break;
    2189             :     }
    2190             :     case kX64I32x4Sub: {
    2191           7 :       __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2192             :       break;
    2193             :     }
    2194             :     case kX64I32x4Mul: {
    2195             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2196           7 :       __ pmulld(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2197             :       break;
    2198             :     }
    2199             :     case kX64I32x4MinS: {
    2200             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2201           7 :       __ pminsd(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2202             :       break;
    2203             :     }
    2204             :     case kX64I32x4MaxS: {
    2205             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2206           7 :       __ pmaxsd(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2207             :       break;
    2208             :     }
    2209             :     case kX64I32x4Eq: {
    2210           7 :       __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2211             :       break;
    2212             :     }
    2213             :     case kX64I32x4Ne: {
    2214          14 :       __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2215             :       __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
    2216          14 :       __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
    2217             :       break;
    2218             :     }
    2219             :     case kX64I32x4ShrU: {
    2220          14 :       __ psrld(i.OutputSimd128Register(), i.InputInt8(1));
    2221           7 :       break;
    2222             :     }
    2223             :     case kX64I32x4MinU: {
    2224             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2225           7 :       __ pminud(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2226             :       break;
    2227             :     }
    2228             :     case kX64I32x4MaxU: {
    2229             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2230           7 :       __ pmaxud(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2231             :       break;
    2232             :     }
    2233             :     case kX64S128Zero: {
    2234           0 :       XMMRegister dst = i.OutputSimd128Register();
    2235           0 :       __ xorps(dst, dst);
    2236             :       break;
    2237             :     }
    2238             :     case kX64I16x8Splat: {
    2239         266 :       XMMRegister dst = i.OutputSimd128Register();
    2240         266 :       __ movd(dst, i.InputRegister(0));
    2241         266 :       __ pshuflw(dst, dst, 0x0);
    2242         266 :       __ pshufhw(dst, dst, 0x0);
    2243         266 :       __ pshufd(dst, dst, 0x0);
    2244             :       break;
    2245             :     }
    2246             :     case kX64I16x8ExtractLane: {
    2247             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2248        1428 :       Register dst = i.OutputRegister();
    2249        2856 :       __ pextrw(dst, i.InputSimd128Register(0), i.InputInt8(1));
    2250        1428 :       __ movsxwl(dst, dst);
    2251             :       break;
    2252             :     }
    2253             :     case kX64I16x8ReplaceLane: {
    2254             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2255         140 :       if (instr->InputAt(2)->IsRegister()) {
    2256             :         __ pinsrw(i.OutputSimd128Register(), i.InputRegister(2),
    2257         140 :                   i.InputInt8(1));
    2258             :       } else {
    2259           0 :         __ pinsrw(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
    2260             :       }
    2261             :       break;
    2262             :     }
    2263             :     case kX64I16x8Shl: {
    2264          14 :       __ psllw(i.OutputSimd128Register(), i.InputInt8(1));
    2265           7 :       break;
    2266             :     }
    2267             :     case kX64I16x8ShrS: {
    2268          14 :       __ psraw(i.OutputSimd128Register(), i.InputInt8(1));
    2269           7 :       break;
    2270             :     }
    2271             :     case kX64I16x8Add: {
    2272           7 :       __ paddw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2273             :       break;
    2274             :     }
    2275             :     case kX64I16x8AddSaturateS: {
    2276           7 :       __ paddsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2277             :       break;
    2278             :     }
    2279             :     case kX64I16x8Sub: {
    2280           7 :       __ psubw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2281             :       break;
    2282             :     }
    2283             :     case kX64I16x8SubSaturateS: {
    2284           7 :       __ psubsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2285             :       break;
    2286             :     }
    2287             :     case kX64I16x8Mul: {
    2288             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2289           7 :       __ pmullw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2290             :       break;
    2291             :     }
    2292             :     case kX64I16x8MinS: {
    2293             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2294           7 :       __ pminsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2295             :       break;
    2296             :     }
    2297             :     case kX64I16x8MaxS: {
    2298             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2299           7 :       __ pmaxsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2300             :       break;
    2301             :     }
    2302             :     case kX64I16x8Eq: {
    2303           7 :       __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2304             :       break;
    2305             :     }
    2306             :     case kX64I16x8Ne: {
    2307          14 :       __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2308             :       __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg);
    2309          14 :       __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
    2310             :       break;
    2311             :     }
    2312             :     case kX64I16x8ShrU: {
    2313          14 :       __ psrlw(i.OutputSimd128Register(), i.InputInt8(1));
    2314           7 :       break;
    2315             :     }
    2316             :     case kX64I16x8AddSaturateU: {
    2317           7 :       __ paddusw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2318             :       break;
    2319             :     }
    2320             :     case kX64I16x8SubSaturateU: {
    2321           7 :       __ psubusw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2322             :       break;
    2323             :     }
    2324             :     case kX64I16x8MinU: {
    2325             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2326           7 :       __ pminuw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2327             :       break;
    2328             :     }
    2329             :     case kX64I16x8MaxU: {
    2330             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2331           7 :       __ pmaxuw(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2332             :       break;
    2333             :     }
    2334             :     case kX64I8x16Splat: {
    2335             :       CpuFeatureScope sse_scope(masm(), SSSE3);
    2336             :       XMMRegister dst = i.OutputSimd128Register();
    2337         231 :       __ movd(dst, i.InputRegister(0));
    2338         231 :       __ xorps(kScratchDoubleReg, kScratchDoubleReg);
    2339             :       __ pshufb(dst, kScratchDoubleReg);
    2340             :       break;
    2341             :     }
    2342             :     case kX64I8x16ExtractLane: {
    2343             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2344        3220 :       Register dst = i.OutputRegister();
    2345        6440 :       __ pextrb(dst, i.InputSimd128Register(0), i.InputInt8(1));
    2346        3220 :       __ movsxbl(dst, dst);
    2347             :       break;
    2348             :     }
    2349             :     case kX64I8x16ReplaceLane: {
    2350             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2351         252 :       if (instr->InputAt(2)->IsRegister()) {
    2352             :         __ pinsrb(i.OutputSimd128Register(), i.InputRegister(2),
    2353         252 :                   i.InputInt8(1));
    2354             :       } else {
    2355           0 :         __ pinsrb(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
    2356             :       }
    2357             :       break;
    2358             :     }
    2359             :     case kX64I8x16Add: {
    2360           7 :       __ paddb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2361             :       break;
    2362             :     }
    2363             :     case kX64I8x16AddSaturateS: {
    2364           7 :       __ paddsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2365             :       break;
    2366             :     }
    2367             :     case kX64I8x16Sub: {
    2368           7 :       __ psubb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2369             :       break;
    2370             :     }
    2371             :     case kX64I8x16SubSaturateS: {
    2372           7 :       __ psubsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2373             :       break;
    2374             :     }
    2375             :     case kX64I8x16MinS: {
    2376             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2377           7 :       __ pminsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2378             :       break;
    2379             :     }
    2380             :     case kX64I8x16MaxS: {
    2381             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2382           7 :       __ pmaxsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2383             :       break;
    2384             :     }
    2385             :     case kX64I8x16Eq: {
    2386           7 :       __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2387             :       break;
    2388             :     }
    2389             :     case kX64I8x16Ne: {
    2390          14 :       __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2391             :       __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg);
    2392          14 :       __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
    2393             :       break;
    2394             :     }
    2395             :     case kX64I8x16AddSaturateU: {
    2396           7 :       __ paddusb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2397             :       break;
    2398             :     }
    2399             :     case kX64I8x16SubSaturateU: {
    2400           7 :       __ psubusb(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2401             :       break;
    2402             :     }
    2403             :     case kX64I8x16MinU: {
    2404             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2405           7 :       __ pminub(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2406             :       break;
    2407             :     }
    2408             :     case kX64I8x16MaxU: {
    2409             :       CpuFeatureScope sse_scope(masm(), SSE4_1);
    2410           7 :       __ pmaxub(i.OutputSimd128Register(), i.InputSimd128Register(1));
    2411             :       break;
    2412             :     }
    2413             :     case kX64S128Select: {
    2414             :       // Mask used here is stored in dst.
    2415          63 :       XMMRegister dst = i.OutputSimd128Register();
    2416          63 :       __ movaps(kScratchDoubleReg, i.InputSimd128Register(1));
    2417         126 :       __ xorps(kScratchDoubleReg, i.InputSimd128Register(2));
    2418          63 :       __ andps(dst, kScratchDoubleReg);
    2419         126 :       __ xorps(dst, i.InputSimd128Register(2));
    2420             :       break;
    2421             :     }
    2422             :     case kCheckedLoadInt8:
    2423       36700 :       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
    2424        3670 :       break;
    2425             :     case kCheckedLoadUint8:
    2426       35100 :       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
    2427        2925 :       break;
    2428             :     case kCheckedLoadInt16:
    2429       12400 :       ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
    2430        1240 :       break;
    2431             :     case kCheckedLoadUint16:
    2432        9204 :       ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
    2433         767 :       break;
    2434             :     case kCheckedLoadWord32:
    2435      670528 :       ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
    2436       55966 :       break;
    2437             :     case kCheckedLoadWord64:
    2438          84 :       ASSEMBLE_CHECKED_LOAD_INTEGER(movq);
    2439           7 :       break;
    2440             :     case kCheckedLoadFloat32:
    2441       71813 :       ASSEMBLE_CHECKED_LOAD_FLOAT(Movss, OutOfLineLoadFloat32NaN);
    2442        9016 :       break;
    2443             :     case kCheckedLoadFloat64:
    2444        4477 :       ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd, OutOfLineLoadFloat64NaN);
    2445         599 :       break;
    2446             :     case kCheckedStoreWord8:
    2447       43709 :       ASSEMBLE_CHECKED_STORE_INTEGER(movb);
    2448             :       break;
    2449             :     case kCheckedStoreWord16:
    2450       20870 :       ASSEMBLE_CHECKED_STORE_INTEGER(movw);
    2451             :       break;
    2452             :     case kCheckedStoreWord32:
    2453      500110 :       ASSEMBLE_CHECKED_STORE_INTEGER(movl);
    2454             :       break;
    2455             :     case kCheckedStoreWord64:
    2456          84 :       ASSEMBLE_CHECKED_STORE_INTEGER(movq);
    2457             :       break;
    2458             :     case kCheckedStoreFloat32:
    2459       49104 :       ASSEMBLE_CHECKED_STORE_FLOAT(Movss);
    2460             :       break;
    2461             :     case kCheckedStoreFloat64:
    2462        4392 :       ASSEMBLE_CHECKED_STORE_FLOAT(Movsd);
    2463             :       break;
    2464             :     case kX64StackCheck:
    2465      519337 :       __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
    2466      519337 :       break;
    2467             :     case kAtomicExchangeInt8: {
    2468         172 :       __ xchgb(i.InputRegister(0), i.MemoryOperand(1));
    2469         172 :       __ movsxbl(i.InputRegister(0), i.InputRegister(0));
    2470          86 :       break;
    2471             :     }
    2472             :     case kAtomicExchangeUint8: {
    2473          86 :       __ xchgb(i.InputRegister(0), i.MemoryOperand(1));
    2474          43 :       __ movzxbl(i.InputRegister(0), i.InputRegister(0));
    2475             :       break;
    2476             :     }
    2477             :     case kAtomicExchangeInt16: {
    2478         172 :       __ xchgw(i.InputRegister(0), i.MemoryOperand(1));
    2479         172 :       __ movsxwl(i.InputRegister(0), i.InputRegister(0));
    2480          86 :       break;
    2481             :     }
    2482             :     case kAtomicExchangeUint16: {
    2483          86 :       __ xchgw(i.InputRegister(0), i.MemoryOperand(1));
    2484          43 :       __ movzxwl(i.InputRegister(0), i.InputRegister(0));
    2485             :       break;
    2486             :     }
    2487             :     case kAtomicExchangeWord32: {
    2488         258 :       __ xchgl(i.InputRegister(0), i.MemoryOperand(1));
    2489             :       break;
    2490             :     }
    2491             :     case kAtomicCompareExchangeInt8: {
    2492          43 :       __ lock();
    2493          86 :       __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1));
    2494          43 :       __ movsxbl(rax, rax);
    2495          43 :       break;
    2496             :     }
    2497             :     case kAtomicCompareExchangeUint8: {
    2498          43 :       __ lock();
    2499          86 :       __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1));
    2500             :       __ movzxbl(rax, rax);
    2501             :       break;
    2502             :     }
    2503             :     case kAtomicCompareExchangeInt16: {
    2504          43 :       __ lock();
    2505          86 :       __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1));
    2506          43 :       __ movsxwl(rax, rax);
    2507          43 :       break;
    2508             :     }
    2509             :     case kAtomicCompareExchangeUint16: {
    2510          43 :       __ lock();
    2511          86 :       __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1));
    2512             :       __ movzxwl(rax, rax);
    2513             :       break;
    2514             :     }
    2515             :     case kAtomicCompareExchangeWord32: {
    2516          86 :       __ lock();
    2517          86 :       __ cmpxchgl(i.MemoryOperand(2), i.InputRegister(1));
    2518             :       break;
    2519             :     }
    2520             : #define ATOMIC_BINOP_CASE(op, inst)              \
    2521             :   case kAtomic##op##Int8:                        \
    2522             :     ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \
    2523             :     __ movsxbl(rax, rax);                        \
    2524             :     break;                                       \
    2525             :   case kAtomic##op##Uint8:                       \
    2526             :     ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \
    2527             :     __ movzxbl(rax, rax);                        \
    2528             :     break;                                       \
    2529             :   case kAtomic##op##Int16:                       \
    2530             :     ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \
    2531             :     __ movsxwl(rax, rax);                        \
    2532             :     break;                                       \
    2533             :   case kAtomic##op##Uint16:                      \
    2534             :     ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \
    2535             :     __ movzxwl(rax, rax);                        \
    2536             :     break;                                       \
    2537             :   case kAtomic##op##Word32:                      \
    2538             :     ASSEMBLE_ATOMIC_BINOP(inst, movl, cmpxchgl); \
    2539             :     break;
    2540        1290 :       ATOMIC_BINOP_CASE(Add, addl)
    2541        1290 :       ATOMIC_BINOP_CASE(Sub, subl)
    2542        1290 :       ATOMIC_BINOP_CASE(And, andl)
    2543        1290 :       ATOMIC_BINOP_CASE(Or, orl)
    2544        1290 :       ATOMIC_BINOP_CASE(Xor, xorl)
    2545             : #undef ATOMIC_BINOP_CASE
    2546             :     case kAtomicLoadInt8:
    2547             :     case kAtomicLoadUint8:
    2548             :     case kAtomicLoadInt16:
    2549             :     case kAtomicLoadUint16:
    2550             :     case kAtomicLoadWord32:
    2551             :     case kAtomicStoreWord8:
    2552             :     case kAtomicStoreWord16:
    2553             :     case kAtomicStoreWord32:
    2554           0 :       UNREACHABLE();  // Won't be generated by instruction selector.
    2555             :       break;
    2556             :   }
    2557             :   return kSuccess;
    2558             : }  // NOLINT(readability/fn_size)
    2559             : 
    2560             : namespace {
    2561             : 
    2562     3783827 : Condition FlagsConditionToCondition(FlagsCondition condition) {
    2563     3783827 :   switch (condition) {
    2564             :     case kUnorderedEqual:
    2565             :     case kEqual:
    2566             :       return equal;
    2567             :     case kUnorderedNotEqual:
    2568             :     case kNotEqual:
    2569      991789 :       return not_equal;
    2570             :     case kSignedLessThan:
    2571       72622 :       return less;
    2572             :     case kSignedGreaterThanOrEqual:
    2573       39951 :       return greater_equal;
    2574             :     case kSignedLessThanOrEqual:
    2575       26990 :       return less_equal;
    2576             :     case kSignedGreaterThan:
    2577       35373 :       return greater;
    2578             :     case kUnsignedLessThan:
    2579       48803 :       return below;
    2580             :     case kUnsignedGreaterThanOrEqual:
    2581       73862 :       return above_equal;
    2582             :     case kUnsignedLessThanOrEqual:
    2583      743768 :       return below_equal;
    2584             :     case kUnsignedGreaterThan:
    2585       37109 :       return above;
    2586             :     case kOverflow:
    2587      195402 :       return overflow;
    2588             :     case kNotOverflow:
    2589         903 :       return no_overflow;
    2590             :     default:
    2591             :       break;
    2592             :   }
    2593           0 :   UNREACHABLE();
    2594             :   return no_condition;
    2595             : }
    2596             : 
    2597             : }  // namespace
    2598             : 
    2599             : // Assembles branches after this instruction.
    2600     3550946 : void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
    2601             :   Label::Distance flabel_distance =
    2602     3550946 :       branch->fallthru ? Label::kNear : Label::kFar;
    2603     3550946 :   Label* tlabel = branch->true_label;
    2604     3550946 :   Label* flabel = branch->false_label;
    2605     3550946 :   if (branch->condition == kUnorderedEqual) {
    2606       40410 :     __ j(parity_even, flabel, flabel_distance);
    2607     3510536 :   } else if (branch->condition == kUnorderedNotEqual) {
    2608       58795 :     __ j(parity_even, tlabel);
    2609             :   }
    2610     3550946 :   __ j(FlagsConditionToCondition(branch->condition), tlabel);
    2611             : 
    2612     3550945 :   if (!branch->fallthru) __ jmp(flabel, flabel_distance);
    2613     3550945 : }
    2614             : 
    2615             : 
    2616     5334404 : void CodeGenerator::AssembleArchJump(RpoNumber target) {
    2617     5334404 :   if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
    2618     3284621 : }
    2619             : 
    2620       16305 : void CodeGenerator::AssembleArchTrap(Instruction* instr,
    2621       16305 :                                      FlagsCondition condition) {
    2622           0 :   class OutOfLineTrap final : public OutOfLineCode {
    2623             :    public:
    2624             :     OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
    2625             :         : OutOfLineCode(gen),
    2626             :           frame_elided_(frame_elided),
    2627             :           instr_(instr),
    2628       16305 :           gen_(gen) {}
    2629             : 
    2630       16305 :     void Generate() final {
    2631       32610 :       X64OperandConverter i(gen_, instr_);
    2632             : 
    2633             :       Builtins::Name trap_id =
    2634       16305 :           static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
    2635       27298 :       bool old_has_frame = __ has_frame();
    2636       16305 :       if (frame_elided_) {
    2637             :         __ set_has_frame(true);
    2638       10993 :         __ EnterFrame(StackFrame::WASM_COMPILED);
    2639             :       }
    2640       16305 :       GenerateCallToTrap(trap_id);
    2641       16305 :       if (frame_elided_) {
    2642             :         __ set_has_frame(old_has_frame);
    2643             :       }
    2644       16305 :     }
    2645             : 
    2646             :    private:
    2647       16305 :     void GenerateCallToTrap(Builtins::Name trap_id) {
    2648       16305 :       if (trap_id == Builtins::builtin_count) {
    2649             :         // We cannot test calls to the runtime in cctest/test-run-wasm.
    2650             :         // Therefore we emit a call to C here instead of a call to the runtime.
    2651       33801 :         __ PrepareCallCFunction(0);
    2652             :         __ CallCFunction(
    2653             :             ExternalReference::wasm_call_trap_callback_for_testing(isolate()),
    2654       11664 :             0);
    2655        5832 :         __ LeaveFrame(StackFrame::WASM_COMPILED);
    2656        5832 :         __ Ret();
    2657             :       } else {
    2658       10473 :         gen_->AssembleSourcePosition(instr_);
    2659             :         __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()),
    2660       10473 :                 RelocInfo::CODE_TARGET);
    2661             :         ReferenceMap* reference_map =
    2662       10473 :             new (gen_->zone()) ReferenceMap(gen_->zone());
    2663             :         gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
    2664       10473 :                               Safepoint::kNoLazyDeopt);
    2665       10473 :         if (FLAG_debug_code) {
    2666           0 :           __ ud2();
    2667             :         }
    2668             :       }
    2669       16305 :     }
    2670             : 
    2671             :     bool frame_elided_;
    2672             :     Instruction* instr_;
    2673             :     CodeGenerator* gen_;
    2674             :   };
    2675       16305 :   bool frame_elided = !frame_access_state()->has_frame();
    2676             :   auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
    2677       16305 :   Label* tlabel = ool->entry();
    2678             :   Label end;
    2679       16305 :   if (condition == kUnorderedEqual) {
    2680           0 :     __ j(parity_even, &end);
    2681       16305 :   } else if (condition == kUnorderedNotEqual) {
    2682         401 :     __ j(parity_even, tlabel);
    2683             :   }
    2684       16305 :   __ j(FlagsConditionToCondition(condition), tlabel);
    2685       16305 :   __ bind(&end);
    2686       16305 : }
    2687             : 
    2688             : // Assembles boolean materializations after this instruction.
    2689      433154 : void CodeGenerator::AssembleArchBoolean(Instruction* instr,
    2690             :                                         FlagsCondition condition) {
    2691             :   X64OperandConverter i(this, instr);
    2692             :   Label done;
    2693             : 
    2694             :   // Materialize a full 64-bit 1 or 0 value. The result register is always the
    2695             :   // last output of the instruction.
    2696             :   Label check;
    2697             :   DCHECK_NE(0u, instr->OutputCount());
    2698      216577 :   Register reg = i.OutputRegister(instr->OutputCount() - 1);
    2699      216577 :   if (condition == kUnorderedEqual) {
    2700        5946 :     __ j(parity_odd, &check, Label::kNear);
    2701             :     __ movl(reg, Immediate(0));
    2702        5946 :     __ jmp(&done, Label::kNear);
    2703      210631 :   } else if (condition == kUnorderedNotEqual) {
    2704        2388 :     __ j(parity_odd, &check, Label::kNear);
    2705             :     __ movl(reg, Immediate(1));
    2706        2388 :     __ jmp(&done, Label::kNear);
    2707             :   }
    2708      216577 :   __ bind(&check);
    2709      216577 :   __ setcc(FlagsConditionToCondition(condition), reg);
    2710             :   __ movzxbl(reg, reg);
    2711      216577 :   __ bind(&done);
    2712      216577 : }
    2713             : 
    2714             : 
    2715      121278 : void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
    2716             :   X64OperandConverter i(this, instr);
    2717       16755 :   Register input = i.InputRegister(0);
    2718      121278 :   for (size_t index = 2; index < instr->InputCount(); index += 2) {
    2719       87768 :     __ cmpl(input, Immediate(i.InputInt32(index + 0)));
    2720       87768 :     __ j(equal, GetLabel(i.InputRpo(index + 1)));
    2721             :   }
    2722       16755 :   AssembleArchJump(i.InputRpo(1));
    2723       16755 : }
    2724             : 
    2725             : 
    2726      117704 : void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
    2727             :   X64OperandConverter i(this, instr);
    2728        6072 :   Register input = i.InputRegister(0);
    2729        6072 :   int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
    2730        6072 :   Label** cases = zone()->NewArray<Label*>(case_count);
    2731      111632 :   for (int32_t index = 0; index < case_count; ++index) {
    2732      211120 :     cases[index] = GetLabel(i.InputRpo(index + 2));
    2733             :   }
    2734        6072 :   Label* const table = AddJumpTable(cases, case_count);
    2735        6072 :   __ cmpl(input, Immediate(case_count));
    2736       12144 :   __ j(above_equal, GetLabel(i.InputRpo(1)));
    2737       12144 :   __ leaq(kScratchRegister, Operand(table));
    2738        6072 :   __ jmp(Operand(kScratchRegister, input, times_8, 0));
    2739        6072 : }
    2740             : 
    2741      277964 : CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
    2742             :     int deoptimization_id, SourcePosition pos) {
    2743      277964 :   DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id);
    2744             :   DeoptimizeReason deoptimization_reason =
    2745      277964 :       GetDeoptimizationReason(deoptimization_id);
    2746             :   Deoptimizer::BailoutType bailout_type =
    2747             :       deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT
    2748      277964 :                                                    : Deoptimizer::EAGER;
    2749             :   Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
    2750      277964 :       isolate(), deoptimization_id, bailout_type);
    2751      277964 :   if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
    2752      277964 :   if (isolate()->NeedsSourcePositionsForProfiling()) {
    2753       19982 :     __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
    2754             :   }
    2755      277964 :   __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
    2756      277964 :   return kSuccess;
    2757             : }
    2758             : 
    2759             : 
    2760             : namespace {
    2761             : 
    2762             : static const int kQuadWordSize = 16;
    2763             : 
    2764             : }  // namespace
    2765             : 
    2766      912392 : void CodeGenerator::FinishFrame(Frame* frame) {
    2767     1824784 :   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
    2768             : 
    2769             :   const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
    2770      912392 :   if (saves_fp != 0) {
    2771             :     frame->AlignSavedCalleeRegisterSlots();
    2772           0 :     if (saves_fp != 0) {  // Save callee-saved XMM registers.
    2773             :       const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
    2774           0 :       frame->AllocateSavedCalleeRegisterSlots(saves_fp_count *
    2775           0 :                                               (kQuadWordSize / kPointerSize));
    2776             :     }
    2777             :   }
    2778             :   const RegList saves = descriptor->CalleeSavedRegisters();
    2779      912392 :   if (saves != 0) {  // Save callee-saved registers.
    2780             :     int count = 0;
    2781     4028928 :     for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
    2782     4028928 :       if (((1 << i) & saves)) {
    2783     1259040 :         ++count;
    2784             :       }
    2785             :     }
    2786             :     frame->AllocateSavedCalleeRegisterSlots(count);
    2787             :   }
    2788      912392 : }
    2789             : 
    2790     4554001 : void CodeGenerator::AssembleConstructFrame() {
    2791     3496796 :   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
    2792      874199 :   if (frame_access_state()->has_frame()) {
    2793      874199 :     int pc_base = __ pc_offset();
    2794             : 
    2795      874199 :     if (descriptor->IsCFunctionCall()) {
    2796      251808 :       __ pushq(rbp);
    2797             :       __ movq(rbp, rsp);
    2798      622391 :     } else if (descriptor->IsJSFunctionCall()) {
    2799      429002 :       __ Prologue(this->info()->GeneratePreagedPrologue());
    2800      429002 :       if (descriptor->PushArgumentCount()) {
    2801        8589 :         __ pushq(kJavaScriptCallArgCountRegister);
    2802             :       }
    2803             :     } else {
    2804      193389 :       __ StubPrologue(info()->GetOutputStackFrameType());
    2805             :     }
    2806             : 
    2807     1303200 :     if (!descriptor->IsJSFunctionCall() || !info()->GeneratePreagedPrologue()) {
    2808      874199 :       unwinding_info_writer_.MarkFrameConstructed(pc_base);
    2809             :     }
    2810             :   }
    2811             :   int shrink_slots =
    2812      874199 :       frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
    2813             : 
    2814      874199 :   if (info()->is_osr()) {
    2815             :     // TurboFan OSR-compiled functions cannot be entered directly.
    2816        5813 :     __ Abort(kShouldNotDirectlyEnterOsrFunction);
    2817             : 
    2818             :     // Unoptimized code jumps directly to this entrypoint while the unoptimized
    2819             :     // frame is still on the stack. Optimized code uses OSR values directly from
    2820             :     // the unoptimized frame. Thus, all that needs to be done is to allocate the
    2821             :     // remaining stack slots.
    2822        5813 :     if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
    2823       11626 :     osr_pc_offset_ = __ pc_offset();
    2824        5813 :     shrink_slots -= static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots());
    2825             :   }
    2826             : 
    2827             :   const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
    2828      874199 :   if (shrink_slots > 0) {
    2829     1190952 :     __ subq(rsp, Immediate(shrink_slots * kPointerSize));
    2830             :   }
    2831             : 
    2832      874199 :   if (saves_fp != 0) {  // Save callee-saved XMM registers.
    2833             :     const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
    2834           0 :     const int stack_size = saves_fp_count * kQuadWordSize;
    2835             :     // Adjust the stack pointer.
    2836           0 :     __ subp(rsp, Immediate(stack_size));
    2837             :     // Store the registers on the stack.
    2838             :     int slot_idx = 0;
    2839           0 :     for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
    2840           0 :       if (!((1 << i) & saves_fp)) continue;
    2841             :       __ movdqu(Operand(rsp, kQuadWordSize * slot_idx),
    2842           0 :                 XMMRegister::from_code(i));
    2843           0 :       slot_idx++;
    2844             :     }
    2845             :   }
    2846             : 
    2847             :   const RegList saves = descriptor->CalleeSavedRegisters();
    2848      874199 :   if (saves != 0) {  // Save callee-saved registers.
    2849     4028928 :     for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
    2850     4028928 :       if (!((1 << i) & saves)) continue;
    2851     1259040 :       __ pushq(Register::from_code(i));
    2852             :     }
    2853             :   }
    2854      874199 : }
    2855             : 
    2856     1908872 : void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
    2857     4350896 :   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
    2858             : 
    2859             :   // Restore registers.
    2860             :   const RegList saves = descriptor->CalleeSavedRegisters();
    2861     1087724 :   if (saves != 0) {
    2862     4265216 :     for (int i = 0; i < Register::kNumRegisters; i++) {
    2863     4265216 :       if (!((1 << i) & saves)) continue;
    2864     1332880 :       __ popq(Register::from_code(i));
    2865             :     }
    2866             :   }
    2867             :   const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
    2868     1087724 :   if (saves_fp != 0) {
    2869             :     const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
    2870           0 :     const int stack_size = saves_fp_count * kQuadWordSize;
    2871             :     // Load the registers from the stack.
    2872             :     int slot_idx = 0;
    2873           0 :     for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
    2874           0 :       if (!((1 << i) & saves_fp)) continue;
    2875             :       __ movdqu(XMMRegister::from_code(i),
    2876           0 :                 Operand(rsp, kQuadWordSize * slot_idx));
    2877           0 :       slot_idx++;
    2878             :     }
    2879             :     // Adjust the stack pointer.
    2880           0 :     __ addp(rsp, Immediate(stack_size));
    2881             :   }
    2882             : 
    2883             :   unwinding_info_writer_.MarkBlockWillExit();
    2884             : 
    2885             :   // Might need rcx for scratch if pop_size is too big or if there is a variable
    2886             :   // pop count.
    2887             :   DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit());
    2888             :   DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rdx.bit());
    2889     1087724 :   size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
    2890             :   X64OperandConverter g(this, nullptr);
    2891     1087724 :   if (descriptor->IsCFunctionCall()) {
    2892      266576 :     AssembleDeconstructFrame();
    2893      821148 :   } else if (frame_access_state()->has_frame()) {
    2894     1344864 :     if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
    2895             :       // Canonicalize JSFunction return sites for now.
    2896      671970 :       if (return_label_.is_bound()) {
    2897      187738 :         __ jmp(&return_label_);
    2898     1087724 :         return;
    2899             :       } else {
    2900      484232 :         __ bind(&return_label_);
    2901      484232 :         AssembleDeconstructFrame();
    2902             :       }
    2903             :     } else {
    2904         917 :       AssembleDeconstructFrame();
    2905             :     }
    2906             :   }
    2907             : 
    2908      899986 :   if (pop->IsImmediate()) {
    2909             :     DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
    2910     1798151 :     pop_size += g.ToConstant(pop).ToInt32() * kPointerSize;
    2911      899075 :     CHECK_LT(pop_size, static_cast<size_t>(std::numeric_limits<int>::max()));
    2912      899075 :     __ Ret(static_cast<int>(pop_size), rcx);
    2913             :   } else {
    2914             :     Register pop_reg = g.ToRegister(pop);
    2915         910 :     Register scratch_reg = pop_reg.is(rcx) ? rdx : rcx;
    2916         910 :     __ popq(scratch_reg);
    2917        1820 :     __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast<int>(pop_size)));
    2918         910 :     __ jmp(scratch_reg);
    2919             :   }
    2920             : }
    2921             : 
    2922      912392 : void CodeGenerator::FinishCode() {}
    2923             : 
    2924    23466664 : void CodeGenerator::AssembleMove(InstructionOperand* source,
    2925             :                                  InstructionOperand* destination) {
    2926             :   X64OperandConverter g(this, nullptr);
    2927             :   // Dispatch on the source and destination operand kinds.  Not all
    2928             :   // combinations are possible.
    2929    23466664 :   if (source->IsRegister()) {
    2930             :     DCHECK(destination->IsRegister() || destination->IsStackSlot());
    2931             :     Register src = g.ToRegister(source);
    2932     5177500 :     if (destination->IsRegister()) {
    2933     2650188 :       __ movq(g.ToRegister(destination), src);
    2934             :     } else {
    2935     2527313 :       __ movq(g.ToOperand(destination), src);
    2936             :     }
    2937    18289164 :   } else if (source->IsStackSlot()) {
    2938             :     DCHECK(destination->IsRegister() || destination->IsStackSlot());
    2939     5688906 :     Operand src = g.ToOperand(source);
    2940     5688906 :     if (destination->IsRegister()) {
    2941             :       Register dst = g.ToRegister(destination);
    2942     5618859 :       __ movq(dst, src);
    2943             :     } else {
    2944             :       // Spill on demand to use a temporary register for memory-to-memory
    2945             :       // moves.
    2946             :       Register tmp = kScratchRegister;
    2947       70047 :       Operand dst = g.ToOperand(destination);
    2948       70047 :       __ movq(tmp, src);
    2949             :       __ movq(dst, tmp);
    2950             :     }
    2951    12600258 :   } else if (source->IsConstant()) {
    2952             :     ConstantOperand* constant_source = ConstantOperand::cast(source);
    2953    12174993 :     Constant src = g.ToConstant(constant_source);
    2954    12542870 :     if (destination->IsRegister() || destination->IsStackSlot()) {
    2955             :       Register dst = destination->IsRegister() ? g.ToRegister(destination)
    2956    11854555 :                                                : kScratchRegister;
    2957    11854555 :       switch (src.type()) {
    2958             :         case Constant::kInt32: {
    2959     5404064 :           if (RelocInfo::IsWasmPtrReference(src.rmode())) {
    2960           0 :             __ movq(dst, src.ToInt64(), src.rmode());
    2961             :           } else {
    2962             :             // TODO(dcarney): don't need scratch in this case.
    2963     2702032 :             int32_t value = src.ToInt32();
    2964     2702032 :             if (value == 0) {
    2965      755677 :               __ xorl(dst, dst);
    2966             :             } else {
    2967     1946355 :               if (RelocInfo::IsWasmSizeReference(src.rmode())) {
    2968           0 :                 __ movl(dst, Immediate(value, src.rmode()));
    2969             :               } else {
    2970     1946355 :                 __ movl(dst, Immediate(value));
    2971             :               }
    2972             :             }
    2973             :           }
    2974             :           break;
    2975             :         }
    2976             :         case Constant::kInt64:
    2977     1316574 :           if (RelocInfo::IsWasmPtrReference(src.rmode())) {
    2978       48009 :             __ movq(dst, src.ToInt64(), src.rmode());
    2979             :           } else {
    2980             :             DCHECK(!RelocInfo::IsWasmSizeReference(src.rmode()));
    2981      610278 :             __ Set(dst, src.ToInt64());
    2982             :           }
    2983             :           break;
    2984             :         case Constant::kFloat32:
    2985             :           __ Move(dst,
    2986           0 :                   isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
    2987           0 :           break;
    2988             :         case Constant::kFloat64:
    2989             :           __ Move(dst,
    2990     2715963 :                   isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
    2991     2715960 :           break;
    2992             :         case Constant::kExternalReference:
    2993     1186125 :           __ Move(dst, src.ToExternalReference());
    2994             :           break;
    2995             :         case Constant::kHeapObject: {
    2996     4592154 :           Handle<HeapObject> src_object = src.ToHeapObject();
    2997             :           Heap::RootListIndex index;
    2998     4592154 :           if (IsMaterializableFromRoot(src_object, &index)) {
    2999     1551156 :             __ LoadRoot(dst, index);
    3000             :           } else {
    3001     3041001 :             __ Move(dst, src_object);
    3002             :           }
    3003             :           break;
    3004             :         }
    3005             :         case Constant::kRpoNumber:
    3006           0 :           UNREACHABLE();  // TODO(dcarney): load of labels on x64.
    3007             :           break;
    3008             :       }
    3009    11854546 :       if (destination->IsStackSlot()) {
    3010       47437 :         __ movq(g.ToOperand(destination), kScratchRegister);
    3011             :       }
    3012      320439 :     } else if (src.type() == Constant::kFloat32) {
    3013             :       // TODO(turbofan): Can we do better here?
    3014        9160 :       uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
    3015        9160 :       if (destination->IsFPRegister()) {
    3016        9160 :         __ Move(g.ToDoubleRegister(destination), src_const);
    3017             :       } else {
    3018             :         DCHECK(destination->IsFPStackSlot());
    3019           0 :         Operand dst = g.ToOperand(destination);
    3020           0 :         __ movl(dst, Immediate(src_const));
    3021             :       }
    3022             :     } else {
    3023             :       DCHECK_EQ(Constant::kFloat64, src.type());
    3024      311279 :       uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
    3025      311279 :       if (destination->IsFPRegister()) {
    3026      310984 :         __ Move(g.ToDoubleRegister(destination), src_const);
    3027             :       } else {
    3028             :         DCHECK(destination->IsFPStackSlot());
    3029         295 :         __ movq(kScratchRegister, src_const);
    3030             :         __ movq(g.ToOperand(destination), kScratchRegister);
    3031             :       }
    3032             :     }
    3033      425265 :   } else if (source->IsFPRegister()) {
    3034      213614 :     XMMRegister src = g.ToDoubleRegister(source);
    3035      213614 :     if (destination->IsFPRegister()) {
    3036       47109 :       XMMRegister dst = g.ToDoubleRegister(destination);
    3037       47109 :       __ Movapd(dst, src);
    3038             :     } else {
    3039             :       DCHECK(destination->IsFPStackSlot());
    3040      166505 :       Operand dst = g.ToOperand(destination);
    3041             :       MachineRepresentation rep =
    3042             :           LocationOperand::cast(source)->representation();
    3043      166505 :       if (rep != MachineRepresentation::kSimd128) {
    3044      166505 :         __ Movsd(dst, src);
    3045             :       } else {
    3046           0 :         __ Movups(dst, src);
    3047             :       }
    3048             :     }
    3049      211651 :   } else if (source->IsFPStackSlot()) {
    3050             :     DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
    3051      211651 :     Operand src = g.ToOperand(source);
    3052             :     MachineRepresentation rep = LocationOperand::cast(source)->representation();
    3053      211651 :     if (destination->IsFPRegister()) {
    3054      204499 :       XMMRegister dst = g.ToDoubleRegister(destination);
    3055      204499 :       if (rep != MachineRepresentation::kSimd128) {
    3056      204499 :         __ Movsd(dst, src);
    3057             :       } else {
    3058           0 :         __ Movups(dst, src);
    3059             :       }
    3060             :     } else {
    3061        7152 :       Operand dst = g.ToOperand(destination);
    3062        7152 :       if (rep != MachineRepresentation::kSimd128) {
    3063        7152 :         __ Movsd(kScratchDoubleReg, src);
    3064        7152 :         __ Movsd(dst, kScratchDoubleReg);
    3065             :       } else {
    3066           0 :         __ Movups(kScratchDoubleReg, src);
    3067           0 :         __ Movups(dst, kScratchDoubleReg);
    3068             :       }
    3069             :     }
    3070             :   } else {
    3071           0 :     UNREACHABLE();
    3072             :   }
    3073    23466658 : }
    3074             : 
    3075             : 
    3076       18462 : void CodeGenerator::AssembleSwap(InstructionOperand* source,
    3077        7884 :                                  InstructionOperand* destination) {
    3078             :   X64OperandConverter g(this, nullptr);
    3079             :   // Dispatch on the source and destination operand kinds.  Not all
    3080             :   // combinations are possible.
    3081       33112 :   if (source->IsRegister() && destination->IsRegister()) {
    3082             :     // Register-register.
    3083             :     Register src = g.ToRegister(source);
    3084             :     Register dst = g.ToRegister(destination);
    3085       14410 :     __ movq(kScratchRegister, src);
    3086             :     __ movq(src, dst);
    3087             :     __ movq(dst, kScratchRegister);
    3088        4292 :   } else if (source->IsRegister() && destination->IsStackSlot()) {
    3089             :     Register src = g.ToRegister(source);
    3090         240 :     __ pushq(src);
    3091             :     frame_access_state()->IncreaseSPDelta(1);
    3092             :     unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    3093         480 :                                                      kPointerSize);
    3094         240 :     Operand dst = g.ToOperand(destination);
    3095             :     __ movq(src, dst);
    3096             :     frame_access_state()->IncreaseSPDelta(-1);
    3097         240 :     dst = g.ToOperand(destination);
    3098         240 :     __ popq(dst);
    3099             :     unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    3100         480 :                                                      -kPointerSize);
    3101       11436 :   } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
    3102          65 :              (source->IsFPStackSlot() && destination->IsFPStackSlot())) {
    3103             :     // Memory-memory.
    3104        3702 :     Operand src = g.ToOperand(source);
    3105        3702 :     Operand dst = g.ToOperand(destination);
    3106             :     MachineRepresentation rep = LocationOperand::cast(source)->representation();
    3107        3702 :     if (rep != MachineRepresentation::kSimd128) {
    3108             :       Register tmp = kScratchRegister;
    3109        3702 :       __ movq(tmp, dst);
    3110        3702 :       __ pushq(src);
    3111             :       unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    3112        7404 :                                                        kPointerSize);
    3113             :       frame_access_state()->IncreaseSPDelta(1);
    3114        3702 :       src = g.ToOperand(source);
    3115             :       __ movq(src, tmp);
    3116             :       frame_access_state()->IncreaseSPDelta(-1);
    3117        3702 :       dst = g.ToOperand(destination);
    3118        3702 :       __ popq(dst);
    3119             :       unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
    3120        7404 :                                                        -kPointerSize);
    3121             :     } else {
    3122             :       // Use the XOR trick to swap without a temporary.
    3123           0 :       __ Movups(kScratchDoubleReg, src);
    3124           0 :       __ Xorps(kScratchDoubleReg, dst);  // scratch contains src ^ dst.
    3125           0 :       __ Movups(src, kScratchDoubleReg);
    3126           0 :       __ Xorps(kScratchDoubleReg, dst);  // scratch contains src.
    3127           0 :       __ Movups(dst, kScratchDoubleReg);
    3128           0 :       __ Xorps(kScratchDoubleReg, src);  // scratch contains dst.
    3129           0 :       __ Movups(src, kScratchDoubleReg);
    3130             :     }
    3131         220 :   } else if (source->IsFPRegister() && destination->IsFPRegister()) {
    3132             :     // XMM register-register swap.
    3133         110 :     XMMRegister src = g.ToDoubleRegister(source);
    3134         110 :     XMMRegister dst = g.ToDoubleRegister(destination);
    3135         110 :     __ Movapd(kScratchDoubleReg, src);
    3136         110 :     __ Movapd(src, dst);
    3137         110 :     __ Movapd(dst, kScratchDoubleReg);
    3138           0 :   } else if (source->IsFPRegister() && destination->IsFPStackSlot()) {
    3139             :     // XMM register-memory swap.
    3140           0 :     XMMRegister src = g.ToDoubleRegister(source);
    3141           0 :     Operand dst = g.ToOperand(destination);
    3142             :     MachineRepresentation rep = LocationOperand::cast(source)->representation();
    3143           0 :     if (rep != MachineRepresentation::kSimd128) {
    3144           0 :       __ Movsd(kScratchDoubleReg, src);
    3145           0 :       __ Movsd(src, dst);
    3146           0 :       __ Movsd(dst, kScratchDoubleReg);
    3147             :     } else {
    3148           0 :       __ Movups(kScratchDoubleReg, src);
    3149           0 :       __ Movups(src, dst);
    3150           0 :       __ Movups(dst, kScratchDoubleReg);
    3151             :     }
    3152             :   } else {
    3153             :     // No other combinations are possible.
    3154           0 :     UNREACHABLE();
    3155             :   }
    3156       18462 : }
    3157             : 
    3158             : 
    3159        6072 : void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
    3160      111632 :   for (size_t index = 0; index < target_count; ++index) {
    3161      105560 :     __ dq(targets[index]);
    3162             :   }
    3163        6072 : }
    3164             : 
    3165             : 
    3166     3608642 : void CodeGenerator::EnsureSpaceForLazyDeopt() {
    3167     3608642 :   if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
    3168     3608642 :     return;
    3169             :   }
    3170             : 
    3171     3114326 :   int space_needed = Deoptimizer::patch_size();
    3172             :   // Ensure that we have enough space after the previous lazy-bailout
    3173             :   // instruction for patching the code here.
    3174     3114326 :   int current_pc = __ pc_offset();
    3175     3114326 :   if (current_pc < last_lazy_deopt_pc_ + space_needed) {
    3176      102207 :     int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
    3177      102207 :     __ Nop(padding_size);
    3178             :   }
    3179             : }
    3180             : 
    3181             : #undef __
    3182             : 
    3183             : }  // namespace compiler
    3184             : }  // namespace internal
    3185             : }  // namespace v8

Generated by: LCOV version 1.10