LCOV - code coverage report
Current view: top level - src/x64 - assembler-x64.h (source / functions) Hit Total Coverage
Test: app.info Lines: 169 198 85.4 %
Date: 2017-04-26 Functions: 63 68 92.6 %

          Line data    Source code
       1             : // Copyright (c) 1994-2006 Sun Microsystems Inc.
       2             : // All Rights Reserved.
       3             : //
       4             : // Redistribution and use in source and binary forms, with or without
       5             : // modification, are permitted provided that the following conditions are
       6             : // met:
       7             : //
       8             : // - Redistributions of source code must retain the above copyright notice,
       9             : // this list of conditions and the following disclaimer.
      10             : //
      11             : // - Redistribution in binary form must reproduce the above copyright
      12             : // notice, this list of conditions and the following disclaimer in the
      13             : // documentation and/or other materials provided with the distribution.
      14             : //
      15             : // - Neither the name of Sun Microsystems or the names of contributors may
      16             : // be used to endorse or promote products derived from this software without
      17             : // specific prior written permission.
      18             : //
      19             : // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
      20             : // IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
      21             : // THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
      22             : // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
      23             : // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
      24             : // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
      25             : // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
      26             : // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
      27             : // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
      28             : // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
      29             : // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
      30             : 
      31             : // The original source code covered by the above license above has been
      32             : // modified significantly by Google Inc.
      33             : // Copyright 2012 the V8 project authors. All rights reserved.
      34             : 
      35             : // A lightweight X64 Assembler.
      36             : 
      37             : #ifndef V8_X64_ASSEMBLER_X64_H_
      38             : #define V8_X64_ASSEMBLER_X64_H_
      39             : 
      40             : #include <deque>
      41             : 
      42             : #include "src/assembler.h"
      43             : #include "src/x64/sse-instr.h"
      44             : 
      45             : namespace v8 {
      46             : namespace internal {
      47             : 
      48             : // Utility functions
      49             : 
      50             : #define GENERAL_REGISTERS(V) \
      51             :   V(rax)                     \
      52             :   V(rcx)                     \
      53             :   V(rdx)                     \
      54             :   V(rbx)                     \
      55             :   V(rsp)                     \
      56             :   V(rbp)                     \
      57             :   V(rsi)                     \
      58             :   V(rdi)                     \
      59             :   V(r8)                      \
      60             :   V(r9)                      \
      61             :   V(r10)                     \
      62             :   V(r11)                     \
      63             :   V(r12)                     \
      64             :   V(r13)                     \
      65             :   V(r14)                     \
      66             :   V(r15)
      67             : 
      68             : #define ALLOCATABLE_GENERAL_REGISTERS(V) \
      69             :   V(rax)                                 \
      70             :   V(rbx)                                 \
      71             :   V(rdx)                                 \
      72             :   V(rcx)                                 \
      73             :   V(rsi)                                 \
      74             :   V(rdi)                                 \
      75             :   V(r8)                                  \
      76             :   V(r9)                                  \
      77             :   V(r11)                                 \
      78             :   V(r12)                                 \
      79             :   V(r14)                                 \
      80             :   V(r15)
      81             : 
      82             : // The length of pushq(rbp), movp(rbp, rsp), Push(rsi) and Push(rdi).
      83             : constexpr int kNoCodeAgeSequenceLength = kPointerSize == kInt64Size ? 6 : 17;
      84             : 
      85             : // CPU Registers.
      86             : //
      87             : // 1) We would prefer to use an enum, but enum values are assignment-
      88             : // compatible with int, which has caused code-generation bugs.
      89             : //
      90             : // 2) We would prefer to use a class instead of a struct but we don't like
      91             : // the register initialization to depend on the particular initialization
      92             : // order (which appears to be different on OS X, Linux, and Windows for the
      93             : // installed versions of C++ we tried). Using a struct permits C-style
      94             : // "initialization". Also, the Register objects cannot be const as this
      95             : // forces initialization stubs in MSVC, making us dependent on initialization
      96             : // order.
      97             : //
      98             : // 3) By not using an enum, we are possibly preventing the compiler from
      99             : // doing certain constant folds, which may significantly reduce the
     100             : // code generated for some assembly instructions (because they boil down
     101             : // to a few constants). If this is a problem, we could change the code
     102             : // such that we use an enum in optimized mode, and the struct in debug
     103             : // mode. This way we get the compile-time error checking in debug mode
     104             : // and best performance in optimized code.
     105             : //
     106             : struct Register {
     107             :   enum Code {
     108             : #define REGISTER_CODE(R) kCode_##R,
     109             :     GENERAL_REGISTERS(REGISTER_CODE)
     110             : #undef REGISTER_CODE
     111             :         kAfterLast,
     112             :     kCode_no_reg = -1
     113             :   };
     114             : 
     115             :   static constexpr int kNumRegisters = Code::kAfterLast;
     116             : 
     117             :   static Register from_code(int code) {
     118             :     DCHECK(code >= 0);
     119             :     DCHECK(code < kNumRegisters);
     120             :     Register r = {code};
     121             :     return r;
     122             :   }
     123      239542 :   bool is_valid() const { return 0 <= reg_code && reg_code < kNumRegisters; }
     124    12245981 :   bool is(Register reg) const { return reg_code == reg.reg_code; }
     125             :   int code() const {
     126             :     DCHECK(is_valid());
     127    16246837 :     return reg_code;
     128             :   }
     129             :   int bit() const {
     130             :     DCHECK(is_valid());
     131         168 :     return 1 << reg_code;
     132             :   }
     133             : 
     134      648713 :   bool is_byte_register() const { return reg_code <= 3; }
     135             :   // Return the high bit of the register code as a 0 or 1.  Used often
     136             :   // when constructing the REX prefix byte.
     137   330724440 :   int high_bit() const { return reg_code >> 3; }
     138             :   // Return the 3 low bits of the register code.  Used when encoding registers
     139             :   // in modR/M, SIB, and opcode bytes.
     140   337705515 :   int low_bits() const { return reg_code & 0x7; }
     141             : 
     142             :   // Unfortunately we can't make this private in a struct when initializing
     143             :   // by assignment.
     144             :   int reg_code;
     145             : };
     146             : 
     147             : #define DECLARE_REGISTER(R) constexpr Register R = {Register::kCode_##R};
     148             : GENERAL_REGISTERS(DECLARE_REGISTER)
     149             : #undef DECLARE_REGISTER
     150             : constexpr Register no_reg = {Register::kCode_no_reg};
     151             : 
     152             : #ifdef _WIN64
     153             :   // Windows calling convention
     154             : constexpr Register arg_reg_1 = {Register::kCode_rcx};
     155             : constexpr Register arg_reg_2 = {Register::kCode_rdx};
     156             : constexpr Register arg_reg_3 = {Register::kCode_r8};
     157             : constexpr Register arg_reg_4 = {Register::kCode_r9};
     158             : #else
     159             :   // AMD64 calling convention
     160             : constexpr Register arg_reg_1 = {Register::kCode_rdi};
     161             : constexpr Register arg_reg_2 = {Register::kCode_rsi};
     162             : constexpr Register arg_reg_3 = {Register::kCode_rdx};
     163             : constexpr Register arg_reg_4 = {Register::kCode_rcx};
     164             : #endif  // _WIN64
     165             : 
     166             : 
     167             : #define DOUBLE_REGISTERS(V) \
     168             :   V(xmm0)                   \
     169             :   V(xmm1)                   \
     170             :   V(xmm2)                   \
     171             :   V(xmm3)                   \
     172             :   V(xmm4)                   \
     173             :   V(xmm5)                   \
     174             :   V(xmm6)                   \
     175             :   V(xmm7)                   \
     176             :   V(xmm8)                   \
     177             :   V(xmm9)                   \
     178             :   V(xmm10)                  \
     179             :   V(xmm11)                  \
     180             :   V(xmm12)                  \
     181             :   V(xmm13)                  \
     182             :   V(xmm14)                  \
     183             :   V(xmm15)
     184             : 
     185             : #define FLOAT_REGISTERS DOUBLE_REGISTERS
     186             : #define SIMD128_REGISTERS DOUBLE_REGISTERS
     187             : 
     188             : #define ALLOCATABLE_DOUBLE_REGISTERS(V) \
     189             :   V(xmm0)                               \
     190             :   V(xmm1)                               \
     191             :   V(xmm2)                               \
     192             :   V(xmm3)                               \
     193             :   V(xmm4)                               \
     194             :   V(xmm5)                               \
     195             :   V(xmm6)                               \
     196             :   V(xmm7)                               \
     197             :   V(xmm8)                               \
     198             :   V(xmm9)                               \
     199             :   V(xmm10)                              \
     200             :   V(xmm11)                              \
     201             :   V(xmm12)                              \
     202             :   V(xmm13)                              \
     203             :   V(xmm14)
     204             : 
     205             : constexpr bool kSimpleFPAliasing = true;
     206             : constexpr bool kSimdMaskRegisters = false;
     207             : 
     208             : struct XMMRegister {
     209             :   enum Code {
     210             : #define REGISTER_CODE(R) kCode_##R,
     211             :     DOUBLE_REGISTERS(REGISTER_CODE)
     212             : #undef REGISTER_CODE
     213             :         kAfterLast,
     214             :     kCode_no_reg = -1
     215             :   };
     216             : 
     217             :   static constexpr int kMaxNumRegisters = Code::kAfterLast;
     218             : 
     219             :   static XMMRegister from_code(int code) {
     220             :     XMMRegister result = {code};
     221             :     return result;
     222             :   }
     223             : 
     224             :   bool is_valid() const { return 0 <= reg_code && reg_code < kMaxNumRegisters; }
     225             :   bool is(XMMRegister reg) const { return reg_code == reg.reg_code; }
     226             :   int code() const {
     227             :     DCHECK(is_valid());
     228      164248 :     return reg_code;
     229             :   }
     230             : 
     231             :   // Return the high bit of the register code as a 0 or 1.  Used often
     232             :   // when constructing the REX prefix byte.
     233    11911342 :   int high_bit() const { return reg_code >> 3; }
     234             :   // Return the 3 low bits of the register code.  Used when encoding registers
     235             :   // in modR/M, SIB, and opcode bytes.
     236     4512864 :   int low_bits() const { return reg_code & 0x7; }
     237             : 
     238             :   // Unfortunately we can't make this private in a struct when initializing
     239             :   // by assignment.
     240             :   int reg_code;
     241             : };
     242             : 
     243             : typedef XMMRegister FloatRegister;
     244             : 
     245             : typedef XMMRegister DoubleRegister;
     246             : 
     247             : typedef XMMRegister Simd128Register;
     248             : 
     249             : #define DECLARE_REGISTER(R) \
     250             :   constexpr DoubleRegister R = {DoubleRegister::kCode_##R};
     251             : DOUBLE_REGISTERS(DECLARE_REGISTER)
     252             : #undef DECLARE_REGISTER
     253             : constexpr DoubleRegister no_double_reg = {DoubleRegister::kCode_no_reg};
     254             : 
     255             : enum Condition {
     256             :   // any value < 0 is considered no_condition
     257             :   no_condition  = -1,
     258             : 
     259             :   overflow      =  0,
     260             :   no_overflow   =  1,
     261             :   below         =  2,
     262             :   above_equal   =  3,
     263             :   equal         =  4,
     264             :   not_equal     =  5,
     265             :   below_equal   =  6,
     266             :   above         =  7,
     267             :   negative      =  8,
     268             :   positive      =  9,
     269             :   parity_even   = 10,
     270             :   parity_odd    = 11,
     271             :   less          = 12,
     272             :   greater_equal = 13,
     273             :   less_equal    = 14,
     274             :   greater       = 15,
     275             : 
     276             :   // Fake conditions that are handled by the
     277             :   // opcodes using them.
     278             :   always        = 16,
     279             :   never         = 17,
     280             :   // aliases
     281             :   carry         = below,
     282             :   not_carry     = above_equal,
     283             :   zero          = equal,
     284             :   not_zero      = not_equal,
     285             :   sign          = negative,
     286             :   not_sign      = positive,
     287             :   last_condition = greater
     288             : };
     289             : 
     290             : 
     291             : // Returns the equivalent of !cc.
     292             : // Negation of the default no_condition (-1) results in a non-default
     293             : // no_condition value (-2). As long as tests for no_condition check
     294             : // for condition < 0, this will work as expected.
     295             : inline Condition NegateCondition(Condition cc) {
     296      690713 :   return static_cast<Condition>(cc ^ 1);
     297             : }
     298             : 
     299             : 
     300             : // Commute a condition such that {a cond b == b cond' a}.
     301        6780 : inline Condition CommuteCondition(Condition cc) {
     302        6780 :   switch (cc) {
     303             :     case below:
     304             :       return above;
     305             :     case above:
     306           0 :       return below;
     307             :     case above_equal:
     308           0 :       return below_equal;
     309             :     case below_equal:
     310        6384 :       return above_equal;
     311             :     case less:
     312          71 :       return greater;
     313             :     case greater:
     314          15 :       return less;
     315             :     case greater_equal:
     316          49 :       return less_equal;
     317             :     case less_equal:
     318          26 :       return greater_equal;
     319             :     default:
     320         235 :       return cc;
     321             :   }
     322             : }
     323             : 
     324             : 
     325             : enum RoundingMode {
     326             :   kRoundToNearest = 0x0,
     327             :   kRoundDown = 0x1,
     328             :   kRoundUp = 0x2,
     329             :   kRoundToZero = 0x3
     330             : };
     331             : 
     332             : 
     333             : // -----------------------------------------------------------------------------
     334             : // Machine instruction Immediates
     335             : 
     336             : class Immediate BASE_EMBEDDED {
     337             :  public:
     338    17855898 :   explicit Immediate(int32_t value) : value_(value) {}
     339             :   explicit Immediate(int32_t value, RelocInfo::Mode rmode)
     340      254150 :       : value_(value), rmode_(rmode) {}
     341             :   explicit Immediate(Smi* value) {
     342             :     DCHECK(SmiValuesAre31Bits());  // Only available for 31-bit SMI.
     343             :     value_ = static_cast<int32_t>(reinterpret_cast<intptr_t>(value));
     344             :   }
     345             : 
     346             :  private:
     347             :   int32_t value_;
     348             :   RelocInfo::Mode rmode_ = RelocInfo::NONE32;
     349             : 
     350             :   friend class Assembler;
     351             : };
     352             : 
     353             : 
     354             : // -----------------------------------------------------------------------------
     355             : // Machine instruction Operands
     356             : 
     357             : enum ScaleFactor {
     358             :   times_1 = 0,
     359             :   times_2 = 1,
     360             :   times_4 = 2,
     361             :   times_8 = 3,
     362             :   times_int_size = times_4,
     363             :   times_pointer_size = (kPointerSize == 8) ? times_8 : times_4
     364             : };
     365             : 
     366             : 
     367             : class Operand BASE_EMBEDDED {
     368             :  public:
     369             :   // [base + disp/r]
     370             :   Operand(Register base, int32_t disp);
     371             : 
     372             :   // [base + index*scale + disp/r]
     373             :   Operand(Register base,
     374             :           Register index,
     375             :           ScaleFactor scale,
     376             :           int32_t disp);
     377             : 
     378             :   // [index*scale + disp/r]
     379             :   Operand(Register index,
     380             :           ScaleFactor scale,
     381             :           int32_t disp);
     382             : 
     383             :   // Offset from existing memory operand.
     384             :   // Offset is added to existing displacement as 32-bit signed values and
     385             :   // this must not overflow.
     386             :   Operand(const Operand& base, int32_t offset);
     387             : 
     388             :   // [rip + disp/r]
     389             :   explicit Operand(Label* label);
     390             : 
     391             :   // Checks whether either base or index register is the given register.
     392             :   // Does not check the "reg" part of the Operand.
     393             :   bool AddressUsesRegister(Register reg) const;
     394             : 
     395             :   // Queries related to the size of the generated instruction.
     396             :   // Whether the generated instruction will have a REX prefix.
     397             :   bool requires_rex() const { return rex_ != 0; }
     398             :   // Size of the ModR/M, SIB and displacement parts of the generated
     399             :   // instruction.
     400        4851 :   int operand_size() const { return len_; }
     401             : 
     402             :  private:
     403             :   byte rex_;
     404             :   byte buf_[9];
     405             :   // The number of bytes of buf_ in use.
     406             :   byte len_;
     407             : 
     408             :   // Set the ModR/M byte without an encoded 'reg' register. The
     409             :   // register is encoded later as part of the emit_operand operation.
     410             :   // set_modrm can be called before or after set_sib and set_disp*.
     411             :   inline void set_modrm(int mod, Register rm);
     412             : 
     413             :   // Set the SIB byte if one is needed. Sets the length to 2 rather than 1.
     414             :   inline void set_sib(ScaleFactor scale, Register index, Register base);
     415             : 
     416             :   // Adds operand displacement fields (offsets added to the memory address).
     417             :   // Needs to be called after set_sib, not before it.
     418             :   inline void set_disp8(int disp);
     419             :   inline void set_disp32(int disp);
     420             :   inline void set_disp64(int64_t disp);  // for labels.
     421             : 
     422             :   friend class Assembler;
     423             : };
     424             : 
     425             : #define ASSEMBLER_INSTRUCTION_LIST(V) \
     426             :   V(add)                              \
     427             :   V(and)                              \
     428             :   V(cmp)                              \
     429             :   V(cmpxchg)                          \
     430             :   V(dec)                              \
     431             :   V(idiv)                             \
     432             :   V(div)                              \
     433             :   V(imul)                             \
     434             :   V(inc)                              \
     435             :   V(lea)                              \
     436             :   V(mov)                              \
     437             :   V(movzxb)                           \
     438             :   V(movzxw)                           \
     439             :   V(neg)                              \
     440             :   V(not)                              \
     441             :   V(or)                               \
     442             :   V(repmovs)                          \
     443             :   V(sbb)                              \
     444             :   V(sub)                              \
     445             :   V(test)                             \
     446             :   V(xchg)                             \
     447             :   V(xor)
     448             : 
     449             : // Shift instructions on operands/registers with kPointerSize, kInt32Size and
     450             : // kInt64Size.
     451             : #define SHIFT_INSTRUCTION_LIST(V)       \
     452             :   V(rol, 0x0)                           \
     453             :   V(ror, 0x1)                           \
     454             :   V(rcl, 0x2)                           \
     455             :   V(rcr, 0x3)                           \
     456             :   V(shl, 0x4)                           \
     457             :   V(shr, 0x5)                           \
     458             :   V(sar, 0x7)                           \
     459             : 
     460             : 
     461             : class Assembler : public AssemblerBase {
     462             :  private:
     463             :   // We check before assembling an instruction that there is sufficient
     464             :   // space to write an instruction and its relocation information.
     465             :   // The relocation writer's position must be kGap bytes above the end of
     466             :   // the generated instructions. This leaves enough space for the
     467             :   // longest possible x64 instruction, 15 bytes, and the longest possible
     468             :   // relocation information encoding, RelocInfoWriter::kMaxLength == 16.
     469             :   // (There is a 15 byte limit on x64 instruction length that rules out some
     470             :   // otherwise valid instructions.)
     471             :   // This allows for a single, fast space check per instruction.
     472             :   static constexpr int kGap = 32;
     473             : 
     474             :  public:
     475             :   // Create an assembler. Instructions and relocation information are emitted
     476             :   // into a buffer, with the instructions starting from the beginning and the
     477             :   // relocation information starting from the end of the buffer. See CodeDesc
     478             :   // for a detailed comment on the layout (globals.h).
     479             :   //
     480             :   // If the provided buffer is NULL, the assembler allocates and grows its own
     481             :   // buffer, and buffer_size determines the initial buffer size. The buffer is
     482             :   // owned by the assembler and deallocated upon destruction of the assembler.
     483             :   //
     484             :   // If the provided buffer is not NULL, the assembler uses the provided buffer
     485             :   // for code generation and assumes its size to be buffer_size. If the buffer
     486             :   // is too small, a fatal error occurs. No deallocation of the buffer is done
     487             :   // upon destruction of the assembler.
     488    13459112 :   Assembler(Isolate* isolate, void* buffer, int buffer_size)
     489    13459112 :       : Assembler(IsolateData(isolate), buffer, buffer_size) {}
     490             :   Assembler(IsolateData isolate_data, void* buffer, int buffer_size);
     491    26916261 :   virtual ~Assembler() { }
     492             : 
     493             :   // GetCode emits any pending (non-emitted) code and fills the descriptor
     494             :   // desc. GetCode() is idempotent; it returns the same result if no other
     495             :   // Assembler functions are invoked in between GetCode() calls.
     496             :   void GetCode(CodeDesc* desc);
     497             : 
     498             :   // Read/Modify the code target in the relative branch/call instruction at pc.
     499             :   // On the x64 architecture, we use relative jumps with a 32-bit displacement
     500             :   // to jump to other Code objects in the Code space in the heap.
     501             :   // Jumps to C functions are done indirectly through a 64-bit register holding
     502             :   // the absolute address of the target.
     503             :   // These functions convert between absolute Addresses of Code objects and
     504             :   // the relative displacements stored in the code.
     505             :   // The isolate argument is unused (and may be nullptr) when skipping flushing.
     506             :   static inline Address target_address_at(Address pc, Address constant_pool);
     507             :   static inline void set_target_address_at(
     508             :       Isolate* isolate, Address pc, Address constant_pool, Address target,
     509             :       ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
     510             :   static inline Address target_address_at(Address pc, Code* code);
     511             :   static inline void set_target_address_at(
     512             :       Isolate* isolate, Address pc, Code* code, Address target,
     513             :       ICacheFlushMode icache_flush_mode = FLUSH_ICACHE_IF_NEEDED);
     514             : 
     515             :   // Return the code target address at a call site from the return address
     516             :   // of that call in the instruction stream.
     517             :   static inline Address target_address_from_return_address(Address pc);
     518             : 
     519             :   // This sets the branch destination (which is in the instruction on x64).
     520             :   // This is for calls and branches within generated code.
     521             :   inline static void deserialization_set_special_target_at(
     522             :       Isolate* isolate, Address instruction_payload, Code* code,
     523             :       Address target);
     524             : 
     525             :   // This sets the internal reference at the pc.
     526             :   inline static void deserialization_set_target_internal_reference_at(
     527             :       Isolate* isolate, Address pc, Address target,
     528             :       RelocInfo::Mode mode = RelocInfo::INTERNAL_REFERENCE);
     529             : 
     530             :   static inline RelocInfo::Mode RelocInfoNone() {
     531             :     if (kPointerSize == kInt64Size) {
     532             :       return RelocInfo::NONE64;
     533             :     } else {
     534             :       DCHECK(kPointerSize == kInt32Size);
     535             :       return RelocInfo::NONE32;
     536             :     }
     537             :   }
     538             : 
     539             :   inline Handle<Code> code_target_object_handle_at(Address pc);
     540             :   inline Address runtime_entry_at(Address pc);
     541             :   // Number of bytes taken up by the branch target in the code.
     542             :   static constexpr int kSpecialTargetSize = 4;  // 32-bit displacement.
     543             :   // Distance between the address of the code target in the call instruction
     544             :   // and the return address pushed on the stack.
     545             :   static constexpr int kCallTargetAddressOffset = 4;  // 32-bit displacement.
     546             :   // The length of call(kScratchRegister).
     547             :   static constexpr int kCallScratchRegisterInstructionLength = 3;
     548             :   // The length of call(Immediate32).
     549             :   static constexpr int kShortCallInstructionLength = 5;
     550             :   // The length of movq(kScratchRegister, address).
     551             :   static constexpr int kMoveAddressIntoScratchRegisterInstructionLength =
     552             :       2 + kPointerSize;
     553             :   // The length of movq(kScratchRegister, address) and call(kScratchRegister).
     554             :   static constexpr int kCallSequenceLength =
     555             :       kMoveAddressIntoScratchRegisterInstructionLength +
     556             :       kCallScratchRegisterInstructionLength;
     557             : 
     558             :   // The debug break slot must be able to contain an indirect call sequence.
     559             :   static constexpr int kDebugBreakSlotLength = kCallSequenceLength;
     560             :   // Distance between start of patched debug break slot and the emitted address
     561             :   // to jump to.
     562             :   static constexpr int kPatchDebugBreakSlotAddressOffset =
     563             :       kMoveAddressIntoScratchRegisterInstructionLength - kPointerSize;
     564             : 
     565             :   // One byte opcode for test eax,0xXXXXXXXX.
     566             :   static constexpr byte kTestEaxByte = 0xA9;
     567             :   // One byte opcode for test al, 0xXX.
     568             :   static constexpr byte kTestAlByte = 0xA8;
     569             :   // One byte opcode for nop.
     570             :   static constexpr byte kNopByte = 0x90;
     571             : 
     572             :   // One byte prefix for a short conditional jump.
     573             :   static constexpr byte kJccShortPrefix = 0x70;
     574             :   static constexpr byte kJncShortOpcode = kJccShortPrefix | not_carry;
     575             :   static constexpr byte kJcShortOpcode = kJccShortPrefix | carry;
     576             :   static constexpr byte kJnzShortOpcode = kJccShortPrefix | not_zero;
     577             :   static constexpr byte kJzShortOpcode = kJccShortPrefix | zero;
     578             : 
     579             :   // VEX prefix encodings.
     580             :   enum SIMDPrefix { kNone = 0x0, k66 = 0x1, kF3 = 0x2, kF2 = 0x3 };
     581             :   enum VectorLength { kL128 = 0x0, kL256 = 0x4, kLIG = kL128, kLZ = kL128 };
     582             :   enum VexW { kW0 = 0x0, kW1 = 0x80, kWIG = kW0 };
     583             :   enum LeadingOpcode { k0F = 0x1, k0F38 = 0x2, k0F3A = 0x3 };
     584             : 
     585             :   // ---------------------------------------------------------------------------
     586             :   // Code generation
     587             :   //
     588             :   // Function names correspond one-to-one to x64 instruction mnemonics.
     589             :   // Unless specified otherwise, instructions operate on 64-bit operands.
     590             :   //
     591             :   // If we need versions of an assembly instruction that operate on different
     592             :   // width arguments, we add a single-letter suffix specifying the width.
     593             :   // This is done for the following instructions: mov, cmp, inc, dec,
     594             :   // add, sub, and test.
     595             :   // There are no versions of these instructions without the suffix.
     596             :   // - Instructions on 8-bit (byte) operands/registers have a trailing 'b'.
     597             :   // - Instructions on 16-bit (word) operands/registers have a trailing 'w'.
     598             :   // - Instructions on 32-bit (doubleword) operands/registers use 'l'.
     599             :   // - Instructions on 64-bit (quadword) operands/registers use 'q'.
     600             :   // - Instructions on operands/registers with pointer size use 'p'.
     601             : 
     602             :   STATIC_ASSERT(kPointerSize == kInt64Size || kPointerSize == kInt32Size);
     603             : 
     604             : #define DECLARE_INSTRUCTION(instruction)                \
     605             :   template<class P1>                                    \
     606             :   void instruction##p(P1 p1) {                          \
     607             :     emit_##instruction(p1, kPointerSize);               \
     608             :   }                                                     \
     609             :                                                         \
     610             :   template<class P1>                                    \
     611             :   void instruction##l(P1 p1) {                          \
     612             :     emit_##instruction(p1, kInt32Size);                 \
     613             :   }                                                     \
     614             :                                                         \
     615             :   template<class P1>                                    \
     616             :   void instruction##q(P1 p1) {                          \
     617             :     emit_##instruction(p1, kInt64Size);                 \
     618             :   }                                                     \
     619             :                                                         \
     620             :   template<class P1, class P2>                          \
     621             :   void instruction##p(P1 p1, P2 p2) {                   \
     622             :     emit_##instruction(p1, p2, kPointerSize);           \
     623             :   }                                                     \
     624             :                                                         \
     625             :   template<class P1, class P2>                          \
     626             :   void instruction##l(P1 p1, P2 p2) {                   \
     627             :     emit_##instruction(p1, p2, kInt32Size);             \
     628             :   }                                                     \
     629             :                                                         \
     630             :   template<class P1, class P2>                          \
     631             :   void instruction##q(P1 p1, P2 p2) {                   \
     632             :     emit_##instruction(p1, p2, kInt64Size);             \
     633             :   }                                                     \
     634             :                                                         \
     635             :   template<class P1, class P2, class P3>                \
     636             :   void instruction##p(P1 p1, P2 p2, P3 p3) {            \
     637             :     emit_##instruction(p1, p2, p3, kPointerSize);       \
     638             :   }                                                     \
     639             :                                                         \
     640             :   template<class P1, class P2, class P3>                \
     641             :   void instruction##l(P1 p1, P2 p2, P3 p3) {            \
     642             :     emit_##instruction(p1, p2, p3, kInt32Size);         \
     643             :   }                                                     \
     644             :                                                         \
     645             :   template<class P1, class P2, class P3>                \
     646             :   void instruction##q(P1 p1, P2 p2, P3 p3) {            \
     647             :     emit_##instruction(p1, p2, p3, kInt64Size);         \
     648             :   }
     649   126247213 :   ASSEMBLER_INSTRUCTION_LIST(DECLARE_INSTRUCTION)
     650             : #undef DECLARE_INSTRUCTION
     651             : 
     652             :   // Insert the smallest number of nop instructions
     653             :   // possible to align the pc offset to a multiple
     654             :   // of m, where m must be a power of 2.
     655             :   void Align(int m);
     656             :   // Insert the smallest number of zero bytes possible to align the pc offset
     657             :   // to a mulitple of m. m must be a power of 2 (>= 2).
     658             :   void DataAlign(int m);
     659             :   void Nop(int bytes = 1);
     660             :   // Aligns code to something that's optimal for a jump target for the platform.
     661             :   void CodeTargetAlign();
     662             : 
     663             :   // Stack
     664             :   void pushfq();
     665             :   void popfq();
     666             : 
     667             :   void pushq(Immediate value);
     668             :   // Push a 32 bit integer, and guarantee that it is actually pushed as a
     669             :   // 32 bit value, the normal push will optimize the 8 bit case.
     670             :   void pushq_imm32(int32_t imm32);
     671             :   void pushq(Register src);
     672             :   void pushq(const Operand& src);
     673             : 
     674             :   void popq(Register dst);
     675             :   void popq(const Operand& dst);
     676             : 
     677             :   void enter(Immediate size);
     678             :   void leave();
     679             : 
     680             :   // Moves
     681             :   void movb(Register dst, const Operand& src);
     682             :   void movb(Register dst, Immediate imm);
     683             :   void movb(const Operand& dst, Register src);
     684             :   void movb(const Operand& dst, Immediate imm);
     685             : 
     686             :   // Move the low 16 bits of a 64-bit register value to a 16-bit
     687             :   // memory location.
     688             :   void movw(Register dst, const Operand& src);
     689             :   void movw(const Operand& dst, Register src);
     690             :   void movw(const Operand& dst, Immediate imm);
     691             : 
     692             :   // Move the offset of the label location relative to the current
     693             :   // position (after the move) to the destination.
     694             :   void movl(const Operand& dst, Label* src);
     695             : 
     696             :   // Loads a pointer into a register with a relocation mode.
     697             :   void movp(Register dst, void* ptr, RelocInfo::Mode rmode);
     698             : 
     699             :   // Loads a 64-bit immediate into a register.
     700             :   void movq(Register dst, int64_t value,
     701             :             RelocInfo::Mode rmode = RelocInfo::NONE64);
     702             :   void movq(Register dst, uint64_t value,
     703             :             RelocInfo::Mode rmode = RelocInfo::NONE64);
     704             : 
     705             :   void movsxbl(Register dst, Register src);
     706             :   void movsxbl(Register dst, const Operand& src);
     707             :   void movsxbq(Register dst, Register src);
     708             :   void movsxbq(Register dst, const Operand& src);
     709             :   void movsxwl(Register dst, Register src);
     710             :   void movsxwl(Register dst, const Operand& src);
     711             :   void movsxwq(Register dst, Register src);
     712             :   void movsxwq(Register dst, const Operand& src);
     713             :   void movsxlq(Register dst, Register src);
     714             :   void movsxlq(Register dst, const Operand& src);
     715             : 
     716             :   // Repeated moves.
     717             : 
     718             :   void repmovsb();
     719             :   void repmovsw();
     720             :   void repmovsp() { emit_repmovs(kPointerSize); }
     721             :   void repmovsl() { emit_repmovs(kInt32Size); }
     722             :   void repmovsq() { emit_repmovs(kInt64Size); }
     723             : 
     724             :   // Instruction to load from an immediate 64-bit pointer into RAX.
     725             :   void load_rax(void* ptr, RelocInfo::Mode rmode);
     726             :   void load_rax(ExternalReference ext);
     727             : 
     728             :   // Conditional moves.
     729             :   void cmovq(Condition cc, Register dst, Register src);
     730             :   void cmovq(Condition cc, Register dst, const Operand& src);
     731             :   void cmovl(Condition cc, Register dst, Register src);
     732             :   void cmovl(Condition cc, Register dst, const Operand& src);
     733             : 
     734             :   void cmpb(Register dst, Immediate src) {
     735       64580 :     immediate_arithmetic_op_8(0x7, dst, src);
     736             :   }
     737             : 
     738             :   void cmpb_al(Immediate src);
     739             : 
     740             :   void cmpb(Register dst, Register src) {
     741        4454 :     arithmetic_op_8(0x3A, dst, src);
     742             :   }
     743             : 
     744             :   void cmpb(Register dst, const Operand& src) {
     745       17427 :     arithmetic_op_8(0x3A, dst, src);
     746             :   }
     747             : 
     748             :   void cmpb(const Operand& dst, Register src) {
     749         101 :     arithmetic_op_8(0x38, src, dst);
     750             :   }
     751             : 
     752             :   void cmpb(const Operand& dst, Immediate src) {
     753      632585 :     immediate_arithmetic_op_8(0x7, dst, src);
     754             :   }
     755             : 
     756             :   void cmpw(const Operand& dst, Immediate src) {
     757           9 :     immediate_arithmetic_op_16(0x7, dst, src);
     758             :   }
     759             : 
     760             :   void cmpw(Register dst, Immediate src) {
     761           0 :     immediate_arithmetic_op_16(0x7, dst, src);
     762             :   }
     763             : 
     764             :   void cmpw(Register dst, const Operand& src) {
     765          92 :     arithmetic_op_16(0x3B, dst, src);
     766             :   }
     767             : 
     768             :   void cmpw(Register dst, Register src) {
     769           0 :     arithmetic_op_16(0x3B, dst, src);
     770             :   }
     771             : 
     772             :   void cmpw(const Operand& dst, Register src) {
     773          15 :     arithmetic_op_16(0x39, src, dst);
     774             :   }
     775             : 
     776           0 :   void testb(Register reg, const Operand& op) { testb(op, reg); }
     777             : 
     778           0 :   void testw(Register reg, const Operand& op) { testw(op, reg); }
     779             : 
     780             :   void andb(Register dst, Immediate src) {
     781       36713 :     immediate_arithmetic_op_8(0x4, dst, src);
     782             :   }
     783             : 
     784             :   void decb(Register dst);
     785             :   void decb(const Operand& dst);
     786             : 
     787             :   // Lock prefix.
     788             :   void lock();
     789             : 
     790             :   void xchgb(Register reg, const Operand& op);
     791             :   void xchgw(Register reg, const Operand& op);
     792             : 
     793             :   void cmpxchgb(const Operand& dst, Register src);
     794             :   void cmpxchgw(const Operand& dst, Register src);
     795             : 
     796             :   // Sign-extends rax into rdx:rax.
     797             :   void cqo();
     798             :   // Sign-extends eax into edx:eax.
     799             :   void cdq();
     800             : 
     801             :   // Multiply eax by src, put the result in edx:eax.
     802             :   void mull(Register src);
     803             :   void mull(const Operand& src);
     804             :   // Multiply rax by src, put the result in rdx:rax.
     805             :   void mulq(Register src);
     806             : 
     807             : #define DECLARE_SHIFT_INSTRUCTION(instruction, subcode)                       \
     808             :   void instruction##p(Register dst, Immediate imm8) {                         \
     809             :     shift(dst, imm8, subcode, kPointerSize);                                  \
     810             :   }                                                                           \
     811             :                                                                               \
     812             :   void instruction##l(Register dst, Immediate imm8) {                         \
     813             :     shift(dst, imm8, subcode, kInt32Size);                                    \
     814             :   }                                                                           \
     815             :                                                                               \
     816             :   void instruction##q(Register dst, Immediate imm8) {                         \
     817             :     shift(dst, imm8, subcode, kInt64Size);                                    \
     818             :   }                                                                           \
     819             :                                                                               \
     820             :   void instruction##p(Operand dst, Immediate imm8) {                          \
     821             :     shift(dst, imm8, subcode, kPointerSize);                                  \
     822             :   }                                                                           \
     823             :                                                                               \
     824             :   void instruction##l(Operand dst, Immediate imm8) {                          \
     825             :     shift(dst, imm8, subcode, kInt32Size);                                    \
     826             :   }                                                                           \
     827             :                                                                               \
     828             :   void instruction##q(Operand dst, Immediate imm8) {                          \
     829             :     shift(dst, imm8, subcode, kInt64Size);                                    \
     830             :   }                                                                           \
     831             :                                                                               \
     832             :   void instruction##p_cl(Register dst) { shift(dst, subcode, kPointerSize); } \
     833             :                                                                               \
     834             :   void instruction##l_cl(Register dst) { shift(dst, subcode, kInt32Size); }   \
     835             :                                                                               \
     836             :   void instruction##q_cl(Register dst) { shift(dst, subcode, kInt64Size); }   \
     837             :                                                                               \
     838             :   void instruction##p_cl(Operand dst) { shift(dst, subcode, kPointerSize); }  \
     839             :                                                                               \
     840             :   void instruction##l_cl(Operand dst) { shift(dst, subcode, kInt32Size); }    \
     841             :                                                                               \
     842             :   void instruction##q_cl(Operand dst) { shift(dst, subcode, kInt64Size); }
     843     1947090 :   SHIFT_INSTRUCTION_LIST(DECLARE_SHIFT_INSTRUCTION)
     844             : #undef DECLARE_SHIFT_INSTRUCTION
     845             : 
     846             :   // Shifts dst:src left by cl bits, affecting only dst.
     847             :   void shld(Register dst, Register src);
     848             : 
     849             :   // Shifts src:dst right by cl bits, affecting only dst.
     850             :   void shrd(Register dst, Register src);
     851             : 
     852             :   void store_rax(void* dst, RelocInfo::Mode mode);
     853             :   void store_rax(ExternalReference ref);
     854             : 
     855             :   void subb(Register dst, Immediate src) {
     856        3930 :     immediate_arithmetic_op_8(0x5, dst, src);
     857             :   }
     858             : 
     859             :   void testb(Register dst, Register src);
     860             :   void testb(Register reg, Immediate mask);
     861             :   void testb(const Operand& op, Immediate mask);
     862             :   void testb(const Operand& op, Register reg);
     863             : 
     864             :   void testw(Register dst, Register src);
     865             :   void testw(Register reg, Immediate mask);
     866             :   void testw(const Operand& op, Immediate mask);
     867             :   void testw(const Operand& op, Register reg);
     868             : 
     869             :   // Bit operations.
     870             :   void bt(const Operand& dst, Register src);
     871             :   void bts(const Operand& dst, Register src);
     872             :   void bsrq(Register dst, Register src);
     873             :   void bsrq(Register dst, const Operand& src);
     874             :   void bsrl(Register dst, Register src);
     875             :   void bsrl(Register dst, const Operand& src);
     876             :   void bsfq(Register dst, Register src);
     877             :   void bsfq(Register dst, const Operand& src);
     878             :   void bsfl(Register dst, Register src);
     879             :   void bsfl(Register dst, const Operand& src);
     880             : 
     881             :   // Miscellaneous
     882             :   void clc();
     883             :   void cld();
     884             :   void cpuid();
     885             :   void hlt();
     886             :   void int3();
     887             :   void nop();
     888             :   void ret(int imm16);
     889             :   void ud2();
     890             :   void setcc(Condition cc, Register reg);
     891             : 
     892             :   void pshufw(XMMRegister dst, XMMRegister src, uint8_t shuffle);
     893             :   void pshufw(XMMRegister dst, const Operand& src, uint8_t shuffle);
     894             : 
     895             :   // Label operations & relative jumps (PPUM Appendix D)
     896             :   //
     897             :   // Takes a branch opcode (cc) and a label (L) and generates
     898             :   // either a backward branch or a forward branch and links it
     899             :   // to the label fixup chain. Usage:
     900             :   //
     901             :   // Label L;    // unbound label
     902             :   // j(cc, &L);  // forward branch to unbound label
     903             :   // bind(&L);   // bind label to the current pc
     904             :   // j(cc, &L);  // backward branch to bound label
     905             :   // bind(&L);   // illegal: a label may be bound only once
     906             :   //
     907             :   // Note: The same Label can be used for forward and backward branches
     908             :   // but it may be bound only once.
     909             : 
     910             :   void bind(Label* L);  // binds an unbound label L to the current code position
     911             : 
     912             :   // Calls
     913             :   // Call near relative 32-bit displacement, relative to next instruction.
     914             :   void call(Label* L);
     915             :   void call(Address entry, RelocInfo::Mode rmode);
     916             :   void call(Handle<Code> target,
     917             :             RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
     918             :             TypeFeedbackId ast_id = TypeFeedbackId::None());
     919             : 
     920             :   // Calls directly to the given address using a relative offset.
     921             :   // Should only ever be used in Code objects for calls within the
     922             :   // same Code object. Should not be used when generating new code (use labels),
     923             :   // but only when patching existing code.
     924             :   void call(Address target);
     925             : 
     926             :   // Call near absolute indirect, address in register
     927             :   void call(Register adr);
     928             : 
     929             :   // Jumps
     930             :   // Jump short or near relative.
     931             :   // Use a 32-bit signed displacement.
     932             :   // Unconditional jump to L
     933             :   void jmp(Label* L, Label::Distance distance = Label::kFar);
     934             :   void jmp(Handle<Code> target, RelocInfo::Mode rmode);
     935             : 
     936             :   // Jump near absolute indirect (r64)
     937             :   void jmp(Register adr);
     938             :   void jmp(const Operand& src);
     939             : 
     940             :   // Conditional jumps
     941             :   void j(Condition cc,
     942             :          Label* L,
     943             :          Label::Distance distance = Label::kFar);
     944             :   void j(Condition cc, Address entry, RelocInfo::Mode rmode);
     945             :   void j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode);
     946             : 
     947             :   // Floating-point operations
     948             :   void fld(int i);
     949             : 
     950             :   void fld1();
     951             :   void fldz();
     952             :   void fldpi();
     953             :   void fldln2();
     954             : 
     955             :   void fld_s(const Operand& adr);
     956             :   void fld_d(const Operand& adr);
     957             : 
     958             :   void fstp_s(const Operand& adr);
     959             :   void fstp_d(const Operand& adr);
     960             :   void fstp(int index);
     961             : 
     962             :   void fild_s(const Operand& adr);
     963             :   void fild_d(const Operand& adr);
     964             : 
     965             :   void fist_s(const Operand& adr);
     966             : 
     967             :   void fistp_s(const Operand& adr);
     968             :   void fistp_d(const Operand& adr);
     969             : 
     970             :   void fisttp_s(const Operand& adr);
     971             :   void fisttp_d(const Operand& adr);
     972             : 
     973             :   void fabs();
     974             :   void fchs();
     975             : 
     976             :   void fadd(int i);
     977             :   void fsub(int i);
     978             :   void fmul(int i);
     979             :   void fdiv(int i);
     980             : 
     981             :   void fisub_s(const Operand& adr);
     982             : 
     983             :   void faddp(int i = 1);
     984             :   void fsubp(int i = 1);
     985             :   void fsubrp(int i = 1);
     986             :   void fmulp(int i = 1);
     987             :   void fdivp(int i = 1);
     988             :   void fprem();
     989             :   void fprem1();
     990             : 
     991             :   void fxch(int i = 1);
     992             :   void fincstp();
     993             :   void ffree(int i = 0);
     994             : 
     995             :   void ftst();
     996             :   void fucomp(int i);
     997             :   void fucompp();
     998             :   void fucomi(int i);
     999             :   void fucomip();
    1000             : 
    1001             :   void fcompp();
    1002             :   void fnstsw_ax();
    1003             :   void fwait();
    1004             :   void fnclex();
    1005             : 
    1006             :   void fsin();
    1007             :   void fcos();
    1008             :   void fptan();
    1009             :   void fyl2x();
    1010             :   void f2xm1();
    1011             :   void fscale();
    1012             :   void fninit();
    1013             : 
    1014             :   void frndint();
    1015             : 
    1016             :   void sahf();
    1017             : 
    1018             :   // SSE instructions
    1019             :   void addss(XMMRegister dst, XMMRegister src);
    1020             :   void addss(XMMRegister dst, const Operand& src);
    1021             :   void subss(XMMRegister dst, XMMRegister src);
    1022             :   void subss(XMMRegister dst, const Operand& src);
    1023             :   void mulss(XMMRegister dst, XMMRegister src);
    1024             :   void mulss(XMMRegister dst, const Operand& src);
    1025             :   void divss(XMMRegister dst, XMMRegister src);
    1026             :   void divss(XMMRegister dst, const Operand& src);
    1027             : 
    1028             :   void maxss(XMMRegister dst, XMMRegister src);
    1029             :   void maxss(XMMRegister dst, const Operand& src);
    1030             :   void minss(XMMRegister dst, XMMRegister src);
    1031             :   void minss(XMMRegister dst, const Operand& src);
    1032             : 
    1033             :   void sqrtss(XMMRegister dst, XMMRegister src);
    1034             :   void sqrtss(XMMRegister dst, const Operand& src);
    1035             : 
    1036             :   void ucomiss(XMMRegister dst, XMMRegister src);
    1037             :   void ucomiss(XMMRegister dst, const Operand& src);
    1038             :   void movaps(XMMRegister dst, XMMRegister src);
    1039             : 
    1040             :   // Don't use this unless it's important to keep the
    1041             :   // top half of the destination register unchanged.
    1042             :   // Use movaps when moving float values and movd for integer
    1043             :   // values in xmm registers.
    1044             :   void movss(XMMRegister dst, XMMRegister src);
    1045             : 
    1046             :   void movss(XMMRegister dst, const Operand& src);
    1047             :   void movss(const Operand& dst, XMMRegister src);
    1048             :   void shufps(XMMRegister dst, XMMRegister src, byte imm8);
    1049             : 
    1050             :   void cvttss2si(Register dst, const Operand& src);
    1051             :   void cvttss2si(Register dst, XMMRegister src);
    1052             :   void cvtlsi2ss(XMMRegister dst, const Operand& src);
    1053             :   void cvtlsi2ss(XMMRegister dst, Register src);
    1054             : 
    1055             :   void andps(XMMRegister dst, XMMRegister src);
    1056             :   void andps(XMMRegister dst, const Operand& src);
    1057             :   void orps(XMMRegister dst, XMMRegister src);
    1058             :   void orps(XMMRegister dst, const Operand& src);
    1059             :   void xorps(XMMRegister dst, XMMRegister src);
    1060             :   void xorps(XMMRegister dst, const Operand& src);
    1061             : 
    1062             :   void addps(XMMRegister dst, XMMRegister src);
    1063             :   void addps(XMMRegister dst, const Operand& src);
    1064             :   void subps(XMMRegister dst, XMMRegister src);
    1065             :   void subps(XMMRegister dst, const Operand& src);
    1066             :   void mulps(XMMRegister dst, XMMRegister src);
    1067             :   void mulps(XMMRegister dst, const Operand& src);
    1068             :   void divps(XMMRegister dst, XMMRegister src);
    1069             :   void divps(XMMRegister dst, const Operand& src);
    1070             : 
    1071             :   void movmskps(Register dst, XMMRegister src);
    1072             : 
    1073             :   void vinstr(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2,
    1074             :               SIMDPrefix pp, LeadingOpcode m, VexW w);
    1075             :   void vinstr(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2,
    1076             :               SIMDPrefix pp, LeadingOpcode m, VexW w);
    1077             : 
    1078             :   // SSE2 instructions
    1079             :   void sse2_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape,
    1080             :                   byte opcode);
    1081             :   void sse2_instr(XMMRegister dst, const Operand& src, byte prefix, byte escape,
    1082             :                   byte opcode);
    1083             : #define DECLARE_SSE2_INSTRUCTION(instruction, prefix, escape, opcode) \
    1084             :   void instruction(XMMRegister dst, XMMRegister src) {                \
    1085             :     sse2_instr(dst, src, 0x##prefix, 0x##escape, 0x##opcode);         \
    1086             :   }                                                                   \
    1087             :   void instruction(XMMRegister dst, const Operand& src) {             \
    1088             :     sse2_instr(dst, src, 0x##prefix, 0x##escape, 0x##opcode);         \
    1089             :   }
    1090             : 
    1091         323 :   SSE2_INSTRUCTION_LIST(DECLARE_SSE2_INSTRUCTION)
    1092             : #undef DECLARE_SSE2_INSTRUCTION
    1093             : 
    1094             : #define DECLARE_SSE2_AVX_INSTRUCTION(instruction, prefix, escape, opcode)    \
    1095             :   void v##instruction(XMMRegister dst, XMMRegister src1, XMMRegister src2) { \
    1096             :     vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kW0);          \
    1097             :   }                                                                          \
    1098             :   void v##instruction(XMMRegister dst, XMMRegister src1,                     \
    1099             :                       const Operand& src2) {                                 \
    1100             :     vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape, kW0);          \
    1101             :   }
    1102             : 
    1103       36135 :   SSE2_INSTRUCTION_LIST(DECLARE_SSE2_AVX_INSTRUCTION)
    1104             : #undef DECLARE_SSE2_AVX_INSTRUCTION
    1105             : 
    1106             :   // SSE3
    1107             :   void lddqu(XMMRegister dst, const Operand& src);
    1108             : 
    1109             :   // SSSE3
    1110             :   void ssse3_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape1,
    1111             :                    byte escape2, byte opcode);
    1112             :   void ssse3_instr(XMMRegister dst, const Operand& src, byte prefix,
    1113             :                    byte escape1, byte escape2, byte opcode);
    1114             : 
    1115             : #define DECLARE_SSSE3_INSTRUCTION(instruction, prefix, escape1, escape2,     \
    1116             :                                   opcode)                                    \
    1117             :   void instruction(XMMRegister dst, XMMRegister src) {                       \
    1118             :     ssse3_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
    1119             :   }                                                                          \
    1120             :   void instruction(XMMRegister dst, const Operand& src) {                    \
    1121             :     ssse3_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
    1122             :   }
    1123             : 
    1124         231 :   SSSE3_INSTRUCTION_LIST(DECLARE_SSSE3_INSTRUCTION)
    1125             : #undef DECLARE_SSSE3_INSTRUCTION
    1126             : 
    1127             :   // SSE4
    1128             :   void sse4_instr(XMMRegister dst, XMMRegister src, byte prefix, byte escape1,
    1129             :                   byte escape2, byte opcode);
    1130             :   void sse4_instr(XMMRegister dst, const Operand& src, byte prefix,
    1131             :                   byte escape1, byte escape2, byte opcode);
    1132             : #define DECLARE_SSE4_INSTRUCTION(instruction, prefix, escape1, escape2,     \
    1133             :                                  opcode)                                    \
    1134             :   void instruction(XMMRegister dst, XMMRegister src) {                      \
    1135             :     sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
    1136             :   }                                                                         \
    1137             :   void instruction(XMMRegister dst, const Operand& src) {                   \
    1138             :     sse4_instr(dst, src, 0x##prefix, 0x##escape1, 0x##escape2, 0x##opcode); \
    1139             :   }
    1140             : 
    1141          63 :   SSE4_INSTRUCTION_LIST(DECLARE_SSE4_INSTRUCTION)
    1142             : #undef DECLARE_SSE4_INSTRUCTION
    1143             : 
    1144             : #define DECLARE_SSE34_AVX_INSTRUCTION(instruction, prefix, escape1, escape2,  \
    1145             :                                       opcode)                                 \
    1146             :   void v##instruction(XMMRegister dst, XMMRegister src1, XMMRegister src2) {  \
    1147             :     vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape1##escape2, kW0); \
    1148             :   }                                                                           \
    1149             :   void v##instruction(XMMRegister dst, XMMRegister src1,                      \
    1150             :                       const Operand& src2) {                                  \
    1151             :     vinstr(0x##opcode, dst, src1, src2, k##prefix, k##escape1##escape2, kW0); \
    1152             :   }
    1153             : 
    1154             :   SSSE3_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
    1155             :   SSE4_INSTRUCTION_LIST(DECLARE_SSE34_AVX_INSTRUCTION)
    1156             : #undef DECLARE_SSE34_AVX_INSTRUCTION
    1157             : 
    1158             :   void movd(XMMRegister dst, Register src);
    1159             :   void movd(XMMRegister dst, const Operand& src);
    1160             :   void movd(Register dst, XMMRegister src);
    1161             :   void movq(XMMRegister dst, Register src);
    1162             :   void movq(Register dst, XMMRegister src);
    1163             :   void movq(XMMRegister dst, XMMRegister src);
    1164             : 
    1165             :   // Don't use this unless it's important to keep the
    1166             :   // top half of the destination register unchanged.
    1167             :   // Use movapd when moving double values and movq for integer
    1168             :   // values in xmm registers.
    1169             :   void movsd(XMMRegister dst, XMMRegister src);
    1170             : 
    1171             :   void movsd(const Operand& dst, XMMRegister src);
    1172             :   void movsd(XMMRegister dst, const Operand& src);
    1173             : 
    1174             :   void movdqa(const Operand& dst, XMMRegister src);
    1175             :   void movdqa(XMMRegister dst, const Operand& src);
    1176             : 
    1177             :   void movdqu(const Operand& dst, XMMRegister src);
    1178             :   void movdqu(XMMRegister dst, const Operand& src);
    1179             : 
    1180             :   void movapd(XMMRegister dst, XMMRegister src);
    1181             :   void movupd(XMMRegister dst, const Operand& src);
    1182             :   void movupd(const Operand& dst, XMMRegister src);
    1183             : 
    1184             :   void psllq(XMMRegister reg, byte imm8);
    1185             :   void psrlq(XMMRegister reg, byte imm8);
    1186             :   void psllw(XMMRegister reg, byte imm8);
    1187             :   void pslld(XMMRegister reg, byte imm8);
    1188             :   void psrlw(XMMRegister reg, byte imm8);
    1189             :   void psrld(XMMRegister reg, byte imm8);
    1190             :   void psraw(XMMRegister reg, byte imm8);
    1191             :   void psrad(XMMRegister reg, byte imm8);
    1192             : 
    1193             :   void cvttsd2si(Register dst, const Operand& src);
    1194             :   void cvttsd2si(Register dst, XMMRegister src);
    1195             :   void cvttss2siq(Register dst, XMMRegister src);
    1196             :   void cvttss2siq(Register dst, const Operand& src);
    1197             :   void cvttsd2siq(Register dst, XMMRegister src);
    1198             :   void cvttsd2siq(Register dst, const Operand& src);
    1199             : 
    1200             :   void cvtlsi2sd(XMMRegister dst, const Operand& src);
    1201             :   void cvtlsi2sd(XMMRegister dst, Register src);
    1202             : 
    1203             :   void cvtqsi2ss(XMMRegister dst, const Operand& src);
    1204             :   void cvtqsi2ss(XMMRegister dst, Register src);
    1205             : 
    1206             :   void cvtqsi2sd(XMMRegister dst, const Operand& src);
    1207             :   void cvtqsi2sd(XMMRegister dst, Register src);
    1208             : 
    1209             : 
    1210             :   void cvtss2sd(XMMRegister dst, XMMRegister src);
    1211             :   void cvtss2sd(XMMRegister dst, const Operand& src);
    1212             :   void cvtsd2ss(XMMRegister dst, XMMRegister src);
    1213             :   void cvtsd2ss(XMMRegister dst, const Operand& src);
    1214             : 
    1215             :   void cvtsd2si(Register dst, XMMRegister src);
    1216             :   void cvtsd2siq(Register dst, XMMRegister src);
    1217             : 
    1218             :   void addsd(XMMRegister dst, XMMRegister src);
    1219             :   void addsd(XMMRegister dst, const Operand& src);
    1220             :   void subsd(XMMRegister dst, XMMRegister src);
    1221             :   void subsd(XMMRegister dst, const Operand& src);
    1222             :   void mulsd(XMMRegister dst, XMMRegister src);
    1223             :   void mulsd(XMMRegister dst, const Operand& src);
    1224             :   void divsd(XMMRegister dst, XMMRegister src);
    1225             :   void divsd(XMMRegister dst, const Operand& src);
    1226             : 
    1227             :   void maxsd(XMMRegister dst, XMMRegister src);
    1228             :   void maxsd(XMMRegister dst, const Operand& src);
    1229             :   void minsd(XMMRegister dst, XMMRegister src);
    1230             :   void minsd(XMMRegister dst, const Operand& src);
    1231             : 
    1232             :   void andpd(XMMRegister dst, XMMRegister src);
    1233             :   void andpd(XMMRegister dst, const Operand& src);
    1234             :   void orpd(XMMRegister dst, XMMRegister src);
    1235             :   void orpd(XMMRegister dst, const Operand& src);
    1236             :   void xorpd(XMMRegister dst, XMMRegister src);
    1237             :   void xorpd(XMMRegister dst, const Operand& src);
    1238             :   void sqrtsd(XMMRegister dst, XMMRegister src);
    1239             :   void sqrtsd(XMMRegister dst, const Operand& src);
    1240             : 
    1241             :   void ucomisd(XMMRegister dst, XMMRegister src);
    1242             :   void ucomisd(XMMRegister dst, const Operand& src);
    1243             :   void cmpltsd(XMMRegister dst, XMMRegister src);
    1244             : 
    1245             :   void movmskpd(Register dst, XMMRegister src);
    1246             : 
    1247             :   void punpckldq(XMMRegister dst, XMMRegister src);
    1248             :   void punpckldq(XMMRegister dst, const Operand& src);
    1249             :   void punpckhdq(XMMRegister dst, XMMRegister src);
    1250             : 
    1251             :   // SSE 4.1 instruction
    1252             :   void insertps(XMMRegister dst, XMMRegister src, byte imm8);
    1253             :   void extractps(Register dst, XMMRegister src, byte imm8);
    1254             :   void pextrb(Register dst, XMMRegister src, int8_t imm8);
    1255             :   void pextrb(const Operand& dst, XMMRegister src, int8_t imm8);
    1256             :   void pextrw(Register dst, XMMRegister src, int8_t imm8);
    1257             :   void pextrw(const Operand& dst, XMMRegister src, int8_t imm8);
    1258             :   void pextrd(Register dst, XMMRegister src, int8_t imm8);
    1259             :   void pextrd(const Operand& dst, XMMRegister src, int8_t imm8);
    1260             :   void pinsrb(XMMRegister dst, Register src, int8_t imm8);
    1261             :   void pinsrb(XMMRegister dst, const Operand& src, int8_t imm8);
    1262             :   void pinsrw(XMMRegister dst, Register src, int8_t imm8);
    1263             :   void pinsrw(XMMRegister dst, const Operand& src, int8_t imm8);
    1264             :   void pinsrd(XMMRegister dst, Register src, int8_t imm8);
    1265             :   void pinsrd(XMMRegister dst, const Operand& src, int8_t imm8);
    1266             : 
    1267             :   void roundss(XMMRegister dst, XMMRegister src, RoundingMode mode);
    1268             :   void roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode);
    1269             : 
    1270             :   void cmpps(XMMRegister dst, XMMRegister src, int8_t cmp);
    1271             :   void cmpps(XMMRegister dst, const Operand& src, int8_t cmp);
    1272             :   void cmppd(XMMRegister dst, XMMRegister src, int8_t cmp);
    1273             :   void cmppd(XMMRegister dst, const Operand& src, int8_t cmp);
    1274             : 
    1275             : #define SSE_CMP_P(instr, imm8)                                                \
    1276             :   void instr##ps(XMMRegister dst, XMMRegister src) { cmpps(dst, src, imm8); } \
    1277             :   void instr##ps(XMMRegister dst, const Operand& src) {                       \
    1278             :     cmpps(dst, src, imm8);                                                    \
    1279             :   }                                                                           \
    1280             :   void instr##pd(XMMRegister dst, XMMRegister src) { cmppd(dst, src, imm8); } \
    1281             :   void instr##pd(XMMRegister dst, const Operand& src) { cmppd(dst, src, imm8); }
    1282             : 
    1283             :   SSE_CMP_P(cmpeq, 0x0);
    1284             :   SSE_CMP_P(cmplt, 0x1);
    1285             :   SSE_CMP_P(cmple, 0x2);
    1286             :   SSE_CMP_P(cmpneq, 0x4);
    1287             :   SSE_CMP_P(cmpnlt, 0x5);
    1288             :   SSE_CMP_P(cmpnle, 0x6);
    1289             : 
    1290             : #undef SSE_CMP_P
    1291             : 
    1292             :   void minps(XMMRegister dst, XMMRegister src);
    1293             :   void minps(XMMRegister dst, const Operand& src);
    1294             :   void maxps(XMMRegister dst, XMMRegister src);
    1295             :   void maxps(XMMRegister dst, const Operand& src);
    1296             :   void rcpps(XMMRegister dst, XMMRegister src);
    1297             :   void rcpps(XMMRegister dst, const Operand& src);
    1298             :   void rsqrtps(XMMRegister dst, XMMRegister src);
    1299             :   void rsqrtps(XMMRegister dst, const Operand& src);
    1300             :   void sqrtps(XMMRegister dst, XMMRegister src);
    1301             :   void sqrtps(XMMRegister dst, const Operand& src);
    1302             :   void movups(XMMRegister dst, XMMRegister src);
    1303             :   void movups(XMMRegister dst, const Operand& src);
    1304             :   void movups(const Operand& dst, XMMRegister src);
    1305             :   void psrldq(XMMRegister dst, uint8_t shift);
    1306             :   void pshufd(XMMRegister dst, XMMRegister src, uint8_t shuffle);
    1307             :   void pshufd(XMMRegister dst, const Operand& src, uint8_t shuffle);
    1308             :   void pshufhw(XMMRegister dst, XMMRegister src, uint8_t shuffle);
    1309             :   void pshuflw(XMMRegister dst, XMMRegister src, uint8_t shuffle);
    1310             :   void cvtdq2ps(XMMRegister dst, XMMRegister src);
    1311             :   void cvtdq2ps(XMMRegister dst, const Operand& src);
    1312             : 
    1313             :   // AVX instruction
    1314             :   void vfmadd132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1315             :     vfmasd(0x99, dst, src1, src2);
    1316             :   }
    1317             :   void vfmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1318             :     vfmasd(0xa9, dst, src1, src2);
    1319             :   }
    1320             :   void vfmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1321             :     vfmasd(0xb9, dst, src1, src2);
    1322             :   }
    1323             :   void vfmadd132sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1324             :     vfmasd(0x99, dst, src1, src2);
    1325             :   }
    1326             :   void vfmadd213sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1327             :     vfmasd(0xa9, dst, src1, src2);
    1328             :   }
    1329             :   void vfmadd231sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1330             :     vfmasd(0xb9, dst, src1, src2);
    1331             :   }
    1332             :   void vfmsub132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1333             :     vfmasd(0x9b, dst, src1, src2);
    1334             :   }
    1335             :   void vfmsub213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1336             :     vfmasd(0xab, dst, src1, src2);
    1337             :   }
    1338             :   void vfmsub231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1339             :     vfmasd(0xbb, dst, src1, src2);
    1340             :   }
    1341             :   void vfmsub132sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1342             :     vfmasd(0x9b, dst, src1, src2);
    1343             :   }
    1344             :   void vfmsub213sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1345             :     vfmasd(0xab, dst, src1, src2);
    1346             :   }
    1347             :   void vfmsub231sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1348             :     vfmasd(0xbb, dst, src1, src2);
    1349             :   }
    1350             :   void vfnmadd132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1351             :     vfmasd(0x9d, dst, src1, src2);
    1352             :   }
    1353             :   void vfnmadd213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1354             :     vfmasd(0xad, dst, src1, src2);
    1355             :   }
    1356             :   void vfnmadd231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1357             :     vfmasd(0xbd, dst, src1, src2);
    1358             :   }
    1359             :   void vfnmadd132sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1360             :     vfmasd(0x9d, dst, src1, src2);
    1361             :   }
    1362             :   void vfnmadd213sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1363             :     vfmasd(0xad, dst, src1, src2);
    1364             :   }
    1365             :   void vfnmadd231sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1366             :     vfmasd(0xbd, dst, src1, src2);
    1367             :   }
    1368             :   void vfnmsub132sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1369             :     vfmasd(0x9f, dst, src1, src2);
    1370             :   }
    1371             :   void vfnmsub213sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1372             :     vfmasd(0xaf, dst, src1, src2);
    1373             :   }
    1374             :   void vfnmsub231sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1375             :     vfmasd(0xbf, dst, src1, src2);
    1376             :   }
    1377             :   void vfnmsub132sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1378             :     vfmasd(0x9f, dst, src1, src2);
    1379             :   }
    1380             :   void vfnmsub213sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1381             :     vfmasd(0xaf, dst, src1, src2);
    1382             :   }
    1383             :   void vfnmsub231sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1384             :     vfmasd(0xbf, dst, src1, src2);
    1385             :   }
    1386             :   void vfmasd(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
    1387             :   void vfmasd(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2);
    1388             : 
    1389             :   void vfmadd132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1390             :     vfmass(0x99, dst, src1, src2);
    1391             :   }
    1392             :   void vfmadd213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1393             :     vfmass(0xa9, dst, src1, src2);
    1394             :   }
    1395             :   void vfmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1396             :     vfmass(0xb9, dst, src1, src2);
    1397             :   }
    1398             :   void vfmadd132ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1399             :     vfmass(0x99, dst, src1, src2);
    1400             :   }
    1401             :   void vfmadd213ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1402             :     vfmass(0xa9, dst, src1, src2);
    1403             :   }
    1404             :   void vfmadd231ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1405             :     vfmass(0xb9, dst, src1, src2);
    1406             :   }
    1407             :   void vfmsub132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1408             :     vfmass(0x9b, dst, src1, src2);
    1409             :   }
    1410             :   void vfmsub213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1411             :     vfmass(0xab, dst, src1, src2);
    1412             :   }
    1413             :   void vfmsub231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1414             :     vfmass(0xbb, dst, src1, src2);
    1415             :   }
    1416             :   void vfmsub132ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1417             :     vfmass(0x9b, dst, src1, src2);
    1418             :   }
    1419             :   void vfmsub213ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1420             :     vfmass(0xab, dst, src1, src2);
    1421             :   }
    1422             :   void vfmsub231ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1423             :     vfmass(0xbb, dst, src1, src2);
    1424             :   }
    1425             :   void vfnmadd132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1426             :     vfmass(0x9d, dst, src1, src2);
    1427             :   }
    1428             :   void vfnmadd213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1429             :     vfmass(0xad, dst, src1, src2);
    1430             :   }
    1431             :   void vfnmadd231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1432             :     vfmass(0xbd, dst, src1, src2);
    1433             :   }
    1434             :   void vfnmadd132ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1435             :     vfmass(0x9d, dst, src1, src2);
    1436             :   }
    1437             :   void vfnmadd213ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1438             :     vfmass(0xad, dst, src1, src2);
    1439             :   }
    1440             :   void vfnmadd231ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1441             :     vfmass(0xbd, dst, src1, src2);
    1442             :   }
    1443             :   void vfnmsub132ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1444             :     vfmass(0x9f, dst, src1, src2);
    1445             :   }
    1446             :   void vfnmsub213ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1447             :     vfmass(0xaf, dst, src1, src2);
    1448             :   }
    1449             :   void vfnmsub231ss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1450             :     vfmass(0xbf, dst, src1, src2);
    1451             :   }
    1452             :   void vfnmsub132ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1453             :     vfmass(0x9f, dst, src1, src2);
    1454             :   }
    1455             :   void vfnmsub213ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1456             :     vfmass(0xaf, dst, src1, src2);
    1457             :   }
    1458             :   void vfnmsub231ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1459             :     vfmass(0xbf, dst, src1, src2);
    1460             :   }
    1461             :   void vfmass(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
    1462             :   void vfmass(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2);
    1463             : 
    1464             :   void vmovd(XMMRegister dst, Register src);
    1465             :   void vmovd(XMMRegister dst, const Operand& src);
    1466             :   void vmovd(Register dst, XMMRegister src);
    1467             :   void vmovq(XMMRegister dst, Register src);
    1468             :   void vmovq(XMMRegister dst, const Operand& src);
    1469             :   void vmovq(Register dst, XMMRegister src);
    1470             : 
    1471        1122 :   void vmovsd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1472             :     vsd(0x10, dst, src1, src2);
    1473        1122 :   }
    1474     3042962 :   void vmovsd(XMMRegister dst, const Operand& src) {
    1475             :     vsd(0x10, dst, xmm0, src);
    1476     3042962 :   }
    1477     2714252 :   void vmovsd(const Operand& dst, XMMRegister src) {
    1478             :     vsd(0x11, src, xmm0, dst);
    1479     2714252 :   }
    1480             : 
    1481             : #define AVX_SP_3(instr, opcode) \
    1482             :   AVX_S_3(instr, opcode)        \
    1483             :   AVX_P_3(instr, opcode)
    1484             : 
    1485             : #define AVX_S_3(instr, opcode)  \
    1486             :   AVX_3(instr##ss, opcode, vss) \
    1487             :   AVX_3(instr##sd, opcode, vsd)
    1488             : 
    1489             : #define AVX_P_3(instr, opcode)  \
    1490             :   AVX_3(instr##ps, opcode, vps) \
    1491             :   AVX_3(instr##pd, opcode, vpd)
    1492             : 
    1493             : #define AVX_3(instr, opcode, impl)                                     \
    1494             :   void instr(XMMRegister dst, XMMRegister src1, XMMRegister src2) {    \
    1495             :     impl(opcode, dst, src1, src2);                                     \
    1496             :   }                                                                    \
    1497             :   void instr(XMMRegister dst, XMMRegister src1, const Operand& src2) { \
    1498             :     impl(opcode, dst, src1, src2);                                     \
    1499             :   }
    1500             : 
    1501         722 :   AVX_SP_3(vsqrt, 0x51);
    1502      203662 :   AVX_SP_3(vadd, 0x58);
    1503       64858 :   AVX_SP_3(vsub, 0x5c);
    1504       57143 :   AVX_SP_3(vmul, 0x59);
    1505      156601 :   AVX_SP_3(vdiv, 0x5e);
    1506             :   AVX_SP_3(vmin, 0x5d);
    1507             :   AVX_SP_3(vmax, 0x5f);
    1508        1131 :   AVX_P_3(vand, 0x54);
    1509         714 :   AVX_P_3(vor, 0x56);
    1510      656962 :   AVX_P_3(vxor, 0x57);
    1511       17696 :   AVX_3(vcvtsd2ss, 0x5a, vsd);
    1512             : 
    1513             : #undef AVX_3
    1514             : #undef AVX_S_3
    1515             : #undef AVX_P_3
    1516             : #undef AVX_SP_3
    1517             : 
    1518             :   void vpsrlq(XMMRegister dst, XMMRegister src, byte imm8) {
    1519         322 :     XMMRegister iop = {2};
    1520         322 :     vpd(0x73, iop, dst, src);
    1521             :     emit(imm8);
    1522             :   }
    1523             :   void vpsllq(XMMRegister dst, XMMRegister src, byte imm8) {
    1524       26324 :     XMMRegister iop = {6};
    1525       26324 :     vpd(0x73, iop, dst, src);
    1526             :     emit(imm8);
    1527             :   }
    1528             :   void vcvtss2sd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1529       10295 :     vinstr(0x5a, dst, src1, src2, kF3, k0F, kWIG);
    1530             :   }
    1531             :   void vcvtss2sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1532        2110 :     vinstr(0x5a, dst, src1, src2, kF3, k0F, kWIG);
    1533             :   }
    1534             :   void vcvtlsi2sd(XMMRegister dst, XMMRegister src1, Register src2) {
    1535      405034 :     XMMRegister isrc2 = {src2.code()};
    1536      405034 :     vinstr(0x2a, dst, src1, isrc2, kF2, k0F, kW0);
    1537             :   }
    1538             :   void vcvtlsi2sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1539        8408 :     vinstr(0x2a, dst, src1, src2, kF2, k0F, kW0);
    1540             :   }
    1541             :   void vcvtlsi2ss(XMMRegister dst, XMMRegister src1, Register src2) {
    1542         309 :     XMMRegister isrc2 = {src2.code()};
    1543         309 :     vinstr(0x2a, dst, src1, isrc2, kF3, k0F, kW0);
    1544             :   }
    1545             :   void vcvtlsi2ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1546           0 :     vinstr(0x2a, dst, src1, src2, kF3, k0F, kW0);
    1547             :   }
    1548             :   void vcvtqsi2ss(XMMRegister dst, XMMRegister src1, Register src2) {
    1549          76 :     XMMRegister isrc2 = {src2.code()};
    1550          76 :     vinstr(0x2a, dst, src1, isrc2, kF3, k0F, kW1);
    1551             :   }
    1552             :   void vcvtqsi2ss(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1553           0 :     vinstr(0x2a, dst, src1, src2, kF3, k0F, kW1);
    1554             :   }
    1555             :   void vcvtqsi2sd(XMMRegister dst, XMMRegister src1, Register src2) {
    1556        4172 :     XMMRegister isrc2 = {src2.code()};
    1557        4172 :     vinstr(0x2a, dst, src1, isrc2, kF2, k0F, kW1);
    1558             :   }
    1559             :   void vcvtqsi2sd(XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1560           0 :     vinstr(0x2a, dst, src1, src2, kF2, k0F, kW1);
    1561             :   }
    1562         132 :   void vcvttss2si(Register dst, XMMRegister src) {
    1563         132 :     XMMRegister idst = {dst.code()};
    1564         132 :     vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW0);
    1565         132 :   }
    1566           0 :   void vcvttss2si(Register dst, const Operand& src) {
    1567           0 :     XMMRegister idst = {dst.code()};
    1568           0 :     vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW0);
    1569           0 :   }
    1570      148862 :   void vcvttsd2si(Register dst, XMMRegister src) {
    1571      148862 :     XMMRegister idst = {dst.code()};
    1572      148862 :     vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW0);
    1573      148862 :   }
    1574        8336 :   void vcvttsd2si(Register dst, const Operand& src) {
    1575        8336 :     XMMRegister idst = {dst.code()};
    1576        8336 :     vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW0);
    1577        8336 :   }
    1578          91 :   void vcvttss2siq(Register dst, XMMRegister src) {
    1579          91 :     XMMRegister idst = {dst.code()};
    1580          91 :     vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW1);
    1581          91 :   }
    1582           0 :   void vcvttss2siq(Register dst, const Operand& src) {
    1583           0 :     XMMRegister idst = {dst.code()};
    1584           0 :     vinstr(0x2c, idst, xmm0, src, kF3, k0F, kW1);
    1585           0 :   }
    1586      103201 :   void vcvttsd2siq(Register dst, XMMRegister src) {
    1587      103201 :     XMMRegister idst = {dst.code()};
    1588      103201 :     vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW1);
    1589      103201 :   }
    1590           0 :   void vcvttsd2siq(Register dst, const Operand& src) {
    1591           0 :     XMMRegister idst = {dst.code()};
    1592           0 :     vinstr(0x2c, idst, xmm0, src, kF2, k0F, kW1);
    1593           0 :   }
    1594         244 :   void vcvtsd2si(Register dst, XMMRegister src) {
    1595         244 :     XMMRegister idst = {dst.code()};
    1596         244 :     vinstr(0x2d, idst, xmm0, src, kF2, k0F, kW0);
    1597         244 :   }
    1598             :   void vucomisd(XMMRegister dst, XMMRegister src) {
    1599      299236 :     vinstr(0x2e, dst, xmm0, src, k66, k0F, kWIG);
    1600             :   }
    1601             :   void vucomisd(XMMRegister dst, const Operand& src) {
    1602       79244 :     vinstr(0x2e, dst, xmm0, src, k66, k0F, kWIG);
    1603             :   }
    1604         239 :   void vroundss(XMMRegister dst, XMMRegister src1, XMMRegister src2,
    1605             :                 RoundingMode mode) {
    1606         239 :     vinstr(0x0a, dst, src1, src2, k66, k0F3A, kWIG);
    1607         239 :     emit(static_cast<byte>(mode) | 0x8);  // Mask precision exception.
    1608         239 :   }
    1609       41881 :   void vroundsd(XMMRegister dst, XMMRegister src1, XMMRegister src2,
    1610             :                 RoundingMode mode) {
    1611       41881 :     vinstr(0x0b, dst, src1, src2, k66, k0F3A, kWIG);
    1612       41881 :     emit(static_cast<byte>(mode) | 0x8);  // Mask precision exception.
    1613       41881 :   }
    1614             : 
    1615             :   void vsd(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1616      245955 :     vinstr(op, dst, src1, src2, kF2, k0F, kWIG);
    1617             :   }
    1618             :   void vsd(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2) {
    1619     5761829 :     vinstr(op, dst, src1, src2, kF2, k0F, kWIG);
    1620             :   }
    1621             : 
    1622             :   void vmovss(XMMRegister dst, XMMRegister src1, XMMRegister src2) {
    1623          58 :     vss(0x10, dst, src1, src2);
    1624             :   }
    1625             :   void vmovss(XMMRegister dst, const Operand& src) {
    1626       17969 :     vss(0x10, dst, xmm0, src);
    1627             :   }
    1628             :   void vmovss(const Operand& dst, XMMRegister src) {
    1629     1288863 :     vss(0x11, src, xmm0, dst);
    1630             :   }
    1631             :   void vucomiss(XMMRegister dst, XMMRegister src);
    1632             :   void vucomiss(XMMRegister dst, const Operand& src);
    1633             :   void vss(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
    1634             :   void vss(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2);
    1635             : 
    1636          44 :   void vmovaps(XMMRegister dst, XMMRegister src) { vps(0x28, dst, xmm0, src); }
    1637           0 :   void vmovups(XMMRegister dst, XMMRegister src) { vps(0x10, dst, xmm0, src); }
    1638             :   void vmovups(XMMRegister dst, const Operand& src) {
    1639          14 :     vps(0x10, dst, xmm0, src);
    1640             :   }
    1641             :   void vmovups(const Operand& dst, XMMRegister src) {
    1642          14 :     vps(0x11, src, xmm0, dst);
    1643             :   }
    1644      116980 :   void vmovapd(XMMRegister dst, XMMRegister src) { vpd(0x28, dst, xmm0, src); }
    1645             :   void vmovupd(XMMRegister dst, const Operand& src) {
    1646          14 :     vpd(0x10, dst, xmm0, src);
    1647             :   }
    1648             :   void vmovupd(const Operand& dst, XMMRegister src) {
    1649          14 :     vpd(0x11, src, xmm0, dst);
    1650             :   }
    1651             :   void vmovmskps(Register dst, XMMRegister src) {
    1652          58 :     XMMRegister idst = {dst.code()};
    1653          58 :     vps(0x50, idst, xmm0, src);
    1654             :   }
    1655             :   void vmovmskpd(Register dst, XMMRegister src) {
    1656        7148 :     XMMRegister idst = {dst.code()};
    1657        7148 :     vpd(0x50, idst, xmm0, src);
    1658             :   }
    1659             :   void vcmpps(XMMRegister dst, XMMRegister src1, XMMRegister src2, int8_t cmp) {
    1660             :     vps(0xC2, dst, src1, src2);
    1661             :     emit(cmp);
    1662             :   }
    1663             :   void vcmpps(XMMRegister dst, XMMRegister src1, const Operand& src2,
    1664             :               int8_t cmp) {
    1665             :     vps(0xC2, dst, src1, src2);
    1666             :     emit(cmp);
    1667             :   }
    1668             :   void vcmppd(XMMRegister dst, XMMRegister src1, XMMRegister src2, int8_t cmp) {
    1669             :     vpd(0xC2, dst, src1, src2);
    1670             :     emit(cmp);
    1671             :   }
    1672             :   void vcmppd(XMMRegister dst, XMMRegister src1, const Operand& src2,
    1673             :               int8_t cmp) {
    1674             :     vpd(0xC2, dst, src1, src2);
    1675             :     emit(cmp);
    1676             :   }
    1677             : 
    1678             : #define AVX_CMP_P(instr, imm8)                                             \
    1679             :   void instr##ps(XMMRegister dst, XMMRegister src1, XMMRegister src2) {    \
    1680             :     vcmpps(dst, src1, src2, imm8);                                         \
    1681             :   }                                                                        \
    1682             :   void instr##ps(XMMRegister dst, XMMRegister src1, const Operand& src2) { \
    1683             :     vcmpps(dst, src1, src2, imm8);                                         \
    1684             :   }                                                                        \
    1685             :   void instr##pd(XMMRegister dst, XMMRegister src1, XMMRegister src2) {    \
    1686             :     vcmppd(dst, src1, src2, imm8);                                         \
    1687             :   }                                                                        \
    1688             :   void instr##pd(XMMRegister dst, XMMRegister src1, const Operand& src2) { \
    1689             :     vcmppd(dst, src1, src2, imm8);                                         \
    1690             :   }
    1691             : 
    1692             :   AVX_CMP_P(vcmpeq, 0x0);
    1693             :   AVX_CMP_P(vcmplt, 0x1);
    1694             :   AVX_CMP_P(vcmple, 0x2);
    1695             :   AVX_CMP_P(vcmpneq, 0x4);
    1696             :   AVX_CMP_P(vcmpnlt, 0x5);
    1697             :   AVX_CMP_P(vcmpnle, 0x6);
    1698             : 
    1699             : #undef AVX_CMP_P
    1700             : 
    1701             :   void vlddqu(XMMRegister dst, const Operand& src) {
    1702             :     vinstr(0xF0, dst, xmm0, src, kF2, k0F, kWIG);
    1703             :   }
    1704             :   void vpsllw(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1705             :     XMMRegister iop = {6};
    1706             :     vinstr(0x71, iop, dst, src, k66, k0F, kWIG);
    1707             :     emit(imm8);
    1708             :   }
    1709             :   void vpsrlw(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1710             :     XMMRegister iop = {2};
    1711             :     vinstr(0x71, iop, dst, src, k66, k0F, kWIG);
    1712             :     emit(imm8);
    1713             :   }
    1714             :   void vpsraw(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1715             :     XMMRegister iop = {4};
    1716             :     vinstr(0x71, iop, dst, src, k66, k0F, kWIG);
    1717             :     emit(imm8);
    1718             :   }
    1719             :   void vpslld(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1720             :     XMMRegister iop = {6};
    1721             :     vinstr(0x72, iop, dst, src, k66, k0F, kWIG);
    1722             :     emit(imm8);
    1723             :   }
    1724             :   void vpsrld(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1725             :     XMMRegister iop = {2};
    1726             :     vinstr(0x72, iop, dst, src, k66, k0F, kWIG);
    1727             :     emit(imm8);
    1728             :   }
    1729             :   void vpsrad(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1730             :     XMMRegister iop = {4};
    1731             :     vinstr(0x72, iop, dst, src, k66, k0F, kWIG);
    1732             :     emit(imm8);
    1733             :   }
    1734             :   void vpextrb(Register dst, XMMRegister src, int8_t imm8) {
    1735             :     XMMRegister idst = {dst.code()};
    1736             :     vinstr(0x14, src, xmm0, idst, k66, k0F3A, kW0);
    1737             :     emit(imm8);
    1738             :   }
    1739             :   void vpextrb(const Operand& dst, XMMRegister src, int8_t imm8) {
    1740             :     vinstr(0x14, src, xmm0, dst, k66, k0F3A, kW0);
    1741             :     emit(imm8);
    1742             :   }
    1743             :   void vpextrw(Register dst, XMMRegister src, int8_t imm8) {
    1744             :     XMMRegister idst = {dst.code()};
    1745             :     vinstr(0xc5, idst, xmm0, src, k66, k0F, kW0);
    1746             :     emit(imm8);
    1747             :   }
    1748             :   void vpextrw(const Operand& dst, XMMRegister src, int8_t imm8) {
    1749             :     vinstr(0x15, src, xmm0, dst, k66, k0F3A, kW0);
    1750             :     emit(imm8);
    1751             :   }
    1752             :   void vpextrd(Register dst, XMMRegister src, int8_t imm8) {
    1753             :     XMMRegister idst = {dst.code()};
    1754             :     vinstr(0x16, src, xmm0, idst, k66, k0F3A, kW0);
    1755             :     emit(imm8);
    1756             :   }
    1757             :   void vpextrd(const Operand& dst, XMMRegister src, int8_t imm8) {
    1758             :     vinstr(0x16, src, xmm0, dst, k66, k0F3A, kW0);
    1759             :     emit(imm8);
    1760             :   }
    1761             :   void vpinsrb(XMMRegister dst, XMMRegister src1, Register src2, int8_t imm8) {
    1762             :     XMMRegister isrc = {src2.code()};
    1763             :     vinstr(0x20, dst, src1, isrc, k66, k0F3A, kW0);
    1764             :     emit(imm8);
    1765             :   }
    1766             :   void vpinsrb(XMMRegister dst, XMMRegister src1, const Operand& src2,
    1767             :                int8_t imm8) {
    1768             :     vinstr(0x20, dst, src1, src2, k66, k0F3A, kW0);
    1769             :     emit(imm8);
    1770             :   }
    1771             :   void vpinsrw(XMMRegister dst, XMMRegister src1, Register src2, int8_t imm8) {
    1772             :     XMMRegister isrc = {src2.code()};
    1773             :     vinstr(0xc4, dst, src1, isrc, k66, k0F, kW0);
    1774             :     emit(imm8);
    1775             :   }
    1776             :   void vpinsrw(XMMRegister dst, XMMRegister src1, const Operand& src2,
    1777             :                int8_t imm8) {
    1778             :     vinstr(0xc4, dst, src1, src2, k66, k0F, kW0);
    1779             :     emit(imm8);
    1780             :   }
    1781             :   void vpinsrd(XMMRegister dst, XMMRegister src1, Register src2, int8_t imm8) {
    1782             :     XMMRegister isrc = {src2.code()};
    1783             :     vinstr(0x22, dst, src1, isrc, k66, k0F3A, kW0);
    1784             :     emit(imm8);
    1785             :   }
    1786             :   void vpinsrd(XMMRegister dst, XMMRegister src1, const Operand& src2,
    1787             :                int8_t imm8) {
    1788             :     vinstr(0x22, dst, src1, src2, k66, k0F3A, kW0);
    1789             :     emit(imm8);
    1790             :   }
    1791             :   void vpshufd(XMMRegister dst, XMMRegister src, int8_t imm8) {
    1792             :     vinstr(0x70, dst, xmm0, src, k66, k0F, kWIG);
    1793             :     emit(imm8);
    1794             :   }
    1795             : 
    1796             :   void vps(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
    1797             :   void vps(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2);
    1798             :   void vpd(byte op, XMMRegister dst, XMMRegister src1, XMMRegister src2);
    1799             :   void vpd(byte op, XMMRegister dst, XMMRegister src1, const Operand& src2);
    1800             : 
    1801             :   // BMI instruction
    1802             :   void andnq(Register dst, Register src1, Register src2) {
    1803             :     bmi1q(0xf2, dst, src1, src2);
    1804             :   }
    1805             :   void andnq(Register dst, Register src1, const Operand& src2) {
    1806             :     bmi1q(0xf2, dst, src1, src2);
    1807             :   }
    1808             :   void andnl(Register dst, Register src1, Register src2) {
    1809             :     bmi1l(0xf2, dst, src1, src2);
    1810             :   }
    1811             :   void andnl(Register dst, Register src1, const Operand& src2) {
    1812             :     bmi1l(0xf2, dst, src1, src2);
    1813             :   }
    1814             :   void bextrq(Register dst, Register src1, Register src2) {
    1815             :     bmi1q(0xf7, dst, src2, src1);
    1816             :   }
    1817             :   void bextrq(Register dst, const Operand& src1, Register src2) {
    1818             :     bmi1q(0xf7, dst, src2, src1);
    1819             :   }
    1820             :   void bextrl(Register dst, Register src1, Register src2) {
    1821             :     bmi1l(0xf7, dst, src2, src1);
    1822             :   }
    1823             :   void bextrl(Register dst, const Operand& src1, Register src2) {
    1824             :     bmi1l(0xf7, dst, src2, src1);
    1825             :   }
    1826             :   void blsiq(Register dst, Register src) {
    1827             :     Register ireg = {3};
    1828             :     bmi1q(0xf3, ireg, dst, src);
    1829             :   }
    1830             :   void blsiq(Register dst, const Operand& src) {
    1831             :     Register ireg = {3};
    1832             :     bmi1q(0xf3, ireg, dst, src);
    1833             :   }
    1834             :   void blsil(Register dst, Register src) {
    1835             :     Register ireg = {3};
    1836             :     bmi1l(0xf3, ireg, dst, src);
    1837             :   }
    1838             :   void blsil(Register dst, const Operand& src) {
    1839             :     Register ireg = {3};
    1840             :     bmi1l(0xf3, ireg, dst, src);
    1841             :   }
    1842             :   void blsmskq(Register dst, Register src) {
    1843             :     Register ireg = {2};
    1844             :     bmi1q(0xf3, ireg, dst, src);
    1845             :   }
    1846             :   void blsmskq(Register dst, const Operand& src) {
    1847             :     Register ireg = {2};
    1848             :     bmi1q(0xf3, ireg, dst, src);
    1849             :   }
    1850             :   void blsmskl(Register dst, Register src) {
    1851             :     Register ireg = {2};
    1852             :     bmi1l(0xf3, ireg, dst, src);
    1853             :   }
    1854             :   void blsmskl(Register dst, const Operand& src) {
    1855             :     Register ireg = {2};
    1856             :     bmi1l(0xf3, ireg, dst, src);
    1857             :   }
    1858             :   void blsrq(Register dst, Register src) {
    1859             :     Register ireg = {1};
    1860             :     bmi1q(0xf3, ireg, dst, src);
    1861             :   }
    1862             :   void blsrq(Register dst, const Operand& src) {
    1863             :     Register ireg = {1};
    1864             :     bmi1q(0xf3, ireg, dst, src);
    1865             :   }
    1866             :   void blsrl(Register dst, Register src) {
    1867             :     Register ireg = {1};
    1868             :     bmi1l(0xf3, ireg, dst, src);
    1869             :   }
    1870             :   void blsrl(Register dst, const Operand& src) {
    1871             :     Register ireg = {1};
    1872             :     bmi1l(0xf3, ireg, dst, src);
    1873             :   }
    1874             :   void tzcntq(Register dst, Register src);
    1875             :   void tzcntq(Register dst, const Operand& src);
    1876             :   void tzcntl(Register dst, Register src);
    1877             :   void tzcntl(Register dst, const Operand& src);
    1878             : 
    1879             :   void lzcntq(Register dst, Register src);
    1880             :   void lzcntq(Register dst, const Operand& src);
    1881             :   void lzcntl(Register dst, Register src);
    1882             :   void lzcntl(Register dst, const Operand& src);
    1883             : 
    1884             :   void popcntq(Register dst, Register src);
    1885             :   void popcntq(Register dst, const Operand& src);
    1886             :   void popcntl(Register dst, Register src);
    1887             :   void popcntl(Register dst, const Operand& src);
    1888             : 
    1889             :   void bzhiq(Register dst, Register src1, Register src2) {
    1890             :     bmi2q(kNone, 0xf5, dst, src2, src1);
    1891             :   }
    1892             :   void bzhiq(Register dst, const Operand& src1, Register src2) {
    1893             :     bmi2q(kNone, 0xf5, dst, src2, src1);
    1894             :   }
    1895             :   void bzhil(Register dst, Register src1, Register src2) {
    1896             :     bmi2l(kNone, 0xf5, dst, src2, src1);
    1897             :   }
    1898             :   void bzhil(Register dst, const Operand& src1, Register src2) {
    1899             :     bmi2l(kNone, 0xf5, dst, src2, src1);
    1900             :   }
    1901             :   void mulxq(Register dst1, Register dst2, Register src) {
    1902             :     bmi2q(kF2, 0xf6, dst1, dst2, src);
    1903             :   }
    1904             :   void mulxq(Register dst1, Register dst2, const Operand& src) {
    1905             :     bmi2q(kF2, 0xf6, dst1, dst2, src);
    1906             :   }
    1907             :   void mulxl(Register dst1, Register dst2, Register src) {
    1908             :     bmi2l(kF2, 0xf6, dst1, dst2, src);
    1909             :   }
    1910             :   void mulxl(Register dst1, Register dst2, const Operand& src) {
    1911             :     bmi2l(kF2, 0xf6, dst1, dst2, src);
    1912             :   }
    1913             :   void pdepq(Register dst, Register src1, Register src2) {
    1914             :     bmi2q(kF2, 0xf5, dst, src1, src2);
    1915             :   }
    1916             :   void pdepq(Register dst, Register src1, const Operand& src2) {
    1917             :     bmi2q(kF2, 0xf5, dst, src1, src2);
    1918             :   }
    1919             :   void pdepl(Register dst, Register src1, Register src2) {
    1920             :     bmi2l(kF2, 0xf5, dst, src1, src2);
    1921             :   }
    1922             :   void pdepl(Register dst, Register src1, const Operand& src2) {
    1923             :     bmi2l(kF2, 0xf5, dst, src1, src2);
    1924             :   }
    1925             :   void pextq(Register dst, Register src1, Register src2) {
    1926             :     bmi2q(kF3, 0xf5, dst, src1, src2);
    1927             :   }
    1928             :   void pextq(Register dst, Register src1, const Operand& src2) {
    1929             :     bmi2q(kF3, 0xf5, dst, src1, src2);
    1930             :   }
    1931             :   void pextl(Register dst, Register src1, Register src2) {
    1932             :     bmi2l(kF3, 0xf5, dst, src1, src2);
    1933             :   }
    1934             :   void pextl(Register dst, Register src1, const Operand& src2) {
    1935             :     bmi2l(kF3, 0xf5, dst, src1, src2);
    1936             :   }
    1937             :   void sarxq(Register dst, Register src1, Register src2) {
    1938             :     bmi2q(kF3, 0xf7, dst, src2, src1);
    1939             :   }
    1940             :   void sarxq(Register dst, const Operand& src1, Register src2) {
    1941             :     bmi2q(kF3, 0xf7, dst, src2, src1);
    1942             :   }
    1943             :   void sarxl(Register dst, Register src1, Register src2) {
    1944             :     bmi2l(kF3, 0xf7, dst, src2, src1);
    1945             :   }
    1946             :   void sarxl(Register dst, const Operand& src1, Register src2) {
    1947             :     bmi2l(kF3, 0xf7, dst, src2, src1);
    1948             :   }
    1949             :   void shlxq(Register dst, Register src1, Register src2) {
    1950             :     bmi2q(k66, 0xf7, dst, src2, src1);
    1951             :   }
    1952             :   void shlxq(Register dst, const Operand& src1, Register src2) {
    1953             :     bmi2q(k66, 0xf7, dst, src2, src1);
    1954             :   }
    1955             :   void shlxl(Register dst, Register src1, Register src2) {
    1956             :     bmi2l(k66, 0xf7, dst, src2, src1);
    1957             :   }
    1958             :   void shlxl(Register dst, const Operand& src1, Register src2) {
    1959             :     bmi2l(k66, 0xf7, dst, src2, src1);
    1960             :   }
    1961             :   void shrxq(Register dst, Register src1, Register src2) {
    1962             :     bmi2q(kF2, 0xf7, dst, src2, src1);
    1963             :   }
    1964             :   void shrxq(Register dst, const Operand& src1, Register src2) {
    1965             :     bmi2q(kF2, 0xf7, dst, src2, src1);
    1966             :   }
    1967             :   void shrxl(Register dst, Register src1, Register src2) {
    1968             :     bmi2l(kF2, 0xf7, dst, src2, src1);
    1969             :   }
    1970             :   void shrxl(Register dst, const Operand& src1, Register src2) {
    1971             :     bmi2l(kF2, 0xf7, dst, src2, src1);
    1972             :   }
    1973             :   void rorxq(Register dst, Register src, byte imm8);
    1974             :   void rorxq(Register dst, const Operand& src, byte imm8);
    1975             :   void rorxl(Register dst, Register src, byte imm8);
    1976             :   void rorxl(Register dst, const Operand& src, byte imm8);
    1977             : 
    1978             :   // Check the code size generated from label to here.
    1979       96490 :   int SizeOfCodeGeneratedSince(Label* label) {
    1980     1119314 :     return pc_offset() - label->pos();
    1981             :   }
    1982             : 
    1983             :   // Mark address of a debug break slot.
    1984             :   void RecordDebugBreakSlot(RelocInfo::Mode mode);
    1985             : 
    1986             :   // Record a comment relocation entry that can be used by a disassembler.
    1987             :   // Use --code-comments to enable.
    1988             :   void RecordComment(const char* msg);
    1989             : 
    1990             :   // Record a deoptimization reason that can be used by a log or cpu profiler.
    1991             :   // Use --trace-deopt to enable.
    1992             :   void RecordDeoptReason(DeoptimizeReason reason, SourcePosition position,
    1993             :                          int id);
    1994             : 
    1995           0 :   void PatchConstantPoolAccessInstruction(int pc_offset, int offset,
    1996             :                                           ConstantPoolEntry::Access access,
    1997             :                                           ConstantPoolEntry::Type type) {
    1998             :     // No embedded constant pool support.
    1999           0 :     UNREACHABLE();
    2000             :   }
    2001             : 
    2002             :   void RecordProtectedInstructionLanding(int pc_offset);
    2003             : 
    2004             :   // Writes a single word of data in the code stream.
    2005             :   // Used for inline tables, e.g., jump-tables.
    2006             :   void db(uint8_t data);
    2007             :   void dd(uint32_t data);
    2008             :   void dq(uint64_t data);
    2009           0 :   void dp(uintptr_t data) { dq(data); }
    2010             :   void dq(Label* label);
    2011             : 
    2012             :   // Check if there is less than kGap bytes available in the buffer.
    2013             :   // If this is the case, we need to grow the buffer before emitting
    2014             :   // an instruction or relocation information.
    2015             :   inline bool buffer_overflow() const {
    2016   367791688 :     return pc_ >= reloc_info_writer.pos() - kGap;
    2017             :   }
    2018             : 
    2019             :   // Get the number of bytes available in the buffer.
    2020             :   inline int available_space() const {
    2021             :     return static_cast<int>(reloc_info_writer.pos() - pc_);
    2022             :   }
    2023             : 
    2024             :   static bool IsNop(Address addr);
    2025             : 
    2026             :   // Avoid overflows for displacements etc.
    2027             :   static constexpr int kMaximalBufferSize = 512 * MB;
    2028             : 
    2029             :   byte byte_at(int pos)  { return buffer_[pos]; }
    2030     7726749 :   void set_byte_at(int pos, byte value) { buffer_[pos] = value; }
    2031             : 
    2032             :   Address pc() const { return pc_; }
    2033             : 
    2034             :  protected:
    2035             :   // Call near indirect
    2036             :   void call(const Operand& operand);
    2037             : 
    2038             :  private:
    2039    86757982 :   byte* addr_at(int pos)  { return buffer_ + pos; }
    2040    52908719 :   uint32_t long_at(int pos)  {
    2041    52908719 :     return *reinterpret_cast<uint32_t*>(addr_at(pos));
    2042             :   }
    2043    26164544 :   void long_at_put(int pos, uint32_t x)  {
    2044    26852087 :     *reinterpret_cast<uint32_t*>(addr_at(pos)) = x;
    2045             :   }
    2046             : 
    2047             :   // code emission
    2048             :   void GrowBuffer();
    2049             : 
    2050   599361105 :   void emit(byte x) { *pc_++ = x; }
    2051             :   inline void emitl(uint32_t x);
    2052             :   inline void emitp(void* x, RelocInfo::Mode rmode);
    2053             :   inline void emitq(uint64_t x);
    2054             :   inline void emitw(uint16_t x);
    2055             :   inline void emit_code_target(Handle<Code> target,
    2056             :                                RelocInfo::Mode rmode,
    2057             :                                TypeFeedbackId ast_id = TypeFeedbackId::None());
    2058             :   inline void emit_runtime_entry(Address entry, RelocInfo::Mode rmode);
    2059             :   inline void emit(Immediate x);
    2060             : 
    2061             :   // Emits a REX prefix that encodes a 64-bit operand size and
    2062             :   // the top bit of both register codes.
    2063             :   // High bit of reg goes to REX.R, high bit of rm_reg goes to REX.B.
    2064             :   // REX.W is set.
    2065             :   inline void emit_rex_64(XMMRegister reg, Register rm_reg);
    2066             :   inline void emit_rex_64(Register reg, XMMRegister rm_reg);
    2067             :   inline void emit_rex_64(Register reg, Register rm_reg);
    2068             : 
    2069             :   // Emits a REX prefix that encodes a 64-bit operand size and
    2070             :   // the top bit of the destination, index, and base register codes.
    2071             :   // The high bit of reg is used for REX.R, the high bit of op's base
    2072             :   // register is used for REX.B, and the high bit of op's index register
    2073             :   // is used for REX.X.  REX.W is set.
    2074             :   inline void emit_rex_64(Register reg, const Operand& op);
    2075             :   inline void emit_rex_64(XMMRegister reg, const Operand& op);
    2076             : 
    2077             :   // Emits a REX prefix that encodes a 64-bit operand size and
    2078             :   // the top bit of the register code.
    2079             :   // The high bit of register is used for REX.B.
    2080             :   // REX.W is set and REX.R and REX.X are clear.
    2081             :   inline void emit_rex_64(Register rm_reg);
    2082             : 
    2083             :   // Emits a REX prefix that encodes a 64-bit operand size and
    2084             :   // the top bit of the index and base register codes.
    2085             :   // The high bit of op's base register is used for REX.B, and the high
    2086             :   // bit of op's index register is used for REX.X.
    2087             :   // REX.W is set and REX.R clear.
    2088             :   inline void emit_rex_64(const Operand& op);
    2089             : 
    2090             :   // Emit a REX prefix that only sets REX.W to choose a 64-bit operand size.
    2091             :   void emit_rex_64() { emit(0x48); }
    2092             : 
    2093             :   // High bit of reg goes to REX.R, high bit of rm_reg goes to REX.B.
    2094             :   // REX.W is clear.
    2095             :   inline void emit_rex_32(Register reg, Register rm_reg);
    2096             : 
    2097             :   // The high bit of reg is used for REX.R, the high bit of op's base
    2098             :   // register is used for REX.B, and the high bit of op's index register
    2099             :   // is used for REX.X.  REX.W is cleared.
    2100             :   inline void emit_rex_32(Register reg, const Operand& op);
    2101             : 
    2102             :   // High bit of rm_reg goes to REX.B.
    2103             :   // REX.W, REX.R and REX.X are clear.
    2104             :   inline void emit_rex_32(Register rm_reg);
    2105             : 
    2106             :   // High bit of base goes to REX.B and high bit of index to REX.X.
    2107             :   // REX.W and REX.R are clear.
    2108             :   inline void emit_rex_32(const Operand& op);
    2109             : 
    2110             :   // High bit of reg goes to REX.R, high bit of rm_reg goes to REX.B.
    2111             :   // REX.W is cleared.  If no REX bits are set, no byte is emitted.
    2112             :   inline void emit_optional_rex_32(Register reg, Register rm_reg);
    2113             : 
    2114             :   // The high bit of reg is used for REX.R, the high bit of op's base
    2115             :   // register is used for REX.B, and the high bit of op's index register
    2116             :   // is used for REX.X.  REX.W is cleared.  If no REX bits are set, nothing
    2117             :   // is emitted.
    2118             :   inline void emit_optional_rex_32(Register reg, const Operand& op);
    2119             : 
    2120             :   // As for emit_optional_rex_32(Register, Register), except that
    2121             :   // the registers are XMM registers.
    2122             :   inline void emit_optional_rex_32(XMMRegister reg, XMMRegister base);
    2123             : 
    2124             :   // As for emit_optional_rex_32(Register, Register), except that
    2125             :   // one of the registers is an XMM registers.
    2126             :   inline void emit_optional_rex_32(XMMRegister reg, Register base);
    2127             : 
    2128             :   // As for emit_optional_rex_32(Register, Register), except that
    2129             :   // one of the registers is an XMM registers.
    2130             :   inline void emit_optional_rex_32(Register reg, XMMRegister base);
    2131             : 
    2132             :   // As for emit_optional_rex_32(Register, const Operand&), except that
    2133             :   // the register is an XMM register.
    2134             :   inline void emit_optional_rex_32(XMMRegister reg, const Operand& op);
    2135             : 
    2136             :   // Optionally do as emit_rex_32(Register) if the register number has
    2137             :   // the high bit set.
    2138             :   inline void emit_optional_rex_32(Register rm_reg);
    2139             :   inline void emit_optional_rex_32(XMMRegister rm_reg);
    2140             : 
    2141             :   // Optionally do as emit_rex_32(const Operand&) if the operand register
    2142             :   // numbers have a high bit set.
    2143             :   inline void emit_optional_rex_32(const Operand& op);
    2144             : 
    2145             :   void emit_rex(int size) {
    2146           0 :     if (size == kInt64Size) {
    2147             :       emit_rex_64();
    2148             :     } else {
    2149             :       DCHECK(size == kInt32Size);
    2150             :     }
    2151             :   }
    2152             : 
    2153             :   template<class P1>
    2154             :   void emit_rex(P1 p1, int size) {
    2155    27245002 :     if (size == kInt64Size) {
    2156             :       emit_rex_64(p1);
    2157             :     } else {
    2158             :       DCHECK(size == kInt32Size);
    2159             :       emit_optional_rex_32(p1);
    2160             :     }
    2161             :   }
    2162             : 
    2163             :   template<class P1, class P2>
    2164    91885873 :   void emit_rex(P1 p1, P2 p2, int size) {
    2165    91885873 :     if (size == kInt64Size) {
    2166             :       emit_rex_64(p1, p2);
    2167             :     } else {
    2168             :       DCHECK(size == kInt32Size);
    2169             :       emit_optional_rex_32(p1, p2);
    2170             :     }
    2171    91885873 :   }
    2172             : 
    2173             :   // Emit vex prefix
    2174             :   void emit_vex2_byte0() { emit(0xc5); }
    2175             :   inline void emit_vex2_byte1(XMMRegister reg, XMMRegister v, VectorLength l,
    2176             :                               SIMDPrefix pp);
    2177             :   void emit_vex3_byte0() { emit(0xc4); }
    2178             :   inline void emit_vex3_byte1(XMMRegister reg, XMMRegister rm, LeadingOpcode m);
    2179             :   inline void emit_vex3_byte1(XMMRegister reg, const Operand& rm,
    2180             :                               LeadingOpcode m);
    2181             :   inline void emit_vex3_byte2(VexW w, XMMRegister v, VectorLength l,
    2182             :                               SIMDPrefix pp);
    2183             :   inline void emit_vex_prefix(XMMRegister reg, XMMRegister v, XMMRegister rm,
    2184             :                               VectorLength l, SIMDPrefix pp, LeadingOpcode m,
    2185             :                               VexW w);
    2186             :   inline void emit_vex_prefix(Register reg, Register v, Register rm,
    2187             :                               VectorLength l, SIMDPrefix pp, LeadingOpcode m,
    2188             :                               VexW w);
    2189             :   inline void emit_vex_prefix(XMMRegister reg, XMMRegister v, const Operand& rm,
    2190             :                               VectorLength l, SIMDPrefix pp, LeadingOpcode m,
    2191             :                               VexW w);
    2192             :   inline void emit_vex_prefix(Register reg, Register v, const Operand& rm,
    2193             :                               VectorLength l, SIMDPrefix pp, LeadingOpcode m,
    2194             :                               VexW w);
    2195             : 
    2196             :   // Emit the ModR/M byte, and optionally the SIB byte and
    2197             :   // 1- or 4-byte offset for a memory operand.  Also encodes
    2198             :   // the second operand of the operation, a register or operation
    2199             :   // subcode, into the reg field of the ModR/M byte.
    2200             :   void emit_operand(Register reg, const Operand& adr) {
    2201    75273578 :     emit_operand(reg.low_bits(), adr);
    2202             :   }
    2203             : 
    2204             :   // Emit the ModR/M byte, and optionally the SIB byte and
    2205             :   // 1- or 4-byte offset for a memory operand.  Also used to encode
    2206             :   // a three-bit opcode extension into the ModR/M byte.
    2207             :   void emit_operand(int rm, const Operand& adr);
    2208             : 
    2209             :   // Emit a ModR/M byte with registers coded in the reg and rm_reg fields.
    2210             :   void emit_modrm(Register reg, Register rm_reg) {
    2211    31491597 :     emit(0xC0 | reg.low_bits() << 3 | rm_reg.low_bits());
    2212             :   }
    2213             : 
    2214             :   // Emit a ModR/M byte with an operation subcode in the reg field and
    2215             :   // a register in the rm_reg field.
    2216             :   void emit_modrm(int code, Register rm_reg) {
    2217             :     DCHECK(is_uint3(code));
    2218    38984914 :     emit(0xC0 | code << 3 | rm_reg.low_bits());
    2219             :   }
    2220             : 
    2221             :   // Emit the code-object-relative offset of the label's position
    2222             :   inline void emit_code_relative_offset(Label* label);
    2223             : 
    2224             :   // The first argument is the reg field, the second argument is the r/m field.
    2225             :   void emit_sse_operand(XMMRegister dst, XMMRegister src);
    2226             :   void emit_sse_operand(XMMRegister reg, const Operand& adr);
    2227             :   void emit_sse_operand(Register reg, const Operand& adr);
    2228             :   void emit_sse_operand(XMMRegister dst, Register src);
    2229             :   void emit_sse_operand(Register dst, XMMRegister src);
    2230             :   void emit_sse_operand(XMMRegister dst);
    2231             : 
    2232             :   // Emit machine code for one of the operations ADD, ADC, SUB, SBC,
    2233             :   // AND, OR, XOR, or CMP.  The encodings of these operations are all
    2234             :   // similar, differing just in the opcode or in the reg field of the
    2235             :   // ModR/M byte.
    2236             :   void arithmetic_op_8(byte opcode, Register reg, Register rm_reg);
    2237             :   void arithmetic_op_8(byte opcode, Register reg, const Operand& rm_reg);
    2238             :   void arithmetic_op_16(byte opcode, Register reg, Register rm_reg);
    2239             :   void arithmetic_op_16(byte opcode, Register reg, const Operand& rm_reg);
    2240             :   // Operate on operands/registers with pointer size, 32-bit or 64-bit size.
    2241             :   void arithmetic_op(byte opcode, Register reg, Register rm_reg, int size);
    2242             :   void arithmetic_op(byte opcode,
    2243             :                      Register reg,
    2244             :                      const Operand& rm_reg,
    2245             :                      int size);
    2246             :   // Operate on a byte in memory or register.
    2247             :   void immediate_arithmetic_op_8(byte subcode,
    2248             :                                  Register dst,
    2249             :                                  Immediate src);
    2250             :   void immediate_arithmetic_op_8(byte subcode,
    2251             :                                  const Operand& dst,
    2252             :                                  Immediate src);
    2253             :   // Operate on a word in memory or register.
    2254             :   void immediate_arithmetic_op_16(byte subcode,
    2255             :                                   Register dst,
    2256             :                                   Immediate src);
    2257             :   void immediate_arithmetic_op_16(byte subcode,
    2258             :                                   const Operand& dst,
    2259             :                                   Immediate src);
    2260             :   // Operate on operands/registers with pointer size, 32-bit or 64-bit size.
    2261             :   void immediate_arithmetic_op(byte subcode,
    2262             :                                Register dst,
    2263             :                                Immediate src,
    2264             :                                int size);
    2265             :   void immediate_arithmetic_op(byte subcode,
    2266             :                                const Operand& dst,
    2267             :                                Immediate src,
    2268             :                                int size);
    2269             : 
    2270             :   // Emit machine code for a shift operation.
    2271             :   void shift(Operand dst, Immediate shift_amount, int subcode, int size);
    2272             :   void shift(Register dst, Immediate shift_amount, int subcode, int size);
    2273             :   // Shift dst by cl % 64 bits.
    2274             :   void shift(Register dst, int subcode, int size);
    2275             :   void shift(Operand dst, int subcode, int size);
    2276             : 
    2277             :   void emit_farith(int b1, int b2, int i);
    2278             : 
    2279             :   // labels
    2280             :   // void print(Label* L);
    2281             :   void bind_to(Label* L, int pos);
    2282             : 
    2283             :   // record reloc info for current pc_
    2284             :   void RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data = 0);
    2285             : 
    2286             :   // Arithmetics
    2287             :   void emit_add(Register dst, Register src, int size) {
    2288     1115121 :     arithmetic_op(0x03, dst, src, size);
    2289             :   }
    2290             : 
    2291             :   void emit_add(Register dst, Immediate src, int size) {
    2292     4187279 :     immediate_arithmetic_op(0x0, dst, src, size);
    2293             :   }
    2294             : 
    2295             :   void emit_add(Register dst, const Operand& src, int size) {
    2296       40463 :     arithmetic_op(0x03, dst, src, size);
    2297             :   }
    2298             : 
    2299             :   void emit_add(const Operand& dst, Register src, int size) {
    2300      184425 :     arithmetic_op(0x1, src, dst, size);
    2301             :   }
    2302             : 
    2303             :   void emit_add(const Operand& dst, Immediate src, int size) {
    2304     2283927 :     immediate_arithmetic_op(0x0, dst, src, size);
    2305             :   }
    2306             : 
    2307             :   void emit_and(Register dst, Register src, int size) {
    2308     1719194 :     arithmetic_op(0x23, dst, src, size);
    2309             :   }
    2310             : 
    2311             :   void emit_and(Register dst, const Operand& src, int size) {
    2312       85791 :     arithmetic_op(0x23, dst, src, size);
    2313             :   }
    2314             : 
    2315             :   void emit_and(const Operand& dst, Register src, int size) {
    2316             :     arithmetic_op(0x21, src, dst, size);
    2317             :   }
    2318             : 
    2319             :   void emit_and(Register dst, Immediate src, int size) {
    2320     2163407 :     immediate_arithmetic_op(0x4, dst, src, size);
    2321             :   }
    2322             : 
    2323             :   void emit_and(const Operand& dst, Immediate src, int size) {
    2324           0 :     immediate_arithmetic_op(0x4, dst, src, size);
    2325             :   }
    2326             : 
    2327             :   void emit_cmp(Register dst, Register src, int size) {
    2328     1763449 :     arithmetic_op(0x3B, dst, src, size);
    2329             :   }
    2330             : 
    2331             :   void emit_cmp(Register dst, const Operand& src, int size) {
    2332     3844828 :     arithmetic_op(0x3B, dst, src, size);
    2333             :   }
    2334             : 
    2335             :   void emit_cmp(const Operand& dst, Register src, int size) {
    2336     1563576 :     arithmetic_op(0x39, src, dst, size);
    2337             :   }
    2338             : 
    2339             :   void emit_cmp(Register dst, Immediate src, int size) {
    2340     2964543 :     immediate_arithmetic_op(0x7, dst, src, size);
    2341             :   }
    2342             : 
    2343             :   void emit_cmp(const Operand& dst, Immediate src, int size) {
    2344      264287 :     immediate_arithmetic_op(0x7, dst, src, size);
    2345             :   }
    2346             : 
    2347             :   // Compare {al,ax,eax,rax} with src.  If equal, set ZF and write dst into
    2348             :   // src. Otherwise clear ZF and write src into {al,ax,eax,rax}.  This
    2349             :   // operation is only atomic if prefixed by the lock instruction.
    2350             :   void emit_cmpxchg(const Operand& dst, Register src, int size);
    2351             : 
    2352             :   void emit_dec(Register dst, int size);
    2353             :   void emit_dec(const Operand& dst, int size);
    2354             : 
    2355             :   // Divide rdx:rax by src.  Quotient in rax, remainder in rdx when size is 64.
    2356             :   // Divide edx:eax by lower 32 bits of src.  Quotient in eax, remainder in edx
    2357             :   // when size is 32.
    2358             :   void emit_idiv(Register src, int size);
    2359             :   void emit_div(Register src, int size);
    2360             : 
    2361             :   // Signed multiply instructions.
    2362             :   // rdx:rax = rax * src when size is 64 or edx:eax = eax * src when size is 32.
    2363             :   void emit_imul(Register src, int size);
    2364             :   void emit_imul(const Operand& src, int size);
    2365             :   void emit_imul(Register dst, Register src, int size);
    2366             :   void emit_imul(Register dst, const Operand& src, int size);
    2367             :   void emit_imul(Register dst, Register src, Immediate imm, int size);
    2368             :   void emit_imul(Register dst, const Operand& src, Immediate imm, int size);
    2369             : 
    2370             :   void emit_inc(Register dst, int size);
    2371             :   void emit_inc(const Operand& dst, int size);
    2372             : 
    2373             :   void emit_lea(Register dst, const Operand& src, int size);
    2374             : 
    2375             :   void emit_mov(Register dst, const Operand& src, int size);
    2376             :   void emit_mov(Register dst, Register src, int size);
    2377             :   void emit_mov(const Operand& dst, Register src, int size);
    2378             :   void emit_mov(Register dst, Immediate value, int size);
    2379             :   void emit_mov(const Operand& dst, Immediate value, int size);
    2380             : 
    2381             :   void emit_movzxb(Register dst, const Operand& src, int size);
    2382             :   void emit_movzxb(Register dst, Register src, int size);
    2383             :   void emit_movzxw(Register dst, const Operand& src, int size);
    2384             :   void emit_movzxw(Register dst, Register src, int size);
    2385             : 
    2386             :   void emit_neg(Register dst, int size);
    2387             :   void emit_neg(const Operand& dst, int size);
    2388             : 
    2389             :   void emit_not(Register dst, int size);
    2390             :   void emit_not(const Operand& dst, int size);
    2391             : 
    2392             :   void emit_or(Register dst, Register src, int size) {
    2393      484026 :     arithmetic_op(0x0B, dst, src, size);
    2394             :   }
    2395             : 
    2396             :   void emit_or(Register dst, const Operand& src, int size) {
    2397        7835 :     arithmetic_op(0x0B, dst, src, size);
    2398             :   }
    2399             : 
    2400             :   void emit_or(const Operand& dst, Register src, int size) {
    2401             :     arithmetic_op(0x9, src, dst, size);
    2402             :   }
    2403             : 
    2404             :   void emit_or(Register dst, Immediate src, int size) {
    2405       10743 :     immediate_arithmetic_op(0x1, dst, src, size);
    2406             :   }
    2407             : 
    2408             :   void emit_or(const Operand& dst, Immediate src, int size) {
    2409           0 :     immediate_arithmetic_op(0x1, dst, src, size);
    2410             :   }
    2411             : 
    2412             :   void emit_repmovs(int size);
    2413             : 
    2414             :   void emit_sbb(Register dst, Register src, int size) {
    2415        3424 :     arithmetic_op(0x1b, dst, src, size);
    2416             :   }
    2417             : 
    2418             :   void emit_sub(Register dst, Register src, int size) {
    2419      381579 :     arithmetic_op(0x2B, dst, src, size);
    2420             :   }
    2421             : 
    2422             :   void emit_sub(Register dst, Immediate src, int size) {
    2423     3496250 :     immediate_arithmetic_op(0x5, dst, src, size);
    2424             :   }
    2425             : 
    2426             :   void emit_sub(Register dst, const Operand& src, int size) {
    2427      193358 :     arithmetic_op(0x2B, dst, src, size);
    2428             :   }
    2429             : 
    2430             :   void emit_sub(const Operand& dst, Register src, int size) {
    2431      184425 :     arithmetic_op(0x29, src, dst, size);
    2432             :   }
    2433             : 
    2434             :   void emit_sub(const Operand& dst, Immediate src, int size) {
    2435        6432 :     immediate_arithmetic_op(0x5, dst, src, size);
    2436             :   }
    2437             : 
    2438             :   void emit_test(Register dst, Register src, int size);
    2439             :   void emit_test(Register reg, Immediate mask, int size);
    2440             :   void emit_test(const Operand& op, Register reg, int size);
    2441             :   void emit_test(const Operand& op, Immediate mask, int size);
    2442             :   void emit_test(Register reg, const Operand& op, int size) {
    2443         377 :     return emit_test(op, reg, size);
    2444             :   }
    2445             : 
    2446             :   void emit_xchg(Register dst, Register src, int size);
    2447             :   void emit_xchg(Register dst, const Operand& src, int size);
    2448             : 
    2449     2797930 :   void emit_xor(Register dst, Register src, int size) {
    2450     2894440 :     if (size == kInt64Size && dst.code() == src.code()) {
    2451             :     // 32 bit operations zero the top 32 bits of 64 bit registers. Therefore
    2452             :     // there is no need to make this a 64 bit operation.
    2453        1412 :       arithmetic_op(0x33, dst, src, kInt32Size);
    2454             :     } else {
    2455     2796518 :       arithmetic_op(0x33, dst, src, size);
    2456             :     }
    2457     2797929 :   }
    2458             : 
    2459             :   void emit_xor(Register dst, const Operand& src, int size) {
    2460        1465 :     arithmetic_op(0x33, dst, src, size);
    2461             :   }
    2462             : 
    2463             :   void emit_xor(Register dst, Immediate src, int size) {
    2464       20328 :     immediate_arithmetic_op(0x6, dst, src, size);
    2465             :   }
    2466             : 
    2467             :   void emit_xor(const Operand& dst, Immediate src, int size) {
    2468           0 :     immediate_arithmetic_op(0x6, dst, src, size);
    2469             :   }
    2470             : 
    2471             :   void emit_xor(const Operand& dst, Register src, int size) {
    2472       20696 :     arithmetic_op(0x31, src, dst, size);
    2473             :   }
    2474             : 
    2475             :   // Most BMI instructions are similiar.
    2476             :   void bmi1q(byte op, Register reg, Register vreg, Register rm);
    2477             :   void bmi1q(byte op, Register reg, Register vreg, const Operand& rm);
    2478             :   void bmi1l(byte op, Register reg, Register vreg, Register rm);
    2479             :   void bmi1l(byte op, Register reg, Register vreg, const Operand& rm);
    2480             :   void bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg, Register rm);
    2481             :   void bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
    2482             :              const Operand& rm);
    2483             :   void bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg, Register rm);
    2484             :   void bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
    2485             :              const Operand& rm);
    2486             : 
    2487             :   friend class CodePatcher;
    2488             :   friend class EnsureSpace;
    2489             :   friend class RegExpMacroAssemblerX64;
    2490             : 
    2491             :   // code generation
    2492             :   RelocInfoWriter reloc_info_writer;
    2493             : 
    2494             :   // Internal reference positions, required for (potential) patching in
    2495             :   // GrowBuffer(); contains only those internal references whose labels
    2496             :   // are already bound.
    2497             :   std::deque<int> internal_reference_positions_;
    2498             : 
    2499             :   List< Handle<Code> > code_targets_;
    2500             : };
    2501             : 
    2502             : 
    2503             : // Helper class that ensures that there is enough space for generating
    2504             : // instructions and relocation information.  The constructor makes
    2505             : // sure that there is enough space and (in debug mode) the destructor
    2506             : // checks that we did not generate too much.
    2507             : class EnsureSpace BASE_EMBEDDED {
    2508             :  public:
    2509             :   explicit EnsureSpace(Assembler* assembler) : assembler_(assembler) {
    2510   367791688 :     if (assembler_->buffer_overflow()) assembler_->GrowBuffer();
    2511             : #ifdef DEBUG
    2512             :     space_before_ = assembler_->available_space();
    2513             : #endif
    2514             :   }
    2515             : 
    2516             : #ifdef DEBUG
    2517             :   ~EnsureSpace() {
    2518             :     int bytes_generated = space_before_ - assembler_->available_space();
    2519             :     DCHECK(bytes_generated < assembler_->kGap);
    2520             :   }
    2521             : #endif
    2522             : 
    2523             :  private:
    2524             :   Assembler* assembler_;
    2525             : #ifdef DEBUG
    2526             :   int space_before_;
    2527             : #endif
    2528             : };
    2529             : 
    2530             : }  // namespace internal
    2531             : }  // namespace v8
    2532             : 
    2533             : #endif  // V8_X64_ASSEMBLER_X64_H_

Generated by: LCOV version 1.10