LCOV - code coverage report
Current view: top level - src/x64 - macro-assembler-x64.cc (source / functions) Hit Total Coverage
Test: app.info Lines: 908 1224 74.2 %
Date: 2019-04-17 Functions: 165 209 78.9 %

          Line data    Source code
       1             : // Copyright 2012 the V8 project authors. All rights reserved.
       2             : // Use of this source code is governed by a BSD-style license that can be
       3             : // found in the LICENSE file.
       4             : 
       5             : #if V8_TARGET_ARCH_X64
       6             : 
       7             : #include "src/base/bits.h"
       8             : #include "src/base/division-by-constant.h"
       9             : #include "src/base/utils/random-number-generator.h"
      10             : #include "src/bootstrapper.h"
      11             : #include "src/callable.h"
      12             : #include "src/code-factory.h"
      13             : #include "src/counters.h"
      14             : #include "src/debug/debug.h"
      15             : #include "src/external-reference-table.h"
      16             : #include "src/frames-inl.h"
      17             : #include "src/globals.h"
      18             : #include "src/heap/heap-inl.h"  // For MemoryChunk.
      19             : #include "src/macro-assembler.h"
      20             : #include "src/objects-inl.h"
      21             : #include "src/objects/smi.h"
      22             : #include "src/register-configuration.h"
      23             : #include "src/snapshot/embedded-data.h"
      24             : #include "src/snapshot/snapshot.h"
      25             : #include "src/string-constants.h"
      26             : #include "src/x64/assembler-x64.h"
      27             : 
      28             : // Satisfy cpplint check, but don't include platform-specific header. It is
      29             : // included recursively via macro-assembler.h.
      30             : #if 0
      31             : #include "src/x64/macro-assembler-x64.h"
      32             : #endif
      33             : 
      34             : namespace v8 {
      35             : namespace internal {
      36             : 
      37        1792 : Operand StackArgumentsAccessor::GetArgumentOperand(int index) {
      38             :   DCHECK_GE(index, 0);
      39        1792 :   int receiver = (receiver_mode_ == ARGUMENTS_CONTAIN_RECEIVER) ? 1 : 0;
      40             :   int displacement_to_last_argument =
      41        1792 :       base_reg_ == rsp ? kPCOnStackSize : kFPOnStackSize + kPCOnStackSize;
      42        1792 :   displacement_to_last_argument += extra_displacement_to_last_argument_;
      43        1792 :   if (argument_count_reg_ == no_reg) {
      44             :     // argument[0] is at base_reg_ + displacement_to_last_argument +
      45             :     // (argument_count_immediate_ + receiver - 1) * kSystemPointerSize.
      46             :     DCHECK_GT(argument_count_immediate_ + receiver, 0);
      47             :     return Operand(base_reg_,
      48             :                    displacement_to_last_argument +
      49           0 :                        (argument_count_immediate_ + receiver - 1 - index) *
      50           0 :                            kSystemPointerSize);
      51             :   } else {
      52             :     // argument[0] is at base_reg_ + displacement_to_last_argument +
      53             :     // argument_count_reg_ * times_system_pointer_size + (receiver - 1) *
      54             :     // kSystemPointerSize.
      55             :     return Operand(base_reg_, argument_count_reg_, times_system_pointer_size,
      56             :                    displacement_to_last_argument +
      57        1792 :                        (receiver - 1 - index) * kSystemPointerSize);
      58             :   }
      59             : }
      60             : 
      61           0 : StackArgumentsAccessor::StackArgumentsAccessor(
      62             :     Register base_reg, const ParameterCount& parameter_count,
      63             :     StackArgumentsAccessorReceiverMode receiver_mode,
      64             :     int extra_displacement_to_last_argument)
      65             :     : base_reg_(base_reg),
      66             :       argument_count_reg_(parameter_count.is_reg() ? parameter_count.reg()
      67             :                                                    : no_reg),
      68             :       argument_count_immediate_(
      69             :           parameter_count.is_immediate() ? parameter_count.immediate() : 0),
      70             :       receiver_mode_(receiver_mode),
      71             :       extra_displacement_to_last_argument_(
      72         672 :           extra_displacement_to_last_argument) {}
      73             : 
      74         392 : void MacroAssembler::Load(Register destination, ExternalReference source) {
      75         392 :   if (root_array_available_ && options().enable_root_array_delta_access) {
      76           0 :     intptr_t delta = RootRegisterOffsetForExternalReference(isolate(), source);
      77           0 :     if (is_int32(delta)) {
      78           0 :       movq(destination, Operand(kRootRegister, static_cast<int32_t>(delta)));
      79           0 :       return;
      80             :     }
      81             :   }
      82             :   // Safe code.
      83         392 :   if (destination == rax && !options().isolate_independent_code) {
      84           0 :     load_rax(source);
      85             :   } else {
      86         392 :     movq(destination, ExternalReferenceAsOperand(source));
      87             :   }
      88             : }
      89             : 
      90             : 
      91       47127 : void MacroAssembler::Store(ExternalReference destination, Register source) {
      92       47127 :   if (root_array_available_ && options().enable_root_array_delta_access) {
      93             :     intptr_t delta =
      94           0 :         RootRegisterOffsetForExternalReference(isolate(), destination);
      95           0 :     if (is_int32(delta)) {
      96           0 :       movq(Operand(kRootRegister, static_cast<int32_t>(delta)), source);
      97           0 :       return;
      98             :     }
      99             :   }
     100             :   // Safe code.
     101       47127 :   if (source == rax && !options().isolate_independent_code) {
     102           0 :     store_rax(destination);
     103             :   } else {
     104       47127 :     movq(ExternalReferenceAsOperand(destination), source);
     105             :   }
     106             : }
     107             : 
     108       40544 : void TurboAssembler::LoadFromConstantsTable(Register destination,
     109             :                                             int constant_index) {
     110             :   DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
     111       40544 :   LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
     112             :   LoadTaggedPointerField(
     113             :       destination,
     114             :       FieldOperand(destination, FixedArray::OffsetOfElementAt(constant_index)));
     115       40544 : }
     116             : 
     117       19264 : void TurboAssembler::LoadRootRegisterOffset(Register destination,
     118             :                                             intptr_t offset) {
     119             :   DCHECK(is_int32(offset));
     120       19264 :   if (offset == 0) {
     121             :     Move(destination, kRootRegister);
     122             :   } else {
     123       36736 :     leaq(destination, Operand(kRootRegister, static_cast<int32_t>(offset)));
     124             :   }
     125       19264 : }
     126             : 
     127      775936 : void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
     128     1551872 :   movq(destination, Operand(kRootRegister, offset));
     129      775936 : }
     130             : 
     131      953836 : void TurboAssembler::LoadAddress(Register destination,
     132             :                                  ExternalReference source) {
     133      953836 :   if (root_array_available_ && options().enable_root_array_delta_access) {
     134        1172 :     intptr_t delta = RootRegisterOffsetForExternalReference(isolate(), source);
     135        1172 :     if (is_int32(delta)) {
     136        2344 :       leaq(destination, Operand(kRootRegister, static_cast<int32_t>(delta)));
     137        1172 :       return;
     138             :     }
     139             :   }
     140             :   // Safe code.
     141             :   if (FLAG_embedded_builtins) {
     142      952664 :     if (root_array_available_ && options().isolate_independent_code) {
     143       45528 :       IndirectLoadExternalReference(destination, source);
     144       45528 :       return;
     145             :     }
     146             :   }
     147      907136 :   Move(destination, source);
     148             : }
     149             : 
     150     1148479 : Operand TurboAssembler::ExternalReferenceAsOperand(ExternalReference reference,
     151             :                                                    Register scratch) {
     152     1148479 :   if (root_array_available_ && options().enable_root_array_delta_access) {
     153             :     int64_t delta =
     154        3596 :         RootRegisterOffsetForExternalReference(isolate(), reference);
     155        3596 :     if (is_int32(delta)) {
     156        3596 :       return Operand(kRootRegister, static_cast<int32_t>(delta));
     157             :     }
     158             :   }
     159     1144883 :   if (root_array_available_ && options().isolate_independent_code) {
     160       91504 :     if (IsAddressableThroughRootRegister(isolate(), reference)) {
     161             :       // Some external references can be efficiently loaded as an offset from
     162             :       // kRootRegister.
     163             :       intptr_t offset =
     164       90888 :           RootRegisterOffsetForExternalReference(isolate(), reference);
     165       90888 :       CHECK(is_int32(offset));
     166       90888 :       return Operand(kRootRegister, static_cast<int32_t>(offset));
     167             :     } else {
     168             :       // Otherwise, do a memory load from the external reference table.
     169        1232 :       movq(scratch, Operand(kRootRegister,
     170             :                             RootRegisterOffsetForExternalReferenceTableEntry(
     171             :                                 isolate(), reference)));
     172         616 :       return Operand(scratch, 0);
     173             :     }
     174             :   }
     175     1053379 :   Move(scratch, reference);
     176     1053380 :   return Operand(scratch, 0);
     177             : }
     178             : 
     179         112 : void MacroAssembler::PushAddress(ExternalReference source) {
     180         112 :   LoadAddress(kScratchRegister, source);
     181             :   Push(kScratchRegister);
     182         112 : }
     183             : 
     184     1946833 : void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
     185             :   DCHECK(root_array_available_);
     186     3893672 :   movq(destination,
     187             :        Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
     188     1946839 : }
     189             : 
     190         672 : void MacroAssembler::PushRoot(RootIndex index) {
     191             :   DCHECK(root_array_available_);
     192        1344 :   Push(Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
     193         672 : }
     194             : 
     195      564316 : void TurboAssembler::CompareRoot(Register with, RootIndex index) {
     196             :   DCHECK(root_array_available_);
     197      564316 :   if (IsInRange(index, RootIndex::kFirstStrongOrReadOnlyRoot,
     198             :                 RootIndex::kLastStrongOrReadOnlyRoot)) {
     199        2016 :     cmp_tagged(with,
     200        1008 :                Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
     201             :   } else {
     202             :     // Some smi roots contain system pointer size values like stack limits.
     203      563308 :     cmpq(with, Operand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
     204             :   }
     205      564321 : }
     206             : 
     207           0 : void TurboAssembler::CompareRoot(Operand with, RootIndex index) {
     208             :   DCHECK(root_array_available_);
     209             :   DCHECK(!with.AddressUsesRegister(kScratchRegister));
     210           0 :   LoadRoot(kScratchRegister, index);
     211           0 :   if (IsInRange(index, RootIndex::kFirstStrongOrReadOnlyRoot,
     212             :                 RootIndex::kLastStrongOrReadOnlyRoot)) {
     213           0 :     cmp_tagged(with, kScratchRegister);
     214             :   } else {
     215             :     // Some smi roots contain system pointer size values like stack limits.
     216           0 :     cmpq(with, kScratchRegister);
     217             :   }
     218           0 : }
     219             : 
     220      593044 : void TurboAssembler::LoadTaggedPointerField(Register destination,
     221             :                                             Operand field_operand) {
     222             : #ifdef V8_COMPRESS_POINTERS
     223             :   DecompressTaggedPointer(destination, field_operand);
     224             : #else
     225      635100 :   mov_tagged(destination, field_operand);
     226             : #endif
     227      593044 : }
     228             : 
     229         336 : void TurboAssembler::LoadAnyTaggedField(Register destination,
     230             :                                         Operand field_operand,
     231             :                                         Register scratch) {
     232             : #ifdef V8_COMPRESS_POINTERS
     233             :   DecompressAnyTagged(destination, field_operand, scratch);
     234             : #else
     235         336 :   mov_tagged(destination, field_operand);
     236             : #endif
     237         336 : }
     238             : 
     239         112 : void TurboAssembler::PushTaggedPointerField(Operand field_operand,
     240             :                                             Register scratch) {
     241             : #ifdef V8_COMPRESS_POINTERS
     242             :   DCHECK(!field_operand.AddressUsesRegister(scratch));
     243             :   DecompressTaggedPointer(scratch, field_operand);
     244             :   Push(scratch);
     245             : #else
     246             :   Push(field_operand);
     247             : #endif
     248         112 : }
     249             : 
     250         112 : void TurboAssembler::PushTaggedAnyField(Operand field_operand,
     251             :                                         Register scratch1, Register scratch2) {
     252             : #ifdef V8_COMPRESS_POINTERS
     253             :   DCHECK(!AreAliased(scratch1, scratch2));
     254             :   DCHECK(!field_operand.AddressUsesRegister(scratch1));
     255             :   DCHECK(!field_operand.AddressUsesRegister(scratch2));
     256             :   DecompressAnyTagged(scratch1, field_operand, scratch2);
     257             :   Push(scratch1);
     258             : #else
     259             :   Push(field_operand);
     260             : #endif
     261         112 : }
     262             : 
     263         280 : void TurboAssembler::SmiUntagField(Register dst, Operand src) {
     264         280 :   SmiUntag(dst, src);
     265         280 : }
     266             : 
     267           0 : void TurboAssembler::StoreTaggedField(Operand dst_field_operand,
     268             :                                       Immediate value) {
     269             : #ifdef V8_COMPRESS_POINTERS
     270             :   RecordComment("[ StoreTagged");
     271             :   movl(dst_field_operand, value);
     272             :   RecordComment("]");
     273             : #else
     274           0 :   movq(dst_field_operand, value);
     275             : #endif
     276           0 : }
     277             : 
     278      321680 : void TurboAssembler::StoreTaggedField(Operand dst_field_operand,
     279             :                                       Register value) {
     280             : #ifdef V8_COMPRESS_POINTERS
     281             :   RecordComment("[ StoreTagged");
     282             :   movl(dst_field_operand, value);
     283             :   RecordComment("]");
     284             : #else
     285      321680 :   movq(dst_field_operand, value);
     286             : #endif
     287      321680 : }
     288             : 
     289           0 : void TurboAssembler::DecompressTaggedSigned(Register destination,
     290             :                                             Operand field_operand) {
     291           0 :   RecordComment("[ DecompressTaggedSigned");
     292           0 :   movsxlq(destination, field_operand);
     293           0 :   RecordComment("]");
     294           0 : }
     295             : 
     296           0 : void TurboAssembler::DecompressTaggedPointer(Register destination,
     297             :                                              Operand field_operand) {
     298           0 :   RecordComment("[ DecompressTaggedPointer");
     299           0 :   movsxlq(destination, field_operand);
     300             :   addq(destination, kRootRegister);
     301           0 :   RecordComment("]");
     302           0 : }
     303             : 
     304           0 : void TurboAssembler::DecompressAnyTagged(Register destination,
     305             :                                          Operand field_operand,
     306             :                                          Register scratch) {
     307             :   DCHECK(!AreAliased(destination, scratch));
     308           0 :   RecordComment("[ DecompressAnyTagged");
     309           0 :   movsxlq(destination, field_operand);
     310             :   if (kUseBranchlessPtrDecompression) {
     311             :     // Branchlessly compute |masked_root|:
     312             :     // masked_root = HAS_SMI_TAG(destination) ? 0 : kRootRegister;
     313             :     STATIC_ASSERT((kSmiTagSize == 1) && (kSmiTag < 32));
     314           0 :     Register masked_root = scratch;
     315             :     movl(masked_root, destination);
     316             :     andl(masked_root, Immediate(kSmiTagMask));
     317             :     negq(masked_root);
     318             :     andq(masked_root, kRootRegister);
     319             :     // Now this add operation will either leave the value unchanged if it is
     320             :     // a smi or add the isolate root if it is a heap object.
     321             :     addq(destination, masked_root);
     322             :   } else {
     323             :     Label done;
     324             :     JumpIfSmi(destination, &done);
     325             :     addq(destination, kRootRegister);
     326             :     bind(&done);
     327             :   }
     328           0 :   RecordComment("]");
     329           0 : }
     330             : 
     331         112 : void MacroAssembler::RecordWriteField(Register object, int offset,
     332             :                                       Register value, Register dst,
     333             :                                       SaveFPRegsMode save_fp,
     334             :                                       RememberedSetAction remembered_set_action,
     335             :                                       SmiCheck smi_check) {
     336             :   // First, check if a write barrier is even needed. The tests below
     337             :   // catch stores of Smis.
     338         112 :   Label done;
     339             : 
     340             :   // Skip barrier if writing a smi.
     341         112 :   if (smi_check == INLINE_SMI_CHECK) {
     342          56 :     JumpIfSmi(value, &done);
     343             :   }
     344             : 
     345             :   // Although the object register is tagged, the offset is relative to the start
     346             :   // of the object, so the offset must be a multiple of kTaggedSize.
     347             :   DCHECK(IsAligned(offset, kTaggedSize));
     348             : 
     349         112 :   leaq(dst, FieldOperand(object, offset));
     350         112 :   if (emit_debug_code()) {
     351           0 :     Label ok;
     352           0 :     testb(dst, Immediate(kTaggedSize - 1));
     353           0 :     j(zero, &ok, Label::kNear);
     354           0 :     int3();
     355           0 :     bind(&ok);
     356             :   }
     357             : 
     358             :   RecordWrite(object, dst, value, save_fp, remembered_set_action,
     359         112 :               OMIT_SMI_CHECK);
     360             : 
     361         112 :   bind(&done);
     362             : 
     363             :   // Clobber clobbered input registers when running with the debug-code flag
     364             :   // turned on to provoke errors.
     365         112 :   if (emit_debug_code()) {
     366             :     Move(value, kZapValue, RelocInfo::NONE);
     367             :     Move(dst, kZapValue, RelocInfo::NONE);
     368             :   }
     369         112 : }
     370             : 
     371      321675 : void TurboAssembler::SaveRegisters(RegList registers) {
     372             :   DCHECK_GT(NumRegs(registers), 0);
     373    10615445 :   for (int i = 0; i < Register::kNumRegisters; ++i) {
     374     5146879 :     if ((registers >> i) & 1u) {
     375     1608399 :       pushq(Register::from_code(i));
     376             :     }
     377             :   }
     378      321681 : }
     379             : 
     380      321683 : void TurboAssembler::RestoreRegisters(RegList registers) {
     381             :   DCHECK_GT(NumRegs(registers), 0);
     382    10615445 :   for (int i = Register::kNumRegisters - 1; i >= 0; --i) {
     383     5146883 :     if ((registers >> i) & 1u) {
     384     1608403 :       popq(Register::from_code(i));
     385             :     }
     386             :   }
     387      321681 : }
     388             : 
     389         112 : void TurboAssembler::CallEphemeronKeyBarrier(Register object, Register address,
     390             :                                              SaveFPRegsMode fp_mode) {
     391             :   EphemeronKeyBarrierDescriptor descriptor;
     392             :   RegList registers = descriptor.allocatable_registers();
     393             : 
     394         112 :   SaveRegisters(registers);
     395             : 
     396             :   Register object_parameter(
     397         112 :       descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
     398             :   Register slot_parameter(descriptor.GetRegisterParameter(
     399         112 :       EphemeronKeyBarrierDescriptor::kSlotAddress));
     400             :   Register fp_mode_parameter(
     401         112 :       descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));
     402             : 
     403         112 :   MovePair(slot_parameter, address, object_parameter, object);
     404         112 :   Smi smi_fm = Smi::FromEnum(fp_mode);
     405         112 :   Move(fp_mode_parameter, smi_fm);
     406         112 :   Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
     407         112 :        RelocInfo::CODE_TARGET);
     408             : 
     409         112 :   RestoreRegisters(registers);
     410         112 : }
     411             : 
     412      321284 : void TurboAssembler::CallRecordWriteStub(
     413             :     Register object, Register address,
     414             :     RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
     415      321284 :   CallRecordWriteStub(
     416             :       object, address, remembered_set_action, fp_mode,
     417             :       isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
     418      321284 :       kNullAddress);
     419      321284 : }
     420             : 
     421         284 : void TurboAssembler::CallRecordWriteStub(
     422             :     Register object, Register address,
     423             :     RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
     424             :     Address wasm_target) {
     425             :   CallRecordWriteStub(object, address, remembered_set_action, fp_mode,
     426         284 :                       Handle<Code>::null(), wasm_target);
     427         286 : }
     428             : 
     429      321568 : void TurboAssembler::CallRecordWriteStub(
     430             :     Register object, Register address,
     431             :     RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
     432             :     Handle<Code> code_target, Address wasm_target) {
     433             :   DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
     434             : 
     435             :   RecordWriteDescriptor descriptor;
     436             :   RegList registers = descriptor.allocatable_registers();
     437             : 
     438      321568 :   SaveRegisters(registers);
     439             : 
     440             :   Register object_parameter(
     441      321570 :       descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
     442             :   Register slot_parameter(
     443      321570 :       descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
     444             :   Register remembered_set_parameter(
     445             :       descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
     446             :   Register fp_mode_parameter(
     447             :       descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
     448             : 
     449             :   // Prepare argument registers for calling RecordWrite
     450             :   // slot_parameter   <= address
     451             :   // object_parameter <= object
     452      321570 :   MovePair(slot_parameter, address, object_parameter, object);
     453             : 
     454             :   Smi smi_rsa = Smi::FromEnum(remembered_set_action);
     455             :   Smi smi_fm = Smi::FromEnum(fp_mode);
     456      321569 :   Move(remembered_set_parameter, smi_rsa);
     457      321570 :   if (smi_rsa != smi_fm) {
     458      278011 :     Move(fp_mode_parameter, smi_fm);
     459             :   } else {
     460       43559 :     movq(fp_mode_parameter, remembered_set_parameter);
     461             :   }
     462      321570 :   if (code_target.is_null()) {
     463             :     // Use {near_call} for direct Wasm call within a module.
     464         286 :     near_call(wasm_target, RelocInfo::WASM_STUB_CALL);
     465             :   } else {
     466      321284 :     Call(code_target, RelocInfo::CODE_TARGET);
     467             :   }
     468             : 
     469      321568 :   RestoreRegisters(registers);
     470      321569 : }
     471             : 
     472         112 : void MacroAssembler::RecordWrite(Register object, Register address,
     473             :                                  Register value, SaveFPRegsMode fp_mode,
     474             :                                  RememberedSetAction remembered_set_action,
     475             :                                  SmiCheck smi_check) {
     476             :   DCHECK(object != value);
     477             :   DCHECK(object != address);
     478             :   DCHECK(value != address);
     479         112 :   AssertNotSmi(object);
     480             : 
     481         168 :   if (remembered_set_action == OMIT_REMEMBERED_SET &&
     482          56 :       !FLAG_incremental_marking) {
     483           0 :     return;
     484             :   }
     485             : 
     486         112 :   if (emit_debug_code()) {
     487           0 :     Label ok;
     488           0 :     cmp_tagged(value, Operand(address, 0));
     489           0 :     j(equal, &ok, Label::kNear);
     490           0 :     int3();
     491           0 :     bind(&ok);
     492             :   }
     493             : 
     494             :   // First, check if a write barrier is even needed. The tests below
     495             :   // catch stores of smis and stores into the young generation.
     496         112 :   Label done;
     497             : 
     498         112 :   if (smi_check == INLINE_SMI_CHECK) {
     499             :     // Skip barrier if writing a smi.
     500           0 :     JumpIfSmi(value, &done);
     501             :   }
     502             : 
     503         112 :   CheckPageFlag(value,
     504             :                 value,  // Used as scratch.
     505             :                 MemoryChunk::kPointersToHereAreInterestingMask, zero, &done,
     506         112 :                 Label::kNear);
     507             : 
     508             :   CheckPageFlag(object,
     509             :                 value,  // Used as scratch.
     510             :                 MemoryChunk::kPointersFromHereAreInterestingMask,
     511             :                 zero,
     512             :                 &done,
     513         112 :                 Label::kNear);
     514             : 
     515         112 :   CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
     516             : 
     517         112 :   bind(&done);
     518             : 
     519             :   // Clobber clobbered registers when running with the debug-code flag
     520             :   // turned on to provoke errors.
     521         112 :   if (emit_debug_code()) {
     522             :     Move(address, kZapValue, RelocInfo::NONE);
     523             :     Move(value, kZapValue, RelocInfo::NONE);
     524             :   }
     525             : }
     526             : 
     527         121 : void TurboAssembler::Assert(Condition cc, AbortReason reason) {
     528         121 :   if (emit_debug_code()) Check(cc, reason);
     529         121 : }
     530             : 
     531      320908 : void TurboAssembler::AssertUnreachable(AbortReason reason) {
     532      320908 :   if (emit_debug_code()) Abort(reason);
     533      320908 : }
     534             : 
     535         180 : void TurboAssembler::Check(Condition cc, AbortReason reason) {
     536         180 :   Label L;
     537         180 :   j(cc, &L, Label::kNear);
     538         180 :   Abort(reason);
     539             :   // Control will not return here.
     540         180 :   bind(&L);
     541         180 : }
     542             : 
     543           0 : void TurboAssembler::CheckStackAlignment() {
     544           0 :   int frame_alignment = base::OS::ActivationFrameAlignment();
     545           0 :   int frame_alignment_mask = frame_alignment - 1;
     546           0 :   if (frame_alignment > kSystemPointerSize) {
     547             :     DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
     548           0 :     Label alignment_as_expected;
     549           0 :     testq(rsp, Immediate(frame_alignment_mask));
     550           0 :     j(zero, &alignment_as_expected, Label::kNear);
     551             :     // Abort if stack is not aligned.
     552           0 :     int3();
     553           0 :     bind(&alignment_as_expected);
     554             :   }
     555           0 : }
     556             : 
     557        4893 : void TurboAssembler::Abort(AbortReason reason) {
     558             : #ifdef DEBUG
     559             :   const char* msg = GetAbortReason(reason);
     560             :   RecordComment("Abort message: ");
     561             :   RecordComment(msg);
     562             : #endif
     563             : 
     564             :   // Avoid emitting call to builtin if requested.
     565        4893 :   if (trap_on_abort()) {
     566           0 :     int3();
     567           0 :     return;
     568             :   }
     569             : 
     570        4893 :   if (should_abort_hard()) {
     571             :     // We don't care if we constructed a frame. Just pretend we did.
     572          10 :     FrameScope assume_frame(this, StackFrame::NONE);
     573          10 :     movl(arg_reg_1, Immediate(static_cast<int>(reason)));
     574          10 :     PrepareCallCFunction(1);
     575          10 :     LoadAddress(rax, ExternalReference::abort_with_reason());
     576          10 :     call(rax);
     577             :     return;
     578             :   }
     579             : 
     580        4883 :   Move(rdx, Smi::FromInt(static_cast<int>(reason)));
     581             : 
     582        4883 :   if (!has_frame()) {
     583             :     // We don't actually want to generate a pile of code for this, so just
     584             :     // claim there is a stack frame, without generating one.
     585          56 :     FrameScope scope(this, StackFrame::NONE);
     586          56 :     Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
     587             :   } else {
     588        4827 :     Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
     589             :   }
     590             :   // Control will not return here.
     591        4883 :   int3();
     592             : }
     593             : 
     594          74 : void TurboAssembler::CallRuntimeWithCEntry(Runtime::FunctionId fid,
     595             :                                            Register centry) {
     596          74 :   const Runtime::Function* f = Runtime::FunctionForId(fid);
     597             :   // TODO(1236192): Most runtime routines don't need the number of
     598             :   // arguments passed in because it is constant. At some point we
     599             :   // should remove this need and make the runtime routine entry code
     600             :   // smarter.
     601          74 :   Set(rax, f->nargs);
     602          74 :   LoadAddress(rbx, ExternalReference::Create(f));
     603             :   DCHECK(!AreAliased(centry, rax, rbx));
     604             :   DCHECK(centry == rcx);
     605          74 :   CallCodeObject(centry);
     606          75 : }
     607             : 
     608        1960 : void MacroAssembler::CallRuntime(const Runtime::Function* f,
     609             :                                  int num_arguments,
     610             :                                  SaveFPRegsMode save_doubles) {
     611             :   // If the expected number of arguments of the runtime function is
     612             :   // constant, we check that the actual number of arguments match the
     613             :   // expectation.
     614        1960 :   CHECK(f->nargs < 0 || f->nargs == num_arguments);
     615             : 
     616             :   // TODO(1236192): Most runtime routines don't need the number of
     617             :   // arguments passed in because it is constant. At some point we
     618             :   // should remove this need and make the runtime routine entry code
     619             :   // smarter.
     620        1960 :   Set(rax, num_arguments);
     621        1960 :   LoadAddress(rbx, ExternalReference::Create(f));
     622             :   Handle<Code> code =
     623        1960 :       CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
     624        1960 :   Call(code, RelocInfo::CODE_TARGET);
     625        1960 : }
     626             : 
     627         840 : void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
     628             :   // ----------- S t a t e -------------
     629             :   //  -- rsp[0]                 : return address
     630             :   //  -- rsp[8]                 : argument num_arguments - 1
     631             :   //  ...
     632             :   //  -- rsp[8 * num_arguments] : argument 0 (receiver)
     633             :   //
     634             :   //  For runtime functions with variable arguments:
     635             :   //  -- rax                    : number of  arguments
     636             :   // -----------------------------------
     637             : 
     638         840 :   const Runtime::Function* function = Runtime::FunctionForId(fid);
     639             :   DCHECK_EQ(1, function->result_size);
     640         840 :   if (function->nargs >= 0) {
     641         840 :     Set(rax, function->nargs);
     642             :   }
     643         840 :   JumpToExternalReference(ExternalReference::Create(fid));
     644         840 : }
     645             : 
     646         840 : void MacroAssembler::JumpToExternalReference(const ExternalReference& ext,
     647             :                                              bool builtin_exit_frame) {
     648             :   // Set the entry point and jump to the C entry runtime stub.
     649         840 :   LoadAddress(rbx, ext);
     650             :   Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
     651         840 :                                           kArgvOnStack, builtin_exit_frame);
     652         840 :   Jump(code, RelocInfo::CODE_TARGET);
     653         840 : }
     654             : 
     655             : static constexpr Register saved_regs[] = {rax, rcx, rdx, rbx, rbp, rsi,
     656             :                                           rdi, r8,  r9,  r10, r11};
     657             : 
     658             : static constexpr int kNumberOfSavedRegs = sizeof(saved_regs) / sizeof(Register);
     659             : 
     660         900 : int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
     661             :                                                     Register exclusion1,
     662             :                                                     Register exclusion2,
     663             :                                                     Register exclusion3) const {
     664             :   int bytes = 0;
     665       20700 :   for (int i = 0; i < kNumberOfSavedRegs; i++) {
     666        9900 :     Register reg = saved_regs[i];
     667        9900 :     if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
     668        9000 :       bytes += kSystemPointerSize;
     669             :     }
     670             :   }
     671             : 
     672             :   // R12 to r15 are callee save on all platforms.
     673         900 :   if (fp_mode == kSaveFPRegs) {
     674         452 :     bytes += kDoubleSize * XMMRegister::kNumRegisters;
     675             :   }
     676             : 
     677         900 :   return bytes;
     678             : }
     679             : 
     680         900 : int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
     681             :                                     Register exclusion2, Register exclusion3) {
     682             :   // We don't allow a GC during a store buffer overflow so there is no need to
     683             :   // store the registers in any particular way, but we do have to store and
     684             :   // restore them.
     685             :   int bytes = 0;
     686       20700 :   for (int i = 0; i < kNumberOfSavedRegs; i++) {
     687        9900 :     Register reg = saved_regs[i];
     688        9900 :     if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
     689        9000 :       pushq(reg);
     690        9000 :       bytes += kSystemPointerSize;
     691             :     }
     692             :   }
     693             : 
     694             :   // R12 to r15 are callee save on all platforms.
     695         900 :   if (fp_mode == kSaveFPRegs) {
     696             :     int delta = kDoubleSize * XMMRegister::kNumRegisters;
     697         452 :     subq(rsp, Immediate(delta));
     698       14916 :     for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
     699             :       XMMRegister reg = XMMRegister::from_code(i);
     700       14464 :       Movsd(Operand(rsp, i * kDoubleSize), reg);
     701             :     }
     702         452 :     bytes += delta;
     703             :   }
     704             : 
     705         900 :   return bytes;
     706             : }
     707             : 
     708         900 : int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
     709             :                                    Register exclusion2, Register exclusion3) {
     710             :   int bytes = 0;
     711         900 :   if (fp_mode == kSaveFPRegs) {
     712       14916 :     for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
     713             :       XMMRegister reg = XMMRegister::from_code(i);
     714       14464 :       Movsd(reg, Operand(rsp, i * kDoubleSize));
     715             :     }
     716             :     int delta = kDoubleSize * XMMRegister::kNumRegisters;
     717         452 :     addq(rsp, Immediate(kDoubleSize * XMMRegister::kNumRegisters));
     718             :     bytes += delta;
     719             :   }
     720             : 
     721       20700 :   for (int i = kNumberOfSavedRegs - 1; i >= 0; i--) {
     722        9900 :     Register reg = saved_regs[i];
     723        9900 :     if (reg != exclusion1 && reg != exclusion2 && reg != exclusion3) {
     724        9000 :       popq(reg);
     725        9000 :       bytes += kSystemPointerSize;
     726             :     }
     727             :   }
     728             : 
     729         900 :   return bytes;
     730             : }
     731             : 
     732        9154 : void TurboAssembler::Cvtss2sd(XMMRegister dst, XMMRegister src) {
     733        9154 :   if (CpuFeatures::IsSupported(AVX)) {
     734             :     CpuFeatureScope scope(this, AVX);
     735        9138 :     vcvtss2sd(dst, src, src);
     736             :   } else {
     737          16 :     cvtss2sd(dst, src);
     738             :   }
     739        9153 : }
     740             : 
     741       11349 : void TurboAssembler::Cvtss2sd(XMMRegister dst, Operand src) {
     742       11349 :   if (CpuFeatures::IsSupported(AVX)) {
     743             :     CpuFeatureScope scope(this, AVX);
     744       11349 :     vcvtss2sd(dst, dst, src);
     745             :   } else {
     746           0 :     cvtss2sd(dst, src);
     747             :   }
     748       11349 : }
     749             : 
     750        6174 : void TurboAssembler::Cvtsd2ss(XMMRegister dst, XMMRegister src) {
     751        6174 :   if (CpuFeatures::IsSupported(AVX)) {
     752             :     CpuFeatureScope scope(this, AVX);
     753        6155 :     vcvtsd2ss(dst, src, src);
     754             :   } else {
     755          19 :     cvtsd2ss(dst, src);
     756             :   }
     757        6174 : }
     758             : 
     759       11852 : void TurboAssembler::Cvtsd2ss(XMMRegister dst, Operand src) {
     760       11852 :   if (CpuFeatures::IsSupported(AVX)) {
     761             :     CpuFeatureScope scope(this, AVX);
     762       11852 :     vcvtsd2ss(dst, dst, src);
     763             :   } else {
     764           0 :     cvtsd2ss(dst, src);
     765             :   }
     766       11852 : }
     767             : 
     768      375418 : void TurboAssembler::Cvtlsi2sd(XMMRegister dst, Register src) {
     769      375418 :   if (CpuFeatures::IsSupported(AVX)) {
     770             :     CpuFeatureScope scope(this, AVX);
     771      373392 :     vxorpd(dst, dst, dst);
     772             :     vcvtlsi2sd(dst, dst, src);
     773             :   } else {
     774        2026 :     xorpd(dst, dst);
     775        2026 :     cvtlsi2sd(dst, src);
     776             :   }
     777      375421 : }
     778             : 
     779        3858 : void TurboAssembler::Cvtlsi2sd(XMMRegister dst, Operand src) {
     780        3858 :   if (CpuFeatures::IsSupported(AVX)) {
     781             :     CpuFeatureScope scope(this, AVX);
     782        3856 :     vxorpd(dst, dst, dst);
     783             :     vcvtlsi2sd(dst, dst, src);
     784             :   } else {
     785           2 :     xorpd(dst, dst);
     786           2 :     cvtlsi2sd(dst, src);
     787             :   }
     788        3858 : }
     789             : 
     790        1095 : void TurboAssembler::Cvtlsi2ss(XMMRegister dst, Register src) {
     791        1095 :   if (CpuFeatures::IsSupported(AVX)) {
     792             :     CpuFeatureScope scope(this, AVX);
     793        1081 :     vxorps(dst, dst, dst);
     794             :     vcvtlsi2ss(dst, dst, src);
     795             :   } else {
     796          14 :     xorps(dst, dst);
     797          14 :     cvtlsi2ss(dst, src);
     798             :   }
     799        1095 : }
     800             : 
     801           8 : void TurboAssembler::Cvtlsi2ss(XMMRegister dst, Operand src) {
     802           8 :   if (CpuFeatures::IsSupported(AVX)) {
     803             :     CpuFeatureScope scope(this, AVX);
     804           8 :     vxorps(dst, dst, dst);
     805             :     vcvtlsi2ss(dst, dst, src);
     806             :   } else {
     807           0 :     xorps(dst, dst);
     808           0 :     cvtlsi2ss(dst, src);
     809             :   }
     810           8 : }
     811             : 
     812         291 : void TurboAssembler::Cvtqsi2ss(XMMRegister dst, Register src) {
     813         291 :   if (CpuFeatures::IsSupported(AVX)) {
     814             :     CpuFeatureScope scope(this, AVX);
     815         291 :     vxorps(dst, dst, dst);
     816             :     vcvtqsi2ss(dst, dst, src);
     817             :   } else {
     818           0 :     xorps(dst, dst);
     819           0 :     cvtqsi2ss(dst, src);
     820             :   }
     821         291 : }
     822             : 
     823           0 : void TurboAssembler::Cvtqsi2ss(XMMRegister dst, Operand src) {
     824           0 :   if (CpuFeatures::IsSupported(AVX)) {
     825             :     CpuFeatureScope scope(this, AVX);
     826           0 :     vxorps(dst, dst, dst);
     827             :     vcvtqsi2ss(dst, dst, src);
     828             :   } else {
     829           0 :     xorps(dst, dst);
     830           0 :     cvtqsi2ss(dst, src);
     831             :   }
     832           0 : }
     833             : 
     834       20519 : void TurboAssembler::Cvtqsi2sd(XMMRegister dst, Register src) {
     835       20519 :   if (CpuFeatures::IsSupported(AVX)) {
     836             :     CpuFeatureScope scope(this, AVX);
     837       20203 :     vxorpd(dst, dst, dst);
     838             :     vcvtqsi2sd(dst, dst, src);
     839             :   } else {
     840         316 :     xorpd(dst, dst);
     841         316 :     cvtqsi2sd(dst, src);
     842             :   }
     843       20533 : }
     844             : 
     845        2000 : void TurboAssembler::Cvtqsi2sd(XMMRegister dst, Operand src) {
     846        2000 :   if (CpuFeatures::IsSupported(AVX)) {
     847             :     CpuFeatureScope scope(this, AVX);
     848        1966 :     vxorpd(dst, dst, dst);
     849             :     vcvtqsi2sd(dst, dst, src);
     850             :   } else {
     851          34 :     xorpd(dst, dst);
     852          34 :     cvtqsi2sd(dst, src);
     853             :   }
     854        2000 : }
     855             : 
     856          88 : void TurboAssembler::Cvtlui2ss(XMMRegister dst, Register src) {
     857             :   // Zero-extend the 32 bit value to 64 bit.
     858          88 :   movl(kScratchRegister, src);
     859          88 :   Cvtqsi2ss(dst, kScratchRegister);
     860          88 : }
     861             : 
     862           0 : void TurboAssembler::Cvtlui2ss(XMMRegister dst, Operand src) {
     863             :   // Zero-extend the 32 bit value to 64 bit.
     864           0 :   movl(kScratchRegister, src);
     865           0 :   Cvtqsi2ss(dst, kScratchRegister);
     866           0 : }
     867             : 
     868         391 : void TurboAssembler::Cvtlui2sd(XMMRegister dst, Register src) {
     869             :   // Zero-extend the 32 bit value to 64 bit.
     870         391 :   movl(kScratchRegister, src);
     871         391 :   Cvtqsi2sd(dst, kScratchRegister);
     872         391 : }
     873             : 
     874       10962 : void TurboAssembler::Cvtlui2sd(XMMRegister dst, Operand src) {
     875             :   // Zero-extend the 32 bit value to 64 bit.
     876       10962 :   movl(kScratchRegister, src);
     877       10962 :   Cvtqsi2sd(dst, kScratchRegister);
     878       10962 : }
     879             : 
     880          45 : void TurboAssembler::Cvtqui2ss(XMMRegister dst, Register src) {
     881          45 :   Label done;
     882          45 :   Cvtqsi2ss(dst, src);
     883          45 :   testq(src, src);
     884          45 :   j(positive, &done, Label::kNear);
     885             : 
     886             :   // Compute {src/2 | (src&1)} (retain the LSB to avoid rounding errors).
     887          45 :   if (src != kScratchRegister) movq(kScratchRegister, src);
     888             :   shrq(kScratchRegister, Immediate(1));
     889             :   // The LSB is shifted into CF. If it is set, set the LSB in {tmp}.
     890          45 :   Label msb_not_set;
     891          45 :   j(not_carry, &msb_not_set, Label::kNear);
     892             :   orq(kScratchRegister, Immediate(1));
     893          45 :   bind(&msb_not_set);
     894          45 :   Cvtqsi2ss(dst, kScratchRegister);
     895          45 :   addss(dst, dst);
     896          45 :   bind(&done);
     897          45 : }
     898             : 
     899           0 : void TurboAssembler::Cvtqui2ss(XMMRegister dst, Operand src) {
     900           0 :   movq(kScratchRegister, src);
     901           0 :   Cvtqui2ss(dst, kScratchRegister);
     902           0 : }
     903             : 
     904        3660 : void TurboAssembler::Cvtqui2sd(XMMRegister dst, Register src) {
     905        3660 :   Label done;
     906        3660 :   Cvtqsi2sd(dst, src);
     907        3659 :   testq(src, src);
     908        3660 :   j(positive, &done, Label::kNear);
     909             : 
     910             :   // Compute {src/2 | (src&1)} (retain the LSB to avoid rounding errors).
     911        3660 :   if (src != kScratchRegister) movq(kScratchRegister, src);
     912             :   shrq(kScratchRegister, Immediate(1));
     913             :   // The LSB is shifted into CF. If it is set, set the LSB in {tmp}.
     914        3660 :   Label msb_not_set;
     915        3660 :   j(not_carry, &msb_not_set, Label::kNear);
     916             :   orq(kScratchRegister, Immediate(1));
     917        3660 :   bind(&msb_not_set);
     918        3660 :   Cvtqsi2sd(dst, kScratchRegister);
     919        3660 :   addsd(dst, dst);
     920        3660 :   bind(&done);
     921        3660 : }
     922             : 
     923        1232 : void TurboAssembler::Cvtqui2sd(XMMRegister dst, Operand src) {
     924        1232 :   movq(kScratchRegister, src);
     925        1232 :   Cvtqui2sd(dst, kScratchRegister);
     926        1232 : }
     927             : 
     928         435 : void TurboAssembler::Cvttss2si(Register dst, XMMRegister src) {
     929         435 :   if (CpuFeatures::IsSupported(AVX)) {
     930             :     CpuFeatureScope scope(this, AVX);
     931         435 :     vcvttss2si(dst, src);
     932             :   } else {
     933           0 :     cvttss2si(dst, src);
     934             :   }
     935         435 : }
     936             : 
     937           0 : void TurboAssembler::Cvttss2si(Register dst, Operand src) {
     938           0 :   if (CpuFeatures::IsSupported(AVX)) {
     939             :     CpuFeatureScope scope(this, AVX);
     940           0 :     vcvttss2si(dst, src);
     941             :   } else {
     942           0 :     cvttss2si(dst, src);
     943             :   }
     944           0 : }
     945             : 
     946      107968 : void TurboAssembler::Cvttsd2si(Register dst, XMMRegister src) {
     947      107968 :   if (CpuFeatures::IsSupported(AVX)) {
     948             :     CpuFeatureScope scope(this, AVX);
     949      107446 :     vcvttsd2si(dst, src);
     950             :   } else {
     951         522 :     cvttsd2si(dst, src);
     952             :   }
     953      107968 : }
     954             : 
     955       20274 : void TurboAssembler::Cvttsd2si(Register dst, Operand src) {
     956       20274 :   if (CpuFeatures::IsSupported(AVX)) {
     957             :     CpuFeatureScope scope(this, AVX);
     958       20274 :     vcvttsd2si(dst, src);
     959             :   } else {
     960           0 :     cvttsd2si(dst, src);
     961             :   }
     962       20274 : }
     963             : 
     964         278 : void TurboAssembler::Cvttss2siq(Register dst, XMMRegister src) {
     965         278 :   if (CpuFeatures::IsSupported(AVX)) {
     966             :     CpuFeatureScope scope(this, AVX);
     967         278 :     vcvttss2siq(dst, src);
     968             :   } else {
     969           0 :     cvttss2siq(dst, src);
     970             :   }
     971         278 : }
     972             : 
     973           0 : void TurboAssembler::Cvttss2siq(Register dst, Operand src) {
     974           0 :   if (CpuFeatures::IsSupported(AVX)) {
     975             :     CpuFeatureScope scope(this, AVX);
     976           0 :     vcvttss2siq(dst, src);
     977             :   } else {
     978           0 :     cvttss2siq(dst, src);
     979             :   }
     980           0 : }
     981             : 
     982       62330 : void TurboAssembler::Cvttsd2siq(Register dst, XMMRegister src) {
     983       62330 :   if (CpuFeatures::IsSupported(AVX)) {
     984             :     CpuFeatureScope scope(this, AVX);
     985       61919 :     vcvttsd2siq(dst, src);
     986             :   } else {
     987         411 :     cvttsd2siq(dst, src);
     988             :   }
     989       62338 : }
     990             : 
     991           1 : void TurboAssembler::Cvttsd2siq(Register dst, Operand src) {
     992           1 :   if (CpuFeatures::IsSupported(AVX)) {
     993             :     CpuFeatureScope scope(this, AVX);
     994           1 :     vcvttsd2siq(dst, src);
     995             :   } else {
     996           0 :     cvttsd2siq(dst, src);
     997             :   }
     998           1 : }
     999             : 
    1000             : namespace {
    1001             : template <typename OperandOrXMMRegister, bool is_double>
    1002        3058 : void ConvertFloatToUint64(TurboAssembler* tasm, Register dst,
    1003             :                           OperandOrXMMRegister src, Label* fail) {
    1004        3058 :   Label success;
    1005             :   // There does not exist a native float-to-uint instruction, so we have to use
    1006             :   // a float-to-int, and postprocess the result.
    1007             :   if (is_double) {
    1008        2991 :     tasm->Cvttsd2siq(dst, src);
    1009             :   } else {
    1010          67 :     tasm->Cvttss2siq(dst, src);
    1011             :   }
    1012             :   // If the result of the conversion is positive, we are already done.
    1013        3058 :   tasm->testq(dst, dst);
    1014        3058 :   tasm->j(positive, &success);
    1015             :   // The result of the first conversion was negative, which means that the
    1016             :   // input value was not within the positive int64 range. We subtract 2^63
    1017             :   // and convert it again to see if it is within the uint64 range.
    1018             :   if (is_double) {
    1019             :     tasm->Move(kScratchDoubleReg, -9223372036854775808.0);
    1020        2991 :     tasm->addsd(kScratchDoubleReg, src);
    1021        2991 :     tasm->Cvttsd2siq(dst, kScratchDoubleReg);
    1022             :   } else {
    1023             :     tasm->Move(kScratchDoubleReg, -9223372036854775808.0f);
    1024          68 :     tasm->addss(kScratchDoubleReg, src);
    1025          68 :     tasm->Cvttss2siq(dst, kScratchDoubleReg);
    1026             :   }
    1027             :   tasm->testq(dst, dst);
    1028             :   // The only possible negative value here is 0x80000000000000000, which is
    1029             :   // used on x64 to indicate an integer overflow.
    1030        3059 :   tasm->j(negative, fail ? fail : &success);
    1031             :   // The input value is within uint64 range and the second conversion worked
    1032             :   // successfully, but we still have to undo the subtraction we did
    1033             :   // earlier.
    1034        3059 :   tasm->Set(kScratchRegister, 0x8000000000000000);
    1035             :   tasm->orq(dst, kScratchRegister);
    1036        3059 :   tasm->bind(&success);
    1037        3059 : }
    1038             : }  // namespace
    1039             : 
    1040           0 : void TurboAssembler::Cvttsd2uiq(Register dst, Operand src, Label* success) {
    1041           0 :   ConvertFloatToUint64<Operand, true>(this, dst, src, success);
    1042           0 : }
    1043             : 
    1044        2991 : void TurboAssembler::Cvttsd2uiq(Register dst, XMMRegister src, Label* success) {
    1045        2991 :   ConvertFloatToUint64<XMMRegister, true>(this, dst, src, success);
    1046        2991 : }
    1047             : 
    1048           0 : void TurboAssembler::Cvttss2uiq(Register dst, Operand src, Label* success) {
    1049           0 :   ConvertFloatToUint64<Operand, false>(this, dst, src, success);
    1050           0 : }
    1051             : 
    1052          68 : void TurboAssembler::Cvttss2uiq(Register dst, XMMRegister src, Label* success) {
    1053          68 :   ConvertFloatToUint64<XMMRegister, false>(this, dst, src, success);
    1054          68 : }
    1055             : 
    1056     2736475 : void TurboAssembler::Set(Register dst, int64_t x) {
    1057     2736475 :   if (x == 0) {
    1058      868130 :     xorl(dst, dst);
    1059     1868345 :   } else if (is_uint32(x)) {
    1060      422368 :     movl(dst, Immediate(static_cast<uint32_t>(x)));
    1061     1445977 :   } else if (is_int32(x)) {
    1062      171430 :     movq(dst, Immediate(static_cast<int32_t>(x)));
    1063             :   } else {
    1064     1274547 :     movq(dst, x);
    1065             :   }
    1066     2736508 : }
    1067             : 
    1068       13279 : void TurboAssembler::Set(Operand dst, intptr_t x) {
    1069       13279 :   if (is_int32(x)) {
    1070        8780 :     movq(dst, Immediate(static_cast<int32_t>(x)));
    1071             :   } else {
    1072        4499 :     Set(kScratchRegister, x);
    1073        4499 :     movq(dst, kScratchRegister);
    1074             :   }
    1075       13279 : }
    1076             : 
    1077             : 
    1078             : // ----------------------------------------------------------------------------
    1079             : // Smi tagging, untagging and tag detection.
    1080             : 
    1081         168 : Register TurboAssembler::GetSmiConstant(Smi source) {
    1082             :   STATIC_ASSERT(kSmiTag == 0);
    1083             :   int value = source->value();
    1084         168 :   if (value == 0) {
    1085           0 :     xorl(kScratchRegister, kScratchRegister);
    1086           0 :     return kScratchRegister;
    1087             :   }
    1088         168 :   Move(kScratchRegister, source);
    1089         168 :   return kScratchRegister;
    1090             : }
    1091             : 
    1092     2174020 : void TurboAssembler::Move(Register dst, Smi source) {
    1093             :   STATIC_ASSERT(kSmiTag == 0);
    1094             :   int value = source->value();
    1095     2174020 :   if (value == 0) {
    1096      770206 :     xorl(dst, dst);
    1097             :   } else {
    1098             :     Move(dst, source.ptr(), RelocInfo::NONE);
    1099             :   }
    1100     2174027 : }
    1101             : 
    1102     5651948 : void TurboAssembler::Move(Register dst, ExternalReference ext) {
    1103             :   if (FLAG_embedded_builtins) {
    1104     5651948 :     if (root_array_available_ && options().isolate_independent_code) {
    1105      740600 :       IndirectLoadExternalReference(dst, ext);
    1106      740600 :       return;
    1107             :     }
    1108             :   }
    1109     4911348 :   movq(dst, Immediate64(ext.address(), RelocInfo::EXTERNAL_REFERENCE));
    1110             : }
    1111             : 
    1112        1246 : void MacroAssembler::SmiTag(Register dst, Register src) {
    1113             :   STATIC_ASSERT(kSmiTag == 0);
    1114        1246 :   if (dst != src) {
    1115         305 :     movq(dst, src);
    1116             :   }
    1117             :   DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
    1118        1246 :   shlq(dst, Immediate(kSmiShift));
    1119        1246 : }
    1120             : 
    1121        5176 : void TurboAssembler::SmiUntag(Register dst, Register src) {
    1122             :   STATIC_ASSERT(kSmiTag == 0);
    1123        5176 :   if (dst != src) {
    1124           0 :     movq(dst, src);
    1125             :   }
    1126             :   DCHECK(SmiValuesAre32Bits() || SmiValuesAre31Bits());
    1127        5176 :   sarq(dst, Immediate(kSmiShift));
    1128        5176 : }
    1129             : 
    1130        1848 : void TurboAssembler::SmiUntag(Register dst, Operand src) {
    1131             :   if (SmiValuesAre32Bits()) {
    1132        3696 :     movl(dst, Operand(src, kSmiShift / kBitsPerByte));
    1133             :     // Sign extend to 64-bit.
    1134        1848 :     movsxlq(dst, dst);
    1135             :   } else {
    1136             :     DCHECK(SmiValuesAre31Bits());
    1137             : #ifdef V8_COMPRESS_POINTERS
    1138             :     movsxlq(dst, src);
    1139             : #else
    1140             :     movq(dst, src);
    1141             : #endif
    1142             :     sarq(dst, Immediate(kSmiShift));
    1143             :   }
    1144        1848 : }
    1145             : 
    1146         165 : void MacroAssembler::SmiCompare(Register smi1, Register smi2) {
    1147         165 :   AssertSmi(smi1);
    1148         165 :   AssertSmi(smi2);
    1149         165 :   cmp_tagged(smi1, smi2);
    1150         165 : }
    1151             : 
    1152         224 : void MacroAssembler::SmiCompare(Register dst, Smi src) {
    1153         224 :   AssertSmi(dst);
    1154         224 :   Cmp(dst, src);
    1155         224 : }
    1156             : 
    1157         224 : void MacroAssembler::Cmp(Register dst, Smi src) {
    1158             :   DCHECK_NE(dst, kScratchRegister);
    1159         224 :   if (src->value() == 0) {
    1160          56 :     test_tagged(dst, dst);
    1161             :   } else {
    1162         168 :     Register constant_reg = GetSmiConstant(src);
    1163         168 :     cmp_tagged(dst, constant_reg);
    1164             :   }
    1165         224 : }
    1166             : 
    1167           0 : void MacroAssembler::SmiCompare(Register dst, Operand src) {
    1168           0 :   AssertSmi(dst);
    1169           0 :   AssertSmi(src);
    1170           0 :   cmp_tagged(dst, src);
    1171           0 : }
    1172             : 
    1173           0 : void MacroAssembler::SmiCompare(Operand dst, Register src) {
    1174           0 :   AssertSmi(dst);
    1175           0 :   AssertSmi(src);
    1176           0 :   cmp_tagged(dst, src);
    1177           0 : }
    1178             : 
    1179           0 : void MacroAssembler::SmiCompare(Operand dst, Smi src) {
    1180           0 :   AssertSmi(dst);
    1181             :   if (SmiValuesAre32Bits()) {
    1182           0 :     cmpl(Operand(dst, kSmiShift / kBitsPerByte), Immediate(src->value()));
    1183             :   } else {
    1184             :     DCHECK(SmiValuesAre31Bits());
    1185             :     cmpl(dst, Immediate(src));
    1186             :   }
    1187           0 : }
    1188             : 
    1189           0 : void MacroAssembler::Cmp(Operand dst, Smi src) {
    1190             :   // The Operand cannot use the smi register.
    1191           0 :   Register smi_reg = GetSmiConstant(src);
    1192             :   DCHECK(!dst.AddressUsesRegister(smi_reg));
    1193           0 :   cmp_tagged(dst, smi_reg);
    1194           0 : }
    1195             : 
    1196             : 
    1197          40 : Condition TurboAssembler::CheckSmi(Register src) {
    1198             :   STATIC_ASSERT(kSmiTag == 0);
    1199      299130 :   testb(src, Immediate(kSmiTagMask));
    1200          40 :   return zero;
    1201             : }
    1202             : 
    1203           0 : Condition TurboAssembler::CheckSmi(Operand src) {
    1204             :   STATIC_ASSERT(kSmiTag == 0);
    1205           0 :   testb(src, Immediate(kSmiTagMask));
    1206           0 :   return zero;
    1207             : }
    1208             : 
    1209      299034 : void TurboAssembler::JumpIfSmi(Register src, Label* on_smi,
    1210             :                                Label::Distance near_jump) {
    1211             :   Condition smi = CheckSmi(src);
    1212      299034 :   j(smi, on_smi, near_jump);
    1213      299034 : }
    1214             : 
    1215          56 : void MacroAssembler::JumpIfNotSmi(Register src,
    1216             :                                   Label* on_not_smi,
    1217             :                                   Label::Distance near_jump) {
    1218             :   Condition smi = CheckSmi(src);
    1219          56 :   j(NegateCondition(smi), on_not_smi, near_jump);
    1220          56 : }
    1221             : 
    1222           0 : void MacroAssembler::JumpIfNotSmi(Operand src, Label* on_not_smi,
    1223             :                                   Label::Distance near_jump) {
    1224             :   Condition smi = CheckSmi(src);
    1225           0 :   j(NegateCondition(smi), on_not_smi, near_jump);
    1226           0 : }
    1227             : 
    1228           0 : void MacroAssembler::SmiAddConstant(Operand dst, Smi constant) {
    1229           0 :   if (constant->value() != 0) {
    1230             :     if (SmiValuesAre32Bits()) {
    1231           0 :       addl(Operand(dst, kSmiShift / kBitsPerByte),
    1232           0 :            Immediate(constant->value()));
    1233             :     } else {
    1234             :       DCHECK(SmiValuesAre31Bits());
    1235             :       if (kTaggedSize == kInt64Size) {
    1236             :         // Sign-extend value after addition
    1237             :         movl(kScratchRegister, dst);
    1238             :         addl(kScratchRegister, Immediate(constant));
    1239             :         movsxlq(kScratchRegister, kScratchRegister);
    1240             :         movq(dst, kScratchRegister);
    1241             :       } else {
    1242             :         DCHECK_EQ(kTaggedSize, kInt32Size);
    1243             :         addl(dst, Immediate(constant));
    1244             :       }
    1245             :     }
    1246             :   }
    1247           0 : }
    1248             : 
    1249         568 : SmiIndex MacroAssembler::SmiToIndex(Register dst,
    1250             :                                     Register src,
    1251             :                                     int shift) {
    1252             :   if (SmiValuesAre32Bits()) {
    1253             :     DCHECK(is_uint6(shift));
    1254             :     // There is a possible optimization if shift is in the range 60-63, but that
    1255             :     // will (and must) never happen.
    1256         568 :     if (dst != src) {
    1257         200 :       movq(dst, src);
    1258             :     }
    1259         568 :     if (shift < kSmiShift) {
    1260         568 :       sarq(dst, Immediate(kSmiShift - shift));
    1261             :     } else {
    1262           0 :       shlq(dst, Immediate(shift - kSmiShift));
    1263             :     }
    1264         568 :     return SmiIndex(dst, times_1);
    1265             :   } else {
    1266             :     DCHECK(SmiValuesAre31Bits());
    1267             :     if (dst != src) {
    1268             :       mov_tagged(dst, src);
    1269             :     }
    1270             :     // We have to sign extend the index register to 64-bit as the SMI might
    1271             :     // be negative.
    1272             :     movsxlq(dst, dst);
    1273             :     if (shift < kSmiShift) {
    1274             :       sarq(dst, Immediate(kSmiShift - shift));
    1275             :     } else if (shift != kSmiShift) {
    1276             :       if (shift - kSmiShift <= static_cast<int>(times_8)) {
    1277             :         return SmiIndex(dst, static_cast<ScaleFactor>(shift - kSmiShift));
    1278             :       }
    1279             :       shlq(dst, Immediate(shift - kSmiShift));
    1280             :     }
    1281             :     return SmiIndex(dst, times_1);
    1282             :   }
    1283             : }
    1284             : 
    1285          56 : void TurboAssembler::Push(Smi source) {
    1286          56 :   intptr_t smi = static_cast<intptr_t>(source.ptr());
    1287          56 :   if (is_int32(smi)) {
    1288          56 :     Push(Immediate(static_cast<int32_t>(smi)));
    1289          56 :     return;
    1290             :   }
    1291           0 :   int first_byte_set = base::bits::CountTrailingZeros64(smi) / 8;
    1292           0 :   int last_byte_set = (63 - base::bits::CountLeadingZeros64(smi)) / 8;
    1293           0 :   if (first_byte_set == last_byte_set) {
    1294             :     // This sequence has only 7 bytes, compared to the 12 bytes below.
    1295             :     Push(Immediate(0));
    1296           0 :     movb(Operand(rsp, first_byte_set),
    1297           0 :          Immediate(static_cast<int8_t>(smi >> (8 * first_byte_set))));
    1298           0 :     return;
    1299             :   }
    1300           0 :   Register constant = GetSmiConstant(source);
    1301             :   Push(constant);
    1302             : }
    1303             : 
    1304             : // ----------------------------------------------------------------------------
    1305             : 
    1306         728 : void TurboAssembler::Move(Register dst, Register src) {
    1307     1052095 :   if (dst != src) {
    1308      633945 :     movq(dst, src);
    1309             :   }
    1310         728 : }
    1311             : 
    1312      321680 : void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
    1313             :                               Register src1) {
    1314      321680 :   if (dst0 != src1) {
    1315             :     // Normal case: Writing to dst0 does not destroy src1.
    1316             :     Move(dst0, src0);
    1317             :     Move(dst1, src1);
    1318        1755 :   } else if (dst1 != src0) {
    1319             :     // Only dst0 and src1 are the same register,
    1320             :     // but writing to dst1 does not destroy src0.
    1321             :     Move(dst1, src1);
    1322             :     Move(dst0, src0);
    1323             :   } else {
    1324             :     // dst0 == src1, and dst1 == src0, a swap is required:
    1325             :     // dst0 \/ src0
    1326             :     // dst1 /\ src1
    1327          14 :     xchgq(dst0, dst1);
    1328             :   }
    1329      321681 : }
    1330             : 
    1331     1523104 : void TurboAssembler::MoveNumber(Register dst, double value) {
    1332             :   int32_t smi;
    1333     1523104 :   if (DoubleToSmiInteger(value, &smi)) {
    1334     1483380 :     Move(dst, Smi::FromInt(smi));
    1335             :   } else {
    1336       39724 :     movq_heap_number(dst, value);
    1337             :   }
    1338     1523109 : }
    1339             : 
    1340      169870 : void TurboAssembler::Move(XMMRegister dst, uint32_t src) {
    1341      169870 :   if (src == 0) {
    1342             :     Xorps(dst, dst);
    1343             :   } else {
    1344             :     unsigned nlz = base::bits::CountLeadingZeros(src);
    1345             :     unsigned ntz = base::bits::CountTrailingZeros(src);
    1346             :     unsigned pop = base::bits::CountPopulation(src);
    1347             :     DCHECK_NE(0u, pop);
    1348      158711 :     if (pop + ntz + nlz == 32) {
    1349             :       Pcmpeqd(dst, dst);
    1350       70825 :       if (ntz) Pslld(dst, static_cast<byte>(ntz + nlz));
    1351       70830 :       if (nlz) Psrld(dst, static_cast<byte>(nlz));
    1352             :     } else {
    1353       87891 :       movl(kScratchRegister, Immediate(src));
    1354             :       Movd(dst, kScratchRegister);
    1355             :     }
    1356             :   }
    1357      169886 : }
    1358             : 
    1359      465355 : void TurboAssembler::Move(XMMRegister dst, uint64_t src) {
    1360      465355 :   if (src == 0) {
    1361             :     Xorpd(dst, dst);
    1362             :   } else {
    1363             :     unsigned nlz = base::bits::CountLeadingZeros(src);
    1364             :     unsigned ntz = base::bits::CountTrailingZeros(src);
    1365             :     unsigned pop = base::bits::CountPopulation(src);
    1366             :     DCHECK_NE(0u, pop);
    1367      359627 :     if (pop + ntz + nlz == 64) {
    1368             :       Pcmpeqd(dst, dst);
    1369      231050 :       if (ntz) Psllq(dst, static_cast<byte>(ntz + nlz));
    1370      231053 :       if (nlz) Psrlq(dst, static_cast<byte>(nlz));
    1371             :     } else {
    1372      128581 :       uint32_t lower = static_cast<uint32_t>(src);
    1373      128581 :       uint32_t upper = static_cast<uint32_t>(src >> 32);
    1374      128581 :       if (upper == 0) {
    1375          40 :         Move(dst, lower);
    1376             :       } else {
    1377      128541 :         movq(kScratchRegister, src);
    1378             :         Movq(dst, kScratchRegister);
    1379             :       }
    1380             :     }
    1381             :   }
    1382      465377 : }
    1383             : 
    1384             : // ----------------------------------------------------------------------------
    1385             : 
    1386           5 : void MacroAssembler::Absps(XMMRegister dst) {
    1387           5 :   Andps(dst, ExternalReferenceAsOperand(
    1388             :                  ExternalReference::address_of_float_abs_constant()));
    1389           5 : }
    1390             : 
    1391           5 : void MacroAssembler::Negps(XMMRegister dst) {
    1392           5 :   Xorps(dst, ExternalReferenceAsOperand(
    1393             :                  ExternalReference::address_of_float_neg_constant()));
    1394           5 : }
    1395             : 
    1396           5 : void MacroAssembler::Abspd(XMMRegister dst) {
    1397           5 :   Andps(dst, ExternalReferenceAsOperand(
    1398             :                  ExternalReference::address_of_double_abs_constant()));
    1399           5 : }
    1400             : 
    1401           5 : void MacroAssembler::Negpd(XMMRegister dst) {
    1402           5 :   Xorps(dst, ExternalReferenceAsOperand(
    1403             :                  ExternalReference::address_of_double_neg_constant()));
    1404           5 : }
    1405             : 
    1406           0 : void MacroAssembler::Cmp(Register dst, Handle<Object> source) {
    1407             :   AllowDeferredHandleDereference smi_check;
    1408           0 :   if (source->IsSmi()) {
    1409           0 :     Cmp(dst, Smi::cast(*source));
    1410             :   } else {
    1411           0 :     Move(kScratchRegister, Handle<HeapObject>::cast(source));
    1412           0 :     cmp_tagged(dst, kScratchRegister);
    1413             :   }
    1414           0 : }
    1415             : 
    1416         112 : void MacroAssembler::Cmp(Operand dst, Handle<Object> source) {
    1417             :   AllowDeferredHandleDereference smi_check;
    1418         112 :   if (source->IsSmi()) {
    1419             :     Cmp(dst, Smi::cast(*source));
    1420             :   } else {
    1421         112 :     Move(kScratchRegister, Handle<HeapObject>::cast(source));
    1422         112 :     cmp_tagged(dst, kScratchRegister);
    1423             :   }
    1424         112 : }
    1425             : 
    1426          56 : void MacroAssembler::JumpIfIsInRange(Register value, unsigned lower_limit,
    1427             :                                      unsigned higher_limit, Label* on_in_range,
    1428             :                                      Label::Distance near_jump) {
    1429          56 :   if (lower_limit != 0) {
    1430         112 :     leal(kScratchRegister, Operand(value, 0u - lower_limit));
    1431          56 :     cmpl(kScratchRegister, Immediate(higher_limit - lower_limit));
    1432             :   } else {
    1433           0 :     cmpl(value, Immediate(higher_limit));
    1434             :   }
    1435          56 :   j(below_equal, on_in_range, near_jump);
    1436          56 : }
    1437             : 
    1438          21 : void TurboAssembler::Push(Handle<HeapObject> source) {
    1439          21 :   Move(kScratchRegister, source);
    1440             :   Push(kScratchRegister);
    1441          21 : }
    1442             : 
    1443     6300431 : void TurboAssembler::Move(Register result, Handle<HeapObject> object,
    1444             :                           RelocInfo::Mode rmode) {
    1445             :   if (FLAG_embedded_builtins) {
    1446     6300431 :     if (root_array_available_ && options().isolate_independent_code) {
    1447       49728 :       IndirectLoadConstant(result, object);
    1448       49728 :       return;
    1449             :     }
    1450             :   }
    1451     6250703 :   movq(result, Immediate64(object.address(), rmode));
    1452             : }
    1453             : 
    1454           0 : void TurboAssembler::Move(Operand dst, Handle<HeapObject> object,
    1455             :                           RelocInfo::Mode rmode) {
    1456           0 :   Move(kScratchRegister, object, rmode);
    1457           0 :   movq(dst, kScratchRegister);
    1458           0 : }
    1459             : 
    1460        2139 : void TurboAssembler::MoveStringConstant(Register result,
    1461             :                                         const StringConstantBase* string,
    1462             :                                         RelocInfo::Mode rmode) {
    1463        2139 :   movq_string(result, string);
    1464        2139 : }
    1465             : 
    1466         288 : void MacroAssembler::Drop(int stack_elements) {
    1467         288 :   if (stack_elements > 0) {
    1468         288 :     addq(rsp, Immediate(stack_elements * kSystemPointerSize));
    1469             :   }
    1470         288 : }
    1471             : 
    1472             : 
    1473          56 : void MacroAssembler::DropUnderReturnAddress(int stack_elements,
    1474             :                                             Register scratch) {
    1475             :   DCHECK_GT(stack_elements, 0);
    1476          56 :   if (stack_elements == 1) {
    1477          56 :     popq(MemOperand(rsp, 0));
    1478          56 :     return;
    1479             :   }
    1480             : 
    1481             :   PopReturnAddressTo(scratch);
    1482           0 :   Drop(stack_elements);
    1483             :   PushReturnAddressFrom(scratch);
    1484             : }
    1485             : 
    1486      674926 : void TurboAssembler::Push(Register src) { pushq(src); }
    1487             : 
    1488       60638 : void TurboAssembler::Push(Operand src) { pushq(src); }
    1489             : 
    1490      805949 : void MacroAssembler::PushQuad(Operand src) { pushq(src); }
    1491             : 
    1492     1738570 : void TurboAssembler::Push(Immediate value) { pushq(value); }
    1493             : 
    1494           0 : void MacroAssembler::PushImm32(int32_t imm32) { pushq_imm32(imm32); }
    1495             : 
    1496        3104 : void MacroAssembler::Pop(Register dst) { popq(dst); }
    1497             : 
    1498       45111 : void MacroAssembler::Pop(Operand dst) { popq(dst); }
    1499             : 
    1500      716400 : void MacroAssembler::PopQuad(Operand dst) { popq(dst); }
    1501             : 
    1502           0 : void TurboAssembler::Jump(ExternalReference ext) {
    1503           0 :   LoadAddress(kScratchRegister, ext);
    1504           0 :   jmp(kScratchRegister);
    1505           0 : }
    1506             : 
    1507           0 : void TurboAssembler::Jump(Operand op) { jmp(op); }
    1508             : 
    1509        1024 : void TurboAssembler::Jump(Address destination, RelocInfo::Mode rmode) {
    1510             :   Move(kScratchRegister, destination, rmode);
    1511        1024 :   jmp(kScratchRegister);
    1512        1024 : }
    1513             : 
    1514      514316 : void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
    1515             :                           Condition cc) {
    1516             :   DCHECK_IMPLIES(options().isolate_independent_code,
    1517             :                  Builtins::IsIsolateIndependentBuiltin(*code_object));
    1518      514316 :   if (options().inline_offheap_trampolines) {
    1519      464367 :     int builtin_index = Builtins::kNoBuiltinId;
    1520      464367 :     if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index) &&
    1521             :         Builtins::IsIsolateIndependent(builtin_index)) {
    1522      464160 :       Label skip;
    1523      464160 :       if (cc != always) {
    1524      464153 :         if (cc == never) return;
    1525      464152 :         j(NegateCondition(cc), &skip, Label::kNear);
    1526             :       }
    1527             :       // Inline the trampoline.
    1528      464154 :       RecordCommentForOffHeapTrampoline(builtin_index);
    1529      464154 :       CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
    1530      464157 :       EmbeddedData d = EmbeddedData::FromBlob();
    1531      464157 :       Address entry = d.InstructionStartOfBuiltin(builtin_index);
    1532             :       Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
    1533      464164 :       jmp(kScratchRegister);
    1534      464163 :       bind(&skip);
    1535      464162 :       return;
    1536             :     }
    1537             :   }
    1538       50157 :   j(cc, code_object, rmode);
    1539             : }
    1540             : 
    1541    36109370 : void MacroAssembler::JumpToInstructionStream(Address entry) {
    1542             :   Move(kOffHeapTrampolineRegister, entry, RelocInfo::OFF_HEAP_TARGET);
    1543    36109367 :   jmp(kOffHeapTrampolineRegister);
    1544    36109363 : }
    1545             : 
    1546           0 : void TurboAssembler::Call(ExternalReference ext) {
    1547           0 :   LoadAddress(kScratchRegister, ext);
    1548           0 :   call(kScratchRegister);
    1549           0 : }
    1550             : 
    1551        3608 : void TurboAssembler::Call(Operand op) {
    1552        3608 :   if (!CpuFeatures::IsSupported(ATOM)) {
    1553        3608 :     call(op);
    1554             :   } else {
    1555           0 :     movq(kScratchRegister, op);
    1556           0 :     call(kScratchRegister);
    1557             :   }
    1558        3608 : }
    1559             : 
    1560           0 : void TurboAssembler::Call(Address destination, RelocInfo::Mode rmode) {
    1561             :   Move(kScratchRegister, destination, rmode);
    1562           0 :   call(kScratchRegister);
    1563           0 : }
    1564             : 
    1565     4838364 : void TurboAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
    1566             :   DCHECK_IMPLIES(options().isolate_independent_code,
    1567             :                  Builtins::IsIsolateIndependentBuiltin(*code_object));
    1568     4838364 :   if (options().inline_offheap_trampolines) {
    1569     3999619 :     int builtin_index = Builtins::kNoBuiltinId;
    1570     3999619 :     if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index) &&
    1571             :         Builtins::IsIsolateIndependent(builtin_index)) {
    1572             :       // Inline the trampoline.
    1573     3983398 :       RecordCommentForOffHeapTrampoline(builtin_index);
    1574     3983399 :       CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
    1575     3983401 :       EmbeddedData d = EmbeddedData::FromBlob();
    1576     3983401 :       Address entry = d.InstructionStartOfBuiltin(builtin_index);
    1577             :       Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
    1578     3983405 :       call(kScratchRegister);
    1579             :       return;
    1580             :     }
    1581             :   }
    1582             :   DCHECK(RelocInfo::IsCodeTarget(rmode));
    1583      854969 :   call(code_object, rmode);
    1584             : }
    1585             : 
    1586        3608 : void TurboAssembler::CallBuiltinPointer(Register builtin_pointer) {
    1587             : #if defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
    1588             :   STATIC_ASSERT(kSmiShiftSize == 0);
    1589             :   STATIC_ASSERT(kSmiTagSize == 1);
    1590             :   STATIC_ASSERT(kSmiTag == 0);
    1591             : 
    1592             :   // The builtin_pointer register contains the builtin index as a Smi.
    1593             :   // Untagging is folded into the indexing operand below (we use times_4 instead
    1594             :   // of times_8 since smis are already shifted by one).
    1595             :   Call(Operand(kRootRegister, builtin_pointer, times_4,
    1596             :                IsolateData::builtin_entry_table_offset()));
    1597             : #else   // defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
    1598             :   STATIC_ASSERT(kSmiShiftSize == 31);
    1599             :   STATIC_ASSERT(kSmiTagSize == 1);
    1600             :   STATIC_ASSERT(kSmiTag == 0);
    1601             : 
    1602             :   // The builtin_pointer register contains the builtin index as a Smi.
    1603        3608 :   SmiUntag(builtin_pointer, builtin_pointer);
    1604        7216 :   Call(Operand(kRootRegister, builtin_pointer, times_8,
    1605        3608 :                IsolateData::builtin_entry_table_offset()));
    1606             : #endif  // defined(V8_COMPRESS_POINTERS) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH)
    1607        3608 : }
    1608             : 
    1609      407082 : void TurboAssembler::LoadCodeObjectEntry(Register destination,
    1610             :                                          Register code_object) {
    1611             :   // Code objects are called differently depending on whether we are generating
    1612             :   // builtin code (which will later be embedded into the binary) or compiling
    1613             :   // user JS code at runtime.
    1614             :   // * Builtin code runs in --jitless mode and thus must not call into on-heap
    1615             :   //   Code targets. Instead, we dispatch through the builtins entry table.
    1616             :   // * Codegen at runtime does not have this restriction and we can use the
    1617             :   //   shorter, branchless instruction sequence. The assumption here is that
    1618             :   //   targets are usually generated code and not builtin Code objects.
    1619             : 
    1620      407082 :   if (options().isolate_independent_code) {
    1621             :     DCHECK(root_array_available());
    1622        9688 :     Label if_code_is_off_heap, out;
    1623             : 
    1624             :     // Check whether the Code object is an off-heap trampoline. If so, call its
    1625             :     // (off-heap) entry point directly without going through the (on-heap)
    1626             :     // trampoline.  Otherwise, just call the Code object as always.
    1627        9688 :     testl(FieldOperand(code_object, Code::kFlagsOffset),
    1628             :           Immediate(Code::IsOffHeapTrampoline::kMask));
    1629        9688 :     j(not_equal, &if_code_is_off_heap);
    1630             : 
    1631             :     // Not an off-heap trampoline, the entry point is at
    1632             :     // Code::raw_instruction_start().
    1633             :     Move(destination, code_object);
    1634             :     addq(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
    1635        9688 :     jmp(&out);
    1636             : 
    1637             :     // An off-heap trampoline, the entry point is loaded from the builtin entry
    1638             :     // table.
    1639        9688 :     bind(&if_code_is_off_heap);
    1640             :     movl(destination, FieldOperand(code_object, Code::kBuiltinIndexOffset));
    1641       19376 :     movq(destination,
    1642             :          Operand(kRootRegister, destination, times_system_pointer_size,
    1643             :                  IsolateData::builtin_entry_table_offset()));
    1644             : 
    1645        9688 :     bind(&out);
    1646             :   } else {
    1647             :     Move(destination, code_object);
    1648      397394 :     addq(destination, Immediate(Code::kHeaderSize - kHeapObjectTag));
    1649             :   }
    1650      407083 : }
    1651             : 
    1652       24224 : void TurboAssembler::CallCodeObject(Register code_object) {
    1653       24224 :   LoadCodeObjectEntry(code_object, code_object);
    1654       24224 :   call(code_object);
    1655       24226 : }
    1656             : 
    1657         728 : void TurboAssembler::JumpCodeObject(Register code_object) {
    1658         728 :   LoadCodeObjectEntry(code_object, code_object);
    1659         728 :   jmp(code_object);
    1660         728 : }
    1661             : 
    1662           0 : void TurboAssembler::RetpolineCall(Register reg) {
    1663           0 :   Label setup_return, setup_target, inner_indirect_branch, capture_spec;
    1664             : 
    1665           0 :   jmp(&setup_return);  // Jump past the entire retpoline below.
    1666             : 
    1667           0 :   bind(&inner_indirect_branch);
    1668           0 :   call(&setup_target);
    1669             : 
    1670           0 :   bind(&capture_spec);
    1671           0 :   pause();
    1672           0 :   jmp(&capture_spec);
    1673             : 
    1674           0 :   bind(&setup_target);
    1675           0 :   movq(Operand(rsp, 0), reg);
    1676           0 :   ret(0);
    1677             : 
    1678           0 :   bind(&setup_return);
    1679           0 :   call(&inner_indirect_branch);  // Callee will return after this instruction.
    1680           0 : }
    1681             : 
    1682           0 : void TurboAssembler::RetpolineCall(Address destination, RelocInfo::Mode rmode) {
    1683             :   Move(kScratchRegister, destination, rmode);
    1684           0 :   RetpolineCall(kScratchRegister);
    1685           0 : }
    1686             : 
    1687           0 : void TurboAssembler::RetpolineJump(Register reg) {
    1688           0 :   Label setup_target, capture_spec;
    1689             : 
    1690           0 :   call(&setup_target);
    1691             : 
    1692           0 :   bind(&capture_spec);
    1693           0 :   pause();
    1694           0 :   jmp(&capture_spec);
    1695             : 
    1696           0 :   bind(&setup_target);
    1697           0 :   movq(Operand(rsp, 0), reg);
    1698           0 :   ret(0);
    1699           0 : }
    1700             : 
    1701       43177 : void TurboAssembler::Pextrd(Register dst, XMMRegister src, int8_t imm8) {
    1702       43177 :   if (imm8 == 0) {
    1703             :     Movd(dst, src);
    1704             :     return;
    1705             :   }
    1706       42683 :   if (CpuFeatures::IsSupported(SSE4_1)) {
    1707             :     CpuFeatureScope sse_scope(this, SSE4_1);
    1708       42351 :     pextrd(dst, src, imm8);
    1709             :     return;
    1710             :   }
    1711             :   DCHECK_EQ(1, imm8);
    1712         332 :   movq(dst, src);
    1713             :   shrq(dst, Immediate(32));
    1714             : }
    1715             : 
    1716         416 : void TurboAssembler::Pinsrd(XMMRegister dst, Register src, int8_t imm8) {
    1717         416 :   if (CpuFeatures::IsSupported(SSE4_1)) {
    1718             :     CpuFeatureScope sse_scope(this, SSE4_1);
    1719         414 :     pinsrd(dst, src, imm8);
    1720             :     return;
    1721             :   }
    1722             :   Movd(kScratchDoubleReg, src);
    1723           2 :   if (imm8 == 1) {
    1724             :     punpckldq(dst, kScratchDoubleReg);
    1725             :   } else {
    1726             :     DCHECK_EQ(0, imm8);
    1727             :     Movss(dst, kScratchDoubleReg);
    1728             :   }
    1729             : }
    1730             : 
    1731        1488 : void TurboAssembler::Pinsrd(XMMRegister dst, Operand src, int8_t imm8) {
    1732        1488 :   if (CpuFeatures::IsSupported(SSE4_1)) {
    1733             :     CpuFeatureScope sse_scope(this, SSE4_1);
    1734        1488 :     pinsrd(dst, src, imm8);
    1735             :     return;
    1736             :   }
    1737             :   Movd(kScratchDoubleReg, src);
    1738           0 :   if (imm8 == 1) {
    1739             :     punpckldq(dst, kScratchDoubleReg);
    1740             :   } else {
    1741             :     DCHECK_EQ(0, imm8);
    1742             :     Movss(dst, kScratchDoubleReg);
    1743             :   }
    1744             : }
    1745             : 
    1746         624 : void TurboAssembler::Lzcntl(Register dst, Register src) {
    1747         624 :   if (CpuFeatures::IsSupported(LZCNT)) {
    1748             :     CpuFeatureScope scope(this, LZCNT);
    1749           0 :     lzcntl(dst, src);
    1750             :     return;
    1751             :   }
    1752         624 :   Label not_zero_src;
    1753         624 :   bsrl(dst, src);
    1754         624 :   j(not_zero, &not_zero_src, Label::kNear);
    1755         624 :   Set(dst, 63);  // 63^31 == 32
    1756         624 :   bind(&not_zero_src);
    1757             :   xorl(dst, Immediate(31));  // for x in [0..31], 31^x == 31 - x
    1758             : }
    1759             : 
    1760          40 : void TurboAssembler::Lzcntl(Register dst, Operand src) {
    1761          40 :   if (CpuFeatures::IsSupported(LZCNT)) {
    1762             :     CpuFeatureScope scope(this, LZCNT);
    1763           0 :     lzcntl(dst, src);
    1764             :     return;
    1765             :   }
    1766          40 :   Label not_zero_src;
    1767          40 :   bsrl(dst, src);
    1768          40 :   j(not_zero, &not_zero_src, Label::kNear);
    1769          40 :   Set(dst, 63);  // 63^31 == 32
    1770          40 :   bind(&not_zero_src);
    1771             :   xorl(dst, Immediate(31));  // for x in [0..31], 31^x == 31 - x
    1772             : }
    1773             : 
    1774          34 : void TurboAssembler::Lzcntq(Register dst, Register src) {
    1775          34 :   if (CpuFeatures::IsSupported(LZCNT)) {
    1776             :     CpuFeatureScope scope(this, LZCNT);
    1777           0 :     lzcntq(dst, src);
    1778             :     return;
    1779             :   }
    1780          34 :   Label not_zero_src;
    1781          34 :   bsrq(dst, src);
    1782          34 :   j(not_zero, &not_zero_src, Label::kNear);
    1783          34 :   Set(dst, 127);  // 127^63 == 64
    1784          34 :   bind(&not_zero_src);
    1785             :   xorl(dst, Immediate(63));  // for x in [0..63], 63^x == 63 - x
    1786             : }
    1787             : 
    1788           0 : void TurboAssembler::Lzcntq(Register dst, Operand src) {
    1789           0 :   if (CpuFeatures::IsSupported(LZCNT)) {
    1790             :     CpuFeatureScope scope(this, LZCNT);
    1791           0 :     lzcntq(dst, src);
    1792             :     return;
    1793             :   }
    1794           0 :   Label not_zero_src;
    1795           0 :   bsrq(dst, src);
    1796           0 :   j(not_zero, &not_zero_src, Label::kNear);
    1797           0 :   Set(dst, 127);  // 127^63 == 64
    1798           0 :   bind(&not_zero_src);
    1799             :   xorl(dst, Immediate(63));  // for x in [0..63], 63^x == 63 - x
    1800             : }
    1801             : 
    1802          34 : void TurboAssembler::Tzcntq(Register dst, Register src) {
    1803          34 :   if (CpuFeatures::IsSupported(BMI1)) {
    1804             :     CpuFeatureScope scope(this, BMI1);
    1805           0 :     tzcntq(dst, src);
    1806             :     return;
    1807             :   }
    1808          34 :   Label not_zero_src;
    1809          34 :   bsfq(dst, src);
    1810          34 :   j(not_zero, &not_zero_src, Label::kNear);
    1811             :   // Define the result of tzcnt(0) separately, because bsf(0) is undefined.
    1812          34 :   Set(dst, 64);
    1813          34 :   bind(&not_zero_src);
    1814             : }
    1815             : 
    1816           0 : void TurboAssembler::Tzcntq(Register dst, Operand src) {
    1817           0 :   if (CpuFeatures::IsSupported(BMI1)) {
    1818             :     CpuFeatureScope scope(this, BMI1);
    1819           0 :     tzcntq(dst, src);
    1820             :     return;
    1821             :   }
    1822           0 :   Label not_zero_src;
    1823           0 :   bsfq(dst, src);
    1824           0 :   j(not_zero, &not_zero_src, Label::kNear);
    1825             :   // Define the result of tzcnt(0) separately, because bsf(0) is undefined.
    1826           0 :   Set(dst, 64);
    1827           0 :   bind(&not_zero_src);
    1828             : }
    1829             : 
    1830         332 : void TurboAssembler::Tzcntl(Register dst, Register src) {
    1831         332 :   if (CpuFeatures::IsSupported(BMI1)) {
    1832             :     CpuFeatureScope scope(this, BMI1);
    1833           0 :     tzcntl(dst, src);
    1834             :     return;
    1835             :   }
    1836         332 :   Label not_zero_src;
    1837         332 :   bsfl(dst, src);
    1838         332 :   j(not_zero, &not_zero_src, Label::kNear);
    1839         332 :   Set(dst, 32);  // The result of tzcnt is 32 if src = 0.
    1840         332 :   bind(&not_zero_src);
    1841             : }
    1842             : 
    1843           0 : void TurboAssembler::Tzcntl(Register dst, Operand src) {
    1844           0 :   if (CpuFeatures::IsSupported(BMI1)) {
    1845             :     CpuFeatureScope scope(this, BMI1);
    1846           0 :     tzcntl(dst, src);
    1847             :     return;
    1848             :   }
    1849           0 :   Label not_zero_src;
    1850           0 :   bsfl(dst, src);
    1851           0 :   j(not_zero, &not_zero_src, Label::kNear);
    1852           0 :   Set(dst, 32);  // The result of tzcnt is 32 if src = 0.
    1853           0 :   bind(&not_zero_src);
    1854             : }
    1855             : 
    1856          80 : void TurboAssembler::Popcntl(Register dst, Register src) {
    1857          80 :   if (CpuFeatures::IsSupported(POPCNT)) {
    1858             :     CpuFeatureScope scope(this, POPCNT);
    1859          80 :     popcntl(dst, src);
    1860          80 :     return;
    1861             :   }
    1862           0 :   UNREACHABLE();
    1863             : }
    1864             : 
    1865           0 : void TurboAssembler::Popcntl(Register dst, Operand src) {
    1866           0 :   if (CpuFeatures::IsSupported(POPCNT)) {
    1867             :     CpuFeatureScope scope(this, POPCNT);
    1868           0 :     popcntl(dst, src);
    1869           0 :     return;
    1870             :   }
    1871           0 :   UNREACHABLE();
    1872             : }
    1873             : 
    1874          41 : void TurboAssembler::Popcntq(Register dst, Register src) {
    1875          41 :   if (CpuFeatures::IsSupported(POPCNT)) {
    1876             :     CpuFeatureScope scope(this, POPCNT);
    1877          41 :     popcntq(dst, src);
    1878          42 :     return;
    1879             :   }
    1880           0 :   UNREACHABLE();
    1881             : }
    1882             : 
    1883           0 : void TurboAssembler::Popcntq(Register dst, Operand src) {
    1884           0 :   if (CpuFeatures::IsSupported(POPCNT)) {
    1885             :     CpuFeatureScope scope(this, POPCNT);
    1886           0 :     popcntq(dst, src);
    1887           0 :     return;
    1888             :   }
    1889           0 :   UNREACHABLE();
    1890             : }
    1891             : 
    1892             : 
    1893           0 : void MacroAssembler::Pushad() {
    1894             :   Push(rax);
    1895             :   Push(rcx);
    1896             :   Push(rdx);
    1897             :   Push(rbx);
    1898             :   // Not pushing rsp or rbp.
    1899             :   Push(rsi);
    1900             :   Push(rdi);
    1901             :   Push(r8);
    1902             :   Push(r9);
    1903             :   // r10 is kScratchRegister.
    1904             :   Push(r11);
    1905             :   Push(r12);
    1906             :   // r13 is kRootRegister.
    1907             :   Push(r14);
    1908             :   Push(r15);
    1909             :   STATIC_ASSERT(12 == kNumSafepointSavedRegisters);
    1910             :   // Use lea for symmetry with Popad.
    1911             :   int sp_delta = (kNumSafepointRegisters - kNumSafepointSavedRegisters) *
    1912             :                  kSystemPointerSize;
    1913           0 :   leaq(rsp, Operand(rsp, -sp_delta));
    1914           0 : }
    1915             : 
    1916             : 
    1917           0 : void MacroAssembler::Popad() {
    1918             :   // Popad must not change the flags, so use lea instead of addq.
    1919             :   int sp_delta = (kNumSafepointRegisters - kNumSafepointSavedRegisters) *
    1920             :                  kSystemPointerSize;
    1921           0 :   leaq(rsp, Operand(rsp, sp_delta));
    1922             :   Pop(r15);
    1923             :   Pop(r14);
    1924             :   Pop(r12);
    1925             :   Pop(r11);
    1926             :   Pop(r9);
    1927             :   Pop(r8);
    1928             :   Pop(rdi);
    1929             :   Pop(rsi);
    1930             :   Pop(rbx);
    1931             :   Pop(rdx);
    1932             :   Pop(rcx);
    1933             :   Pop(rax);
    1934           0 : }
    1935             : 
    1936             : 
    1937             : // Order general registers are pushed by Pushad:
    1938             : // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r14, r15.
    1939             : const int
    1940             : MacroAssembler::kSafepointPushRegisterIndices[Register::kNumRegisters] = {
    1941             :     0,
    1942             :     1,
    1943             :     2,
    1944             :     3,
    1945             :     -1,
    1946             :     -1,
    1947             :     4,
    1948             :     5,
    1949             :     6,
    1950             :     7,
    1951             :     -1,
    1952             :     8,
    1953             :     9,
    1954             :     -1,
    1955             :     10,
    1956             :     11
    1957             : };
    1958             : 
    1959         168 : void MacroAssembler::PushStackHandler() {
    1960             :   // Adjust this code if not the case.
    1961             :   STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kSystemPointerSize);
    1962             :   STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
    1963             : 
    1964             :   Push(Immediate(0));  // Padding.
    1965             : 
    1966             :   // Link the current handler as the next handler.
    1967             :   ExternalReference handler_address =
    1968         168 :       ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
    1969         168 :   Push(ExternalReferenceAsOperand(handler_address));
    1970             : 
    1971             :   // Set this new handler as the current one.
    1972         168 :   movq(ExternalReferenceAsOperand(handler_address), rsp);
    1973         168 : }
    1974             : 
    1975             : 
    1976         168 : void MacroAssembler::PopStackHandler() {
    1977             :   STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
    1978             :   ExternalReference handler_address =
    1979         168 :       ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate());
    1980         168 :   Pop(ExternalReferenceAsOperand(handler_address));
    1981             :   addq(rsp, Immediate(StackHandlerConstants::kSize - kSystemPointerSize));
    1982         168 : }
    1983             : 
    1984        1392 : void TurboAssembler::Ret() { ret(0); }
    1985             : 
    1986     2743846 : void TurboAssembler::Ret(int bytes_dropped, Register scratch) {
    1987     2743846 :   if (is_uint16(bytes_dropped)) {
    1988     2743842 :     ret(bytes_dropped);
    1989             :   } else {
    1990             :     PopReturnAddressTo(scratch);
    1991             :     addq(rsp, Immediate(bytes_dropped));
    1992             :     PushReturnAddressFrom(scratch);
    1993           4 :     ret(0);
    1994             :   }
    1995     2744113 : }
    1996             : 
    1997         560 : void MacroAssembler::CmpObjectType(Register heap_object,
    1998             :                                    InstanceType type,
    1999             :                                    Register map) {
    2000             :   LoadTaggedPointerField(map,
    2001             :                          FieldOperand(heap_object, HeapObject::kMapOffset));
    2002         560 :   CmpInstanceType(map, type);
    2003         560 : }
    2004             : 
    2005             : 
    2006        1120 : void MacroAssembler::CmpInstanceType(Register map, InstanceType type) {
    2007        2240 :   cmpw(FieldOperand(map, Map::kInstanceTypeOffset), Immediate(type));
    2008        1120 : }
    2009             : 
    2010           0 : void MacroAssembler::DoubleToI(Register result_reg, XMMRegister input_reg,
    2011             :                                XMMRegister scratch, Label* lost_precision,
    2012             :                                Label* is_nan, Label::Distance dst) {
    2013           0 :   Cvttsd2si(result_reg, input_reg);
    2014           0 :   Cvtlsi2sd(kScratchDoubleReg, result_reg);
    2015             :   Ucomisd(kScratchDoubleReg, input_reg);
    2016           0 :   j(not_equal, lost_precision, dst);
    2017           0 :   j(parity_even, is_nan, dst);  // NaN.
    2018           0 : }
    2019             : 
    2020             : 
    2021         112 : void MacroAssembler::AssertNotSmi(Register object) {
    2022         112 :   if (emit_debug_code()) {
    2023             :     Condition is_smi = CheckSmi(object);
    2024           0 :     Check(NegateCondition(is_smi), AbortReason::kOperandIsASmi);
    2025             :   }
    2026         112 : }
    2027             : 
    2028             : 
    2029         554 : void MacroAssembler::AssertSmi(Register object) {
    2030         554 :   if (emit_debug_code()) {
    2031             :     Condition is_smi = CheckSmi(object);
    2032           0 :     Check(is_smi, AbortReason::kOperandIsNotASmi);
    2033             :   }
    2034         554 : }
    2035             : 
    2036           0 : void MacroAssembler::AssertSmi(Operand object) {
    2037           0 :   if (emit_debug_code()) {
    2038             :     Condition is_smi = CheckSmi(object);
    2039           0 :     Check(is_smi, AbortReason::kOperandIsNotASmi);
    2040             :   }
    2041           0 : }
    2042             : 
    2043     1721416 : void TurboAssembler::AssertZeroExtended(Register int32_register) {
    2044     1721416 :   if (emit_debug_code()) {
    2045             :     DCHECK_NE(int32_register, kScratchRegister);
    2046          58 :     movq(kScratchRegister, int64_t{0x0000000100000000});
    2047             :     cmpq(kScratchRegister, int32_register);
    2048          58 :     Check(above_equal, AbortReason::k32BitValueInRegisterIsNotZeroExtended);
    2049             :   }
    2050     1721416 : }
    2051             : 
    2052         112 : void MacroAssembler::AssertConstructor(Register object) {
    2053         112 :   if (emit_debug_code()) {
    2054           0 :     testb(object, Immediate(kSmiTagMask));
    2055           0 :     Check(not_equal, AbortReason::kOperandIsASmiAndNotAConstructor);
    2056             :     Push(object);
    2057             :     LoadTaggedPointerField(object,
    2058             :                            FieldOperand(object, HeapObject::kMapOffset));
    2059           0 :     testb(FieldOperand(object, Map::kBitFieldOffset),
    2060           0 :           Immediate(Map::IsConstructorBit::kMask));
    2061             :     Pop(object);
    2062           0 :     Check(not_zero, AbortReason::kOperandIsNotAConstructor);
    2063             :   }
    2064         112 : }
    2065             : 
    2066         280 : void MacroAssembler::AssertFunction(Register object) {
    2067         280 :   if (emit_debug_code()) {
    2068           0 :     testb(object, Immediate(kSmiTagMask));
    2069           0 :     Check(not_equal, AbortReason::kOperandIsASmiAndNotAFunction);
    2070             :     Push(object);
    2071           0 :     CmpObjectType(object, JS_FUNCTION_TYPE, object);
    2072             :     Pop(object);
    2073           0 :     Check(equal, AbortReason::kOperandIsNotAFunction);
    2074             :   }
    2075         280 : }
    2076             : 
    2077             : 
    2078         112 : void MacroAssembler::AssertBoundFunction(Register object) {
    2079         112 :   if (emit_debug_code()) {
    2080           0 :     testb(object, Immediate(kSmiTagMask));
    2081           0 :     Check(not_equal, AbortReason::kOperandIsASmiAndNotABoundFunction);
    2082             :     Push(object);
    2083           0 :     CmpObjectType(object, JS_BOUND_FUNCTION_TYPE, object);
    2084             :     Pop(object);
    2085           0 :     Check(equal, AbortReason::kOperandIsNotABoundFunction);
    2086             :   }
    2087         112 : }
    2088             : 
    2089          56 : void MacroAssembler::AssertGeneratorObject(Register object) {
    2090         112 :   if (!emit_debug_code()) return;
    2091           0 :   testb(object, Immediate(kSmiTagMask));
    2092           0 :   Check(not_equal, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
    2093             : 
    2094             :   // Load map
    2095           0 :   Register map = object;
    2096             :   Push(object);
    2097             :   LoadTaggedPointerField(map, FieldOperand(object, HeapObject::kMapOffset));
    2098             : 
    2099           0 :   Label do_check;
    2100             :   // Check if JSGeneratorObject
    2101           0 :   CmpInstanceType(map, JS_GENERATOR_OBJECT_TYPE);
    2102           0 :   j(equal, &do_check);
    2103             : 
    2104             :   // Check if JSAsyncFunctionObject
    2105           0 :   CmpInstanceType(map, JS_ASYNC_FUNCTION_OBJECT_TYPE);
    2106           0 :   j(equal, &do_check);
    2107             : 
    2108             :   // Check if JSAsyncGeneratorObject
    2109           0 :   CmpInstanceType(map, JS_ASYNC_GENERATOR_OBJECT_TYPE);
    2110             : 
    2111           0 :   bind(&do_check);
    2112             :   // Restore generator object to register and perform assertion
    2113             :   Pop(object);
    2114           0 :   Check(equal, AbortReason::kOperandIsNotAGeneratorObject);
    2115             : }
    2116             : 
    2117         112 : void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
    2118         112 :   if (emit_debug_code()) {
    2119           0 :     Label done_checking;
    2120           0 :     AssertNotSmi(object);
    2121           0 :     Cmp(object, isolate()->factory()->undefined_value());
    2122           0 :     j(equal, &done_checking);
    2123           0 :     Cmp(FieldOperand(object, 0), isolate()->factory()->allocation_site_map());
    2124           0 :     Assert(equal, AbortReason::kExpectedUndefinedOrCell);
    2125           0 :     bind(&done_checking);
    2126             :   }
    2127         112 : }
    2128             : 
    2129          56 : void MacroAssembler::LoadWeakValue(Register in_out, Label* target_if_cleared) {
    2130          56 :   cmpl(in_out, Immediate(kClearedWeakHeapObjectLower32));
    2131          56 :   j(equal, target_if_cleared);
    2132             : 
    2133             :   andq(in_out, Immediate(~static_cast<int32_t>(kWeakHeapObjectMask)));
    2134          56 : }
    2135             : 
    2136          56 : void MacroAssembler::IncrementCounter(StatsCounter* counter, int value) {
    2137             :   DCHECK_GT(value, 0);
    2138          56 :   if (FLAG_native_code_counters && counter->Enabled()) {
    2139             :     Operand counter_operand =
    2140           0 :         ExternalReferenceAsOperand(ExternalReference::Create(counter));
    2141             :     // This operation has to be exactly 32-bit wide in case the external
    2142             :     // reference table redirects the counter to a uint32_t dummy_stats_counter_
    2143             :     // field.
    2144           0 :     if (value == 1) {
    2145           0 :       incl(counter_operand);
    2146             :     } else {
    2147           0 :       addl(counter_operand, Immediate(value));
    2148             :     }
    2149             :   }
    2150          56 : }
    2151             : 
    2152             : 
    2153           0 : void MacroAssembler::DecrementCounter(StatsCounter* counter, int value) {
    2154             :   DCHECK_GT(value, 0);
    2155           0 :   if (FLAG_native_code_counters && counter->Enabled()) {
    2156             :     Operand counter_operand =
    2157           0 :         ExternalReferenceAsOperand(ExternalReference::Create(counter));
    2158             :     // This operation has to be exactly 32-bit wide in case the external
    2159             :     // reference table redirects the counter to a uint32_t dummy_stats_counter_
    2160             :     // field.
    2161           0 :     if (value == 1) {
    2162           0 :       decl(counter_operand);
    2163             :     } else {
    2164           0 :       subl(counter_operand, Immediate(value));
    2165             :     }
    2166             :   }
    2167           0 : }
    2168             : 
    2169          56 : void MacroAssembler::MaybeDropFrames() {
    2170             :   // Check whether we need to drop frames to restart a function on the stack.
    2171             :   ExternalReference restart_fp =
    2172          56 :       ExternalReference::debug_restart_fp_address(isolate());
    2173          56 :   Load(rbx, restart_fp);
    2174          56 :   testq(rbx, rbx);
    2175             : 
    2176          56 :   Label dont_drop;
    2177          56 :   j(zero, &dont_drop, Label::kNear);
    2178          56 :   Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET);
    2179             : 
    2180          56 :   bind(&dont_drop);
    2181          56 : }
    2182             : 
    2183        1288 : void TurboAssembler::PrepareForTailCall(const ParameterCount& callee_args_count,
    2184             :                                         Register caller_args_count_reg,
    2185             :                                         Register scratch0, Register scratch1) {
    2186             : #if DEBUG
    2187             :   if (callee_args_count.is_reg()) {
    2188             :     DCHECK(!AreAliased(callee_args_count.reg(), caller_args_count_reg, scratch0,
    2189             :                        scratch1));
    2190             :   } else {
    2191             :     DCHECK(!AreAliased(caller_args_count_reg, scratch0, scratch1));
    2192             :   }
    2193             : #endif
    2194             : 
    2195             :   // Calculate the destination address where we will put the return address
    2196             :   // after we drop current frame.
    2197        1288 :   Register new_sp_reg = scratch0;
    2198        1288 :   if (callee_args_count.is_reg()) {
    2199        1288 :     subq(caller_args_count_reg, callee_args_count.reg());
    2200        2576 :     leaq(new_sp_reg,
    2201             :          Operand(rbp, caller_args_count_reg, times_system_pointer_size,
    2202             :                  StandardFrameConstants::kCallerPCOffset));
    2203             :   } else {
    2204           0 :     leaq(new_sp_reg,
    2205             :          Operand(rbp, caller_args_count_reg, times_system_pointer_size,
    2206             :                  StandardFrameConstants::kCallerPCOffset -
    2207             :                      callee_args_count.immediate() * kSystemPointerSize));
    2208             :   }
    2209             : 
    2210        1288 :   if (FLAG_debug_code) {
    2211           0 :     cmpq(rsp, new_sp_reg);
    2212           0 :     Check(below, AbortReason::kStackAccessBelowStackPointer);
    2213             :   }
    2214             : 
    2215             :   // Copy return address from caller's frame to current frame's return address
    2216             :   // to avoid its trashing and let the following loop copy it to the right
    2217             :   // place.
    2218        1288 :   Register tmp_reg = scratch1;
    2219        2576 :   movq(tmp_reg, Operand(rbp, StandardFrameConstants::kCallerPCOffset));
    2220        2576 :   movq(Operand(rsp, 0), tmp_reg);
    2221             : 
    2222             :   // Restore caller's frame pointer now as it could be overwritten by
    2223             :   // the copying loop.
    2224        2576 :   movq(rbp, Operand(rbp, StandardFrameConstants::kCallerFPOffset));
    2225             : 
    2226             :   // +2 here is to copy both receiver and return address.
    2227        1288 :   Register count_reg = caller_args_count_reg;
    2228        1288 :   if (callee_args_count.is_reg()) {
    2229        2576 :     leaq(count_reg, Operand(callee_args_count.reg(), 2));
    2230             :   } else {
    2231           0 :     movq(count_reg, Immediate(callee_args_count.immediate() + 2));
    2232             :     // TODO(ishell): Unroll copying loop for small immediate values.
    2233             :   }
    2234             : 
    2235             :   // Now copy callee arguments to the caller frame going backwards to avoid
    2236             :   // callee arguments corruption (source and destination areas could overlap).
    2237        1288 :   Label loop, entry;
    2238        1288 :   jmp(&entry, Label::kNear);
    2239        1288 :   bind(&loop);
    2240             :   decq(count_reg);
    2241        2576 :   movq(tmp_reg, Operand(rsp, count_reg, times_system_pointer_size, 0));
    2242        2576 :   movq(Operand(new_sp_reg, count_reg, times_system_pointer_size, 0), tmp_reg);
    2243        1288 :   bind(&entry);
    2244             :   cmpq(count_reg, Immediate(0));
    2245        1288 :   j(not_equal, &loop, Label::kNear);
    2246             : 
    2247             :   // Leave current frame.
    2248             :   movq(rsp, new_sp_reg);
    2249        1288 : }
    2250             : 
    2251         112 : void MacroAssembler::InvokeFunction(Register function, Register new_target,
    2252             :                                     const ParameterCount& actual,
    2253             :                                     InvokeFlag flag) {
    2254             :   LoadTaggedPointerField(
    2255             :       rbx, FieldOperand(function, JSFunction::kSharedFunctionInfoOffset));
    2256         112 :   movzxwq(rbx,
    2257             :           FieldOperand(rbx, SharedFunctionInfo::kFormalParameterCountOffset));
    2258             : 
    2259             :   ParameterCount expected(rbx);
    2260         112 :   InvokeFunction(function, new_target, expected, actual, flag);
    2261         112 : }
    2262             : 
    2263         168 : void MacroAssembler::InvokeFunction(Register function, Register new_target,
    2264             :                                     const ParameterCount& expected,
    2265             :                                     const ParameterCount& actual,
    2266             :                                     InvokeFlag flag) {
    2267             :   DCHECK(function == rdi);
    2268             :   LoadTaggedPointerField(rsi,
    2269             :                          FieldOperand(function, JSFunction::kContextOffset));
    2270         168 :   InvokeFunctionCode(rdi, new_target, expected, actual, flag);
    2271         168 : }
    2272             : 
    2273         336 : void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
    2274             :                                         const ParameterCount& expected,
    2275             :                                         const ParameterCount& actual,
    2276             :                                         InvokeFlag flag) {
    2277             :   // You can't call a function without a valid frame.
    2278             :   DCHECK(flag == JUMP_FUNCTION || has_frame());
    2279             :   DCHECK(function == rdi);
    2280             :   DCHECK_IMPLIES(new_target.is_valid(), new_target == rdx);
    2281             : 
    2282             :   // On function call, call into the debugger if necessary.
    2283         336 :   CheckDebugHook(function, new_target, expected, actual);
    2284             : 
    2285             :   // Clear the new.target register if not given.
    2286         336 :   if (!new_target.is_valid()) {
    2287         224 :     LoadRoot(rdx, RootIndex::kUndefinedValue);
    2288             :   }
    2289             : 
    2290         336 :   Label done;
    2291         336 :   bool definitely_mismatches = false;
    2292             :   InvokePrologue(expected, actual, &done, &definitely_mismatches, flag,
    2293         336 :                  Label::kNear);
    2294         336 :   if (!definitely_mismatches) {
    2295             :     // We call indirectly through the code field in the function to
    2296             :     // allow recompilation to take effect without changing any of the
    2297             :     // call sites.
    2298             :     static_assert(kJavaScriptCallCodeStartRegister == rcx, "ABI mismatch");
    2299             :     LoadTaggedPointerField(rcx,
    2300             :                            FieldOperand(function, JSFunction::kCodeOffset));
    2301         336 :     if (flag == CALL_FUNCTION) {
    2302         112 :       CallCodeObject(rcx);
    2303             :     } else {
    2304             :       DCHECK(flag == JUMP_FUNCTION);
    2305         224 :       JumpCodeObject(rcx);
    2306             :     }
    2307         336 :     bind(&done);
    2308             :   }
    2309         336 : }
    2310             : 
    2311         336 : void MacroAssembler::InvokePrologue(const ParameterCount& expected,
    2312             :                                     const ParameterCount& actual, Label* done,
    2313             :                                     bool* definitely_mismatches,
    2314             :                                     InvokeFlag flag,
    2315             :                                     Label::Distance near_jump) {
    2316             :   bool definitely_matches = false;
    2317         336 :   *definitely_mismatches = false;
    2318         336 :   Label invoke;
    2319         336 :   if (expected.is_immediate()) {
    2320             :     DCHECK(actual.is_immediate());
    2321           0 :     Set(rax, actual.immediate());
    2322           0 :     if (expected.immediate() == actual.immediate()) {
    2323             :       definitely_matches = true;
    2324             :     } else {
    2325           0 :       if (expected.immediate() ==
    2326             :               SharedFunctionInfo::kDontAdaptArgumentsSentinel) {
    2327             :         // Don't worry about adapting arguments for built-ins that
    2328             :         // don't want that done. Skip adaption code by making it look
    2329             :         // like we have a match between expected and actual number of
    2330             :         // arguments.
    2331             :         definitely_matches = true;
    2332             :       } else {
    2333           0 :         *definitely_mismatches = true;
    2334           0 :         Set(rbx, expected.immediate());
    2335             :       }
    2336             :     }
    2337             :   } else {
    2338         336 :     if (actual.is_immediate()) {
    2339             :       // Expected is in register, actual is immediate. This is the
    2340             :       // case when we invoke function values without going through the
    2341             :       // IC mechanism.
    2342           0 :       Set(rax, actual.immediate());
    2343           0 :       cmpq(expected.reg(), Immediate(actual.immediate()));
    2344           0 :       j(equal, &invoke, Label::kNear);
    2345             :       DCHECK(expected.reg() == rbx);
    2346         336 :     } else if (expected.reg() != actual.reg()) {
    2347             :       // Both expected and actual are in (different) registers. This
    2348             :       // is the case when we invoke functions using call and apply.
    2349         280 :       cmpq(expected.reg(), actual.reg());
    2350         280 :       j(equal, &invoke, Label::kNear);
    2351             :       DCHECK(actual.reg() == rax);
    2352             :       DCHECK(expected.reg() == rbx);
    2353             :     } else {
    2354             :       definitely_matches = true;
    2355             :       Move(rax, actual.reg());
    2356             :     }
    2357             :   }
    2358             : 
    2359         336 :   if (!definitely_matches) {
    2360         280 :     Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
    2361         280 :     if (flag == CALL_FUNCTION) {
    2362         112 :       Call(adaptor, RelocInfo::CODE_TARGET);
    2363         112 :       if (!*definitely_mismatches) {
    2364         112 :         jmp(done, near_jump);
    2365             :       }
    2366             :     } else {
    2367         168 :       Jump(adaptor, RelocInfo::CODE_TARGET);
    2368             :     }
    2369         280 :     bind(&invoke);
    2370             :   }
    2371         336 : }
    2372             : 
    2373         336 : void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
    2374             :                                     const ParameterCount& expected,
    2375             :                                     const ParameterCount& actual) {
    2376         336 :   Label skip_hook;
    2377             :   ExternalReference debug_hook_active =
    2378         336 :       ExternalReference::debug_hook_on_function_call_address(isolate());
    2379             :   Operand debug_hook_active_operand =
    2380         336 :       ExternalReferenceAsOperand(debug_hook_active);
    2381         336 :   cmpb(debug_hook_active_operand, Immediate(0));
    2382         336 :   j(equal, &skip_hook);
    2383             : 
    2384             :   {
    2385             :     FrameScope frame(this,
    2386         672 :                      has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
    2387         336 :     if (expected.is_reg()) {
    2388         336 :       SmiTag(expected.reg(), expected.reg());
    2389             :       Push(expected.reg());
    2390             :     }
    2391         336 :     if (actual.is_reg()) {
    2392         336 :       SmiTag(actual.reg(), actual.reg());
    2393             :       Push(actual.reg());
    2394         336 :       SmiUntag(actual.reg(), actual.reg());
    2395             :     }
    2396         336 :     if (new_target.is_valid()) {
    2397             :       Push(new_target);
    2398             :     }
    2399             :     Push(fun);
    2400             :     Push(fun);
    2401         336 :     Push(StackArgumentsAccessor(rbp, actual).GetReceiverOperand());
    2402         336 :     CallRuntime(Runtime::kDebugOnFunctionCall);
    2403             :     Pop(fun);
    2404         336 :     if (new_target.is_valid()) {
    2405             :       Pop(new_target);
    2406             :     }
    2407         336 :     if (actual.is_reg()) {
    2408             :       Pop(actual.reg());
    2409         336 :       SmiUntag(actual.reg(), actual.reg());
    2410             :     }
    2411         336 :     if (expected.is_reg()) {
    2412             :       Pop(expected.reg());
    2413         336 :       SmiUntag(expected.reg(), expected.reg());
    2414             :     }
    2415             :   }
    2416         336 :   bind(&skip_hook);
    2417         336 : }
    2418             : 
    2419     1110596 : void TurboAssembler::StubPrologue(StackFrame::Type type) {
    2420     1110596 :   pushq(rbp);  // Caller's frame pointer.
    2421             :   movq(rbp, rsp);
    2422             :   Push(Immediate(StackFrame::TypeToMarker(type)));
    2423     1110541 : }
    2424             : 
    2425      646003 : void TurboAssembler::Prologue() {
    2426      646003 :   pushq(rbp);  // Caller's frame pointer.
    2427             :   movq(rbp, rsp);
    2428             :   Push(rsi);  // Callee's context.
    2429             :   Push(rdi);  // Callee's JS function.
    2430      646007 : }
    2431             : 
    2432      461957 : void TurboAssembler::EnterFrame(StackFrame::Type type) {
    2433      461957 :   pushq(rbp);
    2434             :   movq(rbp, rsp);
    2435             :   Push(Immediate(StackFrame::TypeToMarker(type)));
    2436      461961 : }
    2437             : 
    2438      821553 : void TurboAssembler::LeaveFrame(StackFrame::Type type) {
    2439      821553 :   if (emit_debug_code()) {
    2440           0 :     cmpq(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset),
    2441           0 :          Immediate(StackFrame::TypeToMarker(type)));
    2442           0 :     Check(equal, AbortReason::kStackFrameTypesMustMatch);
    2443             :   }
    2444      821553 :   movq(rsp, rbp);
    2445      821558 :   popq(rbp);
    2446      821407 : }
    2447             : 
    2448         672 : void MacroAssembler::EnterExitFramePrologue(bool save_rax,
    2449             :                                             StackFrame::Type frame_type) {
    2450             :   DCHECK(frame_type == StackFrame::EXIT ||
    2451             :          frame_type == StackFrame::BUILTIN_EXIT);
    2452             : 
    2453             :   // Set up the frame structure on the stack.
    2454             :   // All constants are relative to the frame pointer of the exit frame.
    2455             :   DCHECK_EQ(kFPOnStackSize + kPCOnStackSize,
    2456             :             ExitFrameConstants::kCallerSPDisplacement);
    2457             :   DCHECK_EQ(kFPOnStackSize, ExitFrameConstants::kCallerPCOffset);
    2458             :   DCHECK_EQ(0 * kSystemPointerSize, ExitFrameConstants::kCallerFPOffset);
    2459         672 :   pushq(rbp);
    2460             :   movq(rbp, rsp);
    2461             : 
    2462             :   // Reserve room for entry stack pointer.
    2463             :   Push(Immediate(StackFrame::TypeToMarker(frame_type)));
    2464             :   DCHECK_EQ(-2 * kSystemPointerSize, ExitFrameConstants::kSPOffset);
    2465             :   Push(Immediate(0));  // Saved entry sp, patched before call.
    2466             : 
    2467             :   // Save the frame pointer and the context in top.
    2468         672 :   if (save_rax) {
    2469             :     movq(r14, rax);  // Backup rax in callee-save register.
    2470             :   }
    2471             : 
    2472         672 :   Store(
    2473             :       ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()),
    2474         672 :       rbp);
    2475         672 :   Store(ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()),
    2476         672 :         rsi);
    2477         672 :   Store(
    2478             :       ExternalReference::Create(IsolateAddressId::kCFunctionAddress, isolate()),
    2479         672 :       rbx);
    2480         672 : }
    2481             : 
    2482             : 
    2483         672 : void MacroAssembler::EnterExitFrameEpilogue(int arg_stack_space,
    2484             :                                             bool save_doubles) {
    2485             : #ifdef _WIN64
    2486             :   const int kShadowSpace = 4;
    2487             :   arg_stack_space += kShadowSpace;
    2488             : #endif
    2489             :   // Optionally save all XMM registers.
    2490         672 :   if (save_doubles) {
    2491         224 :     int space = XMMRegister::kNumRegisters * kDoubleSize +
    2492         224 :                 arg_stack_space * kSystemPointerSize;
    2493         224 :     subq(rsp, Immediate(space));
    2494             :     int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
    2495         224 :     const RegisterConfiguration* config = RegisterConfiguration::Default();
    2496        6944 :     for (int i = 0; i < config->num_allocatable_double_registers(); ++i) {
    2497             :       DoubleRegister reg =
    2498             :           DoubleRegister::from_code(config->GetAllocatableDoubleCode(i));
    2499        6720 :       Movsd(Operand(rbp, offset - ((i + 1) * kDoubleSize)), reg);
    2500             :     }
    2501         448 :   } else if (arg_stack_space > 0) {
    2502         112 :     subq(rsp, Immediate(arg_stack_space * kSystemPointerSize));
    2503             :   }
    2504             : 
    2505             :   // Get the required frame alignment for the OS.
    2506         672 :   const int kFrameAlignment = base::OS::ActivationFrameAlignment();
    2507         672 :   if (kFrameAlignment > 0) {
    2508             :     DCHECK(base::bits::IsPowerOfTwo(kFrameAlignment));
    2509             :     DCHECK(is_int8(kFrameAlignment));
    2510         672 :     andq(rsp, Immediate(-kFrameAlignment));
    2511             :   }
    2512             : 
    2513             :   // Patch the saved entry sp.
    2514        1344 :   movq(Operand(rbp, ExitFrameConstants::kSPOffset), rsp);
    2515         672 : }
    2516             : 
    2517         448 : void MacroAssembler::EnterExitFrame(int arg_stack_space, bool save_doubles,
    2518             :                                     StackFrame::Type frame_type) {
    2519         448 :   EnterExitFramePrologue(true, frame_type);
    2520             : 
    2521             :   // Set up argv in callee-saved register r15. It is reused in LeaveExitFrame,
    2522             :   // so it must be retained across the C-call.
    2523             :   int offset = StandardFrameConstants::kCallerSPOffset - kSystemPointerSize;
    2524         896 :   leaq(r15, Operand(rbp, r14, times_system_pointer_size, offset));
    2525             : 
    2526         448 :   EnterExitFrameEpilogue(arg_stack_space, save_doubles);
    2527         448 : }
    2528             : 
    2529             : 
    2530         224 : void MacroAssembler::EnterApiExitFrame(int arg_stack_space) {
    2531         224 :   EnterExitFramePrologue(false, StackFrame::EXIT);
    2532         224 :   EnterExitFrameEpilogue(arg_stack_space, false);
    2533         224 : }
    2534             : 
    2535             : 
    2536         560 : void MacroAssembler::LeaveExitFrame(bool save_doubles, bool pop_arguments) {
    2537             :   // Registers:
    2538             :   // r15 : argv
    2539         560 :   if (save_doubles) {
    2540             :     int offset = -ExitFrameConstants::kFixedFrameSizeFromFp;
    2541         224 :     const RegisterConfiguration* config = RegisterConfiguration::Default();
    2542        6944 :     for (int i = 0; i < config->num_allocatable_double_registers(); ++i) {
    2543             :       DoubleRegister reg =
    2544             :           DoubleRegister::from_code(config->GetAllocatableDoubleCode(i));
    2545        6720 :       Movsd(reg, Operand(rbp, offset - ((i + 1) * kDoubleSize)));
    2546             :     }
    2547             :   }
    2548             : 
    2549         560 :   if (pop_arguments) {
    2550             :     // Get the return address from the stack and restore the frame pointer.
    2551         896 :     movq(rcx, Operand(rbp, kFPOnStackSize));
    2552         896 :     movq(rbp, Operand(rbp, 0 * kSystemPointerSize));
    2553             : 
    2554             :     // Drop everything up to and including the arguments and the receiver
    2555             :     // from the caller stack.
    2556         896 :     leaq(rsp, Operand(r15, 1 * kSystemPointerSize));
    2557             : 
    2558             :     PushReturnAddressFrom(rcx);
    2559             :   } else {
    2560             :     // Otherwise just leave the exit frame.
    2561         112 :     leave();
    2562             :   }
    2563             : 
    2564         560 :   LeaveExitFrameEpilogue();
    2565         560 : }
    2566             : 
    2567         112 : void MacroAssembler::LeaveApiExitFrame() {
    2568         112 :   movq(rsp, rbp);
    2569         112 :   popq(rbp);
    2570             : 
    2571         112 :   LeaveExitFrameEpilogue();
    2572         112 : }
    2573             : 
    2574         672 : void MacroAssembler::LeaveExitFrameEpilogue() {
    2575             :   // Restore current context from top and clear it in debug mode.
    2576             :   ExternalReference context_address =
    2577         672 :       ExternalReference::Create(IsolateAddressId::kContextAddress, isolate());
    2578         672 :   Operand context_operand = ExternalReferenceAsOperand(context_address);
    2579         672 :   movq(rsi, context_operand);
    2580             : #ifdef DEBUG
    2581             :   movq(context_operand, Immediate(Context::kInvalidContext));
    2582             : #endif
    2583             : 
    2584             :   // Clear the top frame.
    2585             :   ExternalReference c_entry_fp_address =
    2586         672 :       ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate());
    2587         672 :   Operand c_entry_fp_operand = ExternalReferenceAsOperand(c_entry_fp_address);
    2588             :   movq(c_entry_fp_operand, Immediate(0));
    2589         672 : }
    2590             : 
    2591             : 
    2592             : #ifdef _WIN64
    2593             : static const int kRegisterPassedArguments = 4;
    2594             : #else
    2595             : static const int kRegisterPassedArguments = 6;
    2596             : #endif
    2597             : 
    2598             : 
    2599         336 : void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
    2600         336 :   LoadTaggedPointerField(dst, NativeContextOperand());
    2601             :   LoadTaggedPointerField(dst, ContextOperand(dst, index));
    2602         336 : }
    2603             : 
    2604             : 
    2605           0 : int TurboAssembler::ArgumentStackSlotsForCFunctionCall(int num_arguments) {
    2606             :   // On Windows 64 stack slots are reserved by the caller for all arguments
    2607             :   // including the ones passed in registers, and space is always allocated for
    2608             :   // the four register arguments even if the function takes fewer than four
    2609             :   // arguments.
    2610             :   // On AMD64 ABI (Linux/Mac) the first six arguments are passed in registers
    2611             :   // and the caller does not reserve stack slots for them.
    2612             :   DCHECK_GE(num_arguments, 0);
    2613             : #ifdef _WIN64
    2614             :   const int kMinimumStackSlots = kRegisterPassedArguments;
    2615             :   if (num_arguments < kMinimumStackSlots) return kMinimumStackSlots;
    2616             :   return num_arguments;
    2617             : #else
    2618     1528628 :   if (num_arguments < kRegisterPassedArguments) return 0;
    2619       94502 :   return num_arguments - kRegisterPassedArguments;
    2620             : #endif
    2621             : }
    2622             : 
    2623      764319 : void TurboAssembler::PrepareCallCFunction(int num_arguments) {
    2624      764319 :   int frame_alignment = base::OS::ActivationFrameAlignment();
    2625             :   DCHECK_NE(frame_alignment, 0);
    2626             :   DCHECK_GE(num_arguments, 0);
    2627             : 
    2628             :   // Make stack end at alignment and allocate space for arguments and old rsp.
    2629      764319 :   movq(kScratchRegister, rsp);
    2630             :   DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
    2631             :   int argument_slots_on_stack =
    2632             :       ArgumentStackSlotsForCFunctionCall(num_arguments);
    2633      764319 :   subq(rsp, Immediate((argument_slots_on_stack + 1) * kSystemPointerSize));
    2634      764319 :   andq(rsp, Immediate(-frame_alignment));
    2635     1528638 :   movq(Operand(rsp, argument_slots_on_stack * kSystemPointerSize),
    2636             :        kScratchRegister);
    2637      764319 : }
    2638             : 
    2639      763168 : void TurboAssembler::CallCFunction(ExternalReference function,
    2640             :                                    int num_arguments) {
    2641      763168 :   LoadAddress(rax, function);
    2642      763168 :   CallCFunction(rax, num_arguments);
    2643      763169 : }
    2644             : 
    2645      764308 : void TurboAssembler::CallCFunction(Register function, int num_arguments) {
    2646             :   DCHECK_LE(num_arguments, kMaxCParameters);
    2647             :   DCHECK(has_frame());
    2648             :   // Check stack alignment.
    2649      764308 :   if (emit_debug_code()) {
    2650           0 :     CheckStackAlignment();
    2651             :   }
    2652             : 
    2653             :   // Save the frame pointer and PC so that the stack layout remains iterable,
    2654             :   // even without an ExitFrame which normally exists between JS and C frames.
    2655      764308 :   if (isolate() != nullptr) {
    2656      365281 :     Label get_pc;
    2657             :     DCHECK(!AreAliased(kScratchRegister, function));
    2658      730563 :     leaq(kScratchRegister, Operand(&get_pc, 0));
    2659      365282 :     bind(&get_pc);
    2660      365282 :     movq(ExternalReferenceAsOperand(
    2661             :              ExternalReference::fast_c_call_caller_pc_address(isolate())),
    2662             :          kScratchRegister);
    2663      365282 :     movq(ExternalReferenceAsOperand(
    2664             :              ExternalReference::fast_c_call_caller_fp_address(isolate())),
    2665             :          rbp);
    2666             :   }
    2667             : 
    2668      764309 :   call(function);
    2669             : 
    2670      764309 :   if (isolate() != nullptr) {
    2671             :     // We don't unset the PC; the FP is the source of truth.
    2672      365282 :     movq(ExternalReferenceAsOperand(
    2673             :              ExternalReference::fast_c_call_caller_fp_address(isolate())),
    2674             :          Immediate(0));
    2675             :   }
    2676             : 
    2677             :   DCHECK_NE(base::OS::ActivationFrameAlignment(), 0);
    2678             :   DCHECK_GE(num_arguments, 0);
    2679             :   int argument_slots_on_stack =
    2680             :       ArgumentStackSlotsForCFunctionCall(num_arguments);
    2681     1528618 :   movq(rsp, Operand(rsp, argument_slots_on_stack * kSystemPointerSize));
    2682      764309 : }
    2683             : 
    2684      643360 : void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
    2685             :                                    Condition cc, Label* condition_met,
    2686             :                                    Label::Distance condition_met_distance) {
    2687             :   DCHECK(cc == zero || cc == not_zero);
    2688      643360 :   if (scratch == object) {
    2689         112 :     andq(scratch, Immediate(~kPageAlignmentMask));
    2690             :   } else {
    2691      643248 :     movq(scratch, Immediate(~kPageAlignmentMask));
    2692             :     andq(scratch, object);
    2693             :   }
    2694      643362 :   if (mask < (1 << kBitsPerByte)) {
    2695     1930086 :     testb(Operand(scratch, MemoryChunk::kFlagsOffset),
    2696      643364 :           Immediate(static_cast<uint8_t>(mask)));
    2697             :   } else {
    2698           0 :     testl(Operand(scratch, MemoryChunk::kFlagsOffset), Immediate(mask));
    2699             :   }
    2700      643362 :   j(cc, condition_met, condition_met_distance);
    2701      643361 : }
    2702             : 
    2703         113 : void TurboAssembler::ComputeCodeStartAddress(Register dst) {
    2704         113 :   Label current;
    2705         113 :   bind(&current);
    2706             :   int pc = pc_offset();
    2707             :   // Load effective address to get the address of the current instruction.
    2708         226 :   leaq(dst, Operand(&current, -pc));
    2709         113 : }
    2710             : 
    2711         560 : void TurboAssembler::ResetSpeculationPoisonRegister() {
    2712             :   // TODO(tebbi): Perhaps, we want to put an lfence here.
    2713         560 :   Set(kSpeculationPoisonRegister, -1);
    2714         560 : }
    2715             : 
    2716     3335854 : void TurboAssembler::CallForDeoptimization(Address target, int deopt_id) {
    2717             :   NoRootArrayScope no_root_array(this);
    2718             :   // Save the deopt id in r13 (we don't need the roots array from now on).
    2719     3335854 :   movq(r13, Immediate(deopt_id));
    2720     3335858 :   call(target, RelocInfo::RUNTIME_ENTRY);
    2721     3335854 : }
    2722             : 
    2723             : }  // namespace internal
    2724      122004 : }  // namespace v8
    2725             : 
    2726             : #endif  // V8_TARGET_ARCH_X64

Generated by: LCOV version 1.10