Line data Source code
1 : // Copyright 2012 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
6 : #error This header must be included via macro-assembler.h
7 : #endif
8 :
9 : #ifndef V8_X64_MACRO_ASSEMBLER_X64_H_
10 : #define V8_X64_MACRO_ASSEMBLER_X64_H_
11 :
12 : #include "src/bailout-reason.h"
13 : #include "src/base/flags.h"
14 : #include "src/contexts.h"
15 : #include "src/globals.h"
16 : #include "src/x64/assembler-x64.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 :
21 : // Convenience for platform-independent signatures.
22 : typedef Operand MemOperand;
23 :
24 : class StringConstantBase;
25 :
26 : enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
27 : enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
28 :
29 : struct SmiIndex {
30 : SmiIndex(Register index_register, ScaleFactor scale)
31 : : reg(index_register),
32 : scale(scale) {}
33 : Register reg;
34 : ScaleFactor scale;
35 : };
36 :
37 : enum StackArgumentsAccessorReceiverMode {
38 : ARGUMENTS_CONTAIN_RECEIVER,
39 : ARGUMENTS_DONT_CONTAIN_RECEIVER
40 : };
41 :
42 : class StackArgumentsAccessor {
43 : public:
44 : StackArgumentsAccessor(Register base_reg, int argument_count_immediate,
45 : StackArgumentsAccessorReceiverMode receiver_mode =
46 : ARGUMENTS_CONTAIN_RECEIVER,
47 : int extra_displacement_to_last_argument = 0)
48 : : base_reg_(base_reg),
49 : argument_count_reg_(no_reg),
50 : argument_count_immediate_(argument_count_immediate),
51 : receiver_mode_(receiver_mode),
52 : extra_displacement_to_last_argument_(
53 : extra_displacement_to_last_argument) {}
54 :
55 : StackArgumentsAccessor(Register base_reg, Register argument_count_reg,
56 : StackArgumentsAccessorReceiverMode receiver_mode =
57 : ARGUMENTS_CONTAIN_RECEIVER,
58 : int extra_displacement_to_last_argument = 0)
59 : : base_reg_(base_reg),
60 : argument_count_reg_(argument_count_reg),
61 : argument_count_immediate_(0),
62 : receiver_mode_(receiver_mode),
63 : extra_displacement_to_last_argument_(
64 952 : extra_displacement_to_last_argument) {}
65 :
66 : StackArgumentsAccessor(Register base_reg,
67 : const ParameterCount& parameter_count,
68 : StackArgumentsAccessorReceiverMode receiver_mode =
69 : ARGUMENTS_CONTAIN_RECEIVER,
70 : int extra_displacement_to_last_argument = 0);
71 :
72 : Operand GetArgumentOperand(int index);
73 : Operand GetReceiverOperand() {
74 : DCHECK(receiver_mode_ == ARGUMENTS_CONTAIN_RECEIVER);
75 1008 : return GetArgumentOperand(0);
76 : }
77 :
78 : private:
79 : const Register base_reg_;
80 : const Register argument_count_reg_;
81 : const int argument_count_immediate_;
82 : const StackArgumentsAccessorReceiverMode receiver_mode_;
83 : const int extra_displacement_to_last_argument_;
84 :
85 : DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
86 : };
87 :
88 81812162 : class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
89 : public:
90 122713710 : using TurboAssemblerBase::TurboAssemblerBase;
91 :
92 : template <typename Dst, typename... Args>
93 : struct AvxHelper {
94 : Assembler* assm;
95 : // Call an method where the AVX version expects the dst argument to be
96 : // duplicated.
97 : template <void (Assembler::*avx)(Dst, Dst, Args...),
98 : void (Assembler::*no_avx)(Dst, Args...)>
99 1094999 : void emit(Dst dst, Args... args) {
100 1094999 : if (CpuFeatures::IsSupported(AVX)) {
101 : CpuFeatureScope scope(assm, AVX);
102 1093429 : (assm->*avx)(dst, dst, args...);
103 : } else {
104 1570 : (assm->*no_avx)(dst, args...);
105 : }
106 1095027 : }
107 :
108 : // Call an method where the AVX version expects no duplicated dst argument.
109 : template <void (Assembler::*avx)(Dst, Args...),
110 : void (Assembler::*no_avx)(Dst, Args...)>
111 4203129 : void emit(Dst dst, Args... args) {
112 4203129 : if (CpuFeatures::IsSupported(AVX)) {
113 : CpuFeatureScope scope(assm, AVX);
114 4193453 : (assm->*avx)(dst, args...);
115 : } else {
116 9676 : (assm->*no_avx)(dst, args...);
117 : }
118 4203163 : }
119 : };
120 :
121 : #define AVX_OP(macro_name, name) \
122 : template <typename Dst, typename... Args> \
123 : void macro_name(Dst dst, Args... args) { \
124 : AvxHelper<Dst, Args...>{this} \
125 : .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
126 : }
127 :
128 6471 : AVX_OP(Subsd, subsd)
129 166 : AVX_OP(Divss, divss)
130 629 : AVX_OP(Divsd, divsd)
131 11428 : AVX_OP(Xorps, xorps)
132 112920 : AVX_OP(Xorpd, xorpd)
133 159220 : AVX_OP(Movd, movd)
134 198583 : AVX_OP(Movq, movq)
135 385 : AVX_OP(Movaps, movaps)
136 156402 : AVX_OP(Movapd, movapd)
137 11418 : AVX_OP(Movups, movups)
138 166 : AVX_OP(Movmskps, movmskps)
139 629 : AVX_OP(Movmskpd, movmskpd)
140 736364 : AVX_OP(Movss, movss)
141 2973823 : AVX_OP(Movsd, movsd)
142 301866 : AVX_OP(Pcmpeqd, pcmpeqd)
143 63998 : AVX_OP(Pslld, pslld)
144 224285 : AVX_OP(Psllq, psllq)
145 48177 : AVX_OP(Psrld, psrld)
146 199790 : AVX_OP(Psrlq, psrlq)
147 : AVX_OP(Addsd, addsd)
148 : AVX_OP(Mulsd, mulsd)
149 32 : AVX_OP(Andps, andps)
150 : AVX_OP(Andnps, andnps)
151 22 : AVX_OP(Andpd, andpd)
152 : AVX_OP(Orpd, orpd)
153 : AVX_OP(Cmpeqps, cmpeqps)
154 : AVX_OP(Cmpltps, cmpltps)
155 : AVX_OP(Cmpleps, cmpleps)
156 : AVX_OP(Cmpneqps, cmpneqps)
157 : AVX_OP(Cmpnltps, cmpnltps)
158 : AVX_OP(Cmpnleps, cmpnleps)
159 : AVX_OP(Cmpeqpd, cmpeqpd)
160 : AVX_OP(Cmpltpd, cmpltpd)
161 : AVX_OP(Cmplepd, cmplepd)
162 : AVX_OP(Cmpneqpd, cmpneqpd)
163 : AVX_OP(Cmpnltpd, cmpnltpd)
164 : AVX_OP(Cmpnlepd, cmpnlepd)
165 481 : AVX_OP(Roundss, roundss)
166 45046 : AVX_OP(Roundsd, roundsd)
167 62 : AVX_OP(Sqrtss, sqrtss)
168 469 : AVX_OP(Sqrtsd, sqrtsd)
169 639 : AVX_OP(Ucomiss, ucomiss)
170 4141 : AVX_OP(Ucomisd, ucomisd)
171 :
172 : #undef AVX_OP
173 :
174 1740 : void PushReturnAddressFrom(Register src) { pushq(src); }
175 1460 : void PopReturnAddressTo(Register dst) { popq(dst); }
176 :
177 : void Ret();
178 :
179 : // Return and drop arguments from stack, where the number of arguments
180 : // may be bigger than 2^16 - 1. Requires a scratch register.
181 : void Ret(int bytes_dropped, Register scratch);
182 :
183 : // Load a register with a long value as efficiently as possible.
184 : void Set(Register dst, int64_t x);
185 : void Set(Operand dst, intptr_t x);
186 :
187 : // Operations on roots in the root-array.
188 : void LoadRoot(Register destination, RootIndex index) override;
189 : void LoadRoot(Operand destination, RootIndex index) {
190 : LoadRoot(kScratchRegister, index);
191 : movq(destination, kScratchRegister);
192 : }
193 :
194 : void Push(Register src);
195 : void Push(Operand src);
196 : void Push(Immediate value);
197 : void Push(Smi smi);
198 : void Push(Handle<HeapObject> source);
199 :
200 : // Before calling a C-function from generated code, align arguments on stack.
201 : // After aligning the frame, arguments must be stored in rsp[0], rsp[8],
202 : // etc., not pushed. The argument count assumes all arguments are word sized.
203 : // The number of slots reserved for arguments depends on platform. On Windows
204 : // stack slots are reserved for the arguments passed in registers. On other
205 : // platforms stack slots are only reserved for the arguments actually passed
206 : // on the stack.
207 : void PrepareCallCFunction(int num_arguments);
208 :
209 : // Calls a C function and cleans up the space for arguments allocated
210 : // by PrepareCallCFunction. The called function is not allowed to trigger a
211 : // garbage collection, since that might move the code and invalidate the
212 : // return address (unless this is somehow accounted for by the called
213 : // function).
214 : void CallCFunction(ExternalReference function, int num_arguments);
215 : void CallCFunction(Register function, int num_arguments);
216 :
217 : // Calculate the number of stack slots to reserve for arguments when calling a
218 : // C function.
219 : int ArgumentStackSlotsForCFunctionCall(int num_arguments);
220 :
221 : void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
222 : Label* condition_met,
223 : Label::Distance condition_met_distance = Label::kFar);
224 :
225 : void Cvtss2sd(XMMRegister dst, XMMRegister src);
226 : void Cvtss2sd(XMMRegister dst, Operand src);
227 : void Cvtsd2ss(XMMRegister dst, XMMRegister src);
228 : void Cvtsd2ss(XMMRegister dst, Operand src);
229 : void Cvttsd2si(Register dst, XMMRegister src);
230 : void Cvttsd2si(Register dst, Operand src);
231 : void Cvttsd2siq(Register dst, XMMRegister src);
232 : void Cvttsd2siq(Register dst, Operand src);
233 : void Cvttss2si(Register dst, XMMRegister src);
234 : void Cvttss2si(Register dst, Operand src);
235 : void Cvttss2siq(Register dst, XMMRegister src);
236 : void Cvttss2siq(Register dst, Operand src);
237 : void Cvtqsi2ss(XMMRegister dst, Register src);
238 : void Cvtqsi2ss(XMMRegister dst, Operand src);
239 : void Cvtqsi2sd(XMMRegister dst, Register src);
240 : void Cvtqsi2sd(XMMRegister dst, Operand src);
241 : void Cvtlsi2ss(XMMRegister dst, Register src);
242 : void Cvtlsi2ss(XMMRegister dst, Operand src);
243 : void Cvtlui2ss(XMMRegister dst, Register src);
244 : void Cvtlui2ss(XMMRegister dst, Operand src);
245 : void Cvtlui2sd(XMMRegister dst, Register src);
246 : void Cvtlui2sd(XMMRegister dst, Operand src);
247 : void Cvtqui2ss(XMMRegister dst, Register src);
248 : void Cvtqui2ss(XMMRegister dst, Operand src);
249 : void Cvtqui2sd(XMMRegister dst, Register src);
250 : void Cvtqui2sd(XMMRegister dst, Operand src);
251 : void Cvttsd2uiq(Register dst, Operand src, Label* fail = nullptr);
252 : void Cvttsd2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
253 : void Cvttss2uiq(Register dst, Operand src, Label* fail = nullptr);
254 : void Cvttss2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
255 :
256 : // cvtsi2sd instruction only writes to the low 64-bit of dst register, which
257 : // hinders register renaming and makes dependence chains longer. So we use
258 : // xorpd to clear the dst register before cvtsi2sd to solve this issue.
259 : void Cvtlsi2sd(XMMRegister dst, Register src);
260 : void Cvtlsi2sd(XMMRegister dst, Operand src);
261 :
262 : void Lzcntq(Register dst, Register src);
263 : void Lzcntq(Register dst, Operand src);
264 : void Lzcntl(Register dst, Register src);
265 : void Lzcntl(Register dst, Operand src);
266 : void Tzcntq(Register dst, Register src);
267 : void Tzcntq(Register dst, Operand src);
268 : void Tzcntl(Register dst, Register src);
269 : void Tzcntl(Register dst, Operand src);
270 : void Popcntl(Register dst, Register src);
271 : void Popcntl(Register dst, Operand src);
272 : void Popcntq(Register dst, Register src);
273 : void Popcntq(Register dst, Operand src);
274 :
275 : // Is the value a tagged smi.
276 : Condition CheckSmi(Register src);
277 : Condition CheckSmi(Operand src);
278 :
279 : // Jump to label if the value is a tagged smi.
280 : void JumpIfSmi(Register src, Label* on_smi,
281 : Label::Distance near_jump = Label::kFar);
282 :
283 194227 : void JumpIfEqual(Register a, int32_t b, Label* dest) {
284 194227 : cmpl(a, Immediate(b));
285 194226 : j(equal, dest);
286 194227 : }
287 :
288 41747 : void JumpIfLessThan(Register a, int32_t b, Label* dest) {
289 41747 : cmpl(a, Immediate(b));
290 41747 : j(less, dest);
291 41747 : }
292 :
293 : void Move(Register dst, Smi source);
294 :
295 : void Move(Operand dst, Smi source) {
296 : Register constant = GetSmiConstant(source);
297 : movq(dst, constant);
298 : }
299 :
300 : void Move(Register dst, ExternalReference ext);
301 :
302 : void Move(XMMRegister dst, uint32_t src);
303 : void Move(XMMRegister dst, uint64_t src);
304 196 : void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
305 4046 : void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
306 :
307 : // Move if the registers are not identical.
308 : void Move(Register target, Register source);
309 :
310 : void Move(Register dst, Handle<HeapObject> source,
311 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
312 : void Move(Operand dst, Handle<HeapObject> source,
313 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
314 :
315 : // Loads a pointer into a register with a relocation mode.
316 : void Move(Register dst, Address ptr, RelocInfo::Mode rmode) {
317 : // This method must not be used with heap object references. The stored
318 : // address is not GC safe. Use the handle version instead.
319 : DCHECK(rmode > RelocInfo::LAST_GCED_ENUM);
320 41961780 : movq(dst, Immediate64(ptr, rmode));
321 : }
322 :
323 : // Move src0 to dst0 and src1 to dst1, handling possible overlaps.
324 : void MovePair(Register dst0, Register src0, Register dst1, Register src1);
325 :
326 : void MoveStringConstant(Register result, const StringConstantBase* string,
327 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
328 :
329 : // Convert smi to word-size sign-extended value.
330 : void SmiUntag(Register dst, Register src);
331 : void SmiUntag(Register dst, Operand src);
332 :
333 : // Loads the address of the external reference into the destination
334 : // register.
335 : void LoadAddress(Register destination, ExternalReference source);
336 :
337 : void LoadFromConstantsTable(Register destination,
338 : int constant_index) override;
339 : void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
340 : void LoadRootRelative(Register destination, int32_t offset) override;
341 :
342 : // Operand pointing to an external reference.
343 : // May emit code to set up the scratch register. The operand is
344 : // only guaranteed to be correct as long as the scratch register
345 : // isn't changed.
346 : // If the operand is used more than once, use a scratch register
347 : // that is guaranteed not to be clobbered.
348 : Operand ExternalReferenceAsOperand(ExternalReference reference,
349 : Register scratch = kScratchRegister);
350 :
351 85176 : void Call(Register reg) { call(reg); }
352 : void Call(Operand op);
353 : void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
354 : void Call(Address destination, RelocInfo::Mode rmode);
355 : void Call(ExternalReference ext);
356 : void Call(Label* target) { call(target); }
357 :
358 : void CallBuiltinPointer(Register builtin_pointer) override;
359 :
360 : void LoadCodeObjectEntry(Register destination, Register code_object) override;
361 : void CallCodeObject(Register code_object) override;
362 : void JumpCodeObject(Register code_object) override;
363 :
364 : void RetpolineCall(Register reg);
365 : void RetpolineCall(Address destination, RelocInfo::Mode rmode);
366 :
367 : void Jump(Address destination, RelocInfo::Mode rmode);
368 : void Jump(ExternalReference ext);
369 : void Jump(Operand op);
370 : void Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
371 : Condition cc = always);
372 :
373 : void RetpolineJump(Register reg);
374 :
375 : void CallForDeoptimization(Address target, int deopt_id);
376 :
377 : // Non-SSE2 instructions.
378 : void Pextrd(Register dst, XMMRegister src, int8_t imm8);
379 : void Pinsrd(XMMRegister dst, Register src, int8_t imm8);
380 : void Pinsrd(XMMRegister dst, Operand src, int8_t imm8);
381 :
382 : void CompareRoot(Register with, RootIndex index);
383 : void CompareRoot(Operand with, RootIndex index);
384 :
385 : // Generates function and stub prologue code.
386 : void StubPrologue(StackFrame::Type type);
387 : void Prologue();
388 :
389 : // Calls Abort(msg) if the condition cc is not satisfied.
390 : // Use --debug_code to enable.
391 : void Assert(Condition cc, AbortReason reason);
392 :
393 : // Like Assert(), but without condition.
394 : // Use --debug_code to enable.
395 : void AssertUnreachable(AbortReason reason);
396 :
397 : // Abort execution if a 64 bit register containing a 32 bit payload does not
398 : // have zeros in the top 32 bits, enabled via --debug-code.
399 : void AssertZeroExtended(Register reg);
400 :
401 : // Like Assert(), but always enabled.
402 : void Check(Condition cc, AbortReason reason);
403 :
404 : // Print a message to stdout and abort execution.
405 : void Abort(AbortReason msg);
406 :
407 : // Check that the stack is aligned.
408 : void CheckStackAlignment();
409 :
410 : // Activation support.
411 : void EnterFrame(StackFrame::Type type);
412 : void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
413 : // Out-of-line constant pool not implemented on x64.
414 : UNREACHABLE();
415 : }
416 : void LeaveFrame(StackFrame::Type type);
417 :
418 : // Removes current frame and its arguments from the stack preserving the
419 : // arguments and a return address pushed to the stack for the next call. Both
420 : // |callee_args_count| and |caller_args_count_reg| do not include receiver.
421 : // |callee_args_count| is not modified, |caller_args_count_reg| is trashed.
422 : void PrepareForTailCall(const ParameterCount& callee_args_count,
423 : Register caller_args_count_reg, Register scratch0,
424 : Register scratch1);
425 :
426 : // Call a runtime routine. This expects {centry} to contain a fitting CEntry
427 : // builtin for the target runtime function and uses an indirect call.
428 : void CallRuntimeWithCEntry(Runtime::FunctionId fid, Register centry);
429 :
430 888591 : void InitializeRootRegister() {
431 888591 : ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
432 888591 : Move(kRootRegister, isolate_root);
433 888591 : }
434 :
435 : void SaveRegisters(RegList registers);
436 : void RestoreRegisters(RegList registers);
437 :
438 : void CallRecordWriteStub(Register object, Register address,
439 : RememberedSetAction remembered_set_action,
440 : SaveFPRegsMode fp_mode);
441 : void CallRecordWriteStub(Register object, Register address,
442 : RememberedSetAction remembered_set_action,
443 : SaveFPRegsMode fp_mode, Address wasm_target);
444 : void CallEphemeronKeyBarrier(Register object, Register address,
445 : SaveFPRegsMode fp_mode);
446 :
447 : void MoveNumber(Register dst, double value);
448 : void MoveNonSmi(Register dst, double value);
449 :
450 : // Calculate how much stack space (in bytes) are required to store caller
451 : // registers excluding those specified in the arguments.
452 : int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
453 : Register exclusion1 = no_reg,
454 : Register exclusion2 = no_reg,
455 : Register exclusion3 = no_reg) const;
456 :
457 : // PushCallerSaved and PopCallerSaved do not arrange the registers in any
458 : // particular order so they are not useful for calls that can cause a GC.
459 : // The caller can exclude up to 3 registers that do not need to be saved and
460 : // restored.
461 :
462 : // Push caller saved registers on the stack, and return the number of bytes
463 : // stack pointer is adjusted.
464 : int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
465 : Register exclusion2 = no_reg,
466 : Register exclusion3 = no_reg);
467 : // Restore caller saved registers from the stack, and return the number of
468 : // bytes stack pointer is adjusted.
469 : int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
470 : Register exclusion2 = no_reg,
471 : Register exclusion3 = no_reg);
472 :
473 : // Compute the start of the generated instruction stream from the current PC.
474 : // This is an alternative to embedding the {CodeObject} handle as a reference.
475 : void ComputeCodeStartAddress(Register dst);
476 :
477 : void ResetSpeculationPoisonRegister();
478 :
479 : // ---------------------------------------------------------------------------
480 : // Pointer compression support
481 :
482 : // Loads a field containing a HeapObject and decompresses it if pointer
483 : // compression is enabled.
484 : void LoadTaggedPointerField(Register destination, Operand field_operand);
485 :
486 : // Loads a field containing any tagged value and decompresses it if necessary.
487 : // When pointer compression is enabled, uses |scratch| to decompress the
488 : // value.
489 : void LoadAnyTaggedField(Register destination, Operand field_operand,
490 : Register scratch = kScratchRegister);
491 :
492 : // Loads a field containing a HeapObject, decompresses it if necessary and
493 : // pushes full pointer to the stack. When pointer compression is enabled,
494 : // uses |scratch| to decompress the value.
495 : void PushTaggedPointerField(Operand field_operand, Register scratch);
496 :
497 : // Loads a field containing any tagged value, decompresses it if necessary and
498 : // pushes the full pointer to the stack. When pointer compression is enabled,
499 : // uses |scratch1| and |scratch2| to decompress the value.
500 : void PushTaggedAnyField(Operand field_operand, Register scratch1,
501 : Register scratch2);
502 :
503 : // Loads a field containing smi value and untags it.
504 : void SmiUntagField(Register dst, Operand src);
505 :
506 : // Compresses tagged value if necessary and stores it to given on-heap
507 : // location.
508 : void StoreTaggedField(Operand dst_field_operand, Immediate immediate);
509 : void StoreTaggedField(Operand dst_field_operand, Register value);
510 :
511 : // The following macros work even when pointer compression is not enabled.
512 : void DecompressTaggedSigned(Register destination, Operand field_operand);
513 : void DecompressTaggedPointer(Register destination, Operand field_operand);
514 : void DecompressAnyTagged(Register destination, Operand field_operand,
515 : Register scratch = kScratchRegister);
516 :
517 : protected:
518 : static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
519 : int smi_count = 0;
520 : int heap_object_count = 0;
521 :
522 : // Returns a register holding the smi value. The register MUST NOT be
523 : // modified. It may be the "smi 1 constant" register.
524 : Register GetSmiConstant(Smi value);
525 :
526 : void CallRecordWriteStub(Register object, Register address,
527 : RememberedSetAction remembered_set_action,
528 : SaveFPRegsMode fp_mode, Handle<Code> code_target,
529 : Address wasm_target);
530 : };
531 :
532 : // MacroAssembler implements a collection of frequently used macros.
533 75605484 : class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
534 : public:
535 113407888 : using TurboAssembler::TurboAssembler;
536 :
537 : // Loads and stores the value of an external reference.
538 : // Special case code for load and store to take advantage of
539 : // load_rax/store_rax if possible/necessary.
540 : // For other operations, just use:
541 : // Operand operand = ExternalReferenceAsOperand(extref);
542 : // operation(operand, ..);
543 : void Load(Register destination, ExternalReference source);
544 : void Store(ExternalReference destination, Register source);
545 :
546 : // Pushes the address of the external reference onto the stack.
547 : void PushAddress(ExternalReference source);
548 :
549 : // Operations on roots in the root-array.
550 : // Load a root value where the index (or part of it) is variable.
551 : // The variable_offset register is added to the fixed_offset value
552 : // to get the index into the root-array.
553 : void PushRoot(RootIndex index);
554 :
555 : // Compare the object in a register to a value and jump if they are equal.
556 : void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
557 : Label::Distance if_equal_distance = Label::kFar) {
558 280 : CompareRoot(with, index);
559 280 : j(equal, if_equal, if_equal_distance);
560 : }
561 : void JumpIfRoot(Operand with, RootIndex index, Label* if_equal,
562 : Label::Distance if_equal_distance = Label::kFar) {
563 : CompareRoot(with, index);
564 : j(equal, if_equal, if_equal_distance);
565 : }
566 :
567 : // Compare the object in a register to a value and jump if they are not equal.
568 : void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
569 : Label::Distance if_not_equal_distance = Label::kFar) {
570 56 : CompareRoot(with, index);
571 56 : j(not_equal, if_not_equal, if_not_equal_distance);
572 : }
573 : void JumpIfNotRoot(Operand with, RootIndex index, Label* if_not_equal,
574 : Label::Distance if_not_equal_distance = Label::kFar) {
575 : CompareRoot(with, index);
576 : j(not_equal, if_not_equal, if_not_equal_distance);
577 : }
578 :
579 : // ---------------------------------------------------------------------------
580 : // GC Support
581 :
582 : // Notify the garbage collector that we wrote a pointer into an object.
583 : // |object| is the object being stored into, |value| is the object being
584 : // stored. value and scratch registers are clobbered by the operation.
585 : // The offset is the offset from the start of the object, not the offset from
586 : // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
587 : void RecordWriteField(
588 : Register object, int offset, Register value, Register scratch,
589 : SaveFPRegsMode save_fp,
590 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
591 : SmiCheck smi_check = INLINE_SMI_CHECK);
592 :
593 : // For page containing |object| mark region covering |address|
594 : // dirty. |object| is the object being stored into, |value| is the
595 : // object being stored. The address and value registers are clobbered by the
596 : // operation. RecordWrite filters out smis so it does not update
597 : // the write barrier if the value is a smi.
598 : void RecordWrite(
599 : Register object, Register address, Register value, SaveFPRegsMode save_fp,
600 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
601 : SmiCheck smi_check = INLINE_SMI_CHECK);
602 :
603 : // Frame restart support.
604 : void MaybeDropFrames();
605 :
606 : // Enter specific kind of exit frame; either in normal or
607 : // debug mode. Expects the number of arguments in register rax and
608 : // sets up the number of arguments in register rdi and the pointer
609 : // to the first argument in register rsi.
610 : //
611 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
612 : // stack accessible via StackSpaceOperand.
613 : void EnterExitFrame(int arg_stack_space = 0, bool save_doubles = false,
614 : StackFrame::Type frame_type = StackFrame::EXIT);
615 :
616 : // Enter specific kind of exit frame. Allocates
617 : // (arg_stack_space * kSystemPointerSize) memory (not GCed) on the stack
618 : // accessible via StackSpaceOperand.
619 : void EnterApiExitFrame(int arg_stack_space);
620 :
621 : // Leave the current exit frame. Expects/provides the return value in
622 : // register rax:rdx (untouched) and the pointer to the first
623 : // argument in register rsi (if pop_arguments == true).
624 : void LeaveExitFrame(bool save_doubles = false, bool pop_arguments = true);
625 :
626 : // Leave the current exit frame. Expects/provides the return value in
627 : // register rax (untouched).
628 : void LeaveApiExitFrame();
629 :
630 : // Push and pop the registers that can hold pointers.
631 0 : void PushSafepointRegisters() { Pushad(); }
632 0 : void PopSafepointRegisters() { Popad(); }
633 :
634 : // ---------------------------------------------------------------------------
635 : // JavaScript invokes
636 :
637 : // Invoke the JavaScript function code by either calling or jumping.
638 : void InvokeFunctionCode(Register function, Register new_target,
639 : const ParameterCount& expected,
640 : const ParameterCount& actual, InvokeFlag flag);
641 :
642 : // On function call, call into the debugger if necessary.
643 : void CheckDebugHook(Register fun, Register new_target,
644 : const ParameterCount& expected,
645 : const ParameterCount& actual);
646 :
647 : // Invoke the JavaScript function in the given register. Changes the
648 : // current context to the context in the function before invoking.
649 : void InvokeFunction(Register function, Register new_target,
650 : const ParameterCount& actual, InvokeFlag flag);
651 :
652 : void InvokeFunction(Register function, Register new_target,
653 : const ParameterCount& expected,
654 : const ParameterCount& actual, InvokeFlag flag);
655 :
656 : // ---------------------------------------------------------------------------
657 : // Conversions between tagged smi values and non-tagged integer values.
658 :
659 : // Tag an word-size value. The result must be known to be a valid smi value.
660 : void SmiTag(Register dst, Register src);
661 :
662 : // Simple comparison of smis. Both sides must be known smis to use these,
663 : // otherwise use Cmp.
664 : void SmiCompare(Register smi1, Register smi2);
665 : void SmiCompare(Register dst, Smi src);
666 : void SmiCompare(Register dst, Operand src);
667 : void SmiCompare(Operand dst, Register src);
668 : void SmiCompare(Operand dst, Smi src);
669 :
670 : // Functions performing a check on a known or potential smi. Returns
671 : // a condition that is satisfied if the check is successful.
672 :
673 : // Test-and-jump functions. Typically combines a check function
674 : // above with a conditional jump.
675 :
676 : // Jump to label if the value is not a tagged smi.
677 : void JumpIfNotSmi(Register src,
678 : Label* on_not_smi,
679 : Label::Distance near_jump = Label::kFar);
680 :
681 : // Jump to label if the value is not a tagged smi.
682 : void JumpIfNotSmi(Operand src, Label* on_not_smi,
683 : Label::Distance near_jump = Label::kFar);
684 :
685 : // Operations on tagged smi values.
686 :
687 : // Smis represent a subset of integers. The subset is always equivalent to
688 : // a two's complement interpretation of a fixed number of bits.
689 :
690 : // Add an integer constant to a tagged smi, giving a tagged smi as result.
691 : // No overflow testing on the result is done.
692 : void SmiAddConstant(Operand dst, Smi constant);
693 :
694 : // Specialized operations
695 :
696 : // Converts, if necessary, a smi to a combination of number and
697 : // multiplier to be used as a scaled index.
698 : // The src register contains a *positive* smi value. The shift is the
699 : // power of two to multiply the index value by (e.g. to index by
700 : // smi-value * kSystemPointerSize, pass the smi and kSystemPointerSizeLog2).
701 : // The returned index register may be either src or dst, depending
702 : // on what is most efficient. If src and dst are different registers,
703 : // src is always unchanged.
704 : SmiIndex SmiToIndex(Register dst, Register src, int shift);
705 :
706 : // ---------------------------------------------------------------------------
707 : // Macro instructions.
708 :
709 : void Cmp(Register dst, Handle<Object> source);
710 : void Cmp(Operand dst, Handle<Object> source);
711 : void Cmp(Register dst, Smi src);
712 : void Cmp(Operand dst, Smi src);
713 :
714 : // Checks if value is in range [lower_limit, higher_limit] using a single
715 : // comparison.
716 : void JumpIfIsInRange(Register value, unsigned lower_limit,
717 : unsigned higher_limit, Label* on_in_range,
718 : Label::Distance near_jump = Label::kFar);
719 :
720 : // Emit code to discard a non-negative number of pointer-sized elements
721 : // from the stack, clobbering only the rsp register.
722 : void Drop(int stack_elements);
723 : // Emit code to discard a positive number of pointer-sized elements
724 : // from the stack under the return address which remains on the top,
725 : // clobbering the rsp register.
726 : void DropUnderReturnAddress(int stack_elements,
727 : Register scratch = kScratchRegister);
728 :
729 : void PushQuad(Operand src);
730 : void PushImm32(int32_t imm32);
731 : void Pop(Register dst);
732 : void Pop(Operand dst);
733 : void PopQuad(Operand dst);
734 :
735 : // ---------------------------------------------------------------------------
736 : // SIMD macros.
737 : void Absps(XMMRegister dst);
738 : void Negps(XMMRegister dst);
739 : void Abspd(XMMRegister dst);
740 : void Negpd(XMMRegister dst);
741 : // Generates a trampoline to jump to the off-heap instruction stream.
742 : void JumpToInstructionStream(Address entry);
743 :
744 : // Non-x64 instructions.
745 : // Push/pop all general purpose registers.
746 : // Does not push rsp/rbp nor any of the assembler's special purpose registers
747 : // (kScratchRegister, kRootRegister).
748 : void Pushad();
749 : void Popad();
750 :
751 : // Compare object type for heap object.
752 : // Always use unsigned comparisons: above and below, not less and greater.
753 : // Incoming register is heap_object and outgoing register is map.
754 : // They may be the same register, and may be kScratchRegister.
755 : void CmpObjectType(Register heap_object, InstanceType type, Register map);
756 :
757 : // Compare instance type for map.
758 : // Always use unsigned comparisons: above and below, not less and greater.
759 : void CmpInstanceType(Register map, InstanceType type);
760 :
761 : void DoubleToI(Register result_reg, XMMRegister input_reg,
762 : XMMRegister scratch, Label* lost_precision, Label* is_nan,
763 : Label::Distance dst = Label::kFar);
764 :
765 : template<typename Field>
766 0 : void DecodeField(Register reg) {
767 : static const int shift = Field::kShift;
768 : static const int mask = Field::kMask >> Field::kShift;
769 : if (shift != 0) {
770 0 : shrq(reg, Immediate(shift));
771 : }
772 : andq(reg, Immediate(mask));
773 0 : }
774 :
775 : // Abort execution if argument is a smi, enabled via --debug-code.
776 : void AssertNotSmi(Register object);
777 :
778 : // Abort execution if argument is not a smi, enabled via --debug-code.
779 : void AssertSmi(Register object);
780 : void AssertSmi(Operand object);
781 :
782 : // Abort execution if argument is not a Constructor, enabled via --debug-code.
783 : void AssertConstructor(Register object);
784 :
785 : // Abort execution if argument is not a JSFunction, enabled via --debug-code.
786 : void AssertFunction(Register object);
787 :
788 : // Abort execution if argument is not a JSBoundFunction,
789 : // enabled via --debug-code.
790 : void AssertBoundFunction(Register object);
791 :
792 : // Abort execution if argument is not a JSGeneratorObject (or subclass),
793 : // enabled via --debug-code.
794 : void AssertGeneratorObject(Register object);
795 :
796 : // Abort execution if argument is not undefined or an AllocationSite, enabled
797 : // via --debug-code.
798 : void AssertUndefinedOrAllocationSite(Register object);
799 :
800 : // ---------------------------------------------------------------------------
801 : // Exception handling
802 :
803 : // Push a new stack handler and link it into stack handler chain.
804 : void PushStackHandler();
805 :
806 : // Unlink the stack handler on top of the stack from the stack handler chain.
807 : void PopStackHandler();
808 :
809 : // ---------------------------------------------------------------------------
810 : // Support functions.
811 :
812 : // Load the global proxy from the current context.
813 : void LoadGlobalProxy(Register dst) {
814 112 : LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
815 : }
816 :
817 : // Load the native context slot with the current index.
818 : void LoadNativeContextSlot(int index, Register dst);
819 :
820 : // ---------------------------------------------------------------------------
821 : // Runtime calls
822 :
823 : // Call a runtime routine.
824 : void CallRuntime(const Runtime::Function* f,
825 : int num_arguments,
826 : SaveFPRegsMode save_doubles = kDontSaveFPRegs);
827 :
828 : // Convenience function: Same as above, but takes the fid instead.
829 1568 : void CallRuntime(Runtime::FunctionId fid,
830 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
831 1568 : const Runtime::Function* function = Runtime::FunctionForId(fid);
832 1568 : CallRuntime(function, function->nargs, save_doubles);
833 1568 : }
834 :
835 : // Convenience function: Same as above, but takes the fid instead.
836 : void CallRuntime(Runtime::FunctionId fid, int num_arguments,
837 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
838 392 : CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
839 : }
840 :
841 : // Convenience function: tail call a runtime routine (jump)
842 : void TailCallRuntime(Runtime::FunctionId fid);
843 :
844 : // Jump to a runtime routines
845 : void JumpToExternalReference(const ExternalReference& ext,
846 : bool builtin_exit_frame = false);
847 :
848 : // ---------------------------------------------------------------------------
849 : // StatsCounter support
850 : void IncrementCounter(StatsCounter* counter, int value);
851 : void DecrementCounter(StatsCounter* counter, int value);
852 :
853 : // ---------------------------------------------------------------------------
854 : // In-place weak references.
855 : void LoadWeakValue(Register in_out, Label* target_if_cleared);
856 :
857 : // ---------------------------------------------------------------------------
858 : // Debugging
859 :
860 : static int SafepointRegisterStackIndex(Register reg) {
861 : return SafepointRegisterStackIndex(reg.code());
862 : }
863 :
864 : private:
865 : // Order general registers are pushed by Pushad.
866 : // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15.
867 : static const int kSafepointPushRegisterIndices[Register::kNumRegisters];
868 : static const int kNumSafepointSavedRegisters = 12;
869 :
870 : // Helper functions for generating invokes.
871 : void InvokePrologue(const ParameterCount& expected,
872 : const ParameterCount& actual, Label* done,
873 : bool* definitely_mismatches, InvokeFlag flag,
874 : Label::Distance near_jump);
875 :
876 : void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type);
877 :
878 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
879 : // stack accessible via StackSpaceOperand.
880 : void EnterExitFrameEpilogue(int arg_stack_space, bool save_doubles);
881 :
882 : void LeaveExitFrameEpilogue();
883 :
884 : // Compute memory operands for safepoint stack slots.
885 : static int SafepointRegisterStackIndex(int reg_code) {
886 0 : return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1;
887 : }
888 :
889 : // Needs access to SafepointRegisterStackIndex for compiled frame
890 : // traversal.
891 : friend class StandardFrame;
892 :
893 : DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
894 : };
895 :
896 : // -----------------------------------------------------------------------------
897 : // Static helper functions.
898 :
899 : // Generate an Operand for loading a field from an object.
900 : inline Operand FieldOperand(Register object, int offset) {
901 569740 : return Operand(object, offset - kHeapObjectTag);
902 : }
903 :
904 :
905 : // Generate an Operand for loading an indexed field from an object.
906 : inline Operand FieldOperand(Register object,
907 : Register index,
908 : ScaleFactor scale,
909 : int offset) {
910 9120 : return Operand(object, index, scale, offset - kHeapObjectTag);
911 : }
912 :
913 :
914 : inline Operand ContextOperand(Register context, int index) {
915 672 : return Operand(context, Context::SlotOffset(index));
916 : }
917 :
918 :
919 : inline Operand ContextOperand(Register context, Register index) {
920 : return Operand(context, index, times_system_pointer_size,
921 : Context::SlotOffset(0));
922 : }
923 :
924 :
925 336 : inline Operand NativeContextOperand() {
926 336 : return ContextOperand(rsi, Context::NATIVE_CONTEXT_INDEX);
927 : }
928 :
929 :
930 : // Provides access to exit frame stack space (not GCed).
931 : inline Operand StackSpaceOperand(int index) {
932 : #ifdef _WIN64
933 : const int kShaddowSpace = 4;
934 : return Operand(rsp, (index + kShaddowSpace) * kSystemPointerSize);
935 : #else
936 392 : return Operand(rsp, index * kSystemPointerSize);
937 : #endif
938 : }
939 :
940 :
941 : inline Operand StackOperandForReturnAddress(int32_t disp) {
942 56 : return Operand(rsp, disp);
943 : }
944 :
945 : #define ACCESS_MASM(masm) masm->
946 :
947 : } // namespace internal
948 : } // namespace v8
949 :
950 : #endif // V8_X64_MACRO_ASSEMBLER_X64_H_
|
