Line data Source code
1 : // Copyright 2012 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
6 : #error This header must be included via macro-assembler.h
7 : #endif
8 :
9 : #ifndef V8_X64_MACRO_ASSEMBLER_X64_H_
10 : #define V8_X64_MACRO_ASSEMBLER_X64_H_
11 :
12 : #include "src/bailout-reason.h"
13 : #include "src/base/flags.h"
14 : #include "src/contexts.h"
15 : #include "src/globals.h"
16 : #include "src/x64/assembler-x64.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 :
21 : // Convenience for platform-independent signatures.
22 : typedef Operand MemOperand;
23 :
24 : class StringConstantBase;
25 :
26 : enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
27 : enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
28 :
29 : struct SmiIndex {
30 : SmiIndex(Register index_register, ScaleFactor scale)
31 : : reg(index_register),
32 : scale(scale) {}
33 : Register reg;
34 : ScaleFactor scale;
35 : };
36 :
37 : enum StackArgumentsAccessorReceiverMode {
38 : ARGUMENTS_CONTAIN_RECEIVER,
39 : ARGUMENTS_DONT_CONTAIN_RECEIVER
40 : };
41 :
42 : class StackArgumentsAccessor {
43 : public:
44 : StackArgumentsAccessor(Register base_reg, int argument_count_immediate,
45 : StackArgumentsAccessorReceiverMode receiver_mode =
46 : ARGUMENTS_CONTAIN_RECEIVER,
47 : int extra_displacement_to_last_argument = 0)
48 : : base_reg_(base_reg),
49 : argument_count_reg_(no_reg),
50 : argument_count_immediate_(argument_count_immediate),
51 : receiver_mode_(receiver_mode),
52 : extra_displacement_to_last_argument_(
53 : extra_displacement_to_last_argument) {}
54 :
55 952 : StackArgumentsAccessor(Register base_reg, Register argument_count_reg,
56 : StackArgumentsAccessorReceiverMode receiver_mode =
57 : ARGUMENTS_CONTAIN_RECEIVER,
58 : int extra_displacement_to_last_argument = 0)
59 : : base_reg_(base_reg),
60 : argument_count_reg_(argument_count_reg),
61 : argument_count_immediate_(0),
62 : receiver_mode_(receiver_mode),
63 : extra_displacement_to_last_argument_(
64 952 : extra_displacement_to_last_argument) {}
65 :
66 : StackArgumentsAccessor(Register base_reg,
67 : const ParameterCount& parameter_count,
68 : StackArgumentsAccessorReceiverMode receiver_mode =
69 : ARGUMENTS_CONTAIN_RECEIVER,
70 : int extra_displacement_to_last_argument = 0);
71 :
72 : Operand GetArgumentOperand(int index);
73 672 : Operand GetReceiverOperand() {
74 : DCHECK(receiver_mode_ == ARGUMENTS_CONTAIN_RECEIVER);
75 1008 : return GetArgumentOperand(0);
76 : }
77 :
78 : private:
79 : const Register base_reg_;
80 : const Register argument_count_reg_;
81 : const int argument_count_immediate_;
82 : const StackArgumentsAccessorReceiverMode receiver_mode_;
83 : const int extra_displacement_to_last_argument_;
84 :
85 : DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
86 : };
87 :
88 35149582 : class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
89 : public:
90 : template <typename... Args>
91 35149663 : explicit TurboAssembler(Args&&... args)
92 70278774 : : TurboAssemblerBase(std::forward<Args>(args)...) {}
93 :
94 : template <typename Dst, typename... Args>
95 : struct AvxHelper {
96 : Assembler* assm;
97 : // Call an method where the AVX version expects the dst argument to be
98 : // duplicated.
99 : template <void (Assembler::*avx)(Dst, Dst, Args...),
100 : void (Assembler::*no_avx)(Dst, Args...)>
101 1120389 : void emit(Dst dst, Args... args) {
102 1120389 : if (CpuFeatures::IsSupported(AVX)) {
103 : CpuFeatureScope scope(assm, AVX);
104 1118797 : (assm->*avx)(dst, dst, args...);
105 : } else {
106 1592 : (assm->*no_avx)(dst, args...);
107 : }
108 1120402 : }
109 :
110 : // Call an method where the AVX version expects no duplicated dst argument.
111 : template <void (Assembler::*avx)(Dst, Args...),
112 : void (Assembler::*no_avx)(Dst, Args...)>
113 4370484 : void emit(Dst dst, Args... args) {
114 4370484 : if (CpuFeatures::IsSupported(AVX)) {
115 247237 : CpuFeatureScope scope(assm, AVX);
116 4360808 : (assm->*avx)(dst, args...);
117 : } else {
118 9676 : (assm->*no_avx)(dst, args...);
119 : }
120 4370518 : }
121 : };
122 :
123 : #define AVX_OP(macro_name, name) \
124 : template <typename Dst, typename... Args> \
125 : void macro_name(Dst dst, Args... args) { \
126 : AvxHelper<Dst, Args...>{this} \
127 : .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
128 : }
129 :
130 5907 : AVX_OP(Subsd, subsd)
131 196 : AVX_OP(Divss, divss)
132 657 : AVX_OP(Divsd, divsd)
133 10074 : AVX_OP(Xorps, xorps)
134 114995 : AVX_OP(Xorpd, xorpd)
135 120476 : AVX_OP(Movd, movd)
136 159654 : AVX_OP(Movq, movq)
137 353 : AVX_OP(Movaps, movaps)
138 125461 : AVX_OP(Movapd, movapd)
139 7968 : AVX_OP(Movups, movups)
140 196 : AVX_OP(Movmskps, movmskps)
141 657 : AVX_OP(Movmskpd, movmskpd)
142 805921 : AVX_OP(Movss, movss)
143 3307972 : AVX_OP(Movsd, movsd)
144 265594 : AVX_OP(Pcmpeqd, pcmpeqd)
145 49089 : AVX_OP(Pslld, pslld)
146 206351 : AVX_OP(Psllq, psllq)
147 37172 : AVX_OP(Psrld, psrld)
148 185713 : AVX_OP(Psrlq, psrlq)
149 : AVX_OP(Addsd, addsd)
150 : AVX_OP(Mulsd, mulsd)
151 42 : AVX_OP(Andps, andps)
152 32 : AVX_OP(Andpd, andpd)
153 : AVX_OP(Orpd, orpd)
154 : AVX_OP(Cmpeqps, cmpeqps)
155 : AVX_OP(Cmpltps, cmpltps)
156 : AVX_OP(Cmpleps, cmpleps)
157 : AVX_OP(Cmpneqps, cmpneqps)
158 : AVX_OP(Cmpnltps, cmpnltps)
159 : AVX_OP(Cmpnleps, cmpnleps)
160 : AVX_OP(Cmpeqpd, cmpeqpd)
161 : AVX_OP(Cmpltpd, cmpltpd)
162 : AVX_OP(Cmplepd, cmplepd)
163 : AVX_OP(Cmpneqpd, cmpneqpd)
164 : AVX_OP(Cmpnltpd, cmpnltpd)
165 : AVX_OP(Cmpnlepd, cmpnlepd)
166 592 : AVX_OP(Roundss, roundss)
167 43967 : AVX_OP(Roundsd, roundsd)
168 152 : AVX_OP(Sqrtss, sqrtss)
169 543 : AVX_OP(Sqrtsd, sqrtsd)
170 1095 : AVX_OP(Ucomiss, ucomiss)
171 4403 : AVX_OP(Ucomisd, ucomisd)
172 :
173 : #undef AVX_OP
174 :
175 1740 : void PushReturnAddressFrom(Register src) { pushq(src); }
176 1348 : void PopReturnAddressTo(Register dst) { popq(dst); }
177 :
178 : void Ret();
179 :
180 : // Return and drop arguments from stack, where the number of arguments
181 : // may be bigger than 2^16 - 1. Requires a scratch register.
182 : void Ret(int bytes_dropped, Register scratch);
183 :
184 : // Load a register with a long value as efficiently as possible.
185 : void Set(Register dst, int64_t x);
186 : void Set(Operand dst, intptr_t x);
187 :
188 : // Operations on roots in the root-array.
189 : void LoadRoot(Register destination, RootIndex index) override;
190 : void LoadRoot(Operand destination, RootIndex index) {
191 : LoadRoot(kScratchRegister, index);
192 : movq(destination, kScratchRegister);
193 : }
194 :
195 : void Push(Register src);
196 : void Push(Operand src);
197 : void Push(Immediate value);
198 : void Push(Smi smi);
199 : void Push(Handle<HeapObject> source);
200 :
201 : // Before calling a C-function from generated code, align arguments on stack.
202 : // After aligning the frame, arguments must be stored in rsp[0], rsp[8],
203 : // etc., not pushed. The argument count assumes all arguments are word sized.
204 : // The number of slots reserved for arguments depends on platform. On Windows
205 : // stack slots are reserved for the arguments passed in registers. On other
206 : // platforms stack slots are only reserved for the arguments actually passed
207 : // on the stack.
208 : void PrepareCallCFunction(int num_arguments);
209 :
210 : // Calls a C function and cleans up the space for arguments allocated
211 : // by PrepareCallCFunction. The called function is not allowed to trigger a
212 : // garbage collection, since that might move the code and invalidate the
213 : // return address (unless this is somehow accounted for by the called
214 : // function).
215 : void CallCFunction(ExternalReference function, int num_arguments);
216 : void CallCFunction(Register function, int num_arguments);
217 :
218 : // Calculate the number of stack slots to reserve for arguments when calling a
219 : // C function.
220 : int ArgumentStackSlotsForCFunctionCall(int num_arguments);
221 :
222 : void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
223 : Label* condition_met,
224 : Label::Distance condition_met_distance = Label::kFar);
225 :
226 : void Cvtss2sd(XMMRegister dst, XMMRegister src);
227 : void Cvtss2sd(XMMRegister dst, Operand src);
228 : void Cvtsd2ss(XMMRegister dst, XMMRegister src);
229 : void Cvtsd2ss(XMMRegister dst, Operand src);
230 : void Cvttsd2si(Register dst, XMMRegister src);
231 : void Cvttsd2si(Register dst, Operand src);
232 : void Cvttsd2siq(Register dst, XMMRegister src);
233 : void Cvttsd2siq(Register dst, Operand src);
234 : void Cvttss2si(Register dst, XMMRegister src);
235 : void Cvttss2si(Register dst, Operand src);
236 : void Cvttss2siq(Register dst, XMMRegister src);
237 : void Cvttss2siq(Register dst, Operand src);
238 : void Cvtqsi2ss(XMMRegister dst, Register src);
239 : void Cvtqsi2ss(XMMRegister dst, Operand src);
240 : void Cvtqsi2sd(XMMRegister dst, Register src);
241 : void Cvtqsi2sd(XMMRegister dst, Operand src);
242 : void Cvtlsi2ss(XMMRegister dst, Register src);
243 : void Cvtlsi2ss(XMMRegister dst, Operand src);
244 : void Cvtlui2ss(XMMRegister dst, Register src);
245 : void Cvtlui2ss(XMMRegister dst, Operand src);
246 : void Cvtlui2sd(XMMRegister dst, Register src);
247 : void Cvtlui2sd(XMMRegister dst, Operand src);
248 : void Cvtqui2ss(XMMRegister dst, Register src);
249 : void Cvtqui2ss(XMMRegister dst, Operand src);
250 : void Cvtqui2sd(XMMRegister dst, Register src);
251 : void Cvtqui2sd(XMMRegister dst, Operand src);
252 : void Cvttsd2uiq(Register dst, Operand src, Label* fail = nullptr);
253 : void Cvttsd2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
254 : void Cvttss2uiq(Register dst, Operand src, Label* fail = nullptr);
255 : void Cvttss2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
256 :
257 : // cvtsi2sd instruction only writes to the low 64-bit of dst register, which
258 : // hinders register renaming and makes dependence chains longer. So we use
259 : // xorpd to clear the dst register before cvtsi2sd to solve this issue.
260 : void Cvtlsi2sd(XMMRegister dst, Register src);
261 : void Cvtlsi2sd(XMMRegister dst, Operand src);
262 :
263 : void Lzcntq(Register dst, Register src);
264 : void Lzcntq(Register dst, Operand src);
265 : void Lzcntl(Register dst, Register src);
266 : void Lzcntl(Register dst, Operand src);
267 : void Tzcntq(Register dst, Register src);
268 : void Tzcntq(Register dst, Operand src);
269 : void Tzcntl(Register dst, Register src);
270 : void Tzcntl(Register dst, Operand src);
271 : void Popcntl(Register dst, Register src);
272 : void Popcntl(Register dst, Operand src);
273 : void Popcntq(Register dst, Register src);
274 : void Popcntq(Register dst, Operand src);
275 :
276 : // Is the value a tagged smi.
277 : Condition CheckSmi(Register src);
278 : Condition CheckSmi(Operand src);
279 :
280 : // Jump to label if the value is a tagged smi.
281 : void JumpIfSmi(Register src, Label* on_smi,
282 : Label::Distance near_jump = Label::kFar);
283 :
284 202246 : void JumpIfEqual(Register a, int32_t b, Label* dest) {
285 202246 : cmpl(a, Immediate(b));
286 202246 : j(equal, dest);
287 202246 : }
288 :
289 45138 : void JumpIfLessThan(Register a, int32_t b, Label* dest) {
290 45138 : cmpl(a, Immediate(b));
291 45138 : j(less, dest);
292 45138 : }
293 :
294 : void Move(Register dst, Smi source);
295 :
296 : void Move(Operand dst, Smi source) {
297 : Register constant = GetSmiConstant(source);
298 : movq(dst, constant);
299 : }
300 :
301 : void Move(Register dst, ExternalReference ext);
302 :
303 : void Move(XMMRegister dst, uint32_t src);
304 : void Move(XMMRegister dst, uint64_t src);
305 216 : void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
306 4448 : void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
307 :
308 : // Move if the registers are not identical.
309 : void Move(Register target, Register source);
310 :
311 : void Move(Register dst, Handle<HeapObject> source,
312 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
313 : void Move(Operand dst, Handle<HeapObject> source,
314 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
315 :
316 : // Loads a pointer into a register with a relocation mode.
317 : void Move(Register dst, Address ptr, RelocInfo::Mode rmode) {
318 : // This method must not be used with heap object references. The stored
319 : // address is not GC safe. Use the handle version instead.
320 : DCHECK(rmode > RelocInfo::LAST_GCED_ENUM);
321 37394324 : movq(dst, Immediate64(ptr, rmode));
322 : }
323 :
324 : void MoveStringConstant(Register result, const StringConstantBase* string,
325 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
326 :
327 : // Convert smi to word-size sign-extended value.
328 : void SmiUntag(Register dst, Register src);
329 : void SmiUntag(Register dst, Operand src);
330 :
331 : // Loads the address of the external reference into the destination
332 : // register.
333 : void LoadAddress(Register destination, ExternalReference source);
334 :
335 : void LoadFromConstantsTable(Register destination,
336 : int constant_index) override;
337 : void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
338 : void LoadRootRelative(Register destination, int32_t offset) override;
339 :
340 : // Operand pointing to an external reference.
341 : // May emit code to set up the scratch register. The operand is
342 : // only guaranteed to be correct as long as the scratch register
343 : // isn't changed.
344 : // If the operand is used more than once, use a scratch register
345 : // that is guaranteed not to be clobbered.
346 : Operand ExternalReferenceAsOperand(ExternalReference reference,
347 : Register scratch = kScratchRegister);
348 :
349 84280 : void Call(Register reg) { call(reg); }
350 : void Call(Operand op);
351 : void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
352 : void Call(Address destination, RelocInfo::Mode rmode);
353 : void Call(ExternalReference ext);
354 : void Call(Label* target) { call(target); }
355 :
356 : void CallBuiltinPointer(Register builtin_pointer) override;
357 :
358 : void LoadCodeObjectEntry(Register destination, Register code_object) override;
359 : void CallCodeObject(Register code_object) override;
360 : void JumpCodeObject(Register code_object) override;
361 :
362 : void RetpolineCall(Register reg);
363 : void RetpolineCall(Address destination, RelocInfo::Mode rmode);
364 :
365 : void Jump(Address destination, RelocInfo::Mode rmode);
366 : void Jump(ExternalReference ext);
367 : void Jump(Operand op);
368 : void Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
369 : Condition cc = always);
370 :
371 : void RetpolineJump(Register reg);
372 :
373 : void CallForDeoptimization(Address target, int deopt_id);
374 :
375 : // Non-SSE2 instructions.
376 : void Pextrd(Register dst, XMMRegister src, int8_t imm8);
377 : void Pinsrd(XMMRegister dst, Register src, int8_t imm8);
378 : void Pinsrd(XMMRegister dst, Operand src, int8_t imm8);
379 :
380 : void CompareRoot(Register with, RootIndex index);
381 : void CompareRoot(Operand with, RootIndex index);
382 :
383 : // Generates function and stub prologue code.
384 : void StubPrologue(StackFrame::Type type);
385 : void Prologue();
386 :
387 : // Calls Abort(msg) if the condition cc is not satisfied.
388 : // Use --debug_code to enable.
389 : void Assert(Condition cc, AbortReason reason);
390 :
391 : // Like Assert(), but without condition.
392 : // Use --debug_code to enable.
393 : void AssertUnreachable(AbortReason reason);
394 :
395 : // Abort execution if a 64 bit register containing a 32 bit payload does not
396 : // have zeros in the top 32 bits, enabled via --debug-code.
397 : void AssertZeroExtended(Register reg);
398 :
399 : // Like Assert(), but always enabled.
400 : void Check(Condition cc, AbortReason reason);
401 :
402 : // Print a message to stdout and abort execution.
403 : void Abort(AbortReason msg);
404 :
405 : // Check that the stack is aligned.
406 : void CheckStackAlignment();
407 :
408 : // Activation support.
409 : void EnterFrame(StackFrame::Type type);
410 : void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
411 : // Out-of-line constant pool not implemented on x64.
412 : UNREACHABLE();
413 : }
414 : void LeaveFrame(StackFrame::Type type);
415 :
416 : // Removes current frame and its arguments from the stack preserving the
417 : // arguments and a return address pushed to the stack for the next call. Both
418 : // |callee_args_count| and |caller_args_count_reg| do not include receiver.
419 : // |callee_args_count| is not modified, |caller_args_count_reg| is trashed.
420 : void PrepareForTailCall(const ParameterCount& callee_args_count,
421 : Register caller_args_count_reg, Register scratch0,
422 : Register scratch1);
423 :
424 : // Call a runtime routine. This expects {centry} to contain a fitting CEntry
425 : // builtin for the target runtime function and uses an indirect call.
426 : void CallRuntimeWithCEntry(Runtime::FunctionId fid, Register centry);
427 :
428 888376 : void InitializeRootRegister() {
429 888376 : ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
430 888376 : Move(kRootRegister, isolate_root);
431 888376 : }
432 :
433 : void SaveRegisters(RegList registers);
434 : void RestoreRegisters(RegList registers);
435 :
436 : void CallRecordWriteStub(Register object, Register address,
437 : RememberedSetAction remembered_set_action,
438 : SaveFPRegsMode fp_mode);
439 : void CallRecordWriteStub(Register object, Register address,
440 : RememberedSetAction remembered_set_action,
441 : SaveFPRegsMode fp_mode, Address wasm_target);
442 :
443 : void MoveNumber(Register dst, double value);
444 : void MoveNonSmi(Register dst, double value);
445 :
446 : // Calculate how much stack space (in bytes) are required to store caller
447 : // registers excluding those specified in the arguments.
448 : int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
449 : Register exclusion1 = no_reg,
450 : Register exclusion2 = no_reg,
451 : Register exclusion3 = no_reg) const;
452 :
453 : // PushCallerSaved and PopCallerSaved do not arrange the registers in any
454 : // particular order so they are not useful for calls that can cause a GC.
455 : // The caller can exclude up to 3 registers that do not need to be saved and
456 : // restored.
457 :
458 : // Push caller saved registers on the stack, and return the number of bytes
459 : // stack pointer is adjusted.
460 : int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
461 : Register exclusion2 = no_reg,
462 : Register exclusion3 = no_reg);
463 : // Restore caller saved registers from the stack, and return the number of
464 : // bytes stack pointer is adjusted.
465 : int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
466 : Register exclusion2 = no_reg,
467 : Register exclusion3 = no_reg);
468 :
469 : // Compute the start of the generated instruction stream from the current PC.
470 : // This is an alternative to embedding the {CodeObject} handle as a reference.
471 : void ComputeCodeStartAddress(Register dst);
472 :
473 : void ResetSpeculationPoisonRegister();
474 :
475 : // ---------------------------------------------------------------------------
476 : // Pointer compression support
477 :
478 : // Loads a field containing a HeapObject and decompresses it if pointer
479 : // compression is enabled.
480 : void LoadTaggedPointerField(Register destination, Operand field_operand);
481 :
482 : // Loads a field containing any tagged value and decompresses it if necessary.
483 : // When pointer compression is enabled, uses |scratch| to decompress the
484 : // value.
485 : void LoadAnyTaggedField(Register destination, Operand field_operand,
486 : Register scratch);
487 :
488 : // Loads a field containing a HeapObject, decompresses it if necessary and
489 : // pushes full pointer to the stack. When pointer compression is enabled,
490 : // uses |scratch| to decompress the value.
491 : void PushTaggedPointerField(Operand field_operand, Register scratch);
492 :
493 : // Loads a field containing any tagged value, decompresses it if necessary and
494 : // pushes the full pointer to the stack. When pointer compression is enabled,
495 : // uses |scratch1| and |scratch2| to decompress the value.
496 : void PushTaggedAnyField(Operand field_operand, Register scratch1,
497 : Register scratch2);
498 :
499 : // Loads a field containing smi value and untags it.
500 : void SmiUntagField(Register dst, Operand src);
501 :
502 : // Compresses tagged value if necessary and stores it to given on-heap
503 : // location.
504 : void StoreTaggedField(Operand dst_field_operand, Immediate immediate);
505 : void StoreTaggedField(Operand dst_field_operand, Register value);
506 :
507 : // The following macros work even when pointer compression is not enabled.
508 : void DecompressTaggedSigned(Register destination, Operand field_operand);
509 : void DecompressTaggedPointer(Register destination, Operand field_operand);
510 : void DecompressAnyTagged(Register destination, Operand field_operand,
511 : Register scratch);
512 :
513 : protected:
514 : static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
515 : int smi_count = 0;
516 : int heap_object_count = 0;
517 :
518 : // Returns a register holding the smi value. The register MUST NOT be
519 : // modified. It may be the "smi 1 constant" register.
520 : Register GetSmiConstant(Smi value);
521 :
522 : void CallRecordWriteStub(Register object, Register address,
523 : RememberedSetAction remembered_set_action,
524 : SaveFPRegsMode fp_mode, Handle<Code> code_target,
525 : Address wasm_target);
526 : };
527 :
528 : // MacroAssembler implements a collection of frequently used macros.
529 32517903 : class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
530 : public:
531 : template <typename... Args>
532 103656 : explicit MacroAssembler(Args&&... args)
533 32414250 : : TurboAssembler(std::forward<Args>(args)...) {}
534 :
535 : // Loads and stores the value of an external reference.
536 : // Special case code for load and store to take advantage of
537 : // load_rax/store_rax if possible/necessary.
538 : // For other operations, just use:
539 : // Operand operand = ExternalReferenceAsOperand(extref);
540 : // operation(operand, ..);
541 : void Load(Register destination, ExternalReference source);
542 : void Store(ExternalReference destination, Register source);
543 :
544 : // Pushes the address of the external reference onto the stack.
545 : void PushAddress(ExternalReference source);
546 :
547 : // Operations on roots in the root-array.
548 : // Load a root value where the index (or part of it) is variable.
549 : // The variable_offset register is added to the fixed_offset value
550 : // to get the index into the root-array.
551 : void PushRoot(RootIndex index);
552 :
553 : // Compare the object in a register to a value and jump if they are equal.
554 336 : void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
555 : Label::Distance if_equal_distance = Label::kFar) {
556 336 : CompareRoot(with, index);
557 336 : j(equal, if_equal, if_equal_distance);
558 336 : }
559 : void JumpIfRoot(Operand with, RootIndex index, Label* if_equal,
560 : Label::Distance if_equal_distance = Label::kFar) {
561 : CompareRoot(with, index);
562 : j(equal, if_equal, if_equal_distance);
563 : }
564 :
565 : // Compare the object in a register to a value and jump if they are not equal.
566 56 : void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
567 : Label::Distance if_not_equal_distance = Label::kFar) {
568 56 : CompareRoot(with, index);
569 56 : j(not_equal, if_not_equal, if_not_equal_distance);
570 56 : }
571 : void JumpIfNotRoot(Operand with, RootIndex index, Label* if_not_equal,
572 : Label::Distance if_not_equal_distance = Label::kFar) {
573 : CompareRoot(with, index);
574 : j(not_equal, if_not_equal, if_not_equal_distance);
575 : }
576 :
577 : // ---------------------------------------------------------------------------
578 : // GC Support
579 :
580 : // Notify the garbage collector that we wrote a pointer into an object.
581 : // |object| is the object being stored into, |value| is the object being
582 : // stored. value and scratch registers are clobbered by the operation.
583 : // The offset is the offset from the start of the object, not the offset from
584 : // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
585 : void RecordWriteField(
586 : Register object, int offset, Register value, Register scratch,
587 : SaveFPRegsMode save_fp,
588 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
589 : SmiCheck smi_check = INLINE_SMI_CHECK);
590 :
591 : // For page containing |object| mark region covering |address|
592 : // dirty. |object| is the object being stored into, |value| is the
593 : // object being stored. The address and value registers are clobbered by the
594 : // operation. RecordWrite filters out smis so it does not update
595 : // the write barrier if the value is a smi.
596 : void RecordWrite(
597 : Register object, Register address, Register value, SaveFPRegsMode save_fp,
598 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
599 : SmiCheck smi_check = INLINE_SMI_CHECK);
600 :
601 : // Frame restart support.
602 : void MaybeDropFrames();
603 :
604 : // Enter specific kind of exit frame; either in normal or
605 : // debug mode. Expects the number of arguments in register rax and
606 : // sets up the number of arguments in register rdi and the pointer
607 : // to the first argument in register rsi.
608 : //
609 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
610 : // stack accessible via StackSpaceOperand.
611 : void EnterExitFrame(int arg_stack_space = 0, bool save_doubles = false,
612 : StackFrame::Type frame_type = StackFrame::EXIT);
613 :
614 : // Enter specific kind of exit frame. Allocates
615 : // (arg_stack_space * kSystemPointerSize) memory (not GCed) on the stack
616 : // accessible via StackSpaceOperand.
617 : void EnterApiExitFrame(int arg_stack_space);
618 :
619 : // Leave the current exit frame. Expects/provides the return value in
620 : // register rax:rdx (untouched) and the pointer to the first
621 : // argument in register rsi (if pop_arguments == true).
622 : void LeaveExitFrame(bool save_doubles = false, bool pop_arguments = true);
623 :
624 : // Leave the current exit frame. Expects/provides the return value in
625 : // register rax (untouched).
626 : void LeaveApiExitFrame();
627 :
628 : // Push and pop the registers that can hold pointers.
629 0 : void PushSafepointRegisters() { Pushad(); }
630 0 : void PopSafepointRegisters() { Popad(); }
631 :
632 : // ---------------------------------------------------------------------------
633 : // JavaScript invokes
634 :
635 : // Invoke the JavaScript function code by either calling or jumping.
636 : void InvokeFunctionCode(Register function, Register new_target,
637 : const ParameterCount& expected,
638 : const ParameterCount& actual, InvokeFlag flag);
639 :
640 : // On function call, call into the debugger if necessary.
641 : void CheckDebugHook(Register fun, Register new_target,
642 : const ParameterCount& expected,
643 : const ParameterCount& actual);
644 :
645 : // Invoke the JavaScript function in the given register. Changes the
646 : // current context to the context in the function before invoking.
647 : void InvokeFunction(Register function, Register new_target,
648 : const ParameterCount& actual, InvokeFlag flag);
649 :
650 : void InvokeFunction(Register function, Register new_target,
651 : const ParameterCount& expected,
652 : const ParameterCount& actual, InvokeFlag flag);
653 :
654 : // ---------------------------------------------------------------------------
655 : // Conversions between tagged smi values and non-tagged integer values.
656 :
657 : // Tag an word-size value. The result must be known to be a valid smi value.
658 : void SmiTag(Register dst, Register src);
659 :
660 : // Simple comparison of smis. Both sides must be known smis to use these,
661 : // otherwise use Cmp.
662 : void SmiCompare(Register smi1, Register smi2);
663 : void SmiCompare(Register dst, Smi src);
664 : void SmiCompare(Register dst, Operand src);
665 : void SmiCompare(Operand dst, Register src);
666 : void SmiCompare(Operand dst, Smi src);
667 :
668 : // Functions performing a check on a known or potential smi. Returns
669 : // a condition that is satisfied if the check is successful.
670 :
671 : // Test-and-jump functions. Typically combines a check function
672 : // above with a conditional jump.
673 :
674 : // Jump to label if the value is not a tagged smi.
675 : void JumpIfNotSmi(Register src,
676 : Label* on_not_smi,
677 : Label::Distance near_jump = Label::kFar);
678 :
679 : // Jump to label if the value is not a tagged smi.
680 : void JumpIfNotSmi(Operand src, Label* on_not_smi,
681 : Label::Distance near_jump = Label::kFar);
682 :
683 : // Operations on tagged smi values.
684 :
685 : // Smis represent a subset of integers. The subset is always equivalent to
686 : // a two's complement interpretation of a fixed number of bits.
687 :
688 : // Add an integer constant to a tagged smi, giving a tagged smi as result.
689 : // No overflow testing on the result is done.
690 : void SmiAddConstant(Operand dst, Smi constant);
691 :
692 : // Specialized operations
693 :
694 : // Converts, if necessary, a smi to a combination of number and
695 : // multiplier to be used as a scaled index.
696 : // The src register contains a *positive* smi value. The shift is the
697 : // power of two to multiply the index value by (e.g. to index by
698 : // smi-value * kSystemPointerSize, pass the smi and kSystemPointerSizeLog2).
699 : // The returned index register may be either src or dst, depending
700 : // on what is most efficient. If src and dst are different registers,
701 : // src is always unchanged.
702 : SmiIndex SmiToIndex(Register dst, Register src, int shift);
703 :
704 : // ---------------------------------------------------------------------------
705 : // Macro instructions.
706 :
707 : // Load/store with specific representation.
708 : void Load(Register dst, Operand src, Representation r);
709 : void Store(Operand dst, Register src, Representation r);
710 :
711 : void Cmp(Register dst, Handle<Object> source);
712 : void Cmp(Operand dst, Handle<Object> source);
713 : void Cmp(Register dst, Smi src);
714 : void Cmp(Operand dst, Smi src);
715 :
716 : // Emit code to discard a non-negative number of pointer-sized elements
717 : // from the stack, clobbering only the rsp register.
718 : void Drop(int stack_elements);
719 : // Emit code to discard a positive number of pointer-sized elements
720 : // from the stack under the return address which remains on the top,
721 : // clobbering the rsp register.
722 : void DropUnderReturnAddress(int stack_elements,
723 : Register scratch = kScratchRegister);
724 :
725 : void PushQuad(Operand src);
726 : void PushImm32(int32_t imm32);
727 : void Pop(Register dst);
728 : void Pop(Operand dst);
729 : void PopQuad(Operand dst);
730 :
731 : // ---------------------------------------------------------------------------
732 : // SIMD macros.
733 : void Absps(XMMRegister dst);
734 : void Negps(XMMRegister dst);
735 : void Abspd(XMMRegister dst);
736 : void Negpd(XMMRegister dst);
737 : // Generates a trampoline to jump to the off-heap instruction stream.
738 : void JumpToInstructionStream(Address entry);
739 :
740 : // Non-x64 instructions.
741 : // Push/pop all general purpose registers.
742 : // Does not push rsp/rbp nor any of the assembler's special purpose registers
743 : // (kScratchRegister, kRootRegister).
744 : void Pushad();
745 : void Popad();
746 :
747 : // Compare object type for heap object.
748 : // Always use unsigned comparisons: above and below, not less and greater.
749 : // Incoming register is heap_object and outgoing register is map.
750 : // They may be the same register, and may be kScratchRegister.
751 : void CmpObjectType(Register heap_object, InstanceType type, Register map);
752 :
753 : // Compare instance type for map.
754 : // Always use unsigned comparisons: above and below, not less and greater.
755 : void CmpInstanceType(Register map, InstanceType type);
756 :
757 : void DoubleToI(Register result_reg, XMMRegister input_reg,
758 : XMMRegister scratch, Label* lost_precision, Label* is_nan,
759 : Label::Distance dst = Label::kFar);
760 :
761 : template<typename Field>
762 0 : void DecodeField(Register reg) {
763 : static const int shift = Field::kShift;
764 : static const int mask = Field::kMask >> Field::kShift;
765 : if (shift != 0) {
766 0 : shrq(reg, Immediate(shift));
767 : }
768 0 : andq(reg, Immediate(mask));
769 0 : }
770 :
771 : // Abort execution if argument is a smi, enabled via --debug-code.
772 : void AssertNotSmi(Register object);
773 :
774 : // Abort execution if argument is not a smi, enabled via --debug-code.
775 : void AssertSmi(Register object);
776 : void AssertSmi(Operand object);
777 :
778 : // Abort execution if argument is not a Constructor, enabled via --debug-code.
779 : void AssertConstructor(Register object);
780 :
781 : // Abort execution if argument is not a JSFunction, enabled via --debug-code.
782 : void AssertFunction(Register object);
783 :
784 : // Abort execution if argument is not a JSBoundFunction,
785 : // enabled via --debug-code.
786 : void AssertBoundFunction(Register object);
787 :
788 : // Abort execution if argument is not a JSGeneratorObject (or subclass),
789 : // enabled via --debug-code.
790 : void AssertGeneratorObject(Register object);
791 :
792 : // Abort execution if argument is not undefined or an AllocationSite, enabled
793 : // via --debug-code.
794 : void AssertUndefinedOrAllocationSite(Register object);
795 :
796 : // ---------------------------------------------------------------------------
797 : // Exception handling
798 :
799 : // Push a new stack handler and link it into stack handler chain.
800 : void PushStackHandler();
801 :
802 : // Unlink the stack handler on top of the stack from the stack handler chain.
803 : void PopStackHandler();
804 :
805 : // ---------------------------------------------------------------------------
806 : // Support functions.
807 :
808 : // Load the global proxy from the current context.
809 112 : void LoadGlobalProxy(Register dst) {
810 112 : LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
811 112 : }
812 :
813 : // Load the native context slot with the current index.
814 : void LoadNativeContextSlot(int index, Register dst);
815 :
816 : // ---------------------------------------------------------------------------
817 : // Runtime calls
818 :
819 : // Call a runtime routine.
820 : void CallRuntime(const Runtime::Function* f,
821 : int num_arguments,
822 : SaveFPRegsMode save_doubles = kDontSaveFPRegs);
823 :
824 : // Convenience function: Same as above, but takes the fid instead.
825 1568 : void CallRuntime(Runtime::FunctionId fid,
826 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
827 1568 : const Runtime::Function* function = Runtime::FunctionForId(fid);
828 1568 : CallRuntime(function, function->nargs, save_doubles);
829 1568 : }
830 :
831 : // Convenience function: Same as above, but takes the fid instead.
832 336 : void CallRuntime(Runtime::FunctionId fid, int num_arguments,
833 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
834 392 : CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
835 336 : }
836 :
837 : // Convenience function: tail call a runtime routine (jump)
838 : void TailCallRuntime(Runtime::FunctionId fid);
839 :
840 : // Jump to a runtime routines
841 : void JumpToExternalReference(const ExternalReference& ext,
842 : bool builtin_exit_frame = false);
843 :
844 : // ---------------------------------------------------------------------------
845 : // StatsCounter support
846 : void IncrementCounter(StatsCounter* counter, int value);
847 : void DecrementCounter(StatsCounter* counter, int value);
848 :
849 : // ---------------------------------------------------------------------------
850 : // In-place weak references.
851 : void LoadWeakValue(Register in_out, Label* target_if_cleared);
852 :
853 : // ---------------------------------------------------------------------------
854 : // Debugging
855 :
856 : static int SafepointRegisterStackIndex(Register reg) {
857 : return SafepointRegisterStackIndex(reg.code());
858 : }
859 :
860 : private:
861 : // Order general registers are pushed by Pushad.
862 : // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15.
863 : static const int kSafepointPushRegisterIndices[Register::kNumRegisters];
864 : static const int kNumSafepointSavedRegisters = 12;
865 :
866 : // Helper functions for generating invokes.
867 : void InvokePrologue(const ParameterCount& expected,
868 : const ParameterCount& actual, Label* done,
869 : bool* definitely_mismatches, InvokeFlag flag,
870 : Label::Distance near_jump);
871 :
872 : void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type);
873 :
874 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
875 : // stack accessible via StackSpaceOperand.
876 : void EnterExitFrameEpilogue(int arg_stack_space, bool save_doubles);
877 :
878 : void LeaveExitFrameEpilogue();
879 :
880 : // Compute memory operands for safepoint stack slots.
881 : static int SafepointRegisterStackIndex(int reg_code) {
882 0 : return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1;
883 : }
884 :
885 : // Needs access to SafepointRegisterStackIndex for compiled frame
886 : // traversal.
887 : friend class StandardFrame;
888 :
889 : DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
890 : };
891 :
892 : // -----------------------------------------------------------------------------
893 : // Static helper functions.
894 :
895 : // Generate an Operand for loading a field from an object.
896 5152 : inline Operand FieldOperand(Register object, int offset) {
897 557808 : return Operand(object, offset - kHeapObjectTag);
898 : }
899 :
900 :
901 : // Generate an Operand for loading an indexed field from an object.
902 336 : inline Operand FieldOperand(Register object,
903 : Register index,
904 : ScaleFactor scale,
905 : int offset) {
906 9076 : return Operand(object, index, scale, offset - kHeapObjectTag);
907 : }
908 :
909 :
910 672 : inline Operand ContextOperand(Register context, int index) {
911 672 : return Operand(context, Context::SlotOffset(index));
912 : }
913 :
914 :
915 : inline Operand ContextOperand(Register context, Register index) {
916 : return Operand(context, index, times_system_pointer_size,
917 : Context::SlotOffset(0));
918 : }
919 :
920 :
921 : inline Operand NativeContextOperand() {
922 336 : return ContextOperand(rsi, Context::NATIVE_CONTEXT_INDEX);
923 : }
924 :
925 :
926 : // Provides access to exit frame stack space (not GCed).
927 392 : inline Operand StackSpaceOperand(int index) {
928 : #ifdef _WIN64
929 : const int kShaddowSpace = 4;
930 : return Operand(rsp, (index + kShaddowSpace) * kSystemPointerSize);
931 : #else
932 392 : return Operand(rsp, index * kSystemPointerSize);
933 : #endif
934 : }
935 :
936 :
937 56 : inline Operand StackOperandForReturnAddress(int32_t disp) {
938 56 : return Operand(rsp, disp);
939 : }
940 :
941 : #define ACCESS_MASM(masm) masm->
942 :
943 : } // namespace internal
944 : } // namespace v8
945 :
946 : #endif // V8_X64_MACRO_ASSEMBLER_X64_H_
|
