Line data Source code
1 : // Copyright 2012 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
6 : #error This header must be included via macro-assembler.h
7 : #endif
8 :
9 : #ifndef V8_X64_MACRO_ASSEMBLER_X64_H_
10 : #define V8_X64_MACRO_ASSEMBLER_X64_H_
11 :
12 : #include "src/bailout-reason.h"
13 : #include "src/base/flags.h"
14 : #include "src/contexts.h"
15 : #include "src/globals.h"
16 : #include "src/x64/assembler-x64.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 :
21 : // Convenience for platform-independent signatures.
22 : typedef Operand MemOperand;
23 :
24 : class StringConstantBase;
25 :
26 : enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
27 : enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
28 :
29 : struct SmiIndex {
30 : SmiIndex(Register index_register, ScaleFactor scale)
31 : : reg(index_register),
32 : scale(scale) {}
33 : Register reg;
34 : ScaleFactor scale;
35 : };
36 :
37 : enum StackArgumentsAccessorReceiverMode {
38 : ARGUMENTS_CONTAIN_RECEIVER,
39 : ARGUMENTS_DONT_CONTAIN_RECEIVER
40 : };
41 :
42 : class StackArgumentsAccessor {
43 : public:
44 : StackArgumentsAccessor(Register base_reg, int argument_count_immediate,
45 : StackArgumentsAccessorReceiverMode receiver_mode =
46 : ARGUMENTS_CONTAIN_RECEIVER,
47 : int extra_displacement_to_last_argument = 0)
48 : : base_reg_(base_reg),
49 : argument_count_reg_(no_reg),
50 : argument_count_immediate_(argument_count_immediate),
51 : receiver_mode_(receiver_mode),
52 : extra_displacement_to_last_argument_(
53 : extra_displacement_to_last_argument) {}
54 :
55 952 : StackArgumentsAccessor(Register base_reg, Register argument_count_reg,
56 : StackArgumentsAccessorReceiverMode receiver_mode =
57 : ARGUMENTS_CONTAIN_RECEIVER,
58 : int extra_displacement_to_last_argument = 0)
59 : : base_reg_(base_reg),
60 : argument_count_reg_(argument_count_reg),
61 : argument_count_immediate_(0),
62 : receiver_mode_(receiver_mode),
63 : extra_displacement_to_last_argument_(
64 952 : extra_displacement_to_last_argument) {}
65 :
66 : StackArgumentsAccessor(Register base_reg,
67 : const ParameterCount& parameter_count,
68 : StackArgumentsAccessorReceiverMode receiver_mode =
69 : ARGUMENTS_CONTAIN_RECEIVER,
70 : int extra_displacement_to_last_argument = 0);
71 :
72 : Operand GetArgumentOperand(int index);
73 672 : Operand GetReceiverOperand() {
74 : DCHECK(receiver_mode_ == ARGUMENTS_CONTAIN_RECEIVER);
75 1008 : return GetArgumentOperand(0);
76 : }
77 :
78 : private:
79 : const Register base_reg_;
80 : const Register argument_count_reg_;
81 : const int argument_count_immediate_;
82 : const StackArgumentsAccessorReceiverMode receiver_mode_;
83 : const int extra_displacement_to_last_argument_;
84 :
85 : DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
86 : };
87 :
88 81810283 : class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
89 : public:
90 122668858 : using TurboAssemblerBase::TurboAssemblerBase;
91 :
92 : template <typename Dst, typename... Args>
93 : struct AvxHelper {
94 : Assembler* assm;
95 : // Call an method where the AVX version expects the dst argument to be
96 : // duplicated.
97 : template <void (Assembler::*avx)(Dst, Dst, Args...),
98 : void (Assembler::*no_avx)(Dst, Args...)>
99 1130282 : void emit(Dst dst, Args... args) {
100 1130282 : if (CpuFeatures::IsSupported(AVX)) {
101 : CpuFeatureScope scope(assm, AVX);
102 1128732 : (assm->*avx)(dst, dst, args...);
103 : } else {
104 1550 : (assm->*no_avx)(dst, args...);
105 : }
106 1130323 : }
107 :
108 : // Call an method where the AVX version expects no duplicated dst argument.
109 : template <void (Assembler::*avx)(Dst, Args...),
110 : void (Assembler::*no_avx)(Dst, Args...)>
111 4222813 : void emit(Dst dst, Args... args) {
112 4222813 : if (CpuFeatures::IsSupported(AVX)) {
113 486478 : CpuFeatureScope scope(assm, AVX);
114 4213179 : (assm->*avx)(dst, args...);
115 : } else {
116 9634 : (assm->*no_avx)(dst, args...);
117 : }
118 4222860 : }
119 : };
120 :
121 : #define AVX_OP(macro_name, name) \
122 : template <typename Dst, typename... Args> \
123 : void macro_name(Dst dst, Args... args) { \
124 : AvxHelper<Dst, Args...>{this} \
125 : .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
126 : }
127 :
128 6049 : AVX_OP(Subsd, subsd)
129 193 : AVX_OP(Divss, divss)
130 660 : AVX_OP(Divsd, divsd)
131 10009 : AVX_OP(Xorps, xorps)
132 110921 : AVX_OP(Xorpd, xorpd)
133 120185 : AVX_OP(Movd, movd)
134 159872 : AVX_OP(Movq, movq)
135 353 : AVX_OP(Movaps, movaps)
136 126914 : AVX_OP(Movapd, movapd)
137 13324 : AVX_OP(Movups, movups)
138 195 : AVX_OP(Movmskps, movmskps)
139 661 : AVX_OP(Movmskpd, movmskpd)
140 807339 : AVX_OP(Movss, movss)
141 3152124 : AVX_OP(Movsd, movsd)
142 269431 : AVX_OP(Pcmpeqd, pcmpeqd)
143 49104 : AVX_OP(Pslld, pslld)
144 210171 : AVX_OP(Psllq, psllq)
145 37201 : AVX_OP(Psrld, psrld)
146 189561 : AVX_OP(Psrlq, psrlq)
147 : AVX_OP(Addsd, addsd)
148 : AVX_OP(Mulsd, mulsd)
149 42 : AVX_OP(Andps, andps)
150 32 : AVX_OP(Andpd, andpd)
151 : AVX_OP(Orpd, orpd)
152 : AVX_OP(Cmpeqps, cmpeqps)
153 : AVX_OP(Cmpltps, cmpltps)
154 : AVX_OP(Cmpleps, cmpleps)
155 : AVX_OP(Cmpneqps, cmpneqps)
156 : AVX_OP(Cmpnltps, cmpnltps)
157 : AVX_OP(Cmpnleps, cmpnleps)
158 : AVX_OP(Cmpeqpd, cmpeqpd)
159 : AVX_OP(Cmpltpd, cmpltpd)
160 : AVX_OP(Cmplepd, cmplepd)
161 : AVX_OP(Cmpneqpd, cmpneqpd)
162 : AVX_OP(Cmpnltpd, cmpnltpd)
163 : AVX_OP(Cmpnlepd, cmpnlepd)
164 586 : AVX_OP(Roundss, roundss)
165 46187 : AVX_OP(Roundsd, roundsd)
166 152 : AVX_OP(Sqrtss, sqrtss)
167 543 : AVX_OP(Sqrtsd, sqrtsd)
168 1090 : AVX_OP(Ucomiss, ucomiss)
169 4630 : AVX_OP(Ucomisd, ucomisd)
170 :
171 : #undef AVX_OP
172 :
173 1852 : void PushReturnAddressFrom(Register src) { pushq(src); }
174 1460 : void PopReturnAddressTo(Register dst) { popq(dst); }
175 :
176 : void Ret();
177 :
178 : // Return and drop arguments from stack, where the number of arguments
179 : // may be bigger than 2^16 - 1. Requires a scratch register.
180 : void Ret(int bytes_dropped, Register scratch);
181 :
182 : // Load a register with a long value as efficiently as possible.
183 : void Set(Register dst, int64_t x);
184 : void Set(Operand dst, intptr_t x);
185 :
186 : // Operations on roots in the root-array.
187 : void LoadRoot(Register destination, RootIndex index) override;
188 : void LoadRoot(Operand destination, RootIndex index) {
189 : LoadRoot(kScratchRegister, index);
190 : movq(destination, kScratchRegister);
191 : }
192 :
193 : void Push(Register src);
194 : void Push(Operand src);
195 : void Push(Immediate value);
196 : void Push(Smi smi);
197 : void Push(Handle<HeapObject> source);
198 :
199 : // Before calling a C-function from generated code, align arguments on stack.
200 : // After aligning the frame, arguments must be stored in rsp[0], rsp[8],
201 : // etc., not pushed. The argument count assumes all arguments are word sized.
202 : // The number of slots reserved for arguments depends on platform. On Windows
203 : // stack slots are reserved for the arguments passed in registers. On other
204 : // platforms stack slots are only reserved for the arguments actually passed
205 : // on the stack.
206 : void PrepareCallCFunction(int num_arguments);
207 :
208 : // Calls a C function and cleans up the space for arguments allocated
209 : // by PrepareCallCFunction. The called function is not allowed to trigger a
210 : // garbage collection, since that might move the code and invalidate the
211 : // return address (unless this is somehow accounted for by the called
212 : // function).
213 : void CallCFunction(ExternalReference function, int num_arguments);
214 : void CallCFunction(Register function, int num_arguments);
215 :
216 : // Calculate the number of stack slots to reserve for arguments when calling a
217 : // C function.
218 : int ArgumentStackSlotsForCFunctionCall(int num_arguments);
219 :
220 : void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
221 : Label* condition_met,
222 : Label::Distance condition_met_distance = Label::kFar);
223 :
224 : void Cvtss2sd(XMMRegister dst, XMMRegister src);
225 : void Cvtss2sd(XMMRegister dst, Operand src);
226 : void Cvtsd2ss(XMMRegister dst, XMMRegister src);
227 : void Cvtsd2ss(XMMRegister dst, Operand src);
228 : void Cvttsd2si(Register dst, XMMRegister src);
229 : void Cvttsd2si(Register dst, Operand src);
230 : void Cvttsd2siq(Register dst, XMMRegister src);
231 : void Cvttsd2siq(Register dst, Operand src);
232 : void Cvttss2si(Register dst, XMMRegister src);
233 : void Cvttss2si(Register dst, Operand src);
234 : void Cvttss2siq(Register dst, XMMRegister src);
235 : void Cvttss2siq(Register dst, Operand src);
236 : void Cvtqsi2ss(XMMRegister dst, Register src);
237 : void Cvtqsi2ss(XMMRegister dst, Operand src);
238 : void Cvtqsi2sd(XMMRegister dst, Register src);
239 : void Cvtqsi2sd(XMMRegister dst, Operand src);
240 : void Cvtlsi2ss(XMMRegister dst, Register src);
241 : void Cvtlsi2ss(XMMRegister dst, Operand src);
242 : void Cvtlui2ss(XMMRegister dst, Register src);
243 : void Cvtlui2ss(XMMRegister dst, Operand src);
244 : void Cvtlui2sd(XMMRegister dst, Register src);
245 : void Cvtlui2sd(XMMRegister dst, Operand src);
246 : void Cvtqui2ss(XMMRegister dst, Register src);
247 : void Cvtqui2ss(XMMRegister dst, Operand src);
248 : void Cvtqui2sd(XMMRegister dst, Register src);
249 : void Cvtqui2sd(XMMRegister dst, Operand src);
250 : void Cvttsd2uiq(Register dst, Operand src, Label* fail = nullptr);
251 : void Cvttsd2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
252 : void Cvttss2uiq(Register dst, Operand src, Label* fail = nullptr);
253 : void Cvttss2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
254 :
255 : // cvtsi2sd instruction only writes to the low 64-bit of dst register, which
256 : // hinders register renaming and makes dependence chains longer. So we use
257 : // xorpd to clear the dst register before cvtsi2sd to solve this issue.
258 : void Cvtlsi2sd(XMMRegister dst, Register src);
259 : void Cvtlsi2sd(XMMRegister dst, Operand src);
260 :
261 : void Lzcntq(Register dst, Register src);
262 : void Lzcntq(Register dst, Operand src);
263 : void Lzcntl(Register dst, Register src);
264 : void Lzcntl(Register dst, Operand src);
265 : void Tzcntq(Register dst, Register src);
266 : void Tzcntq(Register dst, Operand src);
267 : void Tzcntl(Register dst, Register src);
268 : void Tzcntl(Register dst, Operand src);
269 : void Popcntl(Register dst, Register src);
270 : void Popcntl(Register dst, Operand src);
271 : void Popcntq(Register dst, Register src);
272 : void Popcntq(Register dst, Operand src);
273 :
274 : // Is the value a tagged smi.
275 : Condition CheckSmi(Register src);
276 : Condition CheckSmi(Operand src);
277 :
278 : // Jump to label if the value is a tagged smi.
279 : void JumpIfSmi(Register src, Label* on_smi,
280 : Label::Distance near_jump = Label::kFar);
281 :
282 188322 : void JumpIfEqual(Register a, int32_t b, Label* dest) {
283 188322 : cmpl(a, Immediate(b));
284 188322 : j(equal, dest);
285 188323 : }
286 :
287 40664 : void JumpIfLessThan(Register a, int32_t b, Label* dest) {
288 40664 : cmpl(a, Immediate(b));
289 40664 : j(less, dest);
290 40664 : }
291 :
292 : void Move(Register dst, Smi source);
293 :
294 : void Move(Operand dst, Smi source) {
295 : Register constant = GetSmiConstant(source);
296 : movq(dst, constant);
297 : }
298 :
299 : void Move(Register dst, ExternalReference ext);
300 :
301 : void Move(XMMRegister dst, uint32_t src);
302 : void Move(XMMRegister dst, uint64_t src);
303 216 : void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
304 3824 : void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
305 :
306 : // Move if the registers are not identical.
307 : void Move(Register target, Register source);
308 :
309 : void Move(Register dst, Handle<HeapObject> source,
310 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
311 : void Move(Operand dst, Handle<HeapObject> source,
312 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
313 :
314 : // Loads a pointer into a register with a relocation mode.
315 : void Move(Register dst, Address ptr, RelocInfo::Mode rmode) {
316 : // This method must not be used with heap object references. The stored
317 : // address is not GC safe. Use the handle version instead.
318 : DCHECK(rmode > RelocInfo::LAST_GCED_ENUM);
319 42486592 : movq(dst, Immediate64(ptr, rmode));
320 : }
321 :
322 : // Move src0 to dst0 and src1 to dst1, handling possible overlaps.
323 : void MovePair(Register dst0, Register src0, Register dst1, Register src1);
324 :
325 : void MoveStringConstant(Register result, const StringConstantBase* string,
326 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
327 :
328 : // Convert smi to word-size sign-extended value.
329 : void SmiUntag(Register dst, Register src);
330 : void SmiUntag(Register dst, Operand src);
331 :
332 : // Loads the address of the external reference into the destination
333 : // register.
334 : void LoadAddress(Register destination, ExternalReference source);
335 :
336 : void LoadFromConstantsTable(Register destination,
337 : int constant_index) override;
338 : void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
339 : void LoadRootRelative(Register destination, int32_t offset) override;
340 :
341 : // Operand pointing to an external reference.
342 : // May emit code to set up the scratch register. The operand is
343 : // only guaranteed to be correct as long as the scratch register
344 : // isn't changed.
345 : // If the operand is used more than once, use a scratch register
346 : // that is guaranteed not to be clobbered.
347 : Operand ExternalReferenceAsOperand(ExternalReference reference,
348 : Register scratch = kScratchRegister);
349 :
350 84840 : void Call(Register reg) { call(reg); }
351 : void Call(Operand op);
352 : void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
353 : void Call(Address destination, RelocInfo::Mode rmode);
354 : void Call(ExternalReference ext);
355 : void Call(Label* target) { call(target); }
356 :
357 : void CallBuiltinPointer(Register builtin_pointer) override;
358 :
359 : void LoadCodeObjectEntry(Register destination, Register code_object) override;
360 : void CallCodeObject(Register code_object) override;
361 : void JumpCodeObject(Register code_object) override;
362 :
363 : void RetpolineCall(Register reg);
364 : void RetpolineCall(Address destination, RelocInfo::Mode rmode);
365 :
366 : void Jump(Address destination, RelocInfo::Mode rmode);
367 : void Jump(ExternalReference ext);
368 : void Jump(Operand op);
369 : void Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
370 : Condition cc = always);
371 :
372 : void RetpolineJump(Register reg);
373 :
374 : void CallForDeoptimization(Address target, int deopt_id);
375 :
376 : // Non-SSE2 instructions.
377 : void Pextrd(Register dst, XMMRegister src, int8_t imm8);
378 : void Pinsrd(XMMRegister dst, Register src, int8_t imm8);
379 : void Pinsrd(XMMRegister dst, Operand src, int8_t imm8);
380 :
381 : void CompareRoot(Register with, RootIndex index);
382 : void CompareRoot(Operand with, RootIndex index);
383 :
384 : // Generates function and stub prologue code.
385 : void StubPrologue(StackFrame::Type type);
386 : void Prologue();
387 :
388 : // Calls Abort(msg) if the condition cc is not satisfied.
389 : // Use --debug_code to enable.
390 : void Assert(Condition cc, AbortReason reason);
391 :
392 : // Like Assert(), but without condition.
393 : // Use --debug_code to enable.
394 : void AssertUnreachable(AbortReason reason);
395 :
396 : // Abort execution if a 64 bit register containing a 32 bit payload does not
397 : // have zeros in the top 32 bits, enabled via --debug-code.
398 : void AssertZeroExtended(Register reg);
399 :
400 : // Like Assert(), but always enabled.
401 : void Check(Condition cc, AbortReason reason);
402 :
403 : // Print a message to stdout and abort execution.
404 : void Abort(AbortReason msg);
405 :
406 : // Check that the stack is aligned.
407 : void CheckStackAlignment();
408 :
409 : // Activation support.
410 : void EnterFrame(StackFrame::Type type);
411 : void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
412 : // Out-of-line constant pool not implemented on x64.
413 : UNREACHABLE();
414 : }
415 : void LeaveFrame(StackFrame::Type type);
416 :
417 : // Removes current frame and its arguments from the stack preserving the
418 : // arguments and a return address pushed to the stack for the next call. Both
419 : // |callee_args_count| and |caller_args_count_reg| do not include receiver.
420 : // |callee_args_count| is not modified, |caller_args_count_reg| is trashed.
421 : void PrepareForTailCall(const ParameterCount& callee_args_count,
422 : Register caller_args_count_reg, Register scratch0,
423 : Register scratch1);
424 :
425 : // Call a runtime routine. This expects {centry} to contain a fitting CEntry
426 : // builtin for the target runtime function and uses an indirect call.
427 : void CallRuntimeWithCEntry(Runtime::FunctionId fid, Register centry);
428 :
429 888607 : void InitializeRootRegister() {
430 888607 : ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
431 888607 : Move(kRootRegister, isolate_root);
432 888607 : }
433 :
434 : void SaveRegisters(RegList registers);
435 : void RestoreRegisters(RegList registers);
436 :
437 : void CallRecordWriteStub(Register object, Register address,
438 : RememberedSetAction remembered_set_action,
439 : SaveFPRegsMode fp_mode);
440 : void CallRecordWriteStub(Register object, Register address,
441 : RememberedSetAction remembered_set_action,
442 : SaveFPRegsMode fp_mode, Address wasm_target);
443 :
444 : void MoveNumber(Register dst, double value);
445 : void MoveNonSmi(Register dst, double value);
446 :
447 : // Calculate how much stack space (in bytes) are required to store caller
448 : // registers excluding those specified in the arguments.
449 : int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
450 : Register exclusion1 = no_reg,
451 : Register exclusion2 = no_reg,
452 : Register exclusion3 = no_reg) const;
453 :
454 : // PushCallerSaved and PopCallerSaved do not arrange the registers in any
455 : // particular order so they are not useful for calls that can cause a GC.
456 : // The caller can exclude up to 3 registers that do not need to be saved and
457 : // restored.
458 :
459 : // Push caller saved registers on the stack, and return the number of bytes
460 : // stack pointer is adjusted.
461 : int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
462 : Register exclusion2 = no_reg,
463 : Register exclusion3 = no_reg);
464 : // Restore caller saved registers from the stack, and return the number of
465 : // bytes stack pointer is adjusted.
466 : int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
467 : Register exclusion2 = no_reg,
468 : Register exclusion3 = no_reg);
469 :
470 : // Compute the start of the generated instruction stream from the current PC.
471 : // This is an alternative to embedding the {CodeObject} handle as a reference.
472 : void ComputeCodeStartAddress(Register dst);
473 :
474 : void ResetSpeculationPoisonRegister();
475 :
476 : // ---------------------------------------------------------------------------
477 : // Pointer compression support
478 :
479 : // Loads a field containing a HeapObject and decompresses it if pointer
480 : // compression is enabled.
481 : void LoadTaggedPointerField(Register destination, Operand field_operand);
482 :
483 : // Loads a field containing any tagged value and decompresses it if necessary.
484 : // When pointer compression is enabled, uses |scratch| to decompress the
485 : // value.
486 : void LoadAnyTaggedField(Register destination, Operand field_operand,
487 : Register scratch = kScratchRegister);
488 :
489 : // Loads a field containing a HeapObject, decompresses it if necessary and
490 : // pushes full pointer to the stack. When pointer compression is enabled,
491 : // uses |scratch| to decompress the value.
492 : void PushTaggedPointerField(Operand field_operand, Register scratch);
493 :
494 : // Loads a field containing any tagged value, decompresses it if necessary and
495 : // pushes the full pointer to the stack. When pointer compression is enabled,
496 : // uses |scratch1| and |scratch2| to decompress the value.
497 : void PushTaggedAnyField(Operand field_operand, Register scratch1,
498 : Register scratch2);
499 :
500 : // Loads a field containing smi value and untags it.
501 : void SmiUntagField(Register dst, Operand src);
502 :
503 : // Compresses tagged value if necessary and stores it to given on-heap
504 : // location.
505 : void StoreTaggedField(Operand dst_field_operand, Immediate immediate);
506 : void StoreTaggedField(Operand dst_field_operand, Register value);
507 :
508 : // The following macros work even when pointer compression is not enabled.
509 : void DecompressTaggedSigned(Register destination, Operand field_operand);
510 : void DecompressTaggedPointer(Register destination, Operand field_operand);
511 : void DecompressAnyTagged(Register destination, Operand field_operand,
512 : Register scratch = kScratchRegister);
513 :
514 : protected:
515 : static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
516 : int smi_count = 0;
517 : int heap_object_count = 0;
518 :
519 : // Returns a register holding the smi value. The register MUST NOT be
520 : // modified. It may be the "smi 1 constant" register.
521 : Register GetSmiConstant(Smi value);
522 :
523 : void CallRecordWriteStub(Register object, Register address,
524 : RememberedSetAction remembered_set_action,
525 : SaveFPRegsMode fp_mode, Handle<Code> code_target,
526 : Address wasm_target);
527 : };
528 :
529 : // MacroAssembler implements a collection of frequently used macros.
530 75586289 : class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
531 : public:
532 113337637 : using TurboAssembler::TurboAssembler;
533 :
534 : // Loads and stores the value of an external reference.
535 : // Special case code for load and store to take advantage of
536 : // load_rax/store_rax if possible/necessary.
537 : // For other operations, just use:
538 : // Operand operand = ExternalReferenceAsOperand(extref);
539 : // operation(operand, ..);
540 : void Load(Register destination, ExternalReference source);
541 : void Store(ExternalReference destination, Register source);
542 :
543 : // Pushes the address of the external reference onto the stack.
544 : void PushAddress(ExternalReference source);
545 :
546 : // Operations on roots in the root-array.
547 : // Load a root value where the index (or part of it) is variable.
548 : // The variable_offset register is added to the fixed_offset value
549 : // to get the index into the root-array.
550 : void PushRoot(RootIndex index);
551 :
552 : // Compare the object in a register to a value and jump if they are equal.
553 280 : void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
554 : Label::Distance if_equal_distance = Label::kFar) {
555 280 : CompareRoot(with, index);
556 280 : j(equal, if_equal, if_equal_distance);
557 280 : }
558 : void JumpIfRoot(Operand with, RootIndex index, Label* if_equal,
559 : Label::Distance if_equal_distance = Label::kFar) {
560 : CompareRoot(with, index);
561 : j(equal, if_equal, if_equal_distance);
562 : }
563 :
564 : // Compare the object in a register to a value and jump if they are not equal.
565 56 : void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
566 : Label::Distance if_not_equal_distance = Label::kFar) {
567 56 : CompareRoot(with, index);
568 56 : j(not_equal, if_not_equal, if_not_equal_distance);
569 56 : }
570 : void JumpIfNotRoot(Operand with, RootIndex index, Label* if_not_equal,
571 : Label::Distance if_not_equal_distance = Label::kFar) {
572 : CompareRoot(with, index);
573 : j(not_equal, if_not_equal, if_not_equal_distance);
574 : }
575 :
576 : // ---------------------------------------------------------------------------
577 : // GC Support
578 :
579 : // Notify the garbage collector that we wrote a pointer into an object.
580 : // |object| is the object being stored into, |value| is the object being
581 : // stored. value and scratch registers are clobbered by the operation.
582 : // The offset is the offset from the start of the object, not the offset from
583 : // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
584 : void RecordWriteField(
585 : Register object, int offset, Register value, Register scratch,
586 : SaveFPRegsMode save_fp,
587 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
588 : SmiCheck smi_check = INLINE_SMI_CHECK);
589 :
590 : // For page containing |object| mark region covering |address|
591 : // dirty. |object| is the object being stored into, |value| is the
592 : // object being stored. The address and value registers are clobbered by the
593 : // operation. RecordWrite filters out smis so it does not update
594 : // the write barrier if the value is a smi.
595 : void RecordWrite(
596 : Register object, Register address, Register value, SaveFPRegsMode save_fp,
597 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
598 : SmiCheck smi_check = INLINE_SMI_CHECK);
599 :
600 : // Frame restart support.
601 : void MaybeDropFrames();
602 :
603 : // Enter specific kind of exit frame; either in normal or
604 : // debug mode. Expects the number of arguments in register rax and
605 : // sets up the number of arguments in register rdi and the pointer
606 : // to the first argument in register rsi.
607 : //
608 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
609 : // stack accessible via StackSpaceOperand.
610 : void EnterExitFrame(int arg_stack_space = 0, bool save_doubles = false,
611 : StackFrame::Type frame_type = StackFrame::EXIT);
612 :
613 : // Enter specific kind of exit frame. Allocates
614 : // (arg_stack_space * kSystemPointerSize) memory (not GCed) on the stack
615 : // accessible via StackSpaceOperand.
616 : void EnterApiExitFrame(int arg_stack_space);
617 :
618 : // Leave the current exit frame. Expects/provides the return value in
619 : // register rax:rdx (untouched) and the pointer to the first
620 : // argument in register rsi (if pop_arguments == true).
621 : void LeaveExitFrame(bool save_doubles = false, bool pop_arguments = true);
622 :
623 : // Leave the current exit frame. Expects/provides the return value in
624 : // register rax (untouched).
625 : void LeaveApiExitFrame();
626 :
627 : // Push and pop the registers that can hold pointers.
628 0 : void PushSafepointRegisters() { Pushad(); }
629 0 : void PopSafepointRegisters() { Popad(); }
630 :
631 : // ---------------------------------------------------------------------------
632 : // JavaScript invokes
633 :
634 : // Invoke the JavaScript function code by either calling or jumping.
635 : void InvokeFunctionCode(Register function, Register new_target,
636 : const ParameterCount& expected,
637 : const ParameterCount& actual, InvokeFlag flag);
638 :
639 : // On function call, call into the debugger if necessary.
640 : void CheckDebugHook(Register fun, Register new_target,
641 : const ParameterCount& expected,
642 : const ParameterCount& actual);
643 :
644 : // Invoke the JavaScript function in the given register. Changes the
645 : // current context to the context in the function before invoking.
646 : void InvokeFunction(Register function, Register new_target,
647 : const ParameterCount& actual, InvokeFlag flag);
648 :
649 : void InvokeFunction(Register function, Register new_target,
650 : const ParameterCount& expected,
651 : const ParameterCount& actual, InvokeFlag flag);
652 :
653 : // ---------------------------------------------------------------------------
654 : // Conversions between tagged smi values and non-tagged integer values.
655 :
656 : // Tag an word-size value. The result must be known to be a valid smi value.
657 : void SmiTag(Register dst, Register src);
658 :
659 : // Simple comparison of smis. Both sides must be known smis to use these,
660 : // otherwise use Cmp.
661 : void SmiCompare(Register smi1, Register smi2);
662 : void SmiCompare(Register dst, Smi src);
663 : void SmiCompare(Register dst, Operand src);
664 : void SmiCompare(Operand dst, Register src);
665 : void SmiCompare(Operand dst, Smi src);
666 :
667 : // Functions performing a check on a known or potential smi. Returns
668 : // a condition that is satisfied if the check is successful.
669 :
670 : // Test-and-jump functions. Typically combines a check function
671 : // above with a conditional jump.
672 :
673 : // Jump to label if the value is not a tagged smi.
674 : void JumpIfNotSmi(Register src,
675 : Label* on_not_smi,
676 : Label::Distance near_jump = Label::kFar);
677 :
678 : // Jump to label if the value is not a tagged smi.
679 : void JumpIfNotSmi(Operand src, Label* on_not_smi,
680 : Label::Distance near_jump = Label::kFar);
681 :
682 : // Operations on tagged smi values.
683 :
684 : // Smis represent a subset of integers. The subset is always equivalent to
685 : // a two's complement interpretation of a fixed number of bits.
686 :
687 : // Add an integer constant to a tagged smi, giving a tagged smi as result.
688 : // No overflow testing on the result is done.
689 : void SmiAddConstant(Operand dst, Smi constant);
690 :
691 : // Specialized operations
692 :
693 : // Converts, if necessary, a smi to a combination of number and
694 : // multiplier to be used as a scaled index.
695 : // The src register contains a *positive* smi value. The shift is the
696 : // power of two to multiply the index value by (e.g. to index by
697 : // smi-value * kSystemPointerSize, pass the smi and kSystemPointerSizeLog2).
698 : // The returned index register may be either src or dst, depending
699 : // on what is most efficient. If src and dst are different registers,
700 : // src is always unchanged.
701 : SmiIndex SmiToIndex(Register dst, Register src, int shift);
702 :
703 : // ---------------------------------------------------------------------------
704 : // Macro instructions.
705 :
706 : void Cmp(Register dst, Handle<Object> source);
707 : void Cmp(Operand dst, Handle<Object> source);
708 : void Cmp(Register dst, Smi src);
709 : void Cmp(Operand dst, Smi src);
710 :
711 : // Checks if value is in range [lower_limit, higher_limit] using a single
712 : // comparison.
713 : void JumpIfIsInRange(Register value, unsigned lower_limit,
714 : unsigned higher_limit, Label* on_in_range,
715 : Label::Distance near_jump = Label::kFar);
716 :
717 : // Emit code to discard a non-negative number of pointer-sized elements
718 : // from the stack, clobbering only the rsp register.
719 : void Drop(int stack_elements);
720 : // Emit code to discard a positive number of pointer-sized elements
721 : // from the stack under the return address which remains on the top,
722 : // clobbering the rsp register.
723 : void DropUnderReturnAddress(int stack_elements,
724 : Register scratch = kScratchRegister);
725 :
726 : void PushQuad(Operand src);
727 : void PushImm32(int32_t imm32);
728 : void Pop(Register dst);
729 : void Pop(Operand dst);
730 : void PopQuad(Operand dst);
731 :
732 : // ---------------------------------------------------------------------------
733 : // SIMD macros.
734 : void Absps(XMMRegister dst);
735 : void Negps(XMMRegister dst);
736 : void Abspd(XMMRegister dst);
737 : void Negpd(XMMRegister dst);
738 : // Generates a trampoline to jump to the off-heap instruction stream.
739 : void JumpToInstructionStream(Address entry);
740 :
741 : // Non-x64 instructions.
742 : // Push/pop all general purpose registers.
743 : // Does not push rsp/rbp nor any of the assembler's special purpose registers
744 : // (kScratchRegister, kRootRegister).
745 : void Pushad();
746 : void Popad();
747 :
748 : // Compare object type for heap object.
749 : // Always use unsigned comparisons: above and below, not less and greater.
750 : // Incoming register is heap_object and outgoing register is map.
751 : // They may be the same register, and may be kScratchRegister.
752 : void CmpObjectType(Register heap_object, InstanceType type, Register map);
753 :
754 : // Compare instance type for map.
755 : // Always use unsigned comparisons: above and below, not less and greater.
756 : void CmpInstanceType(Register map, InstanceType type);
757 :
758 : void DoubleToI(Register result_reg, XMMRegister input_reg,
759 : XMMRegister scratch, Label* lost_precision, Label* is_nan,
760 : Label::Distance dst = Label::kFar);
761 :
762 : template<typename Field>
763 56 : void DecodeField(Register reg) {
764 : static const int shift = Field::kShift;
765 : static const int mask = Field::kMask >> Field::kShift;
766 : if (shift != 0) {
767 0 : shrq(reg, Immediate(shift));
768 : }
769 56 : andq(reg, Immediate(mask));
770 56 : }
771 :
772 : // Abort execution if argument is a smi, enabled via --debug-code.
773 : void AssertNotSmi(Register object);
774 :
775 : // Abort execution if argument is not a smi, enabled via --debug-code.
776 : void AssertSmi(Register object);
777 : void AssertSmi(Operand object);
778 :
779 : // Abort execution if argument is not a Constructor, enabled via --debug-code.
780 : void AssertConstructor(Register object);
781 :
782 : // Abort execution if argument is not a JSFunction, enabled via --debug-code.
783 : void AssertFunction(Register object);
784 :
785 : // Abort execution if argument is not a JSBoundFunction,
786 : // enabled via --debug-code.
787 : void AssertBoundFunction(Register object);
788 :
789 : // Abort execution if argument is not a JSGeneratorObject (or subclass),
790 : // enabled via --debug-code.
791 : void AssertGeneratorObject(Register object);
792 :
793 : // Abort execution if argument is not undefined or an AllocationSite, enabled
794 : // via --debug-code.
795 : void AssertUndefinedOrAllocationSite(Register object);
796 :
797 : // ---------------------------------------------------------------------------
798 : // Exception handling
799 :
800 : // Push a new stack handler and link it into stack handler chain.
801 : void PushStackHandler();
802 :
803 : // Unlink the stack handler on top of the stack from the stack handler chain.
804 : void PopStackHandler();
805 :
806 : // ---------------------------------------------------------------------------
807 : // Support functions.
808 :
809 : // Load the global proxy from the current context.
810 112 : void LoadGlobalProxy(Register dst) {
811 112 : LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
812 112 : }
813 :
814 : // Load the native context slot with the current index.
815 : void LoadNativeContextSlot(int index, Register dst);
816 :
817 : // ---------------------------------------------------------------------------
818 : // Runtime calls
819 :
820 : // Call a runtime routine.
821 : void CallRuntime(const Runtime::Function* f,
822 : int num_arguments,
823 : SaveFPRegsMode save_doubles = kDontSaveFPRegs);
824 :
825 : // Convenience function: Same as above, but takes the fid instead.
826 1568 : void CallRuntime(Runtime::FunctionId fid,
827 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
828 1568 : const Runtime::Function* function = Runtime::FunctionForId(fid);
829 1568 : CallRuntime(function, function->nargs, save_doubles);
830 1568 : }
831 :
832 : // Convenience function: Same as above, but takes the fid instead.
833 336 : void CallRuntime(Runtime::FunctionId fid, int num_arguments,
834 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
835 392 : CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
836 336 : }
837 :
838 : // Convenience function: tail call a runtime routine (jump)
839 : void TailCallRuntime(Runtime::FunctionId fid);
840 :
841 : // Jump to a runtime routines
842 : void JumpToExternalReference(const ExternalReference& ext,
843 : bool builtin_exit_frame = false);
844 :
845 : // ---------------------------------------------------------------------------
846 : // StatsCounter support
847 : void IncrementCounter(StatsCounter* counter, int value);
848 : void DecrementCounter(StatsCounter* counter, int value);
849 :
850 : // ---------------------------------------------------------------------------
851 : // In-place weak references.
852 : void LoadWeakValue(Register in_out, Label* target_if_cleared);
853 :
854 : // ---------------------------------------------------------------------------
855 : // Debugging
856 :
857 : static int SafepointRegisterStackIndex(Register reg) {
858 : return SafepointRegisterStackIndex(reg.code());
859 : }
860 :
861 : private:
862 : // Order general registers are pushed by Pushad.
863 : // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15.
864 : static const int kSafepointPushRegisterIndices[Register::kNumRegisters];
865 : static const int kNumSafepointSavedRegisters = 12;
866 :
867 : // Helper functions for generating invokes.
868 : void InvokePrologue(const ParameterCount& expected,
869 : const ParameterCount& actual, Label* done,
870 : bool* definitely_mismatches, InvokeFlag flag,
871 : Label::Distance near_jump);
872 :
873 : void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type);
874 :
875 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
876 : // stack accessible via StackSpaceOperand.
877 : void EnterExitFrameEpilogue(int arg_stack_space, bool save_doubles);
878 :
879 : void LeaveExitFrameEpilogue();
880 :
881 : // Compute memory operands for safepoint stack slots.
882 : static int SafepointRegisterStackIndex(int reg_code) {
883 0 : return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1;
884 : }
885 :
886 : // Needs access to SafepointRegisterStackIndex for compiled frame
887 : // traversal.
888 : friend class StandardFrame;
889 :
890 : DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
891 : };
892 :
893 : // -----------------------------------------------------------------------------
894 : // Static helper functions.
895 :
896 : // Generate an Operand for loading a field from an object.
897 5320 : inline Operand FieldOperand(Register object, int offset) {
898 567620 : return Operand(object, offset - kHeapObjectTag);
899 : }
900 :
901 :
902 : // Generate an Operand for loading an indexed field from an object.
903 336 : inline Operand FieldOperand(Register object,
904 : Register index,
905 : ScaleFactor scale,
906 : int offset) {
907 9095 : return Operand(object, index, scale, offset - kHeapObjectTag);
908 : }
909 :
910 :
911 : inline Operand ContextOperand(Register context, int index) {
912 672 : return Operand(context, Context::SlotOffset(index));
913 : }
914 :
915 :
916 : inline Operand ContextOperand(Register context, Register index) {
917 : return Operand(context, index, times_system_pointer_size,
918 : Context::SlotOffset(0));
919 : }
920 :
921 :
922 336 : inline Operand NativeContextOperand() {
923 336 : return ContextOperand(rsi, Context::NATIVE_CONTEXT_INDEX);
924 : }
925 :
926 :
927 : // Provides access to exit frame stack space (not GCed).
928 392 : inline Operand StackSpaceOperand(int index) {
929 : #ifdef _WIN64
930 : const int kShaddowSpace = 4;
931 : return Operand(rsp, (index + kShaddowSpace) * kSystemPointerSize);
932 : #else
933 392 : return Operand(rsp, index * kSystemPointerSize);
934 : #endif
935 : }
936 :
937 :
938 56 : inline Operand StackOperandForReturnAddress(int32_t disp) {
939 56 : return Operand(rsp, disp);
940 : }
941 :
942 : #define ACCESS_MASM(masm) masm->
943 :
944 : } // namespace internal
945 : } // namespace v8
946 :
947 : #endif // V8_X64_MACRO_ASSEMBLER_X64_H_
|
