Line data Source code
1 : // Copyright 2012 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
6 : #error This header must be included via macro-assembler.h
7 : #endif
8 :
9 : #ifndef V8_X64_MACRO_ASSEMBLER_X64_H_
10 : #define V8_X64_MACRO_ASSEMBLER_X64_H_
11 :
12 : #include "src/bailout-reason.h"
13 : #include "src/base/flags.h"
14 : #include "src/contexts.h"
15 : #include "src/globals.h"
16 : #include "src/x64/assembler-x64.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 :
21 : // Convenience for platform-independent signatures.
22 : typedef Operand MemOperand;
23 :
24 : class StringConstantBase;
25 :
26 : enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
27 : enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
28 :
29 : struct SmiIndex {
30 : SmiIndex(Register index_register, ScaleFactor scale)
31 : : reg(index_register),
32 : scale(scale) {}
33 : Register reg;
34 : ScaleFactor scale;
35 : };
36 :
37 : enum StackArgumentsAccessorReceiverMode {
38 : ARGUMENTS_CONTAIN_RECEIVER,
39 : ARGUMENTS_DONT_CONTAIN_RECEIVER
40 : };
41 :
42 : class StackArgumentsAccessor {
43 : public:
44 : StackArgumentsAccessor(Register base_reg, int argument_count_immediate,
45 : StackArgumentsAccessorReceiverMode receiver_mode =
46 : ARGUMENTS_CONTAIN_RECEIVER,
47 : int extra_displacement_to_last_argument = 0)
48 : : base_reg_(base_reg),
49 : argument_count_reg_(no_reg),
50 : argument_count_immediate_(argument_count_immediate),
51 : receiver_mode_(receiver_mode),
52 : extra_displacement_to_last_argument_(
53 : extra_displacement_to_last_argument) {}
54 :
55 : StackArgumentsAccessor(Register base_reg, Register argument_count_reg,
56 : StackArgumentsAccessorReceiverMode receiver_mode =
57 : ARGUMENTS_CONTAIN_RECEIVER,
58 : int extra_displacement_to_last_argument = 0)
59 : : base_reg_(base_reg),
60 : argument_count_reg_(argument_count_reg),
61 : argument_count_immediate_(0),
62 : receiver_mode_(receiver_mode),
63 : extra_displacement_to_last_argument_(
64 952 : extra_displacement_to_last_argument) {}
65 :
66 : StackArgumentsAccessor(Register base_reg,
67 : const ParameterCount& parameter_count,
68 : StackArgumentsAccessorReceiverMode receiver_mode =
69 : ARGUMENTS_CONTAIN_RECEIVER,
70 : int extra_displacement_to_last_argument = 0);
71 :
72 : Operand GetArgumentOperand(int index);
73 : Operand GetReceiverOperand() {
74 : DCHECK(receiver_mode_ == ARGUMENTS_CONTAIN_RECEIVER);
75 1008 : return GetArgumentOperand(0);
76 : }
77 :
78 : private:
79 : const Register base_reg_;
80 : const Register argument_count_reg_;
81 : const int argument_count_immediate_;
82 : const StackArgumentsAccessorReceiverMode receiver_mode_;
83 : const int extra_displacement_to_last_argument_;
84 :
85 : DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
86 : };
87 :
88 81736760 : class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
89 : public:
90 122598337 : using TurboAssemblerBase::TurboAssemblerBase;
91 :
92 : template <typename Dst, typename... Args>
93 : struct AvxHelper {
94 : Assembler* assm;
95 : // Call an method where the AVX version expects the dst argument to be
96 : // duplicated.
97 : template <void (Assembler::*avx)(Dst, Dst, Args...),
98 : void (Assembler::*no_avx)(Dst, Args...)>
99 1092924 : void emit(Dst dst, Args... args) {
100 1092924 : if (CpuFeatures::IsSupported(AVX)) {
101 : CpuFeatureScope scope(assm, AVX);
102 1091354 : (assm->*avx)(dst, dst, args...);
103 : } else {
104 1570 : (assm->*no_avx)(dst, args...);
105 : }
106 1092941 : }
107 :
108 : // Call an method where the AVX version expects no duplicated dst argument.
109 : template <void (Assembler::*avx)(Dst, Args...),
110 : void (Assembler::*no_avx)(Dst, Args...)>
111 4200438 : void emit(Dst dst, Args... args) {
112 4200438 : if (CpuFeatures::IsSupported(AVX)) {
113 : CpuFeatureScope scope(assm, AVX);
114 4190772 : (assm->*avx)(dst, args...);
115 : } else {
116 9666 : (assm->*no_avx)(dst, args...);
117 : }
118 4200461 : }
119 : };
120 :
121 : #define AVX_OP(macro_name, name) \
122 : template <typename Dst, typename... Args> \
123 : void macro_name(Dst dst, Args... args) { \
124 : AvxHelper<Dst, Args...>{this} \
125 : .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
126 : }
127 :
128 6465 : AVX_OP(Subsd, subsd)
129 166 : AVX_OP(Divss, divss)
130 629 : AVX_OP(Divsd, divsd)
131 11430 : AVX_OP(Xorps, xorps)
132 112907 : AVX_OP(Xorpd, xorpd)
133 159355 : AVX_OP(Movd, movd)
134 198575 : AVX_OP(Movq, movq)
135 385 : AVX_OP(Movaps, movaps)
136 157024 : AVX_OP(Movapd, movapd)
137 11426 : AVX_OP(Movups, movups)
138 166 : AVX_OP(Movmskps, movmskps)
139 629 : AVX_OP(Movmskpd, movmskpd)
140 736369 : AVX_OP(Movss, movss)
141 2970240 : AVX_OP(Movsd, movsd)
142 301266 : AVX_OP(Pcmpeqd, pcmpeqd)
143 64010 : AVX_OP(Pslld, pslld)
144 223665 : AVX_OP(Psllq, psllq)
145 48178 : AVX_OP(Psrld, psrld)
146 199155 : AVX_OP(Psrlq, psrlq)
147 : AVX_OP(Addsd, addsd)
148 : AVX_OP(Mulsd, mulsd)
149 32 : AVX_OP(Andps, andps)
150 : AVX_OP(Andnps, andnps)
151 22 : AVX_OP(Andpd, andpd)
152 : AVX_OP(Orpd, orpd)
153 : AVX_OP(Cmpeqps, cmpeqps)
154 : AVX_OP(Cmpltps, cmpltps)
155 : AVX_OP(Cmpleps, cmpleps)
156 : AVX_OP(Cmpneqps, cmpneqps)
157 : AVX_OP(Cmpnltps, cmpnltps)
158 : AVX_OP(Cmpnleps, cmpnleps)
159 : AVX_OP(Cmpeqpd, cmpeqpd)
160 : AVX_OP(Cmpltpd, cmpltpd)
161 : AVX_OP(Cmplepd, cmplepd)
162 : AVX_OP(Cmpneqpd, cmpneqpd)
163 : AVX_OP(Cmpnltpd, cmpnltpd)
164 : AVX_OP(Cmpnlepd, cmpnlepd)
165 481 : AVX_OP(Roundss, roundss)
166 45054 : AVX_OP(Roundsd, roundsd)
167 62 : AVX_OP(Sqrtss, sqrtss)
168 468 : AVX_OP(Sqrtsd, sqrtsd)
169 638 : AVX_OP(Ucomiss, ucomiss)
170 4141 : AVX_OP(Ucomisd, ucomisd)
171 :
172 : #undef AVX_OP
173 :
174 1740 : void PushReturnAddressFrom(Register src) { pushq(src); }
175 1460 : void PopReturnAddressTo(Register dst) { popq(dst); }
176 :
177 : void Ret();
178 :
179 : // Return and drop arguments from stack, where the number of arguments
180 : // may be bigger than 2^16 - 1. Requires a scratch register.
181 : void Ret(int bytes_dropped, Register scratch);
182 :
183 : // Load a register with a long value as efficiently as possible.
184 : void Set(Register dst, int64_t x);
185 : void Set(Operand dst, intptr_t x);
186 :
187 : // Operations on roots in the root-array.
188 : void LoadRoot(Register destination, RootIndex index) override;
189 : void LoadRoot(Operand destination, RootIndex index) {
190 : LoadRoot(kScratchRegister, index);
191 : movq(destination, kScratchRegister);
192 : }
193 :
194 : void Push(Register src);
195 : void Push(Operand src);
196 : void Push(Immediate value);
197 : void Push(Smi smi);
198 : void Push(Handle<HeapObject> source);
199 :
200 : // Before calling a C-function from generated code, align arguments on stack.
201 : // After aligning the frame, arguments must be stored in rsp[0], rsp[8],
202 : // etc., not pushed. The argument count assumes all arguments are word sized.
203 : // The number of slots reserved for arguments depends on platform. On Windows
204 : // stack slots are reserved for the arguments passed in registers. On other
205 : // platforms stack slots are only reserved for the arguments actually passed
206 : // on the stack.
207 : void PrepareCallCFunction(int num_arguments);
208 :
209 : // Calls a C function and cleans up the space for arguments allocated
210 : // by PrepareCallCFunction. The called function is not allowed to trigger a
211 : // garbage collection, since that might move the code and invalidate the
212 : // return address (unless this is somehow accounted for by the called
213 : // function).
214 : void CallCFunction(ExternalReference function, int num_arguments);
215 : void CallCFunction(Register function, int num_arguments);
216 :
217 : // Calculate the number of stack slots to reserve for arguments when calling a
218 : // C function.
219 : int ArgumentStackSlotsForCFunctionCall(int num_arguments);
220 :
221 : void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
222 : Label* condition_met,
223 : Label::Distance condition_met_distance = Label::kFar);
224 :
225 : void Cvtss2sd(XMMRegister dst, XMMRegister src);
226 : void Cvtss2sd(XMMRegister dst, Operand src);
227 : void Cvtsd2ss(XMMRegister dst, XMMRegister src);
228 : void Cvtsd2ss(XMMRegister dst, Operand src);
229 : void Cvttsd2si(Register dst, XMMRegister src);
230 : void Cvttsd2si(Register dst, Operand src);
231 : void Cvttsd2siq(Register dst, XMMRegister src);
232 : void Cvttsd2siq(Register dst, Operand src);
233 : void Cvttss2si(Register dst, XMMRegister src);
234 : void Cvttss2si(Register dst, Operand src);
235 : void Cvttss2siq(Register dst, XMMRegister src);
236 : void Cvttss2siq(Register dst, Operand src);
237 : void Cvtqsi2ss(XMMRegister dst, Register src);
238 : void Cvtqsi2ss(XMMRegister dst, Operand src);
239 : void Cvtqsi2sd(XMMRegister dst, Register src);
240 : void Cvtqsi2sd(XMMRegister dst, Operand src);
241 : void Cvtlsi2ss(XMMRegister dst, Register src);
242 : void Cvtlsi2ss(XMMRegister dst, Operand src);
243 : void Cvtlui2ss(XMMRegister dst, Register src);
244 : void Cvtlui2ss(XMMRegister dst, Operand src);
245 : void Cvtlui2sd(XMMRegister dst, Register src);
246 : void Cvtlui2sd(XMMRegister dst, Operand src);
247 : void Cvtqui2ss(XMMRegister dst, Register src);
248 : void Cvtqui2ss(XMMRegister dst, Operand src);
249 : void Cvtqui2sd(XMMRegister dst, Register src);
250 : void Cvtqui2sd(XMMRegister dst, Operand src);
251 : void Cvttsd2uiq(Register dst, Operand src, Label* fail = nullptr);
252 : void Cvttsd2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
253 : void Cvttss2uiq(Register dst, Operand src, Label* fail = nullptr);
254 : void Cvttss2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
255 :
256 : // cvtsi2sd instruction only writes to the low 64-bit of dst register, which
257 : // hinders register renaming and makes dependence chains longer. So we use
258 : // xorpd to clear the dst register before cvtsi2sd to solve this issue.
259 : void Cvtlsi2sd(XMMRegister dst, Register src);
260 : void Cvtlsi2sd(XMMRegister dst, Operand src);
261 :
262 : void Lzcntq(Register dst, Register src);
263 : void Lzcntq(Register dst, Operand src);
264 : void Lzcntl(Register dst, Register src);
265 : void Lzcntl(Register dst, Operand src);
266 : void Tzcntq(Register dst, Register src);
267 : void Tzcntq(Register dst, Operand src);
268 : void Tzcntl(Register dst, Register src);
269 : void Tzcntl(Register dst, Operand src);
270 : void Popcntl(Register dst, Register src);
271 : void Popcntl(Register dst, Operand src);
272 : void Popcntq(Register dst, Register src);
273 : void Popcntq(Register dst, Operand src);
274 :
275 : // Is the value a tagged smi.
276 : Condition CheckSmi(Register src);
277 : Condition CheckSmi(Operand src);
278 :
279 : // Jump to label if the value is a tagged smi.
280 : void JumpIfSmi(Register src, Label* on_smi,
281 : Label::Distance near_jump = Label::kFar);
282 :
283 192496 : void JumpIfEqual(Register a, int32_t b, Label* dest) {
284 192496 : cmpl(a, Immediate(b));
285 192496 : j(equal, dest);
286 192495 : }
287 :
288 41712 : void JumpIfLessThan(Register a, int32_t b, Label* dest) {
289 41712 : cmpl(a, Immediate(b));
290 41712 : j(less, dest);
291 41712 : }
292 :
293 : void Move(Register dst, Smi source);
294 :
295 : void Move(Operand dst, Smi source) {
296 : Register constant = GetSmiConstant(source);
297 : movq(dst, constant);
298 : }
299 :
300 : void Move(Register dst, ExternalReference ext);
301 :
302 : void Move(XMMRegister dst, uint32_t src);
303 : void Move(XMMRegister dst, uint64_t src);
304 196 : void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
305 4052 : void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
306 :
307 : // Move if the registers are not identical.
308 : void Move(Register target, Register source);
309 :
310 : void Move(Register dst, Handle<HeapObject> source,
311 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
312 : void Move(Operand dst, Handle<HeapObject> source,
313 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
314 :
315 : // Loads a pointer into a register with a relocation mode.
316 : void Move(Register dst, Address ptr, RelocInfo::Mode rmode) {
317 : // This method must not be used with heap object references. The stored
318 : // address is not GC safe. Use the handle version instead.
319 : DCHECK(rmode > RelocInfo::LAST_GCED_ENUM);
320 41939723 : movq(dst, Immediate64(ptr, rmode));
321 : }
322 :
323 : // Move src0 to dst0 and src1 to dst1, handling possible overlaps.
324 : void MovePair(Register dst0, Register src0, Register dst1, Register src1);
325 :
326 : void MoveStringConstant(Register result, const StringConstantBase* string,
327 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
328 :
329 : // Convert smi to word-size sign-extended value.
330 : void SmiUntag(Register dst, Register src);
331 : void SmiUntag(Register dst, Operand src);
332 :
333 : // Loads the address of the external reference into the destination
334 : // register.
335 : void LoadAddress(Register destination, ExternalReference source);
336 :
337 : void LoadFromConstantsTable(Register destination,
338 : int constant_index) override;
339 : void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
340 : void LoadRootRelative(Register destination, int32_t offset) override;
341 :
342 : // Operand pointing to an external reference.
343 : // May emit code to set up the scratch register. The operand is
344 : // only guaranteed to be correct as long as the scratch register
345 : // isn't changed.
346 : // If the operand is used more than once, use a scratch register
347 : // that is guaranteed not to be clobbered.
348 : Operand ExternalReferenceAsOperand(ExternalReference reference,
349 : Register scratch = kScratchRegister);
350 :
351 85232 : void Call(Register reg) { call(reg); }
352 : void Call(Operand op);
353 : void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
354 : void Call(Address destination, RelocInfo::Mode rmode);
355 : void Call(ExternalReference ext);
356 : void Call(Label* target) { call(target); }
357 :
358 : void CallBuiltinPointer(Register builtin_pointer) override;
359 :
360 : void LoadCodeObjectEntry(Register destination, Register code_object) override;
361 : void CallCodeObject(Register code_object) override;
362 : void JumpCodeObject(Register code_object) override;
363 :
364 : void RetpolineCall(Register reg);
365 : void RetpolineCall(Address destination, RelocInfo::Mode rmode);
366 :
367 : void Jump(Address destination, RelocInfo::Mode rmode);
368 : void Jump(ExternalReference ext);
369 : void Jump(Operand op);
370 : void Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
371 : Condition cc = always);
372 :
373 : void RetpolineJump(Register reg);
374 :
375 : void CallForDeoptimization(Address target, int deopt_id);
376 :
377 : // Non-SSE2 instructions.
378 : void Pextrd(Register dst, XMMRegister src, int8_t imm8);
379 : void Pinsrd(XMMRegister dst, Register src, int8_t imm8);
380 : void Pinsrd(XMMRegister dst, Operand src, int8_t imm8);
381 :
382 : void CompareRoot(Register with, RootIndex index);
383 : void CompareRoot(Operand with, RootIndex index);
384 :
385 : // Generates function and stub prologue code.
386 : void StubPrologue(StackFrame::Type type);
387 : void Prologue();
388 :
389 : // Calls Abort(msg) if the condition cc is not satisfied.
390 : // Use --debug_code to enable.
391 : void Assert(Condition cc, AbortReason reason);
392 :
393 : // Like Assert(), but without condition.
394 : // Use --debug_code to enable.
395 : void AssertUnreachable(AbortReason reason);
396 :
397 : // Abort execution if a 64 bit register containing a 32 bit payload does not
398 : // have zeros in the top 32 bits, enabled via --debug-code.
399 : void AssertZeroExtended(Register reg);
400 :
401 : // Like Assert(), but always enabled.
402 : void Check(Condition cc, AbortReason reason);
403 :
404 : // Print a message to stdout and abort execution.
405 : void Abort(AbortReason msg);
406 :
407 : // Check that the stack is aligned.
408 : void CheckStackAlignment();
409 :
410 : // Activation support.
411 : void EnterFrame(StackFrame::Type type);
412 : void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
413 : // Out-of-line constant pool not implemented on x64.
414 : UNREACHABLE();
415 : }
416 : void LeaveFrame(StackFrame::Type type);
417 :
418 : // Removes current frame and its arguments from the stack preserving the
419 : // arguments and a return address pushed to the stack for the next call. Both
420 : // |callee_args_count| and |caller_args_count_reg| do not include receiver.
421 : // |callee_args_count| is not modified, |caller_args_count_reg| is trashed.
422 : void PrepareForTailCall(const ParameterCount& callee_args_count,
423 : Register caller_args_count_reg, Register scratch0,
424 : Register scratch1);
425 :
426 : // Call a runtime routine. This expects {centry} to contain a fitting CEntry
427 : // builtin for the target runtime function and uses an indirect call.
428 : void CallRuntimeWithCEntry(Runtime::FunctionId fid, Register centry);
429 :
430 888591 : void InitializeRootRegister() {
431 888591 : ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
432 888591 : Move(kRootRegister, isolate_root);
433 888591 : }
434 :
435 : void SaveRegisters(RegList registers);
436 : void RestoreRegisters(RegList registers);
437 :
438 : void CallRecordWriteStub(Register object, Register address,
439 : RememberedSetAction remembered_set_action,
440 : SaveFPRegsMode fp_mode);
441 : void CallRecordWriteStub(Register object, Register address,
442 : RememberedSetAction remembered_set_action,
443 : SaveFPRegsMode fp_mode, Address wasm_target);
444 : void CallEphemeronKeyBarrier(Register object, Register address,
445 : SaveFPRegsMode fp_mode);
446 :
447 : void MoveNumber(Register dst, double value);
448 : void MoveNonSmi(Register dst, double value);
449 :
450 : // Calculate how much stack space (in bytes) are required to store caller
451 : // registers excluding those specified in the arguments.
452 : int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
453 : Register exclusion1 = no_reg,
454 : Register exclusion2 = no_reg,
455 : Register exclusion3 = no_reg) const;
456 :
457 : // PushCallerSaved and PopCallerSaved do not arrange the registers in any
458 : // particular order so they are not useful for calls that can cause a GC.
459 : // The caller can exclude up to 3 registers that do not need to be saved and
460 : // restored.
461 :
462 : // Push caller saved registers on the stack, and return the number of bytes
463 : // stack pointer is adjusted.
464 : int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
465 : Register exclusion2 = no_reg,
466 : Register exclusion3 = no_reg);
467 : // Restore caller saved registers from the stack, and return the number of
468 : // bytes stack pointer is adjusted.
469 : int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
470 : Register exclusion2 = no_reg,
471 : Register exclusion3 = no_reg);
472 :
473 : // Compute the start of the generated instruction stream from the current PC.
474 : // This is an alternative to embedding the {CodeObject} handle as a reference.
475 : void ComputeCodeStartAddress(Register dst);
476 :
477 : void ResetSpeculationPoisonRegister();
478 :
479 : // ---------------------------------------------------------------------------
480 : // Pointer compression support
481 :
482 : // Loads a field containing a HeapObject and decompresses it if pointer
483 : // compression is enabled.
484 : void LoadTaggedPointerField(Register destination, Operand field_operand);
485 :
486 : // Loads a field containing any tagged value and decompresses it if necessary.
487 : // When pointer compression is enabled, uses |scratch| to decompress the
488 : // value.
489 : void LoadAnyTaggedField(Register destination, Operand field_operand,
490 : Register scratch = kScratchRegister);
491 :
492 : // Loads a field containing a HeapObject, decompresses it if necessary and
493 : // pushes full pointer to the stack. When pointer compression is enabled,
494 : // uses |scratch| to decompress the value.
495 : void PushTaggedPointerField(Operand field_operand, Register scratch);
496 :
497 : // Loads a field containing any tagged value, decompresses it if necessary and
498 : // pushes the full pointer to the stack. When pointer compression is enabled,
499 : // uses |scratch1| and |scratch2| to decompress the value.
500 : void PushTaggedAnyField(Operand field_operand, Register scratch1,
501 : Register scratch2);
502 :
503 : // Loads a field containing smi value and untags it.
504 : void SmiUntagField(Register dst, Operand src);
505 :
506 : // Compresses tagged value if necessary and stores it to given on-heap
507 : // location.
508 : void StoreTaggedField(Operand dst_field_operand, Immediate immediate);
509 : void StoreTaggedField(Operand dst_field_operand, Register value);
510 :
511 : // The following macros work even when pointer compression is not enabled.
512 : void DecompressTaggedSigned(Register destination, Operand field_operand);
513 : void DecompressTaggedPointer(Register destination, Register source);
514 : void DecompressTaggedPointer(Register destination, Operand field_operand);
515 : void DecompressAnyTagged(Register destination, Operand field_operand,
516 : Register scratch = kScratchRegister);
517 :
518 : protected:
519 : static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
520 : int smi_count = 0;
521 : int heap_object_count = 0;
522 :
523 : // Returns a register holding the smi value. The register MUST NOT be
524 : // modified. It may be the "smi 1 constant" register.
525 : Register GetSmiConstant(Smi value);
526 :
527 : void CallRecordWriteStub(Register object, Register address,
528 : RememberedSetAction remembered_set_action,
529 : SaveFPRegsMode fp_mode, Handle<Code> code_target,
530 : Address wasm_target);
531 : };
532 :
533 : // MacroAssembler implements a collection of frequently used macros.
534 75559265 : class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
535 : public:
536 113338657 : using TurboAssembler::TurboAssembler;
537 :
538 : // Loads and stores the value of an external reference.
539 : // Special case code for load and store to take advantage of
540 : // load_rax/store_rax if possible/necessary.
541 : // For other operations, just use:
542 : // Operand operand = ExternalReferenceAsOperand(extref);
543 : // operation(operand, ..);
544 : void Load(Register destination, ExternalReference source);
545 : void Store(ExternalReference destination, Register source);
546 :
547 : // Pushes the address of the external reference onto the stack.
548 : void PushAddress(ExternalReference source);
549 :
550 : // Operations on roots in the root-array.
551 : // Load a root value where the index (or part of it) is variable.
552 : // The variable_offset register is added to the fixed_offset value
553 : // to get the index into the root-array.
554 : void PushRoot(RootIndex index);
555 :
556 : // Compare the object in a register to a value and jump if they are equal.
557 : void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
558 : Label::Distance if_equal_distance = Label::kFar) {
559 280 : CompareRoot(with, index);
560 280 : j(equal, if_equal, if_equal_distance);
561 : }
562 : void JumpIfRoot(Operand with, RootIndex index, Label* if_equal,
563 : Label::Distance if_equal_distance = Label::kFar) {
564 : CompareRoot(with, index);
565 : j(equal, if_equal, if_equal_distance);
566 : }
567 :
568 : // Compare the object in a register to a value and jump if they are not equal.
569 : void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
570 : Label::Distance if_not_equal_distance = Label::kFar) {
571 56 : CompareRoot(with, index);
572 56 : j(not_equal, if_not_equal, if_not_equal_distance);
573 : }
574 : void JumpIfNotRoot(Operand with, RootIndex index, Label* if_not_equal,
575 : Label::Distance if_not_equal_distance = Label::kFar) {
576 : CompareRoot(with, index);
577 : j(not_equal, if_not_equal, if_not_equal_distance);
578 : }
579 :
580 : // ---------------------------------------------------------------------------
581 : // GC Support
582 :
583 : // Notify the garbage collector that we wrote a pointer into an object.
584 : // |object| is the object being stored into, |value| is the object being
585 : // stored. value and scratch registers are clobbered by the operation.
586 : // The offset is the offset from the start of the object, not the offset from
587 : // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
588 : void RecordWriteField(
589 : Register object, int offset, Register value, Register scratch,
590 : SaveFPRegsMode save_fp,
591 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
592 : SmiCheck smi_check = INLINE_SMI_CHECK);
593 :
594 : // For page containing |object| mark region covering |address|
595 : // dirty. |object| is the object being stored into, |value| is the
596 : // object being stored. The address and value registers are clobbered by the
597 : // operation. RecordWrite filters out smis so it does not update
598 : // the write barrier if the value is a smi.
599 : void RecordWrite(
600 : Register object, Register address, Register value, SaveFPRegsMode save_fp,
601 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
602 : SmiCheck smi_check = INLINE_SMI_CHECK);
603 :
604 : // Frame restart support.
605 : void MaybeDropFrames();
606 :
607 : // Enter specific kind of exit frame; either in normal or
608 : // debug mode. Expects the number of arguments in register rax and
609 : // sets up the number of arguments in register rdi and the pointer
610 : // to the first argument in register rsi.
611 : //
612 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
613 : // stack accessible via StackSpaceOperand.
614 : void EnterExitFrame(int arg_stack_space = 0, bool save_doubles = false,
615 : StackFrame::Type frame_type = StackFrame::EXIT);
616 :
617 : // Enter specific kind of exit frame. Allocates
618 : // (arg_stack_space * kSystemPointerSize) memory (not GCed) on the stack
619 : // accessible via StackSpaceOperand.
620 : void EnterApiExitFrame(int arg_stack_space);
621 :
622 : // Leave the current exit frame. Expects/provides the return value in
623 : // register rax:rdx (untouched) and the pointer to the first
624 : // argument in register rsi (if pop_arguments == true).
625 : void LeaveExitFrame(bool save_doubles = false, bool pop_arguments = true);
626 :
627 : // Leave the current exit frame. Expects/provides the return value in
628 : // register rax (untouched).
629 : void LeaveApiExitFrame();
630 :
631 : // Push and pop the registers that can hold pointers.
632 0 : void PushSafepointRegisters() { Pushad(); }
633 0 : void PopSafepointRegisters() { Popad(); }
634 :
635 : // ---------------------------------------------------------------------------
636 : // JavaScript invokes
637 :
638 : // Invoke the JavaScript function code by either calling or jumping.
639 : void InvokeFunctionCode(Register function, Register new_target,
640 : const ParameterCount& expected,
641 : const ParameterCount& actual, InvokeFlag flag);
642 :
643 : // On function call, call into the debugger if necessary.
644 : void CheckDebugHook(Register fun, Register new_target,
645 : const ParameterCount& expected,
646 : const ParameterCount& actual);
647 :
648 : // Invoke the JavaScript function in the given register. Changes the
649 : // current context to the context in the function before invoking.
650 : void InvokeFunction(Register function, Register new_target,
651 : const ParameterCount& actual, InvokeFlag flag);
652 :
653 : void InvokeFunction(Register function, Register new_target,
654 : const ParameterCount& expected,
655 : const ParameterCount& actual, InvokeFlag flag);
656 :
657 : // ---------------------------------------------------------------------------
658 : // Conversions between tagged smi values and non-tagged integer values.
659 :
660 : // Tag an word-size value. The result must be known to be a valid smi value.
661 : void SmiTag(Register dst, Register src);
662 :
663 : // Simple comparison of smis. Both sides must be known smis to use these,
664 : // otherwise use Cmp.
665 : void SmiCompare(Register smi1, Register smi2);
666 : void SmiCompare(Register dst, Smi src);
667 : void SmiCompare(Register dst, Operand src);
668 : void SmiCompare(Operand dst, Register src);
669 : void SmiCompare(Operand dst, Smi src);
670 :
671 : // Functions performing a check on a known or potential smi. Returns
672 : // a condition that is satisfied if the check is successful.
673 :
674 : // Test-and-jump functions. Typically combines a check function
675 : // above with a conditional jump.
676 :
677 : // Jump to label if the value is not a tagged smi.
678 : void JumpIfNotSmi(Register src,
679 : Label* on_not_smi,
680 : Label::Distance near_jump = Label::kFar);
681 :
682 : // Jump to label if the value is not a tagged smi.
683 : void JumpIfNotSmi(Operand src, Label* on_not_smi,
684 : Label::Distance near_jump = Label::kFar);
685 :
686 : // Operations on tagged smi values.
687 :
688 : // Smis represent a subset of integers. The subset is always equivalent to
689 : // a two's complement interpretation of a fixed number of bits.
690 :
691 : // Add an integer constant to a tagged smi, giving a tagged smi as result.
692 : // No overflow testing on the result is done.
693 : void SmiAddConstant(Operand dst, Smi constant);
694 :
695 : // Specialized operations
696 :
697 : // Converts, if necessary, a smi to a combination of number and
698 : // multiplier to be used as a scaled index.
699 : // The src register contains a *positive* smi value. The shift is the
700 : // power of two to multiply the index value by (e.g. to index by
701 : // smi-value * kSystemPointerSize, pass the smi and kSystemPointerSizeLog2).
702 : // The returned index register may be either src or dst, depending
703 : // on what is most efficient. If src and dst are different registers,
704 : // src is always unchanged.
705 : SmiIndex SmiToIndex(Register dst, Register src, int shift);
706 :
707 : // ---------------------------------------------------------------------------
708 : // Macro instructions.
709 :
710 : void Cmp(Register dst, Handle<Object> source);
711 : void Cmp(Operand dst, Handle<Object> source);
712 : void Cmp(Register dst, Smi src);
713 : void Cmp(Operand dst, Smi src);
714 :
715 : // Checks if value is in range [lower_limit, higher_limit] using a single
716 : // comparison.
717 : void JumpIfIsInRange(Register value, unsigned lower_limit,
718 : unsigned higher_limit, Label* on_in_range,
719 : Label::Distance near_jump = Label::kFar);
720 :
721 : // Emit code to discard a non-negative number of pointer-sized elements
722 : // from the stack, clobbering only the rsp register.
723 : void Drop(int stack_elements);
724 : // Emit code to discard a positive number of pointer-sized elements
725 : // from the stack under the return address which remains on the top,
726 : // clobbering the rsp register.
727 : void DropUnderReturnAddress(int stack_elements,
728 : Register scratch = kScratchRegister);
729 :
730 : void PushQuad(Operand src);
731 : void PushImm32(int32_t imm32);
732 : void Pop(Register dst);
733 : void Pop(Operand dst);
734 : void PopQuad(Operand dst);
735 :
736 : // ---------------------------------------------------------------------------
737 : // SIMD macros.
738 : void Absps(XMMRegister dst);
739 : void Negps(XMMRegister dst);
740 : void Abspd(XMMRegister dst);
741 : void Negpd(XMMRegister dst);
742 : // Generates a trampoline to jump to the off-heap instruction stream.
743 : void JumpToInstructionStream(Address entry);
744 :
745 : // Non-x64 instructions.
746 : // Push/pop all general purpose registers.
747 : // Does not push rsp/rbp nor any of the assembler's special purpose registers
748 : // (kScratchRegister, kRootRegister).
749 : void Pushad();
750 : void Popad();
751 :
752 : // Compare object type for heap object.
753 : // Always use unsigned comparisons: above and below, not less and greater.
754 : // Incoming register is heap_object and outgoing register is map.
755 : // They may be the same register, and may be kScratchRegister.
756 : void CmpObjectType(Register heap_object, InstanceType type, Register map);
757 :
758 : // Compare instance type for map.
759 : // Always use unsigned comparisons: above and below, not less and greater.
760 : void CmpInstanceType(Register map, InstanceType type);
761 :
762 : void DoubleToI(Register result_reg, XMMRegister input_reg,
763 : XMMRegister scratch, Label* lost_precision, Label* is_nan,
764 : Label::Distance dst = Label::kFar);
765 :
766 : template<typename Field>
767 0 : void DecodeField(Register reg) {
768 : static const int shift = Field::kShift;
769 : static const int mask = Field::kMask >> Field::kShift;
770 : if (shift != 0) {
771 0 : shrq(reg, Immediate(shift));
772 : }
773 : andq(reg, Immediate(mask));
774 0 : }
775 :
776 : // Abort execution if argument is a smi, enabled via --debug-code.
777 : void AssertNotSmi(Register object);
778 :
779 : // Abort execution if argument is not a smi, enabled via --debug-code.
780 : void AssertSmi(Register object);
781 : void AssertSmi(Operand object);
782 :
783 : // Abort execution if argument is not a Constructor, enabled via --debug-code.
784 : void AssertConstructor(Register object);
785 :
786 : // Abort execution if argument is not a JSFunction, enabled via --debug-code.
787 : void AssertFunction(Register object);
788 :
789 : // Abort execution if argument is not a JSBoundFunction,
790 : // enabled via --debug-code.
791 : void AssertBoundFunction(Register object);
792 :
793 : // Abort execution if argument is not a JSGeneratorObject (or subclass),
794 : // enabled via --debug-code.
795 : void AssertGeneratorObject(Register object);
796 :
797 : // Abort execution if argument is not undefined or an AllocationSite, enabled
798 : // via --debug-code.
799 : void AssertUndefinedOrAllocationSite(Register object);
800 :
801 : // ---------------------------------------------------------------------------
802 : // Exception handling
803 :
804 : // Push a new stack handler and link it into stack handler chain.
805 : void PushStackHandler();
806 :
807 : // Unlink the stack handler on top of the stack from the stack handler chain.
808 : void PopStackHandler();
809 :
810 : // ---------------------------------------------------------------------------
811 : // Support functions.
812 :
813 : // Load the global proxy from the current context.
814 : void LoadGlobalProxy(Register dst) {
815 112 : LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
816 : }
817 :
818 : // Load the native context slot with the current index.
819 : void LoadNativeContextSlot(int index, Register dst);
820 :
821 : // ---------------------------------------------------------------------------
822 : // Runtime calls
823 :
824 : // Call a runtime routine.
825 : void CallRuntime(const Runtime::Function* f,
826 : int num_arguments,
827 : SaveFPRegsMode save_doubles = kDontSaveFPRegs);
828 :
829 : // Convenience function: Same as above, but takes the fid instead.
830 1568 : void CallRuntime(Runtime::FunctionId fid,
831 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
832 1568 : const Runtime::Function* function = Runtime::FunctionForId(fid);
833 1568 : CallRuntime(function, function->nargs, save_doubles);
834 1568 : }
835 :
836 : // Convenience function: Same as above, but takes the fid instead.
837 : void CallRuntime(Runtime::FunctionId fid, int num_arguments,
838 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
839 392 : CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
840 : }
841 :
842 : // Convenience function: tail call a runtime routine (jump)
843 : void TailCallRuntime(Runtime::FunctionId fid);
844 :
845 : // Jump to a runtime routines
846 : void JumpToExternalReference(const ExternalReference& ext,
847 : bool builtin_exit_frame = false);
848 :
849 : // ---------------------------------------------------------------------------
850 : // StatsCounter support
851 : void IncrementCounter(StatsCounter* counter, int value);
852 : void DecrementCounter(StatsCounter* counter, int value);
853 :
854 : // ---------------------------------------------------------------------------
855 : // In-place weak references.
856 : void LoadWeakValue(Register in_out, Label* target_if_cleared);
857 :
858 : // ---------------------------------------------------------------------------
859 : // Debugging
860 :
861 : static int SafepointRegisterStackIndex(Register reg) {
862 : return SafepointRegisterStackIndex(reg.code());
863 : }
864 :
865 : private:
866 : // Order general registers are pushed by Pushad.
867 : // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15.
868 : static const int kSafepointPushRegisterIndices[Register::kNumRegisters];
869 : static const int kNumSafepointSavedRegisters = 12;
870 :
871 : // Helper functions for generating invokes.
872 : void InvokePrologue(const ParameterCount& expected,
873 : const ParameterCount& actual, Label* done,
874 : bool* definitely_mismatches, InvokeFlag flag,
875 : Label::Distance near_jump);
876 :
877 : void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type);
878 :
879 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
880 : // stack accessible via StackSpaceOperand.
881 : void EnterExitFrameEpilogue(int arg_stack_space, bool save_doubles);
882 :
883 : void LeaveExitFrameEpilogue();
884 :
885 : // Compute memory operands for safepoint stack slots.
886 : static int SafepointRegisterStackIndex(int reg_code) {
887 0 : return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1;
888 : }
889 :
890 : // Needs access to SafepointRegisterStackIndex for compiled frame
891 : // traversal.
892 : friend class StandardFrame;
893 :
894 : DISALLOW_IMPLICIT_CONSTRUCTORS(MacroAssembler);
895 : };
896 :
897 : // -----------------------------------------------------------------------------
898 : // Static helper functions.
899 :
900 : // Generate an Operand for loading a field from an object.
901 : inline Operand FieldOperand(Register object, int offset) {
902 569347 : return Operand(object, offset - kHeapObjectTag);
903 : }
904 :
905 :
906 : // Generate an Operand for loading an indexed field from an object.
907 : inline Operand FieldOperand(Register object,
908 : Register index,
909 : ScaleFactor scale,
910 : int offset) {
911 9120 : return Operand(object, index, scale, offset - kHeapObjectTag);
912 : }
913 :
914 :
915 : inline Operand ContextOperand(Register context, int index) {
916 672 : return Operand(context, Context::SlotOffset(index));
917 : }
918 :
919 :
920 : inline Operand ContextOperand(Register context, Register index) {
921 : return Operand(context, index, times_system_pointer_size,
922 : Context::SlotOffset(0));
923 : }
924 :
925 :
926 336 : inline Operand NativeContextOperand() {
927 336 : return ContextOperand(rsi, Context::NATIVE_CONTEXT_INDEX);
928 : }
929 :
930 :
931 : // Provides access to exit frame stack space (not GCed).
932 : inline Operand StackSpaceOperand(int index) {
933 : #ifdef _WIN64
934 : const int kShaddowSpace = 4;
935 : return Operand(rsp, (index + kShaddowSpace) * kSystemPointerSize);
936 : #else
937 392 : return Operand(rsp, index * kSystemPointerSize);
938 : #endif
939 : }
940 :
941 :
942 : inline Operand StackOperandForReturnAddress(int32_t disp) {
943 56 : return Operand(rsp, disp);
944 : }
945 :
946 : #define ACCESS_MASM(masm) masm->
947 :
948 : } // namespace internal
949 : } // namespace v8
950 :
951 : #endif // V8_X64_MACRO_ASSEMBLER_X64_H_
|
