Line data Source code
1 : // Copyright 2012 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #ifndef INCLUDED_FROM_MACRO_ASSEMBLER_H
6 : #error This header must be included via macro-assembler.h
7 : #endif
8 :
9 : #ifndef V8_X64_MACRO_ASSEMBLER_X64_H_
10 : #define V8_X64_MACRO_ASSEMBLER_X64_H_
11 :
12 : #include "src/bailout-reason.h"
13 : #include "src/base/flags.h"
14 : #include "src/contexts.h"
15 : #include "src/globals.h"
16 : #include "src/x64/assembler-x64.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 :
21 : // Convenience for platform-independent signatures.
22 : typedef Operand MemOperand;
23 :
24 : class StringConstantBase;
25 :
26 : enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
27 : enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
28 :
29 : struct SmiIndex {
30 : SmiIndex(Register index_register, ScaleFactor scale)
31 : : reg(index_register),
32 : scale(scale) {}
33 : Register reg;
34 : ScaleFactor scale;
35 : };
36 :
37 : enum StackArgumentsAccessorReceiverMode {
38 : ARGUMENTS_CONTAIN_RECEIVER,
39 : ARGUMENTS_DONT_CONTAIN_RECEIVER
40 : };
41 :
42 : class StackArgumentsAccessor {
43 : public:
44 : StackArgumentsAccessor(Register base_reg, int argument_count_immediate,
45 : StackArgumentsAccessorReceiverMode receiver_mode =
46 : ARGUMENTS_CONTAIN_RECEIVER,
47 : int extra_displacement_to_last_argument = 0)
48 : : base_reg_(base_reg),
49 : argument_count_reg_(no_reg),
50 : argument_count_immediate_(argument_count_immediate),
51 : receiver_mode_(receiver_mode),
52 : extra_displacement_to_last_argument_(
53 : extra_displacement_to_last_argument) {}
54 :
55 : StackArgumentsAccessor(Register base_reg, Register argument_count_reg,
56 : StackArgumentsAccessorReceiverMode receiver_mode =
57 : ARGUMENTS_CONTAIN_RECEIVER,
58 : int extra_displacement_to_last_argument = 0)
59 : : base_reg_(base_reg),
60 : argument_count_reg_(argument_count_reg),
61 : argument_count_immediate_(0),
62 : receiver_mode_(receiver_mode),
63 : extra_displacement_to_last_argument_(
64 952 : extra_displacement_to_last_argument) {}
65 :
66 : StackArgumentsAccessor(Register base_reg,
67 : const ParameterCount& parameter_count,
68 : StackArgumentsAccessorReceiverMode receiver_mode =
69 : ARGUMENTS_CONTAIN_RECEIVER,
70 : int extra_displacement_to_last_argument = 0);
71 :
72 : Operand GetArgumentOperand(int index);
73 : Operand GetReceiverOperand() {
74 : DCHECK(receiver_mode_ == ARGUMENTS_CONTAIN_RECEIVER);
75 1008 : return GetArgumentOperand(0);
76 : }
77 :
78 : private:
79 : const Register base_reg_;
80 : const Register argument_count_reg_;
81 : const int argument_count_immediate_;
82 : const StackArgumentsAccessorReceiverMode receiver_mode_;
83 : const int extra_displacement_to_last_argument_;
84 :
85 : DISALLOW_IMPLICIT_CONSTRUCTORS(StackArgumentsAccessor);
86 : };
87 :
88 5791061 : class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
89 : public:
90 : template <typename... Args>
91 5790589 : explicit TurboAssembler(Args&&... args)
92 11581538 : : TurboAssemblerBase(std::forward<Args>(args)...) {}
93 :
94 : template <typename Dst, typename... Args>
95 : struct AvxHelper {
96 : Assembler* assm;
97 : // Call an method where the AVX version expects the dst argument to be
98 : // duplicated.
99 : template <void (Assembler::*avx)(Dst, Dst, Args...),
100 : void (Assembler::*no_avx)(Dst, Args...)>
101 1221277 : void emit(Dst dst, Args... args) {
102 1221277 : if (CpuFeatures::IsSupported(AVX)) {
103 : CpuFeatureScope scope(assm, AVX);
104 1219681 : (assm->*avx)(dst, dst, args...);
105 : } else {
106 1596 : (assm->*no_avx)(dst, args...);
107 : }
108 1221307 : }
109 :
110 : // Call an method where the AVX version expects no duplicated dst argument.
111 : template <void (Assembler::*avx)(Dst, Args...),
112 : void (Assembler::*no_avx)(Dst, Args...)>
113 4625958 : void emit(Dst dst, Args... args) {
114 4625958 : if (CpuFeatures::IsSupported(AVX)) {
115 : CpuFeatureScope scope(assm, AVX);
116 4616469 : (assm->*avx)(dst, args...);
117 : } else {
118 9489 : (assm->*no_avx)(dst, args...);
119 : }
120 4626055 : }
121 : };
122 :
123 : #define AVX_OP(macro_name, name) \
124 : template <typename Dst, typename... Args> \
125 : void macro_name(Dst dst, Args... args) { \
126 : AvxHelper<Dst, Args...>{this} \
127 : .template emit<&Assembler::v##name, &Assembler::name>(dst, args...); \
128 : }
129 :
130 5818 : AVX_OP(Subsd, subsd)
131 257 : AVX_OP(Divss, divss)
132 785 : AVX_OP(Divsd, divsd)
133 11463 : AVX_OP(Xorps, xorps)
134 110578 : AVX_OP(Xorpd, xorpd)
135 137550 : AVX_OP(Movd, movd)
136 177974 : AVX_OP(Movq, movq)
137 416 : AVX_OP(Movaps, movaps)
138 135985 : AVX_OP(Movapd, movapd)
139 8939 : AVX_OP(Movups, movups)
140 257 : AVX_OP(Movmskps, movmskps)
141 729 : AVX_OP(Movmskpd, movmskpd)
142 844111 : AVX_OP(Movss, movss)
143 3500223 : AVX_OP(Movsd, movsd)
144 292691 : AVX_OP(Pcmpeqd, pcmpeqd)
145 55478 : AVX_OP(Pslld, pslld)
146 225793 : AVX_OP(Psllq, psllq)
147 42066 : AVX_OP(Psrld, psrld)
148 203565 : AVX_OP(Psrlq, psrlq)
149 : AVX_OP(Addsd, addsd)
150 112 : AVX_OP(Mulsd, mulsd)
151 48 : AVX_OP(Andps, andps)
152 38 : AVX_OP(Andpd, andpd)
153 : AVX_OP(Orpd, orpd)
154 : AVX_OP(Cmpeqps, cmpeqps)
155 : AVX_OP(Cmpltps, cmpltps)
156 : AVX_OP(Cmpleps, cmpleps)
157 : AVX_OP(Cmpneqps, cmpneqps)
158 : AVX_OP(Cmpnltps, cmpnltps)
159 : AVX_OP(Cmpnleps, cmpnleps)
160 : AVX_OP(Cmpeqpd, cmpeqpd)
161 : AVX_OP(Cmpltpd, cmpltpd)
162 : AVX_OP(Cmplepd, cmplepd)
163 : AVX_OP(Cmpneqpd, cmpneqpd)
164 : AVX_OP(Cmpnltpd, cmpnltpd)
165 : AVX_OP(Cmpnlepd, cmpnlepd)
166 821 : AVX_OP(Roundss, roundss)
167 44891 : AVX_OP(Roundsd, roundsd)
168 172 : AVX_OP(Sqrtss, sqrtss)
169 622 : AVX_OP(Sqrtsd, sqrtsd)
170 1290 : AVX_OP(Ucomiss, ucomiss)
171 4793 : AVX_OP(Ucomisd, ucomisd)
172 :
173 : #undef AVX_OP
174 :
175 1628 : void PushReturnAddressFrom(Register src) { pushq(src); }
176 1348 : void PopReturnAddressTo(Register dst) { popq(dst); }
177 :
178 : void Ret();
179 :
180 : // Return and drop arguments from stack, where the number of arguments
181 : // may be bigger than 2^16 - 1. Requires a scratch register.
182 : void Ret(int bytes_dropped, Register scratch);
183 :
184 : // Load a register with a long value as efficiently as possible.
185 : void Set(Register dst, int64_t x);
186 : void Set(Operand dst, intptr_t x);
187 :
188 : // Operations on roots in the root-array.
189 : void LoadRoot(Register destination, RootIndex index) override;
190 : void LoadRoot(Operand destination, RootIndex index) {
191 : LoadRoot(kScratchRegister, index);
192 : movp(destination, kScratchRegister);
193 : }
194 :
195 : void Push(Register src);
196 : void Push(Operand src);
197 : void Push(Immediate value);
198 : void Push(Smi smi);
199 : void Push(Handle<HeapObject> source);
200 :
201 : // Before calling a C-function from generated code, align arguments on stack.
202 : // After aligning the frame, arguments must be stored in rsp[0], rsp[8],
203 : // etc., not pushed. The argument count assumes all arguments are word sized.
204 : // The number of slots reserved for arguments depends on platform. On Windows
205 : // stack slots are reserved for the arguments passed in registers. On other
206 : // platforms stack slots are only reserved for the arguments actually passed
207 : // on the stack.
208 : void PrepareCallCFunction(int num_arguments);
209 :
210 : // Calls a C function and cleans up the space for arguments allocated
211 : // by PrepareCallCFunction. The called function is not allowed to trigger a
212 : // garbage collection, since that might move the code and invalidate the
213 : // return address (unless this is somehow accounted for by the called
214 : // function).
215 : void CallCFunction(ExternalReference function, int num_arguments);
216 : void CallCFunction(Register function, int num_arguments);
217 :
218 : // Calculate the number of stack slots to reserve for arguments when calling a
219 : // C function.
220 : int ArgumentStackSlotsForCFunctionCall(int num_arguments);
221 :
222 : void CheckPageFlag(Register object, Register scratch, int mask, Condition cc,
223 : Label* condition_met,
224 : Label::Distance condition_met_distance = Label::kFar);
225 :
226 : void Cvtss2sd(XMMRegister dst, XMMRegister src);
227 : void Cvtss2sd(XMMRegister dst, Operand src);
228 : void Cvtsd2ss(XMMRegister dst, XMMRegister src);
229 : void Cvtsd2ss(XMMRegister dst, Operand src);
230 : void Cvttsd2si(Register dst, XMMRegister src);
231 : void Cvttsd2si(Register dst, Operand src);
232 : void Cvttsd2siq(Register dst, XMMRegister src);
233 : void Cvttsd2siq(Register dst, Operand src);
234 : void Cvttss2si(Register dst, XMMRegister src);
235 : void Cvttss2si(Register dst, Operand src);
236 : void Cvttss2siq(Register dst, XMMRegister src);
237 : void Cvttss2siq(Register dst, Operand src);
238 : void Cvtqsi2ss(XMMRegister dst, Register src);
239 : void Cvtqsi2ss(XMMRegister dst, Operand src);
240 : void Cvtqsi2sd(XMMRegister dst, Register src);
241 : void Cvtqsi2sd(XMMRegister dst, Operand src);
242 : void Cvtlsi2ss(XMMRegister dst, Register src);
243 : void Cvtlsi2ss(XMMRegister dst, Operand src);
244 : void Cvtlui2ss(XMMRegister dst, Register src);
245 : void Cvtlui2ss(XMMRegister dst, Operand src);
246 : void Cvtlui2sd(XMMRegister dst, Register src);
247 : void Cvtlui2sd(XMMRegister dst, Operand src);
248 : void Cvtqui2ss(XMMRegister dst, Register src);
249 : void Cvtqui2ss(XMMRegister dst, Operand src);
250 : void Cvtqui2sd(XMMRegister dst, Register src);
251 : void Cvtqui2sd(XMMRegister dst, Operand src);
252 : void Cvttsd2uiq(Register dst, Operand src, Label* fail = nullptr);
253 : void Cvttsd2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
254 : void Cvttss2uiq(Register dst, Operand src, Label* fail = nullptr);
255 : void Cvttss2uiq(Register dst, XMMRegister src, Label* fail = nullptr);
256 :
257 : // cvtsi2sd instruction only writes to the low 64-bit of dst register, which
258 : // hinders register renaming and makes dependence chains longer. So we use
259 : // xorpd to clear the dst register before cvtsi2sd to solve this issue.
260 : void Cvtlsi2sd(XMMRegister dst, Register src);
261 : void Cvtlsi2sd(XMMRegister dst, Operand src);
262 :
263 : void Lzcntq(Register dst, Register src);
264 : void Lzcntq(Register dst, Operand src);
265 : void Lzcntl(Register dst, Register src);
266 : void Lzcntl(Register dst, Operand src);
267 : void Tzcntq(Register dst, Register src);
268 : void Tzcntq(Register dst, Operand src);
269 : void Tzcntl(Register dst, Register src);
270 : void Tzcntl(Register dst, Operand src);
271 : void Popcntl(Register dst, Register src);
272 : void Popcntl(Register dst, Operand src);
273 : void Popcntq(Register dst, Register src);
274 : void Popcntq(Register dst, Operand src);
275 :
276 : // Is the value a tagged smi.
277 : Condition CheckSmi(Register src);
278 : Condition CheckSmi(Operand src);
279 :
280 : // Jump to label if the value is a tagged smi.
281 : void JumpIfSmi(Register src, Label* on_smi,
282 : Label::Distance near_jump = Label::kFar);
283 :
284 217901 : void JumpIfEqual(Register a, int32_t b, Label* dest) {
285 217901 : cmpl(a, Immediate(b));
286 217902 : j(equal, dest);
287 217901 : }
288 :
289 45187 : void JumpIfLessThan(Register a, int32_t b, Label* dest) {
290 45187 : cmpl(a, Immediate(b));
291 45187 : j(less, dest);
292 45187 : }
293 :
294 : void Move(Register dst, Smi source);
295 :
296 : void Move(Operand dst, Smi source) {
297 : Register constant = GetSmiConstant(source);
298 : movp(dst, constant);
299 : }
300 :
301 : void Move(Register dst, ExternalReference ext);
302 :
303 : void Move(XMMRegister dst, uint32_t src);
304 : void Move(XMMRegister dst, uint64_t src);
305 258 : void Move(XMMRegister dst, float src) { Move(dst, bit_cast<uint32_t>(src)); }
306 4672 : void Move(XMMRegister dst, double src) { Move(dst, bit_cast<uint64_t>(src)); }
307 :
308 : // Move if the registers are not identical.
309 : void Move(Register target, Register source);
310 :
311 : void Move(Register dst, Handle<HeapObject> source,
312 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
313 : void Move(Operand dst, Handle<HeapObject> source,
314 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
315 :
316 : // Loads a pointer into a register with a relocation mode.
317 : void Move(Register dst, Address ptr, RelocInfo::Mode rmode) {
318 : // This method must not be used with heap object references. The stored
319 : // address is not GC safe. Use the handle version instead.
320 : DCHECK(rmode > RelocInfo::LAST_GCED_ENUM);
321 5366347 : movp(dst, ptr, rmode);
322 : }
323 :
324 : void MoveStringConstant(Register result, const StringConstantBase* string,
325 : RelocInfo::Mode rmode = RelocInfo::EMBEDDED_OBJECT);
326 :
327 : // Convert smi to word-size sign-extended value.
328 : void SmiUntag(Register dst, Register src);
329 : void SmiUntag(Register dst, Operand src);
330 :
331 : // Loads the address of the external reference into the destination
332 : // register.
333 : void LoadAddress(Register destination, ExternalReference source);
334 :
335 : void LoadFromConstantsTable(Register destination,
336 : int constant_index) override;
337 : void LoadRootRegisterOffset(Register destination, intptr_t offset) override;
338 : void LoadRootRelative(Register destination, int32_t offset) override;
339 :
340 : // Operand pointing to an external reference.
341 : // May emit code to set up the scratch register. The operand is
342 : // only guaranteed to be correct as long as the scratch register
343 : // isn't changed.
344 : // If the operand is used more than once, use a scratch register
345 : // that is guaranteed not to be clobbered.
346 : Operand ExternalReferenceAsOperand(ExternalReference reference,
347 : Register scratch = kScratchRegister);
348 :
349 84616 : void Call(Register reg) { call(reg); }
350 : void Call(Operand op);
351 : void Call(Handle<Code> code_object, RelocInfo::Mode rmode);
352 : void Call(Address destination, RelocInfo::Mode rmode);
353 : void Call(ExternalReference ext);
354 : void Call(Label* target) { call(target); }
355 :
356 : void CallBuiltinPointer(Register builtin_pointer) override;
357 :
358 : void LoadCodeObjectEntry(Register destination, Register code_object) override;
359 : void CallCodeObject(Register code_object) override;
360 : void JumpCodeObject(Register code_object) override;
361 :
362 : void RetpolineCall(Register reg);
363 : void RetpolineCall(Address destination, RelocInfo::Mode rmode);
364 :
365 : void Jump(Address destination, RelocInfo::Mode rmode);
366 : void Jump(ExternalReference ext);
367 : void Jump(Operand op);
368 : void Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
369 : Condition cc = always);
370 :
371 : void RetpolineJump(Register reg);
372 :
373 : void CallForDeoptimization(Address target, int deopt_id);
374 :
375 : // Non-SSE2 instructions.
376 : void Pextrd(Register dst, XMMRegister src, int8_t imm8);
377 : void Pinsrd(XMMRegister dst, Register src, int8_t imm8);
378 : void Pinsrd(XMMRegister dst, Operand src, int8_t imm8);
379 :
380 : void CompareRoot(Register with, RootIndex index);
381 : void CompareRoot(Operand with, RootIndex index);
382 :
383 : // Generates function and stub prologue code.
384 : void StubPrologue(StackFrame::Type type);
385 : void Prologue();
386 :
387 : // Calls Abort(msg) if the condition cc is not satisfied.
388 : // Use --debug_code to enable.
389 : void Assert(Condition cc, AbortReason reason);
390 :
391 : // Like Assert(), but without condition.
392 : // Use --debug_code to enable.
393 : void AssertUnreachable(AbortReason reason);
394 :
395 : // Abort execution if a 64 bit register containing a 32 bit payload does not
396 : // have zeros in the top 32 bits, enabled via --debug-code.
397 : void AssertZeroExtended(Register reg);
398 :
399 : // Like Assert(), but always enabled.
400 : void Check(Condition cc, AbortReason reason);
401 :
402 : // Print a message to stdout and abort execution.
403 : void Abort(AbortReason msg);
404 :
405 : // Check that the stack is aligned.
406 : void CheckStackAlignment();
407 :
408 : // Activation support.
409 : void EnterFrame(StackFrame::Type type);
410 : void EnterFrame(StackFrame::Type type, bool load_constant_pool_pointer_reg) {
411 : // Out-of-line constant pool not implemented on x64.
412 : UNREACHABLE();
413 : }
414 : void LeaveFrame(StackFrame::Type type);
415 :
416 : // Removes current frame and its arguments from the stack preserving the
417 : // arguments and a return address pushed to the stack for the next call. Both
418 : // |callee_args_count| and |caller_args_count_reg| do not include receiver.
419 : // |callee_args_count| is not modified, |caller_args_count_reg| is trashed.
420 : void PrepareForTailCall(const ParameterCount& callee_args_count,
421 : Register caller_args_count_reg, Register scratch0,
422 : Register scratch1);
423 :
424 : // Call a runtime routine. This expects {centry} to contain a fitting CEntry
425 : // builtin for the target runtime function and uses an indirect call.
426 : void CallRuntimeWithCEntry(Runtime::FunctionId fid, Register centry);
427 :
428 1110350 : void InitializeRootRegister() {
429 1110350 : ExternalReference isolate_root = ExternalReference::isolate_root(isolate());
430 1110350 : Move(kRootRegister, isolate_root);
431 1110350 : }
432 :
433 : void SaveRegisters(RegList registers);
434 : void RestoreRegisters(RegList registers);
435 :
436 : void CallRecordWriteStub(Register object, Register address,
437 : RememberedSetAction remembered_set_action,
438 : SaveFPRegsMode fp_mode);
439 : void CallRecordWriteStub(Register object, Register address,
440 : RememberedSetAction remembered_set_action,
441 : SaveFPRegsMode fp_mode, Address wasm_target);
442 :
443 : void MoveNumber(Register dst, double value);
444 : void MoveNonSmi(Register dst, double value);
445 :
446 : // Calculate how much stack space (in bytes) are required to store caller
447 : // registers excluding those specified in the arguments.
448 : int RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
449 : Register exclusion1 = no_reg,
450 : Register exclusion2 = no_reg,
451 : Register exclusion3 = no_reg) const;
452 :
453 : // PushCallerSaved and PopCallerSaved do not arrange the registers in any
454 : // particular order so they are not useful for calls that can cause a GC.
455 : // The caller can exclude up to 3 registers that do not need to be saved and
456 : // restored.
457 :
458 : // Push caller saved registers on the stack, and return the number of bytes
459 : // stack pointer is adjusted.
460 : int PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
461 : Register exclusion2 = no_reg,
462 : Register exclusion3 = no_reg);
463 : // Restore caller saved registers from the stack, and return the number of
464 : // bytes stack pointer is adjusted.
465 : int PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1 = no_reg,
466 : Register exclusion2 = no_reg,
467 : Register exclusion3 = no_reg);
468 :
469 : // Compute the start of the generated instruction stream from the current PC.
470 : // This is an alternative to embedding the {CodeObject} handle as a reference.
471 : void ComputeCodeStartAddress(Register dst);
472 :
473 : void ResetSpeculationPoisonRegister();
474 :
475 : // ---------------------------------------------------------------------------
476 : // Pointer compression support
477 :
478 : // TODO(ishell): remove |scratch_for_debug| once pointer compression works.
479 :
480 : // Loads a field containing a HeapObject and decompresses it if pointer
481 : // compression is enabled.
482 : void LoadTaggedPointerField(Register destination, Operand field_operand,
483 : Register scratch_for_debug = no_reg);
484 :
485 : // Loads a field containing any tagged value and decompresses it if necessary.
486 : // When pointer compression is enabled, uses |scratch| to decompress the
487 : // value.
488 : void LoadAnyTaggedField(Register destination, Operand field_operand,
489 : Register scratch,
490 : Register scratch_for_debug = no_reg);
491 :
492 : // Loads a field containing a HeapObject, decompresses it if necessary and
493 : // pushes full pointer to the stack. When pointer compression is enabled,
494 : // uses |scratch| to decompress the value.
495 : void PushTaggedPointerField(Operand field_operand, Register scratch,
496 : Register scratch_for_debug = no_reg);
497 :
498 : // Loads a field containing any tagged value, decompresses it if necessary and
499 : // pushes the full pointer to the stack. When pointer compression is enabled,
500 : // uses |scratch1| and |scratch2| to decompress the value.
501 : void PushTaggedAnyField(Operand field_operand, Register scratch1,
502 : Register scratch2,
503 : Register scratch_for_debug = no_reg);
504 :
505 : // Loads a field containing smi value and untags it.
506 : void SmiUntagField(Register dst, Operand src);
507 :
508 : // Compresses and stores tagged value to given on-heap location.
509 : // TODO(ishell): drop once mov_tagged() can be used.
510 : void StoreTaggedField(Operand dst_field_operand, Immediate immediate);
511 : void StoreTaggedField(Operand dst_field_operand, Register value);
512 :
513 : void DecompressTaggedSigned(Register destination, Operand field_operand,
514 : Register scratch_for_debug);
515 : void DecompressTaggedPointer(Register destination, Operand field_operand,
516 : Register scratch_for_debug);
517 : void DecompressAnyTagged(Register destination, Operand field_operand,
518 : Register scratch, Register scratch_for_debug);
519 :
520 : protected:
521 : static const int kSmiShift = kSmiTagSize + kSmiShiftSize;
522 : int smi_count = 0;
523 : int heap_object_count = 0;
524 :
525 : // Returns a register holding the smi value. The register MUST NOT be
526 : // modified. It may be the "smi 1 constant" register.
527 : Register GetSmiConstant(Smi value);
528 :
529 : void CallRecordWriteStub(Register object, Register address,
530 : RememberedSetAction remembered_set_action,
531 : SaveFPRegsMode fp_mode, Handle<Code> code_target,
532 : Address wasm_target);
533 : };
534 :
535 : // MacroAssembler implements a collection of frequently used macros.
536 320968 : class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
537 : public:
538 : template <typename... Args>
539 : explicit MacroAssembler(Args&&... args)
540 320968 : : TurboAssembler(std::forward<Args>(args)...) {}
541 :
542 : // Loads and stores the value of an external reference.
543 : // Special case code for load and store to take advantage of
544 : // load_rax/store_rax if possible/necessary.
545 : // For other operations, just use:
546 : // Operand operand = ExternalReferenceAsOperand(extref);
547 : // operation(operand, ..);
548 : void Load(Register destination, ExternalReference source);
549 : void Store(ExternalReference destination, Register source);
550 :
551 : // Pushes the address of the external reference onto the stack.
552 : void PushAddress(ExternalReference source);
553 :
554 : // Operations on roots in the root-array.
555 : // Load a root value where the index (or part of it) is variable.
556 : // The variable_offset register is added to the fixed_offset value
557 : // to get the index into the root-array.
558 : void PushRoot(RootIndex index);
559 :
560 : // Compare the object in a register to a value and jump if they are equal.
561 : void JumpIfRoot(Register with, RootIndex index, Label* if_equal,
562 : Label::Distance if_equal_distance = Label::kFar) {
563 336 : CompareRoot(with, index);
564 336 : j(equal, if_equal, if_equal_distance);
565 : }
566 : void JumpIfRoot(Operand with, RootIndex index, Label* if_equal,
567 : Label::Distance if_equal_distance = Label::kFar) {
568 : CompareRoot(with, index);
569 : j(equal, if_equal, if_equal_distance);
570 : }
571 :
572 : // Compare the object in a register to a value and jump if they are not equal.
573 : void JumpIfNotRoot(Register with, RootIndex index, Label* if_not_equal,
574 : Label::Distance if_not_equal_distance = Label::kFar) {
575 56 : CompareRoot(with, index);
576 56 : j(not_equal, if_not_equal, if_not_equal_distance);
577 : }
578 : void JumpIfNotRoot(Operand with, RootIndex index, Label* if_not_equal,
579 : Label::Distance if_not_equal_distance = Label::kFar) {
580 : CompareRoot(with, index);
581 : j(not_equal, if_not_equal, if_not_equal_distance);
582 : }
583 :
584 : // ---------------------------------------------------------------------------
585 : // GC Support
586 :
587 : // Notify the garbage collector that we wrote a pointer into an object.
588 : // |object| is the object being stored into, |value| is the object being
589 : // stored. value and scratch registers are clobbered by the operation.
590 : // The offset is the offset from the start of the object, not the offset from
591 : // the tagged HeapObject pointer. For use with FieldOperand(reg, off).
592 : void RecordWriteField(
593 : Register object, int offset, Register value, Register scratch,
594 : SaveFPRegsMode save_fp,
595 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
596 : SmiCheck smi_check = INLINE_SMI_CHECK);
597 :
598 : // For page containing |object| mark region covering |address|
599 : // dirty. |object| is the object being stored into, |value| is the
600 : // object being stored. The address and value registers are clobbered by the
601 : // operation. RecordWrite filters out smis so it does not update
602 : // the write barrier if the value is a smi.
603 : void RecordWrite(
604 : Register object, Register address, Register value, SaveFPRegsMode save_fp,
605 : RememberedSetAction remembered_set_action = EMIT_REMEMBERED_SET,
606 : SmiCheck smi_check = INLINE_SMI_CHECK);
607 :
608 : // Frame restart support.
609 : void MaybeDropFrames();
610 :
611 : // Enter specific kind of exit frame; either in normal or
612 : // debug mode. Expects the number of arguments in register rax and
613 : // sets up the number of arguments in register rdi and the pointer
614 : // to the first argument in register rsi.
615 : //
616 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
617 : // stack accessible via StackSpaceOperand.
618 : void EnterExitFrame(int arg_stack_space = 0, bool save_doubles = false,
619 : StackFrame::Type frame_type = StackFrame::EXIT);
620 :
621 : // Enter specific kind of exit frame. Allocates
622 : // (arg_stack_space * kSystemPointerSize) memory (not GCed) on the stack
623 : // accessible via StackSpaceOperand.
624 : void EnterApiExitFrame(int arg_stack_space);
625 :
626 : // Leave the current exit frame. Expects/provides the return value in
627 : // register rax:rdx (untouched) and the pointer to the first
628 : // argument in register rsi (if pop_arguments == true).
629 : void LeaveExitFrame(bool save_doubles = false, bool pop_arguments = true);
630 :
631 : // Leave the current exit frame. Expects/provides the return value in
632 : // register rax (untouched).
633 : void LeaveApiExitFrame();
634 :
635 : // Push and pop the registers that can hold pointers.
636 0 : void PushSafepointRegisters() { Pushad(); }
637 0 : void PopSafepointRegisters() { Popad(); }
638 :
639 : // ---------------------------------------------------------------------------
640 : // JavaScript invokes
641 :
642 : // Invoke the JavaScript function code by either calling or jumping.
643 : void InvokeFunctionCode(Register function, Register new_target,
644 : const ParameterCount& expected,
645 : const ParameterCount& actual, InvokeFlag flag);
646 :
647 : // On function call, call into the debugger if necessary.
648 : void CheckDebugHook(Register fun, Register new_target,
649 : const ParameterCount& expected,
650 : const ParameterCount& actual);
651 :
652 : // Invoke the JavaScript function in the given register. Changes the
653 : // current context to the context in the function before invoking.
654 : void InvokeFunction(Register function, Register new_target,
655 : const ParameterCount& actual, InvokeFlag flag);
656 :
657 : void InvokeFunction(Register function, Register new_target,
658 : const ParameterCount& expected,
659 : const ParameterCount& actual, InvokeFlag flag);
660 :
661 : // ---------------------------------------------------------------------------
662 : // Conversions between tagged smi values and non-tagged integer values.
663 :
664 : // Tag an word-size value. The result must be known to be a valid smi value.
665 : void SmiTag(Register dst, Register src);
666 :
667 : // Simple comparison of smis. Both sides must be known smis to use these,
668 : // otherwise use Cmp.
669 : void SmiCompare(Register smi1, Register smi2);
670 : void SmiCompare(Register dst, Smi src);
671 : void SmiCompare(Register dst, Operand src);
672 : void SmiCompare(Operand dst, Register src);
673 : void SmiCompare(Operand dst, Smi src);
674 :
675 : // Functions performing a check on a known or potential smi. Returns
676 : // a condition that is satisfied if the check is successful.
677 :
678 : // Test-and-jump functions. Typically combines a check function
679 : // above with a conditional jump.
680 :
681 : // Jump to label if the value is not a tagged smi.
682 : void JumpIfNotSmi(Register src,
683 : Label* on_not_smi,
684 : Label::Distance near_jump = Label::kFar);
685 :
686 : // Jump to label if the value is not a tagged smi.
687 : void JumpIfNotSmi(Operand src, Label* on_not_smi,
688 : Label::Distance near_jump = Label::kFar);
689 :
690 : // Operations on tagged smi values.
691 :
692 : // Smis represent a subset of integers. The subset is always equivalent to
693 : // a two's complement interpretation of a fixed number of bits.
694 :
695 : // Add an integer constant to a tagged smi, giving a tagged smi as result.
696 : // No overflow testing on the result is done.
697 : void SmiAddConstant(Operand dst, Smi constant);
698 :
699 : // Specialized operations
700 :
701 : // Converts, if necessary, a smi to a combination of number and
702 : // multiplier to be used as a scaled index.
703 : // The src register contains a *positive* smi value. The shift is the
704 : // power of two to multiply the index value by (e.g. to index by
705 : // smi-value * kSystemPointerSize, pass the smi and kSystemPointerSizeLog2).
706 : // The returned index register may be either src or dst, depending
707 : // on what is most efficient. If src and dst are different registers,
708 : // src is always unchanged.
709 : SmiIndex SmiToIndex(Register dst, Register src, int shift);
710 :
711 : // ---------------------------------------------------------------------------
712 : // Macro instructions.
713 :
714 : // Load/store with specific representation.
715 : void Load(Register dst, Operand src, Representation r);
716 : void Store(Operand dst, Register src, Representation r);
717 :
718 : void Cmp(Register dst, Handle<Object> source);
719 : void Cmp(Operand dst, Handle<Object> source);
720 : void Cmp(Register dst, Smi src);
721 : void Cmp(Operand dst, Smi src);
722 :
723 : // Emit code to discard a non-negative number of pointer-sized elements
724 : // from the stack, clobbering only the rsp register.
725 : void Drop(int stack_elements);
726 : // Emit code to discard a positive number of pointer-sized elements
727 : // from the stack under the return address which remains on the top,
728 : // clobbering the rsp register.
729 : void DropUnderReturnAddress(int stack_elements,
730 : Register scratch = kScratchRegister);
731 :
732 : void PushQuad(Operand src);
733 : void PushImm32(int32_t imm32);
734 : void Pop(Register dst);
735 : void Pop(Operand dst);
736 : void PopQuad(Operand dst);
737 :
738 : // ---------------------------------------------------------------------------
739 : // SIMD macros.
740 : void Absps(XMMRegister dst);
741 : void Negps(XMMRegister dst);
742 : void Abspd(XMMRegister dst);
743 : void Negpd(XMMRegister dst);
744 : // Generates a trampoline to jump to the off-heap instruction stream.
745 : void JumpToInstructionStream(Address entry);
746 :
747 : // Non-x64 instructions.
748 : // Push/pop all general purpose registers.
749 : // Does not push rsp/rbp nor any of the assembler's special purpose registers
750 : // (kScratchRegister, kRootRegister).
751 : void Pushad();
752 : void Popad();
753 :
754 : // Compare object type for heap object.
755 : // Always use unsigned comparisons: above and below, not less and greater.
756 : // Incoming register is heap_object and outgoing register is map.
757 : // They may be the same register, and may be kScratchRegister.
758 : void CmpObjectType(Register heap_object, InstanceType type, Register map);
759 :
760 : // Compare instance type for map.
761 : // Always use unsigned comparisons: above and below, not less and greater.
762 : void CmpInstanceType(Register map, InstanceType type);
763 :
764 : void DoubleToI(Register result_reg, XMMRegister input_reg,
765 : XMMRegister scratch, Label* lost_precision, Label* is_nan,
766 : Label::Distance dst = Label::kFar);
767 :
768 : template<typename Field>
769 0 : void DecodeField(Register reg) {
770 : static const int shift = Field::kShift;
771 : static const int mask = Field::kMask >> Field::kShift;
772 : if (shift != 0) {
773 0 : shrp(reg, Immediate(shift));
774 : }
775 0 : andp(reg, Immediate(mask));
776 0 : }
777 :
778 : // Abort execution if argument is a smi, enabled via --debug-code.
779 : void AssertNotSmi(Register object);
780 :
781 : // Abort execution if argument is not a smi, enabled via --debug-code.
782 : void AssertSmi(Register object);
783 : void AssertSmi(Operand object);
784 :
785 : // Abort execution if argument is not a Constructor, enabled via --debug-code.
786 : void AssertConstructor(Register object);
787 :
788 : // Abort execution if argument is not a JSFunction, enabled via --debug-code.
789 : void AssertFunction(Register object);
790 :
791 : // Abort execution if argument is not a JSBoundFunction,
792 : // enabled via --debug-code.
793 : void AssertBoundFunction(Register object);
794 :
795 : // Abort execution if argument is not a JSGeneratorObject (or subclass),
796 : // enabled via --debug-code.
797 : void AssertGeneratorObject(Register object);
798 :
799 : // Abort execution if argument is not undefined or an AllocationSite, enabled
800 : // via --debug-code.
801 : void AssertUndefinedOrAllocationSite(Register object);
802 :
803 : // ---------------------------------------------------------------------------
804 : // Exception handling
805 :
806 : // Push a new stack handler and link it into stack handler chain.
807 : void PushStackHandler();
808 :
809 : // Unlink the stack handler on top of the stack from the stack handler chain.
810 : void PopStackHandler();
811 :
812 : // ---------------------------------------------------------------------------
813 : // Support functions.
814 :
815 : // Load the global proxy from the current context.
816 : void LoadGlobalProxy(Register dst) {
817 112 : LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
818 : }
819 :
820 : // Load the native context slot with the current index.
821 : void LoadNativeContextSlot(int index, Register dst);
822 :
823 : // ---------------------------------------------------------------------------
824 : // Runtime calls
825 :
826 : // Call a runtime routine.
827 : void CallRuntime(const Runtime::Function* f,
828 : int num_arguments,
829 : SaveFPRegsMode save_doubles = kDontSaveFPRegs);
830 :
831 : // Convenience function: Same as above, but takes the fid instead.
832 1568 : void CallRuntime(Runtime::FunctionId fid,
833 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
834 1568 : const Runtime::Function* function = Runtime::FunctionForId(fid);
835 1568 : CallRuntime(function, function->nargs, save_doubles);
836 1568 : }
837 :
838 : // Convenience function: Same as above, but takes the fid instead.
839 : void CallRuntime(Runtime::FunctionId fid, int num_arguments,
840 : SaveFPRegsMode save_doubles = kDontSaveFPRegs) {
841 392 : CallRuntime(Runtime::FunctionForId(fid), num_arguments, save_doubles);
842 : }
843 :
844 : // Convenience function: tail call a runtime routine (jump)
845 : void TailCallRuntime(Runtime::FunctionId fid);
846 :
847 : // Jump to a runtime routines
848 : void JumpToExternalReference(const ExternalReference& ext,
849 : bool builtin_exit_frame = false);
850 :
851 : // ---------------------------------------------------------------------------
852 : // StatsCounter support
853 : void IncrementCounter(StatsCounter* counter, int value);
854 : void DecrementCounter(StatsCounter* counter, int value);
855 :
856 : // ---------------------------------------------------------------------------
857 : // In-place weak references.
858 : void LoadWeakValue(Register in_out, Label* target_if_cleared);
859 :
860 : // ---------------------------------------------------------------------------
861 : // Debugging
862 :
863 : static int SafepointRegisterStackIndex(Register reg) {
864 : return SafepointRegisterStackIndex(reg.code());
865 : }
866 :
867 : private:
868 : // Order general registers are pushed by Pushad.
869 : // rax, rcx, rdx, rbx, rsi, rdi, r8, r9, r11, r12, r14, r15.
870 : static const int kSafepointPushRegisterIndices[Register::kNumRegisters];
871 : static const int kNumSafepointSavedRegisters = 12;
872 :
873 : // Helper functions for generating invokes.
874 : void InvokePrologue(const ParameterCount& expected,
875 : const ParameterCount& actual, Label* done,
876 : bool* definitely_mismatches, InvokeFlag flag,
877 : Label::Distance near_jump);
878 :
879 : void EnterExitFramePrologue(bool save_rax, StackFrame::Type frame_type);
880 :
881 : // Allocates arg_stack_space * kSystemPointerSize memory (not GCed) on the
882 : // stack accessible via StackSpaceOperand.
883 : void EnterExitFrameEpilogue(int arg_stack_space, bool save_doubles);
884 :
885 : void LeaveExitFrameEpilogue();
886 :
887 : // Helper for implementing JumpIfNotInNewSpace and JumpIfInNewSpace.
888 : void InNewSpace(Register object,
889 : Register scratch,
890 : Condition cc,
891 : Label* branch,
892 : Label::Distance distance = Label::kFar);
893 :
894 : // Compute memory operands for safepoint stack slots.
895 : static int SafepointRegisterStackIndex(int reg_code) {
896 0 : return kNumSafepointRegisters - kSafepointPushRegisterIndices[reg_code] - 1;
897 : }
898 :
899 : // Needs access to SafepointRegisterStackIndex for compiled frame
900 : // traversal.
901 : friend class StandardFrame;
902 : };
903 :
904 : // -----------------------------------------------------------------------------
905 : // Static helper functions.
906 :
907 : // Generate an Operand for loading a field from an object.
908 : inline Operand FieldOperand(Register object, int offset) {
909 642056 : return Operand(object, offset - kHeapObjectTag);
910 : }
911 :
912 :
913 : // Generate an Operand for loading an indexed field from an object.
914 : inline Operand FieldOperand(Register object,
915 : Register index,
916 : ScaleFactor scale,
917 : int offset) {
918 10638 : return Operand(object, index, scale, offset - kHeapObjectTag);
919 : }
920 :
921 :
922 672 : inline Operand ContextOperand(Register context, int index) {
923 672 : return Operand(context, Context::SlotOffset(index));
924 : }
925 :
926 :
927 : inline Operand ContextOperand(Register context, Register index) {
928 : return Operand(context, index, times_pointer_size, Context::SlotOffset(0));
929 : }
930 :
931 :
932 : inline Operand NativeContextOperand() {
933 336 : return ContextOperand(rsi, Context::NATIVE_CONTEXT_INDEX);
934 : }
935 :
936 :
937 : // Provides access to exit frame stack space (not GCed).
938 : inline Operand StackSpaceOperand(int index) {
939 : #ifdef _WIN64
940 : const int kShaddowSpace = 4;
941 : return Operand(rsp, (index + kShaddowSpace) * kSystemPointerSize);
942 : #else
943 392 : return Operand(rsp, index * kSystemPointerSize);
944 : #endif
945 : }
946 :
947 :
948 : inline Operand StackOperandForReturnAddress(int32_t disp) {
949 56 : return Operand(rsp, disp);
950 : }
951 :
952 : #define ACCESS_MASM(masm) masm->
953 :
954 : } // namespace internal
955 : } // namespace v8
956 :
957 : #endif // V8_X64_MACRO_ASSEMBLER_X64_H_
|
