Line data Source code
1 : // Copyright 2013 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-generator.h"
6 :
7 : #include <limits>
8 :
9 : #include "src/compilation-info.h"
10 : #include "src/compiler/code-generator-impl.h"
11 : #include "src/compiler/gap-resolver.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/osr.h"
14 : #include "src/heap/heap-inl.h"
15 : #include "src/wasm/wasm-module.h"
16 : #include "src/x64/assembler-x64.h"
17 : #include "src/x64/macro-assembler-x64.h"
18 :
19 : namespace v8 {
20 : namespace internal {
21 : namespace compiler {
22 :
23 : #define __ masm()->
24 :
25 : // Adds X64 specific methods for decoding operands.
26 : class X64OperandConverter : public InstructionOperandConverter {
27 : public:
28 : X64OperandConverter(CodeGenerator* gen, Instruction* instr)
29 : : InstructionOperandConverter(gen, instr) {}
30 :
31 : Immediate InputImmediate(size_t index) {
32 2577061 : return ToImmediate(instr_->InputAt(index));
33 : }
34 :
35 525602 : Operand InputOperand(size_t index, int extra = 0) {
36 1576807 : return ToOperand(instr_->InputAt(index), extra);
37 : }
38 :
39 0 : Operand OutputOperand() { return ToOperand(instr_->Output()); }
40 :
41 2139456 : Immediate ToImmediate(InstructionOperand* operand) {
42 2139456 : Constant constant = ToConstant(operand);
43 2139456 : if (constant.type() == Constant::kFloat64) {
44 : DCHECK_EQ(0, bit_cast<int64_t>(constant.ToFloat64()));
45 105799 : return Immediate(0);
46 : }
47 2033657 : if (RelocInfo::IsWasmReference(constant.rmode())) {
48 12563 : return Immediate(constant.ToInt32(), constant.rmode());
49 : }
50 2021094 : return Immediate(constant.ToInt32());
51 : }
52 :
53 : Operand ToOperand(InstructionOperand* op, int extra = 0) {
54 : DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
55 9260195 : return SlotToOperand(AllocatedOperand::cast(op)->index(), extra);
56 : }
57 :
58 9269799 : Operand SlotToOperand(int slot_index, int extra = 0) {
59 18539598 : FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index);
60 : return Operand(offset.from_stack_pointer() ? rsp : rbp,
61 18539598 : offset.offset() + extra);
62 : }
63 :
64 : static size_t NextOffset(size_t* offset) {
65 7513281 : size_t i = *offset;
66 13563676 : (*offset)++;
67 : return i;
68 : }
69 :
70 : static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
71 : STATIC_ASSERT(0 == static_cast<int>(times_1));
72 : STATIC_ASSERT(1 == static_cast<int>(times_2));
73 : STATIC_ASSERT(2 == static_cast<int>(times_4));
74 : STATIC_ASSERT(3 == static_cast<int>(times_8));
75 606275 : int scale = static_cast<int>(mode - one);
76 : DCHECK(scale >= 0 && scale < 4);
77 606275 : return static_cast<ScaleFactor>(scale);
78 : }
79 :
80 7513281 : Operand MemoryOperand(size_t* offset) {
81 7513281 : AddressingMode mode = AddressingModeField::decode(instr_->opcode());
82 7513281 : switch (mode) {
83 : case kMode_MR: {
84 351444 : Register base = InputRegister(NextOffset(offset));
85 : int32_t disp = 0;
86 351444 : return Operand(base, disp);
87 : }
88 : case kMode_MRI: {
89 5131650 : Register base = InputRegister(NextOffset(offset));
90 5131650 : int32_t disp = InputInt32(NextOffset(offset));
91 5131650 : return Operand(base, disp);
92 : }
93 : case kMode_MR1:
94 : case kMode_MR2:
95 : case kMode_MR4:
96 : case kMode_MR8: {
97 241508 : Register base = InputRegister(NextOffset(offset));
98 241508 : Register index = InputRegister(NextOffset(offset));
99 : ScaleFactor scale = ScaleFor(kMode_MR1, mode);
100 : int32_t disp = 0;
101 241508 : return Operand(base, index, scale, disp);
102 : }
103 : case kMode_MR1I:
104 : case kMode_MR2I:
105 : case kMode_MR4I:
106 : case kMode_MR8I: {
107 314106 : Register base = InputRegister(NextOffset(offset));
108 314106 : Register index = InputRegister(NextOffset(offset));
109 : ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
110 314106 : int32_t disp = InputInt32(NextOffset(offset));
111 314106 : return Operand(base, index, scale, disp);
112 : }
113 : case kMode_M1: {
114 0 : Register base = InputRegister(NextOffset(offset));
115 : int32_t disp = 0;
116 0 : return Operand(base, disp);
117 : }
118 : case kMode_M2:
119 0 : UNREACHABLE(); // Should use kModeMR with more compact encoding instead
120 : return Operand(no_reg, 0);
121 : case kMode_M4:
122 : case kMode_M8: {
123 1636 : Register index = InputRegister(NextOffset(offset));
124 : ScaleFactor scale = ScaleFor(kMode_M1, mode);
125 : int32_t disp = 0;
126 1636 : return Operand(index, scale, disp);
127 : }
128 : case kMode_M1I:
129 : case kMode_M2I:
130 : case kMode_M4I:
131 : case kMode_M8I: {
132 49025 : Register index = InputRegister(NextOffset(offset));
133 : ScaleFactor scale = ScaleFor(kMode_M1I, mode);
134 49025 : int32_t disp = InputInt32(NextOffset(offset));
135 49025 : return Operand(index, scale, disp);
136 : }
137 : case kMode_Root: {
138 1423912 : Register base = kRootRegister;
139 1423912 : int32_t disp = InputInt32(NextOffset(offset));
140 1423912 : return Operand(base, disp);
141 : }
142 : case kMode_None:
143 0 : UNREACHABLE();
144 : return Operand(no_reg, 0);
145 : }
146 0 : UNREACHABLE();
147 : return Operand(no_reg, 0);
148 : }
149 :
150 : Operand MemoryOperand(size_t first_input = 0) {
151 3522244 : return MemoryOperand(&first_input);
152 : }
153 : };
154 :
155 :
156 : namespace {
157 :
158 : bool HasImmediateInput(Instruction* instr, size_t index) {
159 12873963 : return instr->InputAt(index)->IsImmediate();
160 : }
161 :
162 :
163 0 : class OutOfLineLoadZero final : public OutOfLineCode {
164 : public:
165 : OutOfLineLoadZero(CodeGenerator* gen, Register result)
166 133 : : OutOfLineCode(gen), result_(result) {}
167 :
168 266 : void Generate() final { __ xorl(result_, result_); }
169 :
170 : private:
171 : Register const result_;
172 : };
173 :
174 0 : class OutOfLineLoadFloat32NaN final : public OutOfLineCode {
175 : public:
176 : OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result)
177 121 : : OutOfLineCode(gen), result_(result) {}
178 :
179 121 : void Generate() final {
180 121 : __ Xorps(result_, result_);
181 121 : __ Divss(result_, result_);
182 121 : }
183 :
184 : private:
185 : XMMRegister const result_;
186 : };
187 :
188 0 : class OutOfLineLoadFloat64NaN final : public OutOfLineCode {
189 : public:
190 : OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result)
191 313 : : OutOfLineCode(gen), result_(result) {}
192 :
193 313 : void Generate() final {
194 313 : __ Xorpd(result_, result_);
195 313 : __ Divsd(result_, result_);
196 313 : }
197 :
198 : private:
199 : XMMRegister const result_;
200 : };
201 :
202 0 : class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
203 : public:
204 : OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
205 : XMMRegister input,
206 : UnwindingInfoWriter* unwinding_info_writer)
207 : : OutOfLineCode(gen),
208 : result_(result),
209 : input_(input),
210 70260 : unwinding_info_writer_(unwinding_info_writer) {}
211 :
212 70260 : void Generate() final {
213 351300 : __ subp(rsp, Immediate(kDoubleSize));
214 : unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
215 140520 : kDoubleSize);
216 140520 : __ Movsd(MemOperand(rsp, 0), input_);
217 70260 : __ SlowTruncateToI(result_, rsp, 0);
218 70260 : __ addp(rsp, Immediate(kDoubleSize));
219 : unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
220 140520 : -kDoubleSize);
221 70260 : }
222 :
223 : private:
224 : Register const result_;
225 : XMMRegister const input_;
226 : UnwindingInfoWriter* const unwinding_info_writer_;
227 : };
228 :
229 :
230 0 : class OutOfLineRecordWrite final : public OutOfLineCode {
231 : public:
232 : OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand,
233 : Register value, Register scratch0, Register scratch1,
234 : RecordWriteMode mode)
235 : : OutOfLineCode(gen),
236 : object_(object),
237 : operand_(operand),
238 : value_(value),
239 : scratch0_(scratch0),
240 : scratch1_(scratch1),
241 345680 : mode_(mode) {}
242 :
243 345680 : void Generate() final {
244 345680 : if (mode_ > RecordWriteMode::kValueIsPointer) {
245 1670844 : __ JumpIfSmi(value_, exit());
246 : }
247 : __ CheckPageFlag(value_, scratch0_,
248 : MemoryChunk::kPointersToHereAreInterestingMask, zero,
249 691360 : exit());
250 : RememberedSetAction const remembered_set_action =
251 : mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
252 345680 : : OMIT_REMEMBERED_SET;
253 : SaveFPRegsMode const save_fp_mode =
254 691360 : frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
255 : RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
256 345680 : remembered_set_action, save_fp_mode);
257 345680 : __ leap(scratch1_, operand_);
258 345680 : __ CallStub(&stub);
259 345680 : }
260 :
261 : private:
262 : Register const object_;
263 : Operand const operand_;
264 : Register const value_;
265 : Register const scratch0_;
266 : Register const scratch1_;
267 : RecordWriteMode const mode_;
268 : };
269 :
270 0 : class WasmOutOfLineTrap final : public OutOfLineCode {
271 : public:
272 : WasmOutOfLineTrap(CodeGenerator* gen, int pc, bool frame_elided,
273 : int32_t position)
274 : : OutOfLineCode(gen),
275 : gen_(gen),
276 : pc_(pc),
277 : frame_elided_(frame_elided),
278 1650 : position_(position) {}
279 :
280 : // TODO(eholk): Refactor this method to take the code generator as a
281 : // parameter.
282 1650 : void Generate() final {
283 8798 : __ RecordProtectedInstructionLanding(pc_);
284 :
285 1650 : if (frame_elided_) {
286 548 : __ EnterFrame(StackFrame::WASM_COMPILED);
287 : }
288 :
289 : wasm::TrapReason trap_id = wasm::kTrapMemOutOfBounds;
290 1650 : int trap_reason = wasm::WasmOpcodes::TrapReasonToMessageId(trap_id);
291 1650 : __ Push(Smi::FromInt(trap_reason));
292 : // TODO(eholk): use AssembleSourcePosition instead of passing in position_
293 : // as a parameter. See AssembleArchTrap as an example. Consider sharing code
294 : // with AssembleArchTrap.
295 3300 : __ Push(Smi::FromInt(position_));
296 : __ Move(rsi, Smi::kZero);
297 1650 : __ CallRuntime(Runtime::kThrowWasmError);
298 :
299 : ReferenceMap* reference_map =
300 1650 : new (gen_->code()->zone()) ReferenceMap(gen_->code()->zone());
301 : gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
302 1650 : Safepoint::kNoLazyDeopt);
303 1650 : }
304 :
305 : private:
306 : CodeGenerator* gen_;
307 : int pc_;
308 : bool frame_elided_;
309 : int32_t position_;
310 : };
311 :
312 4924485 : void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
313 : InstructionCode opcode, size_t input_count,
314 : X64OperandConverter& i, int pc) {
315 : const X64MemoryProtection protection =
316 4922835 : static_cast<X64MemoryProtection>(MiscField::decode(opcode));
317 4922835 : if (protection == X64MemoryProtection::kProtected) {
318 1650 : const bool frame_elided = !codegen->frame_access_state()->has_frame();
319 1650 : const int32_t position = i.InputInt32(input_count - 1);
320 : new (zone) WasmOutOfLineTrap(codegen, pc, frame_elided, position);
321 : }
322 4922835 : }
323 : } // namespace
324 :
325 :
326 : #define ASSEMBLE_UNOP(asm_instr) \
327 : do { \
328 : if (instr->Output()->IsRegister()) { \
329 : __ asm_instr(i.OutputRegister()); \
330 : } else { \
331 : __ asm_instr(i.OutputOperand()); \
332 : } \
333 : } while (0)
334 :
335 : #define ASSEMBLE_BINOP(asm_instr) \
336 : do { \
337 : if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
338 : size_t index = 1; \
339 : Operand right = i.MemoryOperand(&index); \
340 : __ asm_instr(i.InputRegister(0), right); \
341 : } else { \
342 : if (HasImmediateInput(instr, 1)) { \
343 : if (instr->InputAt(0)->IsRegister()) { \
344 : __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
345 : } else { \
346 : __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
347 : } \
348 : } else { \
349 : if (instr->InputAt(1)->IsRegister()) { \
350 : __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
351 : } else { \
352 : __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
353 : } \
354 : } \
355 : } \
356 : } while (0)
357 :
358 : #define ASSEMBLE_COMPARE(asm_instr) \
359 : do { \
360 : if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
361 : size_t index = 0; \
362 : Operand left = i.MemoryOperand(&index); \
363 : if (HasImmediateInput(instr, index)) { \
364 : __ asm_instr(left, i.InputImmediate(index)); \
365 : } else { \
366 : __ asm_instr(left, i.InputRegister(index)); \
367 : } \
368 : } else { \
369 : if (HasImmediateInput(instr, 1)) { \
370 : if (instr->InputAt(0)->IsRegister()) { \
371 : __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
372 : } else { \
373 : __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
374 : } \
375 : } else { \
376 : if (instr->InputAt(1)->IsRegister()) { \
377 : __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
378 : } else { \
379 : __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
380 : } \
381 : } \
382 : } \
383 : } while (0)
384 :
385 : #define ASSEMBLE_MULT(asm_instr) \
386 : do { \
387 : if (HasImmediateInput(instr, 1)) { \
388 : if (instr->InputAt(0)->IsRegister()) { \
389 : __ asm_instr(i.OutputRegister(), i.InputRegister(0), \
390 : i.InputImmediate(1)); \
391 : } else { \
392 : __ asm_instr(i.OutputRegister(), i.InputOperand(0), \
393 : i.InputImmediate(1)); \
394 : } \
395 : } else { \
396 : if (instr->InputAt(1)->IsRegister()) { \
397 : __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
398 : } else { \
399 : __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \
400 : } \
401 : } \
402 : } while (0)
403 :
404 :
405 : #define ASSEMBLE_SHIFT(asm_instr, width) \
406 : do { \
407 : if (HasImmediateInput(instr, 1)) { \
408 : if (instr->Output()->IsRegister()) { \
409 : __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
410 : } else { \
411 : __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \
412 : } \
413 : } else { \
414 : if (instr->Output()->IsRegister()) { \
415 : __ asm_instr##_cl(i.OutputRegister()); \
416 : } else { \
417 : __ asm_instr##_cl(i.OutputOperand()); \
418 : } \
419 : } \
420 : } while (0)
421 :
422 :
423 : #define ASSEMBLE_MOVX(asm_instr) \
424 : do { \
425 : if (instr->addressing_mode() != kMode_None) { \
426 : __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \
427 : } else if (instr->InputAt(0)->IsRegister()) { \
428 : __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
429 : } else { \
430 : __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \
431 : } \
432 : } while (0)
433 :
434 : #define ASSEMBLE_SSE_BINOP(asm_instr) \
435 : do { \
436 : if (instr->InputAt(1)->IsFPRegister()) { \
437 : __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
438 : } else { \
439 : __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \
440 : } \
441 : } while (0)
442 :
443 : #define ASSEMBLE_SSE_UNOP(asm_instr) \
444 : do { \
445 : if (instr->InputAt(0)->IsFPRegister()) { \
446 : __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
447 : } else { \
448 : __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \
449 : } \
450 : } while (0)
451 :
452 : #define ASSEMBLE_AVX_BINOP(asm_instr) \
453 : do { \
454 : CpuFeatureScope avx_scope(masm(), AVX); \
455 : if (instr->InputAt(1)->IsFPRegister()) { \
456 : __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
457 : i.InputDoubleRegister(1)); \
458 : } else { \
459 : __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
460 : i.InputOperand(1)); \
461 : } \
462 : } while (0)
463 :
464 : #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN) \
465 : do { \
466 : auto result = i.OutputDoubleRegister(); \
467 : auto buffer = i.InputRegister(0); \
468 : auto index1 = i.InputRegister(1); \
469 : auto index2 = i.InputUint32(2); \
470 : OutOfLineCode* ool; \
471 : if (instr->InputAt(3)->IsRegister()) { \
472 : auto length = i.InputRegister(3); \
473 : DCHECK_EQ(0u, index2); \
474 : __ cmpl(index1, length); \
475 : ool = new (zone()) OutOfLineLoadNaN(this, result); \
476 : } else { \
477 : auto length = i.InputUint32(3); \
478 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
479 : DCHECK_LE(index2, length); \
480 : __ cmpl(index1, Immediate(length - index2, rmode)); \
481 : class OutOfLineLoadFloat final : public OutOfLineCode { \
482 : public: \
483 : OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \
484 : Register buffer, Register index1, int32_t index2, \
485 : int32_t length, RelocInfo::Mode rmode) \
486 : : OutOfLineCode(gen), \
487 : result_(result), \
488 : buffer_(buffer), \
489 : index1_(index1), \
490 : index2_(index2), \
491 : length_(length), \
492 : rmode_(rmode) {} \
493 : \
494 : void Generate() final { \
495 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
496 : __ Pcmpeqd(result_, result_); \
497 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
498 : __ j(above_equal, exit()); \
499 : __ asm_instr(result_, \
500 : Operand(buffer_, kScratchRegister, times_1, 0)); \
501 : } \
502 : \
503 : private: \
504 : XMMRegister const result_; \
505 : Register const buffer_; \
506 : Register const index1_; \
507 : int32_t const index2_; \
508 : int32_t const length_; \
509 : RelocInfo::Mode rmode_; \
510 : }; \
511 : ool = new (zone()) OutOfLineLoadFloat(this, result, buffer, index1, \
512 : index2, length, rmode); \
513 : } \
514 : __ j(above_equal, ool->entry()); \
515 : __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
516 : __ bind(ool->exit()); \
517 : } while (false)
518 :
519 : #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
520 : do { \
521 : auto result = i.OutputRegister(); \
522 : auto buffer = i.InputRegister(0); \
523 : auto index1 = i.InputRegister(1); \
524 : auto index2 = i.InputUint32(2); \
525 : OutOfLineCode* ool; \
526 : if (instr->InputAt(3)->IsRegister()) { \
527 : auto length = i.InputRegister(3); \
528 : DCHECK_EQ(0u, index2); \
529 : __ cmpl(index1, length); \
530 : ool = new (zone()) OutOfLineLoadZero(this, result); \
531 : } else { \
532 : auto length = i.InputUint32(3); \
533 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
534 : DCHECK_LE(index2, length); \
535 : __ cmpl(index1, Immediate(length - index2, rmode)); \
536 : class OutOfLineLoadInteger final : public OutOfLineCode { \
537 : public: \
538 : OutOfLineLoadInteger(CodeGenerator* gen, Register result, \
539 : Register buffer, Register index1, int32_t index2, \
540 : int32_t length, RelocInfo::Mode rmode) \
541 : : OutOfLineCode(gen), \
542 : result_(result), \
543 : buffer_(buffer), \
544 : index1_(index1), \
545 : index2_(index2), \
546 : length_(length), \
547 : rmode_(rmode) {} \
548 : \
549 : void Generate() final { \
550 : Label oob; \
551 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
552 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
553 : __ j(above_equal, &oob, Label::kNear); \
554 : __ asm_instr(result_, \
555 : Operand(buffer_, kScratchRegister, times_1, 0)); \
556 : __ jmp(exit()); \
557 : __ bind(&oob); \
558 : __ xorl(result_, result_); \
559 : } \
560 : \
561 : private: \
562 : Register const result_; \
563 : Register const buffer_; \
564 : Register const index1_; \
565 : int32_t const index2_; \
566 : int32_t const length_; \
567 : RelocInfo::Mode const rmode_; \
568 : }; \
569 : ool = new (zone()) OutOfLineLoadInteger(this, result, buffer, index1, \
570 : index2, length, rmode); \
571 : } \
572 : __ j(above_equal, ool->entry()); \
573 : __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
574 : __ bind(ool->exit()); \
575 : } while (false)
576 :
577 : #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \
578 : do { \
579 : auto buffer = i.InputRegister(0); \
580 : auto index1 = i.InputRegister(1); \
581 : auto index2 = i.InputUint32(2); \
582 : auto value = i.InputDoubleRegister(4); \
583 : if (instr->InputAt(3)->IsRegister()) { \
584 : auto length = i.InputRegister(3); \
585 : DCHECK_EQ(0u, index2); \
586 : Label done; \
587 : __ cmpl(index1, length); \
588 : __ j(above_equal, &done, Label::kNear); \
589 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
590 : __ bind(&done); \
591 : } else { \
592 : auto length = i.InputUint32(3); \
593 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
594 : DCHECK_LE(index2, length); \
595 : __ cmpl(index1, Immediate(length - index2, rmode)); \
596 : class OutOfLineStoreFloat final : public OutOfLineCode { \
597 : public: \
598 : OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \
599 : Register index1, int32_t index2, int32_t length, \
600 : XMMRegister value, RelocInfo::Mode rmode) \
601 : : OutOfLineCode(gen), \
602 : buffer_(buffer), \
603 : index1_(index1), \
604 : index2_(index2), \
605 : length_(length), \
606 : value_(value), \
607 : rmode_(rmode) {} \
608 : \
609 : void Generate() final { \
610 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
611 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
612 : __ j(above_equal, exit()); \
613 : __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
614 : value_); \
615 : } \
616 : \
617 : private: \
618 : Register const buffer_; \
619 : Register const index1_; \
620 : int32_t const index2_; \
621 : int32_t const length_; \
622 : XMMRegister const value_; \
623 : RelocInfo::Mode rmode_; \
624 : }; \
625 : auto ool = new (zone()) OutOfLineStoreFloat( \
626 : this, buffer, index1, index2, length, value, rmode); \
627 : __ j(above_equal, ool->entry()); \
628 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
629 : __ bind(ool->exit()); \
630 : } \
631 : } while (false)
632 :
633 : #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \
634 : do { \
635 : auto buffer = i.InputRegister(0); \
636 : auto index1 = i.InputRegister(1); \
637 : auto index2 = i.InputUint32(2); \
638 : if (instr->InputAt(3)->IsRegister()) { \
639 : auto length = i.InputRegister(3); \
640 : DCHECK_EQ(0u, index2); \
641 : Label done; \
642 : __ cmpl(index1, length); \
643 : __ j(above_equal, &done, Label::kNear); \
644 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
645 : __ bind(&done); \
646 : } else { \
647 : auto length = i.InputUint32(3); \
648 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
649 : DCHECK_LE(index2, length); \
650 : __ cmpl(index1, Immediate(length - index2, rmode)); \
651 : class OutOfLineStoreInteger final : public OutOfLineCode { \
652 : public: \
653 : OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \
654 : Register index1, int32_t index2, int32_t length, \
655 : Value value, RelocInfo::Mode rmode) \
656 : : OutOfLineCode(gen), \
657 : buffer_(buffer), \
658 : index1_(index1), \
659 : index2_(index2), \
660 : length_(length), \
661 : value_(value), \
662 : rmode_(rmode) {} \
663 : \
664 : void Generate() final { \
665 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
666 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
667 : __ j(above_equal, exit()); \
668 : __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
669 : value_); \
670 : } \
671 : \
672 : private: \
673 : Register const buffer_; \
674 : Register const index1_; \
675 : int32_t const index2_; \
676 : int32_t const length_; \
677 : Value const value_; \
678 : RelocInfo::Mode rmode_; \
679 : }; \
680 : auto ool = new (zone()) OutOfLineStoreInteger( \
681 : this, buffer, index1, index2, length, value, rmode); \
682 : __ j(above_equal, ool->entry()); \
683 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
684 : __ bind(ool->exit()); \
685 : } \
686 : } while (false)
687 :
688 : #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
689 : do { \
690 : if (instr->InputAt(4)->IsRegister()) { \
691 : Register value = i.InputRegister(4); \
692 : ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \
693 : } else { \
694 : Immediate value = i.InputImmediate(4); \
695 : ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
696 : } \
697 : } while (false)
698 :
699 : #define ASSEMBLE_IEEE754_BINOP(name) \
700 : do { \
701 : __ PrepareCallCFunction(2); \
702 : __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
703 : 2); \
704 : } while (false)
705 :
706 : #define ASSEMBLE_IEEE754_UNOP(name) \
707 : do { \
708 : __ PrepareCallCFunction(1); \
709 : __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
710 : 1); \
711 : } while (false)
712 :
713 : #define ASSEMBLE_ATOMIC_BINOP(bin_inst, mov_inst, cmpxchg_inst) \
714 : do { \
715 : Label binop; \
716 : __ bind(&binop); \
717 : __ mov_inst(rax, i.MemoryOperand(1)); \
718 : __ movl(i.TempRegister(0), rax); \
719 : __ bin_inst(i.TempRegister(0), i.InputRegister(0)); \
720 : __ lock(); \
721 : __ cmpxchg_inst(i.MemoryOperand(1), i.TempRegister(0)); \
722 : __ j(not_equal, &binop); \
723 : } while (false)
724 :
725 869523 : void CodeGenerator::AssembleDeconstructFrame() {
726 1739046 : unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
727 869523 : __ movq(rsp, rbp);
728 869523 : __ popq(rbp);
729 869523 : }
730 :
731 298452 : void CodeGenerator::AssemblePrepareTailCall() {
732 149226 : if (frame_access_state()->has_frame()) {
733 31366 : __ movq(rbp, MemOperand(rbp, 0));
734 : }
735 : frame_access_state()->SetFrameAccessToSP();
736 149226 : }
737 :
738 1041 : void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
739 : Register scratch1,
740 : Register scratch2,
741 : Register scratch3) {
742 : DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
743 : Label done;
744 :
745 : // Check if current frame is an arguments adaptor frame.
746 : __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset),
747 2082 : Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
748 1041 : __ j(not_equal, &done, Label::kNear);
749 :
750 : // Load arguments count from current arguments adaptor frame (note, it
751 : // does not include receiver).
752 1041 : Register caller_args_count_reg = scratch1;
753 : __ SmiToInteger32(
754 : caller_args_count_reg,
755 1041 : Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset));
756 :
757 : ParameterCount callee_args_count(args_reg);
758 : __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
759 1041 : scratch3, ReturnAddressState::kOnStack);
760 1041 : __ bind(&done);
761 1041 : }
762 :
763 : namespace {
764 :
765 340283 : void AdjustStackPointerForTailCall(Assembler* assembler,
766 : FrameAccessState* state,
767 : int new_slot_above_sp,
768 : bool allow_shrinkage = true) {
769 : int current_sp_offset = state->GetSPToFPSlotCount() +
770 340283 : StandardFrameConstants::kFixedSlotCountAboveFp;
771 340283 : int stack_slot_delta = new_slot_above_sp - current_sp_offset;
772 340283 : if (stack_slot_delta > 0) {
773 100 : assembler->subq(rsp, Immediate(stack_slot_delta * kPointerSize));
774 : state->IncreaseSPDelta(stack_slot_delta);
775 340233 : } else if (allow_shrinkage && stack_slot_delta < 0) {
776 30936 : assembler->addq(rsp, Immediate(-stack_slot_delta * kPointerSize));
777 : state->IncreaseSPDelta(stack_slot_delta);
778 : }
779 340283 : }
780 :
781 : } // namespace
782 :
783 149226 : void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
784 232888 : int first_unused_stack_slot) {
785 : CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush);
786 : ZoneVector<MoveOperands*> pushes(zone());
787 149226 : GetPushCompatibleMoves(instr, flags, &pushes);
788 :
789 158837 : if (!pushes.empty() &&
790 19222 : (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
791 : first_unused_stack_slot)) {
792 : X64OperandConverter g(this, instr);
793 61053 : for (auto move : pushes) {
794 : LocationOperand destination_location(
795 : LocationOperand::cast(move->destination()));
796 41831 : InstructionOperand source(move->source());
797 41831 : AdjustStackPointerForTailCall(masm(), frame_access_state(),
798 41831 : destination_location.index());
799 41831 : if (source.IsStackSlot()) {
800 : LocationOperand source_location(LocationOperand::cast(source));
801 9611 : __ Push(g.SlotToOperand(source_location.index()));
802 32220 : } else if (source.IsRegister()) {
803 : LocationOperand source_location(LocationOperand::cast(source));
804 32220 : __ Push(source_location.GetRegister());
805 0 : } else if (source.IsImmediate()) {
806 0 : __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
807 : } else {
808 : // Pushes of non-scalar data types is not supported.
809 0 : UNIMPLEMENTED();
810 : }
811 : frame_access_state()->IncreaseSPDelta(1);
812 : move->Eliminate();
813 : }
814 : }
815 149226 : AdjustStackPointerForTailCall(masm(), frame_access_state(),
816 149226 : first_unused_stack_slot, false);
817 149226 : }
818 :
819 149226 : void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
820 149226 : int first_unused_stack_slot) {
821 149226 : AdjustStackPointerForTailCall(masm(), frame_access_state(),
822 149226 : first_unused_stack_slot);
823 149226 : }
824 :
825 : // Assembles an instruction after register allocation, producing machine code.
826 38922581 : CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
827 51256074 : Instruction* instr) {
828 : X64OperandConverter i(this, instr);
829 : InstructionCode opcode = instr->opcode();
830 38922581 : ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
831 38922581 : switch (arch_opcode) {
832 : case kArchCallCodeObject: {
833 3228606 : EnsureSpaceForLazyDeopt();
834 3228605 : if (HasImmediateInput(instr, 0)) {
835 3225591 : Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
836 3225591 : __ Call(code, RelocInfo::CODE_TARGET);
837 : } else {
838 6030 : Register reg = i.InputRegister(0);
839 3015 : __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
840 3015 : __ call(reg);
841 : }
842 3228606 : RecordCallPosition(instr);
843 : frame_access_state()->ClearSPDelta();
844 : break;
845 : }
846 : case kArchTailCallCodeObjectFromJSFunction:
847 : case kArchTailCallCodeObject: {
848 125313 : if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
849 : AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
850 : i.TempRegister(0), i.TempRegister(1),
851 864 : i.TempRegister(2));
852 : }
853 125313 : if (HasImmediateInput(instr, 0)) {
854 124625 : Handle<Code> code = Handle<Code>::cast(i.InputHeapObject(0));
855 124625 : __ jmp(code, RelocInfo::CODE_TARGET);
856 : } else {
857 1376 : Register reg = i.InputRegister(0);
858 688 : __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
859 688 : __ jmp(reg);
860 : }
861 : unwinding_info_writer_.MarkBlockWillExit();
862 : frame_access_state()->ClearSPDelta();
863 125313 : frame_access_state()->SetFrameAccessToDefault();
864 125313 : break;
865 : }
866 : case kArchTailCallAddress: {
867 23736 : CHECK(!HasImmediateInput(instr, 0));
868 23736 : Register reg = i.InputRegister(0);
869 23736 : __ jmp(reg);
870 : unwinding_info_writer_.MarkBlockWillExit();
871 : frame_access_state()->ClearSPDelta();
872 23736 : frame_access_state()->SetFrameAccessToDefault();
873 : break;
874 : }
875 : case kArchCallJSFunction: {
876 150311 : EnsureSpaceForLazyDeopt();
877 150311 : Register func = i.InputRegister(0);
878 150311 : if (FLAG_debug_code) {
879 : // Check the function's context matches the context argument.
880 16 : __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
881 16 : __ Assert(equal, kWrongFunctionContext);
882 : }
883 150311 : __ Call(FieldOperand(func, JSFunction::kCodeEntryOffset));
884 : frame_access_state()->ClearSPDelta();
885 150311 : RecordCallPosition(instr);
886 : break;
887 : }
888 : case kArchTailCallJSFunctionFromJSFunction: {
889 : Register func = i.InputRegister(0);
890 177 : if (FLAG_debug_code) {
891 : // Check the function's context matches the context argument.
892 0 : __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
893 0 : __ Assert(equal, kWrongFunctionContext);
894 : }
895 : AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
896 : i.TempRegister(0), i.TempRegister(1),
897 354 : i.TempRegister(2));
898 177 : __ jmp(FieldOperand(func, JSFunction::kCodeEntryOffset));
899 : frame_access_state()->ClearSPDelta();
900 177 : frame_access_state()->SetFrameAccessToDefault();
901 : break;
902 : }
903 : case kArchPrepareCallCFunction: {
904 : // Frame alignment requires using FP-relative frame addressing.
905 : frame_access_state()->SetFrameAccessToFP();
906 : int const num_parameters = MiscField::decode(instr->opcode());
907 2084 : __ PrepareCallCFunction(num_parameters);
908 2084 : break;
909 : }
910 : case kArchPrepareTailCall:
911 149226 : AssemblePrepareTailCall();
912 149226 : break;
913 : case kArchCallCFunction: {
914 : int const num_parameters = MiscField::decode(instr->opcode());
915 2084 : if (HasImmediateInput(instr, 0)) {
916 1624 : ExternalReference ref = i.InputExternalReference(0);
917 1624 : __ CallCFunction(ref, num_parameters);
918 : } else {
919 460 : Register func = i.InputRegister(0);
920 460 : __ CallCFunction(func, num_parameters);
921 : }
922 2084 : frame_access_state()->SetFrameAccessToDefault();
923 : frame_access_state()->ClearSPDelta();
924 : break;
925 : }
926 : case kArchJmp:
927 3267552 : AssembleArchJump(i.InputRpo(0));
928 3267553 : break;
929 : case kArchLookupSwitch:
930 16755 : AssembleArchLookupSwitch(instr);
931 16755 : break;
932 : case kArchTableSwitch:
933 6072 : AssembleArchTableSwitch(instr);
934 6072 : break;
935 : case kArchComment: {
936 : Address comment_string = i.InputExternalReference(0).address();
937 0 : __ RecordComment(reinterpret_cast<const char*>(comment_string));
938 0 : break;
939 : }
940 : case kArchDebugBreak:
941 9087 : __ int3();
942 9087 : break;
943 : case kArchNop:
944 : case kArchThrowTerminator:
945 : // don't emit code for nops.
946 : break;
947 : case kArchDeoptimize: {
948 : int deopt_state_id =
949 15564 : BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
950 : CodeGenResult result =
951 15564 : AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
952 15564 : if (result != kSuccess) return result;
953 : break;
954 : }
955 : case kArchRet:
956 1087724 : AssembleReturn(instr->InputAt(0));
957 1087723 : break;
958 : case kArchStackPointer:
959 43 : __ movq(i.OutputRegister(), rsp);
960 : break;
961 : case kArchFramePointer:
962 4760 : __ movq(i.OutputRegister(), rbp);
963 : break;
964 : case kArchParentFramePointer:
965 53041 : if (frame_access_state()->has_frame()) {
966 38847 : __ movq(i.OutputRegister(), Operand(rbp, 0));
967 : } else {
968 40092 : __ movq(i.OutputRegister(), rbp);
969 : }
970 : break;
971 : case kArchTruncateDoubleToI: {
972 : auto result = i.OutputRegister();
973 : auto input = i.InputDoubleRegister(0);
974 : auto ool = new (zone()) OutOfLineTruncateDoubleToI(
975 70260 : this, result, input, &unwinding_info_writer_);
976 : // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The
977 : // use of Cvttsd2siq requires the movl below to avoid sign extension.
978 70260 : __ Cvttsd2siq(result, input);
979 70260 : __ cmpq(result, Immediate(1));
980 70260 : __ j(overflow, ool->entry());
981 70260 : __ bind(ool->exit());
982 : __ movl(result, result);
983 : break;
984 : }
985 : case kArchStoreWithWriteBarrier: {
986 : RecordWriteMode mode =
987 : static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
988 : Register object = i.InputRegister(0);
989 345680 : size_t index = 0;
990 345680 : Operand operand = i.MemoryOperand(&index);
991 345680 : Register value = i.InputRegister(index);
992 : Register scratch0 = i.TempRegister(0);
993 : Register scratch1 = i.TempRegister(1);
994 : auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value,
995 691360 : scratch0, scratch1, mode);
996 345680 : __ movp(operand, value);
997 : __ CheckPageFlag(object, scratch0,
998 : MemoryChunk::kPointersFromHereAreInterestingMask,
999 345680 : not_zero, ool->entry());
1000 345680 : __ bind(ool->exit());
1001 : break;
1002 : }
1003 : case kArchStackSlot: {
1004 : FrameOffset offset =
1005 2836 : frame_access_state()->GetFrameOffset(i.InputInt32(0));
1006 : Register base;
1007 1418 : if (offset.from_stack_pointer()) {
1008 0 : base = rsp;
1009 : } else {
1010 1418 : base = rbp;
1011 : }
1012 4254 : __ leaq(i.OutputRegister(), Operand(base, offset.offset()));
1013 : break;
1014 : }
1015 : case kIeee754Float64Acos:
1016 50 : ASSEMBLE_IEEE754_UNOP(acos);
1017 50 : break;
1018 : case kIeee754Float64Acosh:
1019 49 : ASSEMBLE_IEEE754_UNOP(acosh);
1020 49 : break;
1021 : case kIeee754Float64Asin:
1022 50 : ASSEMBLE_IEEE754_UNOP(asin);
1023 50 : break;
1024 : case kIeee754Float64Asinh:
1025 49 : ASSEMBLE_IEEE754_UNOP(asinh);
1026 49 : break;
1027 : case kIeee754Float64Atan:
1028 66 : ASSEMBLE_IEEE754_UNOP(atan);
1029 66 : break;
1030 : case kIeee754Float64Atanh:
1031 49 : ASSEMBLE_IEEE754_UNOP(atanh);
1032 49 : break;
1033 : case kIeee754Float64Atan2:
1034 73 : ASSEMBLE_IEEE754_BINOP(atan2);
1035 73 : break;
1036 : case kIeee754Float64Cbrt:
1037 49 : ASSEMBLE_IEEE754_UNOP(cbrt);
1038 49 : break;
1039 : case kIeee754Float64Cos:
1040 293 : ASSEMBLE_IEEE754_UNOP(cos);
1041 293 : break;
1042 : case kIeee754Float64Cosh:
1043 58 : ASSEMBLE_IEEE754_UNOP(cosh);
1044 58 : break;
1045 : case kIeee754Float64Exp:
1046 66 : ASSEMBLE_IEEE754_UNOP(exp);
1047 66 : break;
1048 : case kIeee754Float64Expm1:
1049 49 : ASSEMBLE_IEEE754_UNOP(expm1);
1050 49 : break;
1051 : case kIeee754Float64Log:
1052 168 : ASSEMBLE_IEEE754_UNOP(log);
1053 168 : break;
1054 : case kIeee754Float64Log1p:
1055 49 : ASSEMBLE_IEEE754_UNOP(log1p);
1056 49 : break;
1057 : case kIeee754Float64Log2:
1058 49 : ASSEMBLE_IEEE754_UNOP(log2);
1059 49 : break;
1060 : case kIeee754Float64Log10:
1061 49 : ASSEMBLE_IEEE754_UNOP(log10);
1062 49 : break;
1063 : case kIeee754Float64Pow: {
1064 : // TODO(bmeurer): Improve integration of the stub.
1065 105 : __ Movsd(xmm2, xmm0);
1066 105 : MathPowStub stub(isolate(), MathPowStub::DOUBLE);
1067 105 : __ CallStub(&stub);
1068 105 : __ Movsd(xmm0, xmm3);
1069 : break;
1070 : }
1071 : case kIeee754Float64Sin:
1072 293 : ASSEMBLE_IEEE754_UNOP(sin);
1073 293 : break;
1074 : case kIeee754Float64Sinh:
1075 58 : ASSEMBLE_IEEE754_UNOP(sinh);
1076 58 : break;
1077 : case kIeee754Float64Tan:
1078 66 : ASSEMBLE_IEEE754_UNOP(tan);
1079 66 : break;
1080 : case kIeee754Float64Tanh:
1081 58 : ASSEMBLE_IEEE754_UNOP(tanh);
1082 58 : break;
1083 : case kX64Add32:
1084 296210 : ASSEMBLE_BINOP(addl);
1085 : break;
1086 : case kX64Add:
1087 240376 : ASSEMBLE_BINOP(addq);
1088 : break;
1089 : case kX64Sub32:
1090 268198 : ASSEMBLE_BINOP(subl);
1091 : break;
1092 : case kX64Sub:
1093 196402 : ASSEMBLE_BINOP(subq);
1094 : break;
1095 : case kX64And32:
1096 388266 : ASSEMBLE_BINOP(andl);
1097 : break;
1098 : case kX64And:
1099 658037 : ASSEMBLE_BINOP(andq);
1100 : break;
1101 : case kX64Cmp8:
1102 444944 : ASSEMBLE_COMPARE(cmpb);
1103 : break;
1104 : case kX64Cmp16:
1105 87 : ASSEMBLE_COMPARE(cmpw);
1106 : break;
1107 : case kX64Cmp32:
1108 2744441 : ASSEMBLE_COMPARE(cmpl);
1109 : break;
1110 : case kX64Cmp:
1111 6517852 : ASSEMBLE_COMPARE(cmpq);
1112 : break;
1113 : case kX64Test8:
1114 351317 : ASSEMBLE_COMPARE(testb);
1115 : break;
1116 : case kX64Test16:
1117 8344 : ASSEMBLE_COMPARE(testw);
1118 : break;
1119 : case kX64Test32:
1120 122255 : ASSEMBLE_COMPARE(testl);
1121 : break;
1122 : case kX64Test:
1123 2073968 : ASSEMBLE_COMPARE(testq);
1124 : break;
1125 : case kX64Imul32:
1126 189986 : ASSEMBLE_MULT(imull);
1127 : break;
1128 : case kX64Imul:
1129 16387 : ASSEMBLE_MULT(imulq);
1130 : break;
1131 : case kX64ImulHigh32:
1132 10242 : if (instr->InputAt(1)->IsRegister()) {
1133 5121 : __ imull(i.InputRegister(1));
1134 : } else {
1135 0 : __ imull(i.InputOperand(1));
1136 : }
1137 : break;
1138 : case kX64UmulHigh32:
1139 1708 : if (instr->InputAt(1)->IsRegister()) {
1140 854 : __ mull(i.InputRegister(1));
1141 : } else {
1142 0 : __ mull(i.InputOperand(1));
1143 : }
1144 : break;
1145 : case kX64Idiv32:
1146 2755 : __ cdq();
1147 2755 : __ idivl(i.InputRegister(1));
1148 : break;
1149 : case kX64Idiv:
1150 894 : __ cqo();
1151 894 : __ idivq(i.InputRegister(1));
1152 : break;
1153 : case kX64Udiv32:
1154 1084 : __ xorl(rdx, rdx);
1155 1084 : __ divl(i.InputRegister(1));
1156 : break;
1157 : case kX64Udiv:
1158 824 : __ xorq(rdx, rdx);
1159 824 : __ divq(i.InputRegister(1));
1160 : break;
1161 : case kX64Not:
1162 132 : ASSEMBLE_UNOP(notq);
1163 : break;
1164 : case kX64Not32:
1165 7262 : ASSEMBLE_UNOP(notl);
1166 : break;
1167 : case kX64Neg:
1168 470 : ASSEMBLE_UNOP(negq);
1169 : break;
1170 : case kX64Neg32:
1171 10314 : ASSEMBLE_UNOP(negl);
1172 : break;
1173 : case kX64Or32:
1174 61038 : ASSEMBLE_BINOP(orl);
1175 : break;
1176 : case kX64Or:
1177 58234 : ASSEMBLE_BINOP(orq);
1178 : break;
1179 : case kX64Xor32:
1180 41210 : ASSEMBLE_BINOP(xorl);
1181 : break;
1182 : case kX64Xor:
1183 1662 : ASSEMBLE_BINOP(xorq);
1184 : break;
1185 : case kX64Shl32:
1186 35394 : ASSEMBLE_SHIFT(shll, 5);
1187 : break;
1188 : case kX64Shl:
1189 1101651 : ASSEMBLE_SHIFT(shlq, 6);
1190 : break;
1191 : case kX64Shr32:
1192 139505 : ASSEMBLE_SHIFT(shrl, 5);
1193 : break;
1194 : case kX64Shr:
1195 1296795 : ASSEMBLE_SHIFT(shrq, 6);
1196 : break;
1197 : case kX64Sar32:
1198 48719 : ASSEMBLE_SHIFT(sarl, 5);
1199 : break;
1200 : case kX64Sar:
1201 89735 : ASSEMBLE_SHIFT(sarq, 6);
1202 : break;
1203 : case kX64Ror32:
1204 3664 : ASSEMBLE_SHIFT(rorl, 5);
1205 : break;
1206 : case kX64Ror:
1207 238 : ASSEMBLE_SHIFT(rorq, 6);
1208 : break;
1209 : case kX64Lzcnt:
1210 56 : if (instr->InputAt(0)->IsRegister()) {
1211 28 : __ Lzcntq(i.OutputRegister(), i.InputRegister(0));
1212 : } else {
1213 0 : __ Lzcntq(i.OutputRegister(), i.InputOperand(0));
1214 : }
1215 : break;
1216 : case kX64Lzcnt32:
1217 1694 : if (instr->InputAt(0)->IsRegister()) {
1218 802 : __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
1219 : } else {
1220 90 : __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
1221 : }
1222 : break;
1223 : case kX64Tzcnt:
1224 26 : if (instr->InputAt(0)->IsRegister()) {
1225 13 : __ Tzcntq(i.OutputRegister(), i.InputRegister(0));
1226 : } else {
1227 0 : __ Tzcntq(i.OutputRegister(), i.InputOperand(0));
1228 : }
1229 : break;
1230 : case kX64Tzcnt32:
1231 936 : if (instr->InputAt(0)->IsRegister()) {
1232 468 : __ Tzcntl(i.OutputRegister(), i.InputRegister(0));
1233 : } else {
1234 0 : __ Tzcntl(i.OutputRegister(), i.InputOperand(0));
1235 : }
1236 : break;
1237 : case kX64Popcnt:
1238 70 : if (instr->InputAt(0)->IsRegister()) {
1239 35 : __ Popcntq(i.OutputRegister(), i.InputRegister(0));
1240 : } else {
1241 0 : __ Popcntq(i.OutputRegister(), i.InputOperand(0));
1242 : }
1243 : break;
1244 : case kX64Popcnt32:
1245 196 : if (instr->InputAt(0)->IsRegister()) {
1246 98 : __ Popcntl(i.OutputRegister(), i.InputRegister(0));
1247 : } else {
1248 0 : __ Popcntl(i.OutputRegister(), i.InputOperand(0));
1249 : }
1250 : break;
1251 : case kSSEFloat32Cmp:
1252 0 : ASSEMBLE_SSE_BINOP(Ucomiss);
1253 : break;
1254 : case kSSEFloat32Add:
1255 0 : ASSEMBLE_SSE_BINOP(addss);
1256 : break;
1257 : case kSSEFloat32Sub:
1258 0 : ASSEMBLE_SSE_BINOP(subss);
1259 : break;
1260 : case kSSEFloat32Mul:
1261 0 : ASSEMBLE_SSE_BINOP(mulss);
1262 : break;
1263 : case kSSEFloat32Div:
1264 0 : ASSEMBLE_SSE_BINOP(divss);
1265 : // Don't delete this mov. It may improve performance on some CPUs,
1266 : // when there is a (v)mulss depending on the result.
1267 0 : __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1268 0 : break;
1269 : case kSSEFloat32Abs: {
1270 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1271 0 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1272 0 : __ psrlq(kScratchDoubleReg, 33);
1273 0 : __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
1274 0 : break;
1275 : }
1276 : case kSSEFloat32Neg: {
1277 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1278 0 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1279 0 : __ psllq(kScratchDoubleReg, 31);
1280 0 : __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
1281 0 : break;
1282 : }
1283 : case kSSEFloat32Sqrt:
1284 90 : ASSEMBLE_SSE_UNOP(sqrtss);
1285 : break;
1286 : case kSSEFloat32ToFloat64:
1287 33264 : ASSEMBLE_SSE_UNOP(Cvtss2sd);
1288 : break;
1289 : case kSSEFloat32Round: {
1290 : CpuFeatureScope sse_scope(masm(), SSE4_1);
1291 : RoundingMode const mode =
1292 : static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1293 239 : __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1294 : break;
1295 : }
1296 : case kSSEFloat32ToInt32:
1297 264 : if (instr->InputAt(0)->IsFPRegister()) {
1298 132 : __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0));
1299 : } else {
1300 0 : __ Cvttss2si(i.OutputRegister(), i.InputOperand(0));
1301 : }
1302 : break;
1303 : case kSSEFloat32ToUint32: {
1304 26 : if (instr->InputAt(0)->IsFPRegister()) {
1305 13 : __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1306 : } else {
1307 0 : __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1308 : }
1309 : break;
1310 : }
1311 : case kSSEFloat64Cmp:
1312 1932 : ASSEMBLE_SSE_BINOP(Ucomisd);
1313 : break;
1314 : case kSSEFloat64Add:
1315 333 : ASSEMBLE_SSE_BINOP(addsd);
1316 : break;
1317 : case kSSEFloat64Sub:
1318 498 : ASSEMBLE_SSE_BINOP(subsd);
1319 : break;
1320 : case kSSEFloat64Mul:
1321 51 : ASSEMBLE_SSE_BINOP(mulsd);
1322 : break;
1323 : case kSSEFloat64Div:
1324 15 : ASSEMBLE_SSE_BINOP(divsd);
1325 : // Don't delete this mov. It may improve performance on some CPUs,
1326 : // when there is a (v)mulsd depending on the result.
1327 10 : __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1328 5 : break;
1329 : case kSSEFloat64Mod: {
1330 1014 : __ subq(rsp, Immediate(kDoubleSize));
1331 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1332 2028 : kDoubleSize);
1333 : // Move values to st(0) and st(1).
1334 2028 : __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
1335 1014 : __ fld_d(Operand(rsp, 0));
1336 2028 : __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
1337 1014 : __ fld_d(Operand(rsp, 0));
1338 : // Loop while fprem isn't done.
1339 : Label mod_loop;
1340 1014 : __ bind(&mod_loop);
1341 : // This instructions traps on all kinds inputs, but we are assuming the
1342 : // floating point control word is set to ignore them all.
1343 1014 : __ fprem();
1344 : // The following 2 instruction implicitly use rax.
1345 1014 : __ fnstsw_ax();
1346 1014 : if (CpuFeatures::IsSupported(SAHF)) {
1347 : CpuFeatureScope sahf_scope(masm(), SAHF);
1348 1005 : __ sahf();
1349 : } else {
1350 : __ shrl(rax, Immediate(8));
1351 9 : __ andl(rax, Immediate(0xFF));
1352 9 : __ pushq(rax);
1353 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1354 18 : kPointerSize);
1355 9 : __ popfq();
1356 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1357 18 : -kPointerSize);
1358 : }
1359 1014 : __ j(parity_even, &mod_loop);
1360 : // Move output to stack and clean up.
1361 1014 : __ fstp(1);
1362 1014 : __ fstp_d(Operand(rsp, 0));
1363 2028 : __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
1364 1014 : __ addq(rsp, Immediate(kDoubleSize));
1365 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1366 2028 : -kDoubleSize);
1367 : break;
1368 : }
1369 : case kSSEFloat32Max: {
1370 : Label compare_nan, compare_swap, done_compare;
1371 58 : if (instr->InputAt(1)->IsFPRegister()) {
1372 29 : __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1373 : } else {
1374 0 : __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1375 : }
1376 : auto ool =
1377 29 : new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1378 29 : __ j(parity_even, ool->entry());
1379 29 : __ j(above, &done_compare, Label::kNear);
1380 29 : __ j(below, &compare_swap, Label::kNear);
1381 58 : __ Movmskps(kScratchRegister, i.InputDoubleRegister(0));
1382 : __ testl(kScratchRegister, Immediate(1));
1383 29 : __ j(zero, &done_compare, Label::kNear);
1384 29 : __ bind(&compare_swap);
1385 58 : if (instr->InputAt(1)->IsFPRegister()) {
1386 58 : __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1387 : } else {
1388 0 : __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1389 : }
1390 29 : __ bind(&done_compare);
1391 29 : __ bind(ool->exit());
1392 : break;
1393 : }
1394 : case kSSEFloat32Min: {
1395 : Label compare_swap, done_compare;
1396 58 : if (instr->InputAt(1)->IsFPRegister()) {
1397 29 : __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1398 : } else {
1399 0 : __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1400 : }
1401 : auto ool =
1402 29 : new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1403 29 : __ j(parity_even, ool->entry());
1404 29 : __ j(below, &done_compare, Label::kNear);
1405 29 : __ j(above, &compare_swap, Label::kNear);
1406 58 : if (instr->InputAt(1)->IsFPRegister()) {
1407 58 : __ Movmskps(kScratchRegister, i.InputDoubleRegister(1));
1408 : } else {
1409 0 : __ Movss(kScratchDoubleReg, i.InputOperand(1));
1410 0 : __ Movmskps(kScratchRegister, kScratchDoubleReg);
1411 : }
1412 : __ testl(kScratchRegister, Immediate(1));
1413 29 : __ j(zero, &done_compare, Label::kNear);
1414 29 : __ bind(&compare_swap);
1415 58 : if (instr->InputAt(1)->IsFPRegister()) {
1416 58 : __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1417 : } else {
1418 0 : __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1419 : }
1420 29 : __ bind(&done_compare);
1421 29 : __ bind(ool->exit());
1422 : break;
1423 : }
1424 : case kSSEFloat64Max: {
1425 : Label compare_nan, compare_swap, done_compare;
1426 194 : if (instr->InputAt(1)->IsFPRegister()) {
1427 97 : __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1428 : } else {
1429 0 : __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1430 : }
1431 : auto ool =
1432 97 : new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1433 97 : __ j(parity_even, ool->entry());
1434 97 : __ j(above, &done_compare, Label::kNear);
1435 97 : __ j(below, &compare_swap, Label::kNear);
1436 194 : __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0));
1437 : __ testl(kScratchRegister, Immediate(1));
1438 97 : __ j(zero, &done_compare, Label::kNear);
1439 97 : __ bind(&compare_swap);
1440 194 : if (instr->InputAt(1)->IsFPRegister()) {
1441 194 : __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1442 : } else {
1443 0 : __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1444 : }
1445 97 : __ bind(&done_compare);
1446 97 : __ bind(ool->exit());
1447 : break;
1448 : }
1449 : case kSSEFloat64Min: {
1450 : Label compare_swap, done_compare;
1451 306 : if (instr->InputAt(1)->IsFPRegister()) {
1452 153 : __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1453 : } else {
1454 0 : __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1455 : }
1456 : auto ool =
1457 153 : new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1458 153 : __ j(parity_even, ool->entry());
1459 153 : __ j(below, &done_compare, Label::kNear);
1460 153 : __ j(above, &compare_swap, Label::kNear);
1461 306 : if (instr->InputAt(1)->IsFPRegister()) {
1462 306 : __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1));
1463 : } else {
1464 0 : __ Movsd(kScratchDoubleReg, i.InputOperand(1));
1465 0 : __ Movmskpd(kScratchRegister, kScratchDoubleReg);
1466 : }
1467 : __ testl(kScratchRegister, Immediate(1));
1468 153 : __ j(zero, &done_compare, Label::kNear);
1469 153 : __ bind(&compare_swap);
1470 306 : if (instr->InputAt(1)->IsFPRegister()) {
1471 306 : __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1472 : } else {
1473 0 : __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1474 : }
1475 153 : __ bind(&done_compare);
1476 153 : __ bind(ool->exit());
1477 : break;
1478 : }
1479 : case kSSEFloat64Abs: {
1480 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1481 3 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1482 3 : __ psrlq(kScratchDoubleReg, 1);
1483 6 : __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1484 3 : break;
1485 : }
1486 : case kSSEFloat64Neg: {
1487 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1488 78 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1489 78 : __ psllq(kScratchDoubleReg, 63);
1490 156 : __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1491 78 : break;
1492 : }
1493 : case kSSEFloat64Sqrt:
1494 774 : ASSEMBLE_SSE_UNOP(Sqrtsd);
1495 : break;
1496 : case kSSEFloat64Round: {
1497 : CpuFeatureScope sse_scope(masm(), SSE4_1);
1498 : RoundingMode const mode =
1499 : static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1500 18490 : __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1501 : break;
1502 : }
1503 : case kSSEFloat64ToFloat32:
1504 23808 : ASSEMBLE_SSE_UNOP(Cvtsd2ss);
1505 : break;
1506 : case kSSEFloat64ToInt32:
1507 164542 : if (instr->InputAt(0)->IsFPRegister()) {
1508 73935 : __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
1509 : } else {
1510 16672 : __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0));
1511 : }
1512 : break;
1513 : case kSSEFloat64ToUint32: {
1514 774 : if (instr->InputAt(0)->IsFPRegister()) {
1515 387 : __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1516 : } else {
1517 0 : __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1518 : }
1519 774 : if (MiscField::decode(instr->opcode())) {
1520 760 : __ AssertZeroExtended(i.OutputRegister());
1521 : }
1522 : break;
1523 : }
1524 : case kSSEFloat32ToInt64:
1525 52 : if (instr->InputAt(0)->IsFPRegister()) {
1526 26 : __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1527 : } else {
1528 0 : __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1529 : }
1530 26 : if (instr->OutputCount() > 1) {
1531 40 : __ Set(i.OutputRegister(1), 1);
1532 : Label done;
1533 : Label fail;
1534 : __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN));
1535 40 : if (instr->InputAt(0)->IsFPRegister()) {
1536 40 : __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0));
1537 : } else {
1538 0 : __ Ucomiss(kScratchDoubleReg, i.InputOperand(0));
1539 : }
1540 : // If the input is NaN, then the conversion fails.
1541 20 : __ j(parity_even, &fail);
1542 : // If the input is INT64_MIN, then the conversion succeeds.
1543 20 : __ j(equal, &done);
1544 40 : __ cmpq(i.OutputRegister(0), Immediate(1));
1545 : // If the conversion results in INT64_MIN, but the input was not
1546 : // INT64_MIN, then the conversion fails.
1547 20 : __ j(no_overflow, &done);
1548 20 : __ bind(&fail);
1549 40 : __ Set(i.OutputRegister(1), 0);
1550 20 : __ bind(&done);
1551 : }
1552 : break;
1553 : case kSSEFloat64ToInt64:
1554 98 : if (instr->InputAt(0)->IsFPRegister()) {
1555 49 : __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0));
1556 : } else {
1557 0 : __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0));
1558 : }
1559 49 : if (instr->OutputCount() > 1) {
1560 86 : __ Set(i.OutputRegister(1), 1);
1561 : Label done;
1562 : Label fail;
1563 : __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN));
1564 86 : if (instr->InputAt(0)->IsFPRegister()) {
1565 86 : __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0));
1566 : } else {
1567 0 : __ Ucomisd(kScratchDoubleReg, i.InputOperand(0));
1568 : }
1569 : // If the input is NaN, then the conversion fails.
1570 43 : __ j(parity_even, &fail);
1571 : // If the input is INT64_MIN, then the conversion succeeds.
1572 43 : __ j(equal, &done);
1573 86 : __ cmpq(i.OutputRegister(0), Immediate(1));
1574 : // If the conversion results in INT64_MIN, but the input was not
1575 : // INT64_MIN, then the conversion fails.
1576 43 : __ j(no_overflow, &done);
1577 43 : __ bind(&fail);
1578 86 : __ Set(i.OutputRegister(1), 0);
1579 43 : __ bind(&done);
1580 : }
1581 : break;
1582 : case kSSEFloat32ToUint64: {
1583 : Label done;
1584 : Label success;
1585 26 : if (instr->OutputCount() > 1) {
1586 20 : __ Set(i.OutputRegister(1), 0);
1587 : }
1588 : // There does not exist a Float32ToUint64 instruction, so we have to use
1589 : // the Float32ToInt64 instruction.
1590 52 : if (instr->InputAt(0)->IsFPRegister()) {
1591 52 : __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1592 : } else {
1593 0 : __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1594 : }
1595 : // Check if the result of the Float32ToInt64 conversion is positive, we
1596 : // are already done.
1597 52 : __ testq(i.OutputRegister(), i.OutputRegister());
1598 26 : __ j(positive, &success);
1599 : // The result of the first conversion was negative, which means that the
1600 : // input value was not within the positive int64 range. We subtract 2^64
1601 : // and convert it again to see if it is within the uint64 range.
1602 26 : __ Move(kScratchDoubleReg, -9223372036854775808.0f);
1603 52 : if (instr->InputAt(0)->IsFPRegister()) {
1604 52 : __ addss(kScratchDoubleReg, i.InputDoubleRegister(0));
1605 : } else {
1606 0 : __ addss(kScratchDoubleReg, i.InputOperand(0));
1607 : }
1608 52 : __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg);
1609 26 : __ testq(i.OutputRegister(), i.OutputRegister());
1610 : // The only possible negative value here is 0x80000000000000000, which is
1611 : // used on x64 to indicate an integer overflow.
1612 26 : __ j(negative, &done);
1613 : // The input value is within uint64 range and the second conversion worked
1614 : // successfully, but we still have to undo the subtraction we did
1615 : // earlier.
1616 26 : __ Set(kScratchRegister, 0x8000000000000000);
1617 52 : __ orq(i.OutputRegister(), kScratchRegister);
1618 26 : __ bind(&success);
1619 26 : if (instr->OutputCount() > 1) {
1620 40 : __ Set(i.OutputRegister(1), 1);
1621 : }
1622 26 : __ bind(&done);
1623 : break;
1624 : }
1625 : case kSSEFloat64ToUint64: {
1626 : Label done;
1627 : Label success;
1628 155 : if (instr->OutputCount() > 1) {
1629 20 : __ Set(i.OutputRegister(1), 0);
1630 : }
1631 : // There does not exist a Float64ToUint64 instruction, so we have to use
1632 : // the Float64ToInt64 instruction.
1633 310 : if (instr->InputAt(0)->IsFPRegister()) {
1634 310 : __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1635 : } else {
1636 0 : __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1637 : }
1638 : // Check if the result of the Float64ToInt64 conversion is positive, we
1639 : // are already done.
1640 310 : __ testq(i.OutputRegister(), i.OutputRegister());
1641 155 : __ j(positive, &success);
1642 : // The result of the first conversion was negative, which means that the
1643 : // input value was not within the positive int64 range. We subtract 2^64
1644 : // and convert it again to see if it is within the uint64 range.
1645 155 : __ Move(kScratchDoubleReg, -9223372036854775808.0);
1646 310 : if (instr->InputAt(0)->IsFPRegister()) {
1647 310 : __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0));
1648 : } else {
1649 0 : __ addsd(kScratchDoubleReg, i.InputOperand(0));
1650 : }
1651 310 : __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg);
1652 155 : __ testq(i.OutputRegister(), i.OutputRegister());
1653 : // The only possible negative value here is 0x80000000000000000, which is
1654 : // used on x64 to indicate an integer overflow.
1655 155 : __ j(negative, &done);
1656 : // The input value is within uint64 range and the second conversion worked
1657 : // successfully, but we still have to undo the subtraction we did
1658 : // earlier.
1659 155 : __ Set(kScratchRegister, 0x8000000000000000);
1660 310 : __ orq(i.OutputRegister(), kScratchRegister);
1661 155 : __ bind(&success);
1662 155 : if (instr->OutputCount() > 1) {
1663 40 : __ Set(i.OutputRegister(1), 1);
1664 : }
1665 155 : __ bind(&done);
1666 : break;
1667 : }
1668 : case kSSEInt32ToFloat64:
1669 498344 : if (instr->InputAt(0)->IsRegister()) {
1670 245122 : __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1671 : } else {
1672 8100 : __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1673 : }
1674 : break;
1675 : case kSSEInt32ToFloat32:
1676 618 : if (instr->InputAt(0)->IsRegister()) {
1677 309 : __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1678 : } else {
1679 0 : __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1680 : }
1681 : break;
1682 : case kSSEInt64ToFloat32:
1683 26 : if (instr->InputAt(0)->IsRegister()) {
1684 13 : __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1685 : } else {
1686 0 : __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1687 : }
1688 : break;
1689 : case kSSEInt64ToFloat64:
1690 1518 : if (instr->InputAt(0)->IsRegister()) {
1691 759 : __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1692 : } else {
1693 0 : __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1694 : }
1695 : break;
1696 : case kSSEUint64ToFloat32:
1697 26 : if (instr->InputAt(0)->IsRegister()) {
1698 13 : __ movq(kScratchRegister, i.InputRegister(0));
1699 : } else {
1700 0 : __ movq(kScratchRegister, i.InputOperand(0));
1701 : }
1702 : __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister,
1703 26 : i.TempRegister(0));
1704 13 : break;
1705 : case kSSEUint64ToFloat64:
1706 26 : if (instr->InputAt(0)->IsRegister()) {
1707 13 : __ movq(kScratchRegister, i.InputRegister(0));
1708 : } else {
1709 0 : __ movq(kScratchRegister, i.InputOperand(0));
1710 : }
1711 : __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister,
1712 26 : i.TempRegister(0));
1713 13 : break;
1714 : case kSSEUint32ToFloat64:
1715 4002 : if (instr->InputAt(0)->IsRegister()) {
1716 1368 : __ movl(kScratchRegister, i.InputRegister(0));
1717 : } else {
1718 633 : __ movl(kScratchRegister, i.InputOperand(0));
1719 : }
1720 4002 : __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
1721 2001 : break;
1722 : case kSSEUint32ToFloat32:
1723 74 : if (instr->InputAt(0)->IsRegister()) {
1724 37 : __ movl(kScratchRegister, i.InputRegister(0));
1725 : } else {
1726 0 : __ movl(kScratchRegister, i.InputOperand(0));
1727 : }
1728 74 : __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister);
1729 37 : break;
1730 : case kSSEFloat64ExtractLowWord32:
1731 12 : if (instr->InputAt(0)->IsFPStackSlot()) {
1732 0 : __ movl(i.OutputRegister(), i.InputOperand(0));
1733 : } else {
1734 6 : __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
1735 : }
1736 : break;
1737 : case kSSEFloat64ExtractHighWord32:
1738 87714 : if (instr->InputAt(0)->IsFPStackSlot()) {
1739 63786 : __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
1740 : } else {
1741 11964 : __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
1742 : }
1743 : break;
1744 : case kSSEFloat64InsertLowWord32:
1745 12 : if (instr->InputAt(1)->IsRegister()) {
1746 6 : __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
1747 : } else {
1748 0 : __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
1749 : }
1750 : break;
1751 : case kSSEFloat64InsertHighWord32:
1752 12 : if (instr->InputAt(1)->IsRegister()) {
1753 6 : __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
1754 : } else {
1755 0 : __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
1756 : }
1757 : break;
1758 : case kSSEFloat64LoadLowWord32:
1759 0 : if (instr->InputAt(0)->IsRegister()) {
1760 0 : __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
1761 : } else {
1762 0 : __ Movd(i.OutputDoubleRegister(), i.InputOperand(0));
1763 : }
1764 : break;
1765 : case kAVXFloat32Cmp: {
1766 : CpuFeatureScope avx_scope(masm(), AVX);
1767 2380 : if (instr->InputAt(1)->IsFPRegister()) {
1768 1172 : __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1769 : } else {
1770 36 : __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1771 : }
1772 : break;
1773 : }
1774 : case kAVXFloat32Add:
1775 354 : ASSEMBLE_AVX_BINOP(vaddss);
1776 : break;
1777 : case kAVXFloat32Sub:
1778 4368 : ASSEMBLE_AVX_BINOP(vsubss);
1779 : break;
1780 : case kAVXFloat32Mul:
1781 141 : ASSEMBLE_AVX_BINOP(vmulss);
1782 : break;
1783 : case kAVXFloat32Div:
1784 132 : ASSEMBLE_AVX_BINOP(vdivss);
1785 : // Don't delete this mov. It may improve performance on some CPUs,
1786 : // when there is a (v)mulss depending on the result.
1787 88 : __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1788 44 : break;
1789 : case kAVXFloat64Cmp: {
1790 : CpuFeatureScope avx_scope(masm(), AVX);
1791 401940 : if (instr->InputAt(1)->IsFPRegister()) {
1792 191629 : __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1793 : } else {
1794 28023 : __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1795 : }
1796 : break;
1797 : }
1798 : case kAVXFloat64Add:
1799 163488 : ASSEMBLE_AVX_BINOP(vaddsd);
1800 : break;
1801 : case kAVXFloat64Sub:
1802 75417 : ASSEMBLE_AVX_BINOP(vsubsd);
1803 : break;
1804 : case kAVXFloat64Mul:
1805 46617 : ASSEMBLE_AVX_BINOP(vmulsd);
1806 : break;
1807 : case kAVXFloat64Div:
1808 36705 : ASSEMBLE_AVX_BINOP(vdivsd);
1809 : // Don't delete this mov. It may improve performance on some CPUs,
1810 : // when there is a (v)mulsd depending on the result.
1811 24470 : __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1812 12235 : break;
1813 : case kAVXFloat32Abs: {
1814 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1815 : CpuFeatureScope avx_scope(masm(), AVX);
1816 50 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1817 : __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33);
1818 100 : if (instr->InputAt(0)->IsFPRegister()) {
1819 : __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1820 50 : i.InputDoubleRegister(0));
1821 : } else {
1822 : __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1823 0 : i.InputOperand(0));
1824 : }
1825 : break;
1826 : }
1827 : case kAVXFloat32Neg: {
1828 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1829 : CpuFeatureScope avx_scope(masm(), AVX);
1830 28 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1831 : __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31);
1832 56 : if (instr->InputAt(0)->IsFPRegister()) {
1833 : __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1834 28 : i.InputDoubleRegister(0));
1835 : } else {
1836 : __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1837 0 : i.InputOperand(0));
1838 : }
1839 : break;
1840 : }
1841 : case kAVXFloat64Abs: {
1842 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1843 : CpuFeatureScope avx_scope(masm(), AVX);
1844 240 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1845 : __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1);
1846 480 : if (instr->InputAt(0)->IsFPRegister()) {
1847 : __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1848 240 : i.InputDoubleRegister(0));
1849 : } else {
1850 : __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1851 0 : i.InputOperand(0));
1852 : }
1853 : break;
1854 : }
1855 : case kAVXFloat64Neg: {
1856 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1857 : CpuFeatureScope avx_scope(masm(), AVX);
1858 9734 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1859 : __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63);
1860 19468 : if (instr->InputAt(0)->IsFPRegister()) {
1861 : __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1862 9657 : i.InputDoubleRegister(0));
1863 : } else {
1864 : __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1865 154 : i.InputOperand(0));
1866 : }
1867 : break;
1868 : }
1869 : case kSSEFloat64SilenceNaN:
1870 1757 : __ Xorpd(kScratchDoubleReg, kScratchDoubleReg);
1871 3514 : __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg);
1872 1757 : break;
1873 : case kX64Movsxbl:
1874 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1875 14672 : __ pc_offset());
1876 22438 : ASSEMBLE_MOVX(movsxbl);
1877 14672 : __ AssertZeroExtended(i.OutputRegister());
1878 7336 : break;
1879 : case kX64Movzxbl:
1880 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1881 183234 : __ pc_offset());
1882 276393 : ASSEMBLE_MOVX(movzxbl);
1883 183234 : __ AssertZeroExtended(i.OutputRegister());
1884 91617 : break;
1885 : case kX64Movsxbq:
1886 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1887 10860 : __ pc_offset());
1888 16290 : ASSEMBLE_MOVX(movsxbq);
1889 : break;
1890 : case kX64Movzxbq:
1891 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1892 172 : __ pc_offset());
1893 258 : ASSEMBLE_MOVX(movzxbq);
1894 172 : __ AssertZeroExtended(i.OutputRegister());
1895 86 : break;
1896 : case kX64Movb: {
1897 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1898 15330 : __ pc_offset());
1899 7665 : size_t index = 0;
1900 7665 : Operand operand = i.MemoryOperand(&index);
1901 15330 : if (HasImmediateInput(instr, index)) {
1902 2770 : __ movb(operand, Immediate(i.InputInt8(index)));
1903 : } else {
1904 12560 : __ movb(operand, i.InputRegister(index));
1905 : }
1906 : break;
1907 : }
1908 : case kX64Movsxwl:
1909 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1910 4086 : __ pc_offset());
1911 6427 : ASSEMBLE_MOVX(movsxwl);
1912 4086 : __ AssertZeroExtended(i.OutputRegister());
1913 2043 : break;
1914 : case kX64Movzxwl:
1915 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1916 31364 : __ pc_offset());
1917 51606 : ASSEMBLE_MOVX(movzxwl);
1918 31364 : __ AssertZeroExtended(i.OutputRegister());
1919 15682 : break;
1920 : case kX64Movsxwq:
1921 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1922 8194 : __ pc_offset());
1923 12291 : ASSEMBLE_MOVX(movsxwq);
1924 : break;
1925 : case kX64Movzxwq:
1926 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1927 0 : __ pc_offset());
1928 0 : ASSEMBLE_MOVX(movzxwq);
1929 0 : __ AssertZeroExtended(i.OutputRegister());
1930 0 : break;
1931 : case kX64Movw: {
1932 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1933 10858 : __ pc_offset());
1934 5429 : size_t index = 0;
1935 5429 : Operand operand = i.MemoryOperand(&index);
1936 10858 : if (HasImmediateInput(instr, index)) {
1937 478 : __ movw(operand, Immediate(i.InputInt16(index)));
1938 : } else {
1939 10380 : __ movw(operand, i.InputRegister(index));
1940 : }
1941 : break;
1942 : }
1943 : case kX64Movl:
1944 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1945 1040092 : __ pc_offset());
1946 520046 : if (instr->HasOutput()) {
1947 318410 : if (instr->addressing_mode() == kMode_None) {
1948 76424 : if (instr->InputAt(0)->IsRegister()) {
1949 74310 : __ movl(i.OutputRegister(), i.InputRegister(0));
1950 : } else {
1951 2114 : __ movl(i.OutputRegister(), i.InputOperand(0));
1952 : }
1953 : } else {
1954 560396 : __ movl(i.OutputRegister(), i.MemoryOperand());
1955 : }
1956 636820 : __ AssertZeroExtended(i.OutputRegister());
1957 : } else {
1958 201636 : size_t index = 0;
1959 201636 : Operand operand = i.MemoryOperand(&index);
1960 403272 : if (HasImmediateInput(instr, index)) {
1961 43076 : __ movl(operand, i.InputImmediate(index));
1962 : } else {
1963 317120 : __ movl(operand, i.InputRegister(index));
1964 : }
1965 : }
1966 : break;
1967 : case kX64Movsxlq:
1968 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1969 162884 : __ pc_offset());
1970 288199 : ASSEMBLE_MOVX(movsxlq);
1971 : break;
1972 : case kX64Movq:
1973 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1974 7571340 : __ pc_offset());
1975 3785670 : if (instr->HasOutput()) {
1976 4244484 : __ movq(i.OutputRegister(), i.MemoryOperand());
1977 : } else {
1978 1663429 : size_t index = 0;
1979 1663429 : Operand operand = i.MemoryOperand(&index);
1980 3326858 : if (HasImmediateInput(instr, index)) {
1981 144988 : __ movq(operand, i.InputImmediate(index));
1982 : } else {
1983 3036882 : __ movq(operand, i.InputRegister(index));
1984 : }
1985 : }
1986 : break;
1987 : case kX64Movss:
1988 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
1989 36760 : __ pc_offset());
1990 18380 : if (instr->HasOutput()) {
1991 21014 : __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
1992 : } else {
1993 7873 : size_t index = 0;
1994 7873 : Operand operand = i.MemoryOperand(&index);
1995 15746 : __ movss(operand, i.InputDoubleRegister(index));
1996 : }
1997 : break;
1998 : case kX64Movsd:
1999 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr->InputCount(), i,
2000 755828 : __ pc_offset());
2001 377914 : if (instr->HasOutput()) {
2002 575356 : __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand());
2003 : } else {
2004 90236 : size_t index = 0;
2005 90236 : Operand operand = i.MemoryOperand(&index);
2006 180472 : __ Movsd(operand, i.InputDoubleRegister(index));
2007 : }
2008 : break;
2009 : case kX64BitcastFI:
2010 228 : if (instr->InputAt(0)->IsFPStackSlot()) {
2011 0 : __ movl(i.OutputRegister(), i.InputOperand(0));
2012 : } else {
2013 114 : __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
2014 : }
2015 : break;
2016 : case kX64BitcastDL:
2017 514 : if (instr->InputAt(0)->IsFPStackSlot()) {
2018 0 : __ movq(i.OutputRegister(), i.InputOperand(0));
2019 : } else {
2020 257 : __ Movq(i.OutputRegister(), i.InputDoubleRegister(0));
2021 : }
2022 : break;
2023 : case kX64BitcastIF:
2024 104 : if (instr->InputAt(0)->IsRegister()) {
2025 52 : __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
2026 : } else {
2027 0 : __ movss(i.OutputDoubleRegister(), i.InputOperand(0));
2028 : }
2029 : break;
2030 : case kX64BitcastLD:
2031 262 : if (instr->InputAt(0)->IsRegister()) {
2032 131 : __ Movq(i.OutputDoubleRegister(), i.InputRegister(0));
2033 : } else {
2034 0 : __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0));
2035 : }
2036 : break;
2037 : case kX64Lea32: {
2038 : AddressingMode mode = AddressingModeField::decode(instr->opcode());
2039 : // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
2040 : // and addressing mode just happens to work out. The "addl"/"subl" forms
2041 : // in these cases are faster based on measurements.
2042 208214 : if (i.InputRegister(0).is(i.OutputRegister())) {
2043 90290 : if (mode == kMode_MRI) {
2044 67971 : int32_t constant_summand = i.InputInt32(1);
2045 67971 : if (constant_summand > 0) {
2046 82768 : __ addl(i.OutputRegister(), Immediate(constant_summand));
2047 26587 : } else if (constant_summand < 0) {
2048 79635 : __ subl(i.OutputRegister(), Immediate(-constant_summand));
2049 : }
2050 22319 : } else if (mode == kMode_MR1) {
2051 10971 : if (i.InputRegister(1).is(i.OutputRegister())) {
2052 521 : __ shll(i.OutputRegister(), Immediate(1));
2053 : } else {
2054 10450 : __ addl(i.OutputRegister(), i.InputRegister(1));
2055 : }
2056 11348 : } else if (mode == kMode_M2) {
2057 0 : __ shll(i.OutputRegister(), Immediate(1));
2058 11348 : } else if (mode == kMode_M4) {
2059 247 : __ shll(i.OutputRegister(), Immediate(2));
2060 11101 : } else if (mode == kMode_M8) {
2061 177 : __ shll(i.OutputRegister(), Immediate(3));
2062 : } else {
2063 21848 : __ leal(i.OutputRegister(), i.MemoryOperand());
2064 : }
2065 139365 : } else if (mode == kMode_MR1 &&
2066 : i.InputRegister(1).is(i.OutputRegister())) {
2067 14373 : __ addl(i.OutputRegister(), i.InputRegister(0));
2068 : } else {
2069 207102 : __ leal(i.OutputRegister(), i.MemoryOperand());
2070 : }
2071 416428 : __ AssertZeroExtended(i.OutputRegister());
2072 208214 : break;
2073 : }
2074 : case kX64Lea: {
2075 : AddressingMode mode = AddressingModeField::decode(instr->opcode());
2076 : // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation
2077 : // and addressing mode just happens to work out. The "addq"/"subq" forms
2078 : // in these cases are faster based on measurements.
2079 1010125 : if (i.InputRegister(0).is(i.OutputRegister())) {
2080 464070 : if (mode == kMode_MRI) {
2081 433909 : int32_t constant_summand = i.InputInt32(1);
2082 433909 : if (constant_summand > 0) {
2083 630516 : __ addq(i.OutputRegister(), Immediate(constant_summand));
2084 118651 : } else if (constant_summand < 0) {
2085 355863 : __ subq(i.OutputRegister(), Immediate(-constant_summand));
2086 : }
2087 30161 : } else if (mode == kMode_MR1) {
2088 15194 : if (i.InputRegister(1).is(i.OutputRegister())) {
2089 1591 : __ shlq(i.OutputRegister(), Immediate(1));
2090 : } else {
2091 13603 : __ addq(i.OutputRegister(), i.InputRegister(1));
2092 : }
2093 14967 : } else if (mode == kMode_M2) {
2094 0 : __ shlq(i.OutputRegister(), Immediate(1));
2095 14967 : } else if (mode == kMode_M4) {
2096 129 : __ shlq(i.OutputRegister(), Immediate(2));
2097 14838 : } else if (mode == kMode_M8) {
2098 1118 : __ shlq(i.OutputRegister(), Immediate(3));
2099 : } else {
2100 27440 : __ leaq(i.OutputRegister(), i.MemoryOperand());
2101 : }
2102 583867 : } else if (mode == kMode_MR1 &&
2103 : i.InputRegister(1).is(i.OutputRegister())) {
2104 13119 : __ addq(i.OutputRegister(), i.InputRegister(0));
2105 : } else {
2106 1065872 : __ leaq(i.OutputRegister(), i.MemoryOperand());
2107 : }
2108 : break;
2109 : }
2110 : case kX64Dec32:
2111 0 : __ decl(i.OutputRegister());
2112 : break;
2113 : case kX64Inc32:
2114 0 : __ incl(i.OutputRegister());
2115 : break;
2116 : case kX64Push:
2117 3671754 : if (AddressingModeField::decode(instr->opcode()) != kMode_None) {
2118 111410 : size_t index = 0;
2119 111410 : Operand operand = i.MemoryOperand(&index);
2120 111410 : __ pushq(operand);
2121 : frame_access_state()->IncreaseSPDelta(1);
2122 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2123 222820 : kPointerSize);
2124 3560344 : } else if (HasImmediateInput(instr, 0)) {
2125 238406 : __ pushq(i.InputImmediate(0));
2126 : frame_access_state()->IncreaseSPDelta(1);
2127 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2128 476812 : kPointerSize);
2129 3321938 : } else if (instr->InputAt(0)->IsRegister()) {
2130 2913864 : __ pushq(i.InputRegister(0));
2131 : frame_access_state()->IncreaseSPDelta(1);
2132 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2133 5827728 : kPointerSize);
2134 408074 : } else if (instr->InputAt(0)->IsFPRegister()) {
2135 : // TODO(titzer): use another machine instruction?
2136 12622 : __ subq(rsp, Immediate(kDoubleSize));
2137 : frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
2138 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2139 25244 : kDoubleSize);
2140 25244 : __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
2141 : } else {
2142 395452 : __ pushq(i.InputOperand(0));
2143 : frame_access_state()->IncreaseSPDelta(1);
2144 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2145 790904 : kPointerSize);
2146 : }
2147 : break;
2148 : case kX64Poke: {
2149 : int const slot = MiscField::decode(instr->opcode());
2150 1348 : if (HasImmediateInput(instr, 0)) {
2151 902 : __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
2152 : } else {
2153 1794 : __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
2154 : }
2155 : break;
2156 : }
2157 : case kX64I32x4Splat: {
2158 210 : XMMRegister dst = i.OutputSimd128Register();
2159 210 : __ movd(dst, i.InputRegister(0));
2160 210 : __ pshufd(dst, dst, 0x0);
2161 : break;
2162 : }
2163 : case kX64I32x4ExtractLane: {
2164 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2165 1008 : __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1));
2166 : break;
2167 : }
2168 : case kX64I32x4ReplaceLane: {
2169 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2170 84 : if (instr->InputAt(2)->IsRegister()) {
2171 : __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2),
2172 84 : i.InputInt8(1));
2173 : } else {
2174 0 : __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2175 : }
2176 : break;
2177 : }
2178 : case kX64I32x4Shl: {
2179 14 : __ pslld(i.OutputSimd128Register(), i.InputInt8(1));
2180 7 : break;
2181 : }
2182 : case kX64I32x4ShrS: {
2183 14 : __ psrad(i.OutputSimd128Register(), i.InputInt8(1));
2184 7 : break;
2185 : }
2186 : case kX64I32x4Add: {
2187 7 : __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2188 : break;
2189 : }
2190 : case kX64I32x4Sub: {
2191 7 : __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2192 : break;
2193 : }
2194 : case kX64I32x4Mul: {
2195 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2196 7 : __ pmulld(i.OutputSimd128Register(), i.InputSimd128Register(1));
2197 : break;
2198 : }
2199 : case kX64I32x4MinS: {
2200 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2201 7 : __ pminsd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2202 : break;
2203 : }
2204 : case kX64I32x4MaxS: {
2205 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2206 7 : __ pmaxsd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2207 : break;
2208 : }
2209 : case kX64I32x4Eq: {
2210 7 : __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2211 : break;
2212 : }
2213 : case kX64I32x4Ne: {
2214 14 : __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2215 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
2216 14 : __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
2217 : break;
2218 : }
2219 : case kX64I32x4ShrU: {
2220 14 : __ psrld(i.OutputSimd128Register(), i.InputInt8(1));
2221 7 : break;
2222 : }
2223 : case kX64I32x4MinU: {
2224 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2225 7 : __ pminud(i.OutputSimd128Register(), i.InputSimd128Register(1));
2226 : break;
2227 : }
2228 : case kX64I32x4MaxU: {
2229 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2230 7 : __ pmaxud(i.OutputSimd128Register(), i.InputSimd128Register(1));
2231 : break;
2232 : }
2233 : case kX64S128Zero: {
2234 0 : XMMRegister dst = i.OutputSimd128Register();
2235 0 : __ xorps(dst, dst);
2236 : break;
2237 : }
2238 : case kX64I16x8Splat: {
2239 266 : XMMRegister dst = i.OutputSimd128Register();
2240 266 : __ movd(dst, i.InputRegister(0));
2241 266 : __ pshuflw(dst, dst, 0x0);
2242 266 : __ pshufhw(dst, dst, 0x0);
2243 266 : __ pshufd(dst, dst, 0x0);
2244 : break;
2245 : }
2246 : case kX64I16x8ExtractLane: {
2247 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2248 1428 : Register dst = i.OutputRegister();
2249 2856 : __ pextrw(dst, i.InputSimd128Register(0), i.InputInt8(1));
2250 1428 : __ movsxwl(dst, dst);
2251 : break;
2252 : }
2253 : case kX64I16x8ReplaceLane: {
2254 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2255 140 : if (instr->InputAt(2)->IsRegister()) {
2256 : __ pinsrw(i.OutputSimd128Register(), i.InputRegister(2),
2257 140 : i.InputInt8(1));
2258 : } else {
2259 0 : __ pinsrw(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2260 : }
2261 : break;
2262 : }
2263 : case kX64I16x8Shl: {
2264 14 : __ psllw(i.OutputSimd128Register(), i.InputInt8(1));
2265 7 : break;
2266 : }
2267 : case kX64I16x8ShrS: {
2268 14 : __ psraw(i.OutputSimd128Register(), i.InputInt8(1));
2269 7 : break;
2270 : }
2271 : case kX64I16x8Add: {
2272 7 : __ paddw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2273 : break;
2274 : }
2275 : case kX64I16x8AddSaturateS: {
2276 7 : __ paddsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2277 : break;
2278 : }
2279 : case kX64I16x8Sub: {
2280 7 : __ psubw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2281 : break;
2282 : }
2283 : case kX64I16x8SubSaturateS: {
2284 7 : __ psubsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2285 : break;
2286 : }
2287 : case kX64I16x8Mul: {
2288 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2289 7 : __ pmullw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2290 : break;
2291 : }
2292 : case kX64I16x8MinS: {
2293 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2294 7 : __ pminsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2295 : break;
2296 : }
2297 : case kX64I16x8MaxS: {
2298 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2299 7 : __ pmaxsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2300 : break;
2301 : }
2302 : case kX64I16x8Eq: {
2303 7 : __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2304 : break;
2305 : }
2306 : case kX64I16x8Ne: {
2307 14 : __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2308 : __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg);
2309 14 : __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
2310 : break;
2311 : }
2312 : case kX64I16x8ShrU: {
2313 14 : __ psrlw(i.OutputSimd128Register(), i.InputInt8(1));
2314 7 : break;
2315 : }
2316 : case kX64I16x8AddSaturateU: {
2317 7 : __ paddusw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2318 : break;
2319 : }
2320 : case kX64I16x8SubSaturateU: {
2321 7 : __ psubusw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2322 : break;
2323 : }
2324 : case kX64I16x8MinU: {
2325 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2326 7 : __ pminuw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2327 : break;
2328 : }
2329 : case kX64I16x8MaxU: {
2330 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2331 7 : __ pmaxuw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2332 : break;
2333 : }
2334 : case kX64I8x16Splat: {
2335 : CpuFeatureScope sse_scope(masm(), SSSE3);
2336 : XMMRegister dst = i.OutputSimd128Register();
2337 231 : __ movd(dst, i.InputRegister(0));
2338 231 : __ xorps(kScratchDoubleReg, kScratchDoubleReg);
2339 : __ pshufb(dst, kScratchDoubleReg);
2340 : break;
2341 : }
2342 : case kX64I8x16ExtractLane: {
2343 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2344 3220 : Register dst = i.OutputRegister();
2345 6440 : __ pextrb(dst, i.InputSimd128Register(0), i.InputInt8(1));
2346 3220 : __ movsxbl(dst, dst);
2347 : break;
2348 : }
2349 : case kX64I8x16ReplaceLane: {
2350 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2351 252 : if (instr->InputAt(2)->IsRegister()) {
2352 : __ pinsrb(i.OutputSimd128Register(), i.InputRegister(2),
2353 252 : i.InputInt8(1));
2354 : } else {
2355 0 : __ pinsrb(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2356 : }
2357 : break;
2358 : }
2359 : case kX64I8x16Add: {
2360 7 : __ paddb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2361 : break;
2362 : }
2363 : case kX64I8x16AddSaturateS: {
2364 7 : __ paddsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2365 : break;
2366 : }
2367 : case kX64I8x16Sub: {
2368 7 : __ psubb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2369 : break;
2370 : }
2371 : case kX64I8x16SubSaturateS: {
2372 7 : __ psubsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2373 : break;
2374 : }
2375 : case kX64I8x16MinS: {
2376 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2377 7 : __ pminsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2378 : break;
2379 : }
2380 : case kX64I8x16MaxS: {
2381 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2382 7 : __ pmaxsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2383 : break;
2384 : }
2385 : case kX64I8x16Eq: {
2386 7 : __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2387 : break;
2388 : }
2389 : case kX64I8x16Ne: {
2390 14 : __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2391 : __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg);
2392 14 : __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
2393 : break;
2394 : }
2395 : case kX64I8x16AddSaturateU: {
2396 7 : __ paddusb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2397 : break;
2398 : }
2399 : case kX64I8x16SubSaturateU: {
2400 7 : __ psubusb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2401 : break;
2402 : }
2403 : case kX64I8x16MinU: {
2404 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2405 7 : __ pminub(i.OutputSimd128Register(), i.InputSimd128Register(1));
2406 : break;
2407 : }
2408 : case kX64I8x16MaxU: {
2409 : CpuFeatureScope sse_scope(masm(), SSE4_1);
2410 7 : __ pmaxub(i.OutputSimd128Register(), i.InputSimd128Register(1));
2411 : break;
2412 : }
2413 : case kX64S128Select: {
2414 : // Mask used here is stored in dst.
2415 63 : XMMRegister dst = i.OutputSimd128Register();
2416 63 : __ movaps(kScratchDoubleReg, i.InputSimd128Register(1));
2417 126 : __ xorps(kScratchDoubleReg, i.InputSimd128Register(2));
2418 63 : __ andps(dst, kScratchDoubleReg);
2419 126 : __ xorps(dst, i.InputSimd128Register(2));
2420 : break;
2421 : }
2422 : case kCheckedLoadInt8:
2423 36700 : ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
2424 3670 : break;
2425 : case kCheckedLoadUint8:
2426 35100 : ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
2427 2925 : break;
2428 : case kCheckedLoadInt16:
2429 12400 : ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
2430 1240 : break;
2431 : case kCheckedLoadUint16:
2432 9204 : ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
2433 767 : break;
2434 : case kCheckedLoadWord32:
2435 670528 : ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
2436 55966 : break;
2437 : case kCheckedLoadWord64:
2438 84 : ASSEMBLE_CHECKED_LOAD_INTEGER(movq);
2439 7 : break;
2440 : case kCheckedLoadFloat32:
2441 71813 : ASSEMBLE_CHECKED_LOAD_FLOAT(Movss, OutOfLineLoadFloat32NaN);
2442 9016 : break;
2443 : case kCheckedLoadFloat64:
2444 4477 : ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd, OutOfLineLoadFloat64NaN);
2445 599 : break;
2446 : case kCheckedStoreWord8:
2447 43709 : ASSEMBLE_CHECKED_STORE_INTEGER(movb);
2448 : break;
2449 : case kCheckedStoreWord16:
2450 20870 : ASSEMBLE_CHECKED_STORE_INTEGER(movw);
2451 : break;
2452 : case kCheckedStoreWord32:
2453 500110 : ASSEMBLE_CHECKED_STORE_INTEGER(movl);
2454 : break;
2455 : case kCheckedStoreWord64:
2456 84 : ASSEMBLE_CHECKED_STORE_INTEGER(movq);
2457 : break;
2458 : case kCheckedStoreFloat32:
2459 49104 : ASSEMBLE_CHECKED_STORE_FLOAT(Movss);
2460 : break;
2461 : case kCheckedStoreFloat64:
2462 4392 : ASSEMBLE_CHECKED_STORE_FLOAT(Movsd);
2463 : break;
2464 : case kX64StackCheck:
2465 519337 : __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
2466 519337 : break;
2467 : case kAtomicExchangeInt8: {
2468 172 : __ xchgb(i.InputRegister(0), i.MemoryOperand(1));
2469 172 : __ movsxbl(i.InputRegister(0), i.InputRegister(0));
2470 86 : break;
2471 : }
2472 : case kAtomicExchangeUint8: {
2473 86 : __ xchgb(i.InputRegister(0), i.MemoryOperand(1));
2474 43 : __ movzxbl(i.InputRegister(0), i.InputRegister(0));
2475 : break;
2476 : }
2477 : case kAtomicExchangeInt16: {
2478 172 : __ xchgw(i.InputRegister(0), i.MemoryOperand(1));
2479 172 : __ movsxwl(i.InputRegister(0), i.InputRegister(0));
2480 86 : break;
2481 : }
2482 : case kAtomicExchangeUint16: {
2483 86 : __ xchgw(i.InputRegister(0), i.MemoryOperand(1));
2484 43 : __ movzxwl(i.InputRegister(0), i.InputRegister(0));
2485 : break;
2486 : }
2487 : case kAtomicExchangeWord32: {
2488 258 : __ xchgl(i.InputRegister(0), i.MemoryOperand(1));
2489 : break;
2490 : }
2491 : case kAtomicCompareExchangeInt8: {
2492 43 : __ lock();
2493 86 : __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1));
2494 43 : __ movsxbl(rax, rax);
2495 43 : break;
2496 : }
2497 : case kAtomicCompareExchangeUint8: {
2498 43 : __ lock();
2499 86 : __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1));
2500 : __ movzxbl(rax, rax);
2501 : break;
2502 : }
2503 : case kAtomicCompareExchangeInt16: {
2504 43 : __ lock();
2505 86 : __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1));
2506 43 : __ movsxwl(rax, rax);
2507 43 : break;
2508 : }
2509 : case kAtomicCompareExchangeUint16: {
2510 43 : __ lock();
2511 86 : __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1));
2512 : __ movzxwl(rax, rax);
2513 : break;
2514 : }
2515 : case kAtomicCompareExchangeWord32: {
2516 86 : __ lock();
2517 86 : __ cmpxchgl(i.MemoryOperand(2), i.InputRegister(1));
2518 : break;
2519 : }
2520 : #define ATOMIC_BINOP_CASE(op, inst) \
2521 : case kAtomic##op##Int8: \
2522 : ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \
2523 : __ movsxbl(rax, rax); \
2524 : break; \
2525 : case kAtomic##op##Uint8: \
2526 : ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \
2527 : __ movzxbl(rax, rax); \
2528 : break; \
2529 : case kAtomic##op##Int16: \
2530 : ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \
2531 : __ movsxwl(rax, rax); \
2532 : break; \
2533 : case kAtomic##op##Uint16: \
2534 : ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \
2535 : __ movzxwl(rax, rax); \
2536 : break; \
2537 : case kAtomic##op##Word32: \
2538 : ASSEMBLE_ATOMIC_BINOP(inst, movl, cmpxchgl); \
2539 : break;
2540 1290 : ATOMIC_BINOP_CASE(Add, addl)
2541 1290 : ATOMIC_BINOP_CASE(Sub, subl)
2542 1290 : ATOMIC_BINOP_CASE(And, andl)
2543 1290 : ATOMIC_BINOP_CASE(Or, orl)
2544 1290 : ATOMIC_BINOP_CASE(Xor, xorl)
2545 : #undef ATOMIC_BINOP_CASE
2546 : case kAtomicLoadInt8:
2547 : case kAtomicLoadUint8:
2548 : case kAtomicLoadInt16:
2549 : case kAtomicLoadUint16:
2550 : case kAtomicLoadWord32:
2551 : case kAtomicStoreWord8:
2552 : case kAtomicStoreWord16:
2553 : case kAtomicStoreWord32:
2554 0 : UNREACHABLE(); // Won't be generated by instruction selector.
2555 : break;
2556 : }
2557 : return kSuccess;
2558 : } // NOLINT(readability/fn_size)
2559 :
2560 : namespace {
2561 :
2562 3783827 : Condition FlagsConditionToCondition(FlagsCondition condition) {
2563 3783827 : switch (condition) {
2564 : case kUnorderedEqual:
2565 : case kEqual:
2566 : return equal;
2567 : case kUnorderedNotEqual:
2568 : case kNotEqual:
2569 991789 : return not_equal;
2570 : case kSignedLessThan:
2571 72622 : return less;
2572 : case kSignedGreaterThanOrEqual:
2573 39951 : return greater_equal;
2574 : case kSignedLessThanOrEqual:
2575 26990 : return less_equal;
2576 : case kSignedGreaterThan:
2577 35373 : return greater;
2578 : case kUnsignedLessThan:
2579 48803 : return below;
2580 : case kUnsignedGreaterThanOrEqual:
2581 73862 : return above_equal;
2582 : case kUnsignedLessThanOrEqual:
2583 743768 : return below_equal;
2584 : case kUnsignedGreaterThan:
2585 37109 : return above;
2586 : case kOverflow:
2587 195402 : return overflow;
2588 : case kNotOverflow:
2589 903 : return no_overflow;
2590 : default:
2591 : break;
2592 : }
2593 0 : UNREACHABLE();
2594 : return no_condition;
2595 : }
2596 :
2597 : } // namespace
2598 :
2599 : // Assembles branches after this instruction.
2600 3550946 : void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2601 : Label::Distance flabel_distance =
2602 3550946 : branch->fallthru ? Label::kNear : Label::kFar;
2603 3550946 : Label* tlabel = branch->true_label;
2604 3550946 : Label* flabel = branch->false_label;
2605 3550946 : if (branch->condition == kUnorderedEqual) {
2606 40410 : __ j(parity_even, flabel, flabel_distance);
2607 3510536 : } else if (branch->condition == kUnorderedNotEqual) {
2608 58795 : __ j(parity_even, tlabel);
2609 : }
2610 3550946 : __ j(FlagsConditionToCondition(branch->condition), tlabel);
2611 :
2612 3550945 : if (!branch->fallthru) __ jmp(flabel, flabel_distance);
2613 3550945 : }
2614 :
2615 :
2616 5334404 : void CodeGenerator::AssembleArchJump(RpoNumber target) {
2617 5334404 : if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
2618 3284621 : }
2619 :
2620 16305 : void CodeGenerator::AssembleArchTrap(Instruction* instr,
2621 16305 : FlagsCondition condition) {
2622 0 : class OutOfLineTrap final : public OutOfLineCode {
2623 : public:
2624 : OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
2625 : : OutOfLineCode(gen),
2626 : frame_elided_(frame_elided),
2627 : instr_(instr),
2628 16305 : gen_(gen) {}
2629 :
2630 16305 : void Generate() final {
2631 32610 : X64OperandConverter i(gen_, instr_);
2632 :
2633 : Builtins::Name trap_id =
2634 16305 : static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
2635 27298 : bool old_has_frame = __ has_frame();
2636 16305 : if (frame_elided_) {
2637 : __ set_has_frame(true);
2638 10993 : __ EnterFrame(StackFrame::WASM_COMPILED);
2639 : }
2640 16305 : GenerateCallToTrap(trap_id);
2641 16305 : if (frame_elided_) {
2642 : __ set_has_frame(old_has_frame);
2643 : }
2644 16305 : }
2645 :
2646 : private:
2647 16305 : void GenerateCallToTrap(Builtins::Name trap_id) {
2648 16305 : if (trap_id == Builtins::builtin_count) {
2649 : // We cannot test calls to the runtime in cctest/test-run-wasm.
2650 : // Therefore we emit a call to C here instead of a call to the runtime.
2651 33801 : __ PrepareCallCFunction(0);
2652 : __ CallCFunction(
2653 : ExternalReference::wasm_call_trap_callback_for_testing(isolate()),
2654 11664 : 0);
2655 5832 : __ LeaveFrame(StackFrame::WASM_COMPILED);
2656 5832 : __ Ret();
2657 : } else {
2658 10473 : gen_->AssembleSourcePosition(instr_);
2659 : __ Call(handle(isolate()->builtins()->builtin(trap_id), isolate()),
2660 10473 : RelocInfo::CODE_TARGET);
2661 : ReferenceMap* reference_map =
2662 10473 : new (gen_->zone()) ReferenceMap(gen_->zone());
2663 : gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2664 10473 : Safepoint::kNoLazyDeopt);
2665 10473 : if (FLAG_debug_code) {
2666 0 : __ ud2();
2667 : }
2668 : }
2669 16305 : }
2670 :
2671 : bool frame_elided_;
2672 : Instruction* instr_;
2673 : CodeGenerator* gen_;
2674 : };
2675 16305 : bool frame_elided = !frame_access_state()->has_frame();
2676 : auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
2677 16305 : Label* tlabel = ool->entry();
2678 : Label end;
2679 16305 : if (condition == kUnorderedEqual) {
2680 0 : __ j(parity_even, &end);
2681 16305 : } else if (condition == kUnorderedNotEqual) {
2682 401 : __ j(parity_even, tlabel);
2683 : }
2684 16305 : __ j(FlagsConditionToCondition(condition), tlabel);
2685 16305 : __ bind(&end);
2686 16305 : }
2687 :
2688 : // Assembles boolean materializations after this instruction.
2689 433154 : void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2690 : FlagsCondition condition) {
2691 : X64OperandConverter i(this, instr);
2692 : Label done;
2693 :
2694 : // Materialize a full 64-bit 1 or 0 value. The result register is always the
2695 : // last output of the instruction.
2696 : Label check;
2697 : DCHECK_NE(0u, instr->OutputCount());
2698 216577 : Register reg = i.OutputRegister(instr->OutputCount() - 1);
2699 216577 : if (condition == kUnorderedEqual) {
2700 5946 : __ j(parity_odd, &check, Label::kNear);
2701 : __ movl(reg, Immediate(0));
2702 5946 : __ jmp(&done, Label::kNear);
2703 210631 : } else if (condition == kUnorderedNotEqual) {
2704 2388 : __ j(parity_odd, &check, Label::kNear);
2705 : __ movl(reg, Immediate(1));
2706 2388 : __ jmp(&done, Label::kNear);
2707 : }
2708 216577 : __ bind(&check);
2709 216577 : __ setcc(FlagsConditionToCondition(condition), reg);
2710 : __ movzxbl(reg, reg);
2711 216577 : __ bind(&done);
2712 216577 : }
2713 :
2714 :
2715 121278 : void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2716 : X64OperandConverter i(this, instr);
2717 16755 : Register input = i.InputRegister(0);
2718 121278 : for (size_t index = 2; index < instr->InputCount(); index += 2) {
2719 87768 : __ cmpl(input, Immediate(i.InputInt32(index + 0)));
2720 87768 : __ j(equal, GetLabel(i.InputRpo(index + 1)));
2721 : }
2722 16755 : AssembleArchJump(i.InputRpo(1));
2723 16755 : }
2724 :
2725 :
2726 117704 : void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2727 : X64OperandConverter i(this, instr);
2728 6072 : Register input = i.InputRegister(0);
2729 6072 : int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
2730 6072 : Label** cases = zone()->NewArray<Label*>(case_count);
2731 111632 : for (int32_t index = 0; index < case_count; ++index) {
2732 211120 : cases[index] = GetLabel(i.InputRpo(index + 2));
2733 : }
2734 6072 : Label* const table = AddJumpTable(cases, case_count);
2735 6072 : __ cmpl(input, Immediate(case_count));
2736 12144 : __ j(above_equal, GetLabel(i.InputRpo(1)));
2737 12144 : __ leaq(kScratchRegister, Operand(table));
2738 6072 : __ jmp(Operand(kScratchRegister, input, times_8, 0));
2739 6072 : }
2740 :
2741 277964 : CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
2742 : int deoptimization_id, SourcePosition pos) {
2743 277964 : DeoptimizeKind deoptimization_kind = GetDeoptimizationKind(deoptimization_id);
2744 : DeoptimizeReason deoptimization_reason =
2745 277964 : GetDeoptimizationReason(deoptimization_id);
2746 : Deoptimizer::BailoutType bailout_type =
2747 : deoptimization_kind == DeoptimizeKind::kSoft ? Deoptimizer::SOFT
2748 277964 : : Deoptimizer::EAGER;
2749 : Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
2750 277964 : isolate(), deoptimization_id, bailout_type);
2751 277964 : if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
2752 277964 : if (isolate()->NeedsSourcePositionsForProfiling()) {
2753 19982 : __ RecordDeoptReason(deoptimization_reason, pos, deoptimization_id);
2754 : }
2755 277964 : __ call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
2756 277964 : return kSuccess;
2757 : }
2758 :
2759 :
2760 : namespace {
2761 :
2762 : static const int kQuadWordSize = 16;
2763 :
2764 : } // namespace
2765 :
2766 912392 : void CodeGenerator::FinishFrame(Frame* frame) {
2767 1824784 : CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2768 :
2769 : const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2770 912392 : if (saves_fp != 0) {
2771 : frame->AlignSavedCalleeRegisterSlots();
2772 0 : if (saves_fp != 0) { // Save callee-saved XMM registers.
2773 : const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2774 0 : frame->AllocateSavedCalleeRegisterSlots(saves_fp_count *
2775 0 : (kQuadWordSize / kPointerSize));
2776 : }
2777 : }
2778 : const RegList saves = descriptor->CalleeSavedRegisters();
2779 912392 : if (saves != 0) { // Save callee-saved registers.
2780 : int count = 0;
2781 4028928 : for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2782 4028928 : if (((1 << i) & saves)) {
2783 1259040 : ++count;
2784 : }
2785 : }
2786 : frame->AllocateSavedCalleeRegisterSlots(count);
2787 : }
2788 912392 : }
2789 :
2790 4554001 : void CodeGenerator::AssembleConstructFrame() {
2791 3496796 : CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2792 874199 : if (frame_access_state()->has_frame()) {
2793 874199 : int pc_base = __ pc_offset();
2794 :
2795 874199 : if (descriptor->IsCFunctionCall()) {
2796 251808 : __ pushq(rbp);
2797 : __ movq(rbp, rsp);
2798 622391 : } else if (descriptor->IsJSFunctionCall()) {
2799 429002 : __ Prologue(this->info()->GeneratePreagedPrologue());
2800 429002 : if (descriptor->PushArgumentCount()) {
2801 8589 : __ pushq(kJavaScriptCallArgCountRegister);
2802 : }
2803 : } else {
2804 193389 : __ StubPrologue(info()->GetOutputStackFrameType());
2805 : }
2806 :
2807 1303200 : if (!descriptor->IsJSFunctionCall() || !info()->GeneratePreagedPrologue()) {
2808 874199 : unwinding_info_writer_.MarkFrameConstructed(pc_base);
2809 : }
2810 : }
2811 : int shrink_slots =
2812 874199 : frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
2813 :
2814 874199 : if (info()->is_osr()) {
2815 : // TurboFan OSR-compiled functions cannot be entered directly.
2816 5813 : __ Abort(kShouldNotDirectlyEnterOsrFunction);
2817 :
2818 : // Unoptimized code jumps directly to this entrypoint while the unoptimized
2819 : // frame is still on the stack. Optimized code uses OSR values directly from
2820 : // the unoptimized frame. Thus, all that needs to be done is to allocate the
2821 : // remaining stack slots.
2822 5813 : if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
2823 11626 : osr_pc_offset_ = __ pc_offset();
2824 5813 : shrink_slots -= static_cast<int>(OsrHelper(info()).UnoptimizedFrameSlots());
2825 : }
2826 :
2827 : const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2828 874199 : if (shrink_slots > 0) {
2829 1190952 : __ subq(rsp, Immediate(shrink_slots * kPointerSize));
2830 : }
2831 :
2832 874199 : if (saves_fp != 0) { // Save callee-saved XMM registers.
2833 : const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2834 0 : const int stack_size = saves_fp_count * kQuadWordSize;
2835 : // Adjust the stack pointer.
2836 0 : __ subp(rsp, Immediate(stack_size));
2837 : // Store the registers on the stack.
2838 : int slot_idx = 0;
2839 0 : for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
2840 0 : if (!((1 << i) & saves_fp)) continue;
2841 : __ movdqu(Operand(rsp, kQuadWordSize * slot_idx),
2842 0 : XMMRegister::from_code(i));
2843 0 : slot_idx++;
2844 : }
2845 : }
2846 :
2847 : const RegList saves = descriptor->CalleeSavedRegisters();
2848 874199 : if (saves != 0) { // Save callee-saved registers.
2849 4028928 : for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
2850 4028928 : if (!((1 << i) & saves)) continue;
2851 1259040 : __ pushq(Register::from_code(i));
2852 : }
2853 : }
2854 874199 : }
2855 :
2856 1908872 : void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
2857 4350896 : CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
2858 :
2859 : // Restore registers.
2860 : const RegList saves = descriptor->CalleeSavedRegisters();
2861 1087724 : if (saves != 0) {
2862 4265216 : for (int i = 0; i < Register::kNumRegisters; i++) {
2863 4265216 : if (!((1 << i) & saves)) continue;
2864 1332880 : __ popq(Register::from_code(i));
2865 : }
2866 : }
2867 : const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
2868 1087724 : if (saves_fp != 0) {
2869 : const uint32_t saves_fp_count = base::bits::CountPopulation32(saves_fp);
2870 0 : const int stack_size = saves_fp_count * kQuadWordSize;
2871 : // Load the registers from the stack.
2872 : int slot_idx = 0;
2873 0 : for (int i = 0; i < XMMRegister::kMaxNumRegisters; i++) {
2874 0 : if (!((1 << i) & saves_fp)) continue;
2875 : __ movdqu(XMMRegister::from_code(i),
2876 0 : Operand(rsp, kQuadWordSize * slot_idx));
2877 0 : slot_idx++;
2878 : }
2879 : // Adjust the stack pointer.
2880 0 : __ addp(rsp, Immediate(stack_size));
2881 : }
2882 :
2883 : unwinding_info_writer_.MarkBlockWillExit();
2884 :
2885 : // Might need rcx for scratch if pop_size is too big or if there is a variable
2886 : // pop count.
2887 : DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit());
2888 : DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rdx.bit());
2889 1087724 : size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
2890 : X64OperandConverter g(this, nullptr);
2891 1087724 : if (descriptor->IsCFunctionCall()) {
2892 266576 : AssembleDeconstructFrame();
2893 821148 : } else if (frame_access_state()->has_frame()) {
2894 1344864 : if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
2895 : // Canonicalize JSFunction return sites for now.
2896 671970 : if (return_label_.is_bound()) {
2897 187738 : __ jmp(&return_label_);
2898 1087724 : return;
2899 : } else {
2900 484232 : __ bind(&return_label_);
2901 484232 : AssembleDeconstructFrame();
2902 : }
2903 : } else {
2904 917 : AssembleDeconstructFrame();
2905 : }
2906 : }
2907 :
2908 899986 : if (pop->IsImmediate()) {
2909 : DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
2910 1798151 : pop_size += g.ToConstant(pop).ToInt32() * kPointerSize;
2911 899075 : CHECK_LT(pop_size, static_cast<size_t>(std::numeric_limits<int>::max()));
2912 899075 : __ Ret(static_cast<int>(pop_size), rcx);
2913 : } else {
2914 : Register pop_reg = g.ToRegister(pop);
2915 910 : Register scratch_reg = pop_reg.is(rcx) ? rdx : rcx;
2916 910 : __ popq(scratch_reg);
2917 1820 : __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast<int>(pop_size)));
2918 910 : __ jmp(scratch_reg);
2919 : }
2920 : }
2921 :
2922 912392 : void CodeGenerator::FinishCode() {}
2923 :
2924 23466664 : void CodeGenerator::AssembleMove(InstructionOperand* source,
2925 : InstructionOperand* destination) {
2926 : X64OperandConverter g(this, nullptr);
2927 : // Dispatch on the source and destination operand kinds. Not all
2928 : // combinations are possible.
2929 23466664 : if (source->IsRegister()) {
2930 : DCHECK(destination->IsRegister() || destination->IsStackSlot());
2931 : Register src = g.ToRegister(source);
2932 5177500 : if (destination->IsRegister()) {
2933 2650188 : __ movq(g.ToRegister(destination), src);
2934 : } else {
2935 2527313 : __ movq(g.ToOperand(destination), src);
2936 : }
2937 18289164 : } else if (source->IsStackSlot()) {
2938 : DCHECK(destination->IsRegister() || destination->IsStackSlot());
2939 5688906 : Operand src = g.ToOperand(source);
2940 5688906 : if (destination->IsRegister()) {
2941 : Register dst = g.ToRegister(destination);
2942 5618859 : __ movq(dst, src);
2943 : } else {
2944 : // Spill on demand to use a temporary register for memory-to-memory
2945 : // moves.
2946 : Register tmp = kScratchRegister;
2947 70047 : Operand dst = g.ToOperand(destination);
2948 70047 : __ movq(tmp, src);
2949 : __ movq(dst, tmp);
2950 : }
2951 12600258 : } else if (source->IsConstant()) {
2952 : ConstantOperand* constant_source = ConstantOperand::cast(source);
2953 12174993 : Constant src = g.ToConstant(constant_source);
2954 12542870 : if (destination->IsRegister() || destination->IsStackSlot()) {
2955 : Register dst = destination->IsRegister() ? g.ToRegister(destination)
2956 11854555 : : kScratchRegister;
2957 11854555 : switch (src.type()) {
2958 : case Constant::kInt32: {
2959 5404064 : if (RelocInfo::IsWasmPtrReference(src.rmode())) {
2960 0 : __ movq(dst, src.ToInt64(), src.rmode());
2961 : } else {
2962 : // TODO(dcarney): don't need scratch in this case.
2963 2702032 : int32_t value = src.ToInt32();
2964 2702032 : if (value == 0) {
2965 755677 : __ xorl(dst, dst);
2966 : } else {
2967 1946355 : if (RelocInfo::IsWasmSizeReference(src.rmode())) {
2968 0 : __ movl(dst, Immediate(value, src.rmode()));
2969 : } else {
2970 1946355 : __ movl(dst, Immediate(value));
2971 : }
2972 : }
2973 : }
2974 : break;
2975 : }
2976 : case Constant::kInt64:
2977 1316574 : if (RelocInfo::IsWasmPtrReference(src.rmode())) {
2978 48009 : __ movq(dst, src.ToInt64(), src.rmode());
2979 : } else {
2980 : DCHECK(!RelocInfo::IsWasmSizeReference(src.rmode()));
2981 610278 : __ Set(dst, src.ToInt64());
2982 : }
2983 : break;
2984 : case Constant::kFloat32:
2985 : __ Move(dst,
2986 0 : isolate()->factory()->NewNumber(src.ToFloat32(), TENURED));
2987 0 : break;
2988 : case Constant::kFloat64:
2989 : __ Move(dst,
2990 2715963 : isolate()->factory()->NewNumber(src.ToFloat64(), TENURED));
2991 2715960 : break;
2992 : case Constant::kExternalReference:
2993 1186125 : __ Move(dst, src.ToExternalReference());
2994 : break;
2995 : case Constant::kHeapObject: {
2996 4592154 : Handle<HeapObject> src_object = src.ToHeapObject();
2997 : Heap::RootListIndex index;
2998 4592154 : if (IsMaterializableFromRoot(src_object, &index)) {
2999 1551156 : __ LoadRoot(dst, index);
3000 : } else {
3001 3041001 : __ Move(dst, src_object);
3002 : }
3003 : break;
3004 : }
3005 : case Constant::kRpoNumber:
3006 0 : UNREACHABLE(); // TODO(dcarney): load of labels on x64.
3007 : break;
3008 : }
3009 11854546 : if (destination->IsStackSlot()) {
3010 47437 : __ movq(g.ToOperand(destination), kScratchRegister);
3011 : }
3012 320439 : } else if (src.type() == Constant::kFloat32) {
3013 : // TODO(turbofan): Can we do better here?
3014 9160 : uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
3015 9160 : if (destination->IsFPRegister()) {
3016 9160 : __ Move(g.ToDoubleRegister(destination), src_const);
3017 : } else {
3018 : DCHECK(destination->IsFPStackSlot());
3019 0 : Operand dst = g.ToOperand(destination);
3020 0 : __ movl(dst, Immediate(src_const));
3021 : }
3022 : } else {
3023 : DCHECK_EQ(Constant::kFloat64, src.type());
3024 311279 : uint64_t src_const = bit_cast<uint64_t>(src.ToFloat64());
3025 311279 : if (destination->IsFPRegister()) {
3026 310984 : __ Move(g.ToDoubleRegister(destination), src_const);
3027 : } else {
3028 : DCHECK(destination->IsFPStackSlot());
3029 295 : __ movq(kScratchRegister, src_const);
3030 : __ movq(g.ToOperand(destination), kScratchRegister);
3031 : }
3032 : }
3033 425265 : } else if (source->IsFPRegister()) {
3034 213614 : XMMRegister src = g.ToDoubleRegister(source);
3035 213614 : if (destination->IsFPRegister()) {
3036 47109 : XMMRegister dst = g.ToDoubleRegister(destination);
3037 47109 : __ Movapd(dst, src);
3038 : } else {
3039 : DCHECK(destination->IsFPStackSlot());
3040 166505 : Operand dst = g.ToOperand(destination);
3041 : MachineRepresentation rep =
3042 : LocationOperand::cast(source)->representation();
3043 166505 : if (rep != MachineRepresentation::kSimd128) {
3044 166505 : __ Movsd(dst, src);
3045 : } else {
3046 0 : __ Movups(dst, src);
3047 : }
3048 : }
3049 211651 : } else if (source->IsFPStackSlot()) {
3050 : DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
3051 211651 : Operand src = g.ToOperand(source);
3052 : MachineRepresentation rep = LocationOperand::cast(source)->representation();
3053 211651 : if (destination->IsFPRegister()) {
3054 204499 : XMMRegister dst = g.ToDoubleRegister(destination);
3055 204499 : if (rep != MachineRepresentation::kSimd128) {
3056 204499 : __ Movsd(dst, src);
3057 : } else {
3058 0 : __ Movups(dst, src);
3059 : }
3060 : } else {
3061 7152 : Operand dst = g.ToOperand(destination);
3062 7152 : if (rep != MachineRepresentation::kSimd128) {
3063 7152 : __ Movsd(kScratchDoubleReg, src);
3064 7152 : __ Movsd(dst, kScratchDoubleReg);
3065 : } else {
3066 0 : __ Movups(kScratchDoubleReg, src);
3067 0 : __ Movups(dst, kScratchDoubleReg);
3068 : }
3069 : }
3070 : } else {
3071 0 : UNREACHABLE();
3072 : }
3073 23466658 : }
3074 :
3075 :
3076 18462 : void CodeGenerator::AssembleSwap(InstructionOperand* source,
3077 7884 : InstructionOperand* destination) {
3078 : X64OperandConverter g(this, nullptr);
3079 : // Dispatch on the source and destination operand kinds. Not all
3080 : // combinations are possible.
3081 33112 : if (source->IsRegister() && destination->IsRegister()) {
3082 : // Register-register.
3083 : Register src = g.ToRegister(source);
3084 : Register dst = g.ToRegister(destination);
3085 14410 : __ movq(kScratchRegister, src);
3086 : __ movq(src, dst);
3087 : __ movq(dst, kScratchRegister);
3088 4292 : } else if (source->IsRegister() && destination->IsStackSlot()) {
3089 : Register src = g.ToRegister(source);
3090 240 : __ pushq(src);
3091 : frame_access_state()->IncreaseSPDelta(1);
3092 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3093 480 : kPointerSize);
3094 240 : Operand dst = g.ToOperand(destination);
3095 : __ movq(src, dst);
3096 : frame_access_state()->IncreaseSPDelta(-1);
3097 240 : dst = g.ToOperand(destination);
3098 240 : __ popq(dst);
3099 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3100 480 : -kPointerSize);
3101 11436 : } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
3102 65 : (source->IsFPStackSlot() && destination->IsFPStackSlot())) {
3103 : // Memory-memory.
3104 3702 : Operand src = g.ToOperand(source);
3105 3702 : Operand dst = g.ToOperand(destination);
3106 : MachineRepresentation rep = LocationOperand::cast(source)->representation();
3107 3702 : if (rep != MachineRepresentation::kSimd128) {
3108 : Register tmp = kScratchRegister;
3109 3702 : __ movq(tmp, dst);
3110 3702 : __ pushq(src);
3111 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3112 7404 : kPointerSize);
3113 : frame_access_state()->IncreaseSPDelta(1);
3114 3702 : src = g.ToOperand(source);
3115 : __ movq(src, tmp);
3116 : frame_access_state()->IncreaseSPDelta(-1);
3117 3702 : dst = g.ToOperand(destination);
3118 3702 : __ popq(dst);
3119 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3120 7404 : -kPointerSize);
3121 : } else {
3122 : // Use the XOR trick to swap without a temporary.
3123 0 : __ Movups(kScratchDoubleReg, src);
3124 0 : __ Xorps(kScratchDoubleReg, dst); // scratch contains src ^ dst.
3125 0 : __ Movups(src, kScratchDoubleReg);
3126 0 : __ Xorps(kScratchDoubleReg, dst); // scratch contains src.
3127 0 : __ Movups(dst, kScratchDoubleReg);
3128 0 : __ Xorps(kScratchDoubleReg, src); // scratch contains dst.
3129 0 : __ Movups(src, kScratchDoubleReg);
3130 : }
3131 220 : } else if (source->IsFPRegister() && destination->IsFPRegister()) {
3132 : // XMM register-register swap.
3133 110 : XMMRegister src = g.ToDoubleRegister(source);
3134 110 : XMMRegister dst = g.ToDoubleRegister(destination);
3135 110 : __ Movapd(kScratchDoubleReg, src);
3136 110 : __ Movapd(src, dst);
3137 110 : __ Movapd(dst, kScratchDoubleReg);
3138 0 : } else if (source->IsFPRegister() && destination->IsFPStackSlot()) {
3139 : // XMM register-memory swap.
3140 0 : XMMRegister src = g.ToDoubleRegister(source);
3141 0 : Operand dst = g.ToOperand(destination);
3142 : MachineRepresentation rep = LocationOperand::cast(source)->representation();
3143 0 : if (rep != MachineRepresentation::kSimd128) {
3144 0 : __ Movsd(kScratchDoubleReg, src);
3145 0 : __ Movsd(src, dst);
3146 0 : __ Movsd(dst, kScratchDoubleReg);
3147 : } else {
3148 0 : __ Movups(kScratchDoubleReg, src);
3149 0 : __ Movups(src, dst);
3150 0 : __ Movups(dst, kScratchDoubleReg);
3151 : }
3152 : } else {
3153 : // No other combinations are possible.
3154 0 : UNREACHABLE();
3155 : }
3156 18462 : }
3157 :
3158 :
3159 6072 : void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
3160 111632 : for (size_t index = 0; index < target_count; ++index) {
3161 105560 : __ dq(targets[index]);
3162 : }
3163 6072 : }
3164 :
3165 :
3166 3608642 : void CodeGenerator::EnsureSpaceForLazyDeopt() {
3167 3608642 : if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
3168 3608642 : return;
3169 : }
3170 :
3171 3114326 : int space_needed = Deoptimizer::patch_size();
3172 : // Ensure that we have enough space after the previous lazy-bailout
3173 : // instruction for patching the code here.
3174 3114326 : int current_pc = __ pc_offset();
3175 3114326 : if (current_pc < last_lazy_deopt_pc_ + space_needed) {
3176 102207 : int padding_size = last_lazy_deopt_pc_ + space_needed - current_pc;
3177 102207 : __ Nop(padding_size);
3178 : }
3179 : }
3180 :
3181 : #undef __
3182 :
3183 : } // namespace compiler
3184 : } // namespace internal
3185 : } // namespace v8
|
