Line data Source code
1 : // Copyright 2013 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/code-generator.h"
6 :
7 : #include <limits>
8 :
9 : #include "src/compilation-info.h"
10 : #include "src/compiler/code-generator-impl.h"
11 : #include "src/compiler/gap-resolver.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/osr.h"
14 : #include "src/heap/heap-inl.h"
15 : #include "src/x64/assembler-x64.h"
16 : #include "src/x64/macro-assembler-x64.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 : namespace compiler {
21 :
22 : #define __ tasm()->
23 :
24 : // Adds X64 specific methods for decoding operands.
25 : class X64OperandConverter : public InstructionOperandConverter {
26 : public:
27 : X64OperandConverter(CodeGenerator* gen, Instruction* instr)
28 : : InstructionOperandConverter(gen, instr) {}
29 :
30 : Immediate InputImmediate(size_t index) {
31 2806631 : return ToImmediate(instr_->InputAt(index));
32 : }
33 :
34 635483 : Operand InputOperand(size_t index, int extra = 0) {
35 1906449 : return ToOperand(instr_->InputAt(index), extra);
36 : }
37 :
38 0 : Operand OutputOperand() { return ToOperand(instr_->Output()); }
39 :
40 2459051 : Immediate ToImmediate(InstructionOperand* operand) {
41 2459051 : Constant constant = ToConstant(operand);
42 2459071 : if (constant.type() == Constant::kFloat64) {
43 : DCHECK_EQ(0, constant.ToFloat64().AsUint64());
44 81262 : return Immediate(0);
45 : }
46 2377809 : if (RelocInfo::IsWasmReference(constant.rmode())) {
47 3251 : return Immediate(constant.ToInt32(), constant.rmode());
48 : }
49 2374558 : return Immediate(constant.ToInt32());
50 : }
51 :
52 : Operand ToOperand(InstructionOperand* op, int extra = 0) {
53 : DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
54 10675407 : return SlotToOperand(AllocatedOperand::cast(op)->index(), extra);
55 : }
56 :
57 10685925 : Operand SlotToOperand(int slot_index, int extra = 0) {
58 21371850 : FrameOffset offset = frame_access_state()->GetFrameOffset(slot_index);
59 : return Operand(offset.from_stack_pointer() ? rsp : rbp,
60 21371960 : offset.offset() + extra);
61 : }
62 :
63 : static size_t NextOffset(size_t* offset) {
64 7977413 : size_t i = *offset;
65 14628428 : (*offset)++;
66 : return i;
67 : }
68 :
69 : static ScaleFactor ScaleFor(AddressingMode one, AddressingMode mode) {
70 : STATIC_ASSERT(0 == static_cast<int>(times_1));
71 : STATIC_ASSERT(1 == static_cast<int>(times_2));
72 : STATIC_ASSERT(2 == static_cast<int>(times_4));
73 : STATIC_ASSERT(3 == static_cast<int>(times_8));
74 640545 : int scale = static_cast<int>(mode - one);
75 : DCHECK(scale >= 0 && scale < 4);
76 640545 : return static_cast<ScaleFactor>(scale);
77 : }
78 :
79 7977413 : Operand MemoryOperand(size_t* offset) {
80 7977413 : AddressingMode mode = AddressingModeField::decode(instr_->opcode());
81 7977413 : switch (mode) {
82 : case kMode_MR: {
83 7801679 : Register base = InputRegister(NextOffset(offset));
84 : int32_t disp = 0;
85 438247 : return Operand(base, disp);
86 : }
87 : case kMode_MRI: {
88 5706978 : Register base = InputRegister(NextOffset(offset));
89 5706978 : int32_t disp = InputInt32(NextOffset(offset));
90 5706981 : return Operand(base, disp);
91 : }
92 : case kMode_MR1:
93 : case kMode_MR2:
94 : case kMode_MR4:
95 : case kMode_MR8: {
96 272395 : Register base = InputRegister(NextOffset(offset));
97 272395 : Register index = InputRegister(NextOffset(offset));
98 : ScaleFactor scale = ScaleFor(kMode_MR1, mode);
99 : int32_t disp = 0;
100 272395 : return Operand(base, index, scale, disp);
101 : }
102 : case kMode_MR1I:
103 : case kMode_MR2I:
104 : case kMode_MR4I:
105 : case kMode_MR8I: {
106 305267 : Register base = InputRegister(NextOffset(offset));
107 305267 : Register index = InputRegister(NextOffset(offset));
108 : ScaleFactor scale = ScaleFor(kMode_MR1I, mode);
109 305267 : int32_t disp = InputInt32(NextOffset(offset));
110 305267 : return Operand(base, index, scale, disp);
111 : }
112 : case kMode_M1: {
113 0 : Register base = InputRegister(NextOffset(offset));
114 : int32_t disp = 0;
115 0 : return Operand(base, disp);
116 : }
117 : case kMode_M2:
118 0 : UNREACHABLE(); // Should use kModeMR with more compact encoding instead
119 : return Operand(no_reg, 0);
120 : case kMode_M4:
121 : case kMode_M8: {
122 1775 : Register index = InputRegister(NextOffset(offset));
123 : ScaleFactor scale = ScaleFor(kMode_M1, mode);
124 : int32_t disp = 0;
125 1775 : return Operand(index, scale, disp);
126 : }
127 : case kMode_M1I:
128 : case kMode_M2I:
129 : case kMode_M4I:
130 : case kMode_M8I: {
131 61108 : Register index = InputRegister(NextOffset(offset));
132 : ScaleFactor scale = ScaleFor(kMode_M1I, mode);
133 61108 : int32_t disp = InputInt32(NextOffset(offset));
134 61109 : return Operand(index, scale, disp);
135 : }
136 : case kMode_Root: {
137 1191643 : Register base = kRootRegister;
138 1191643 : int32_t disp = InputInt32(NextOffset(offset));
139 1191645 : return Operand(base, disp);
140 : }
141 : case kMode_None:
142 0 : UNREACHABLE();
143 : }
144 0 : UNREACHABLE();
145 : }
146 :
147 : Operand MemoryOperand(size_t first_input = 0) {
148 4325518 : return MemoryOperand(&first_input);
149 : }
150 : };
151 :
152 :
153 : namespace {
154 :
155 : bool HasImmediateInput(Instruction* instr, size_t index) {
156 14437261 : return instr->InputAt(index)->IsImmediate();
157 : }
158 :
159 :
160 0 : class OutOfLineLoadZero final : public OutOfLineCode {
161 : public:
162 : OutOfLineLoadZero(CodeGenerator* gen, Register result)
163 67341 : : OutOfLineCode(gen), result_(result) {}
164 :
165 134682 : void Generate() final { __ xorl(result_, result_); }
166 :
167 : private:
168 : Register const result_;
169 : };
170 :
171 0 : class OutOfLineLoadFloat32NaN final : public OutOfLineCode {
172 : public:
173 : OutOfLineLoadFloat32NaN(CodeGenerator* gen, XMMRegister result)
174 7734 : : OutOfLineCode(gen), result_(result) {}
175 :
176 7734 : void Generate() final {
177 7734 : __ Xorps(result_, result_);
178 7734 : __ Divss(result_, result_);
179 7734 : }
180 :
181 : private:
182 : XMMRegister const result_;
183 : };
184 :
185 0 : class OutOfLineLoadFloat64NaN final : public OutOfLineCode {
186 : public:
187 : OutOfLineLoadFloat64NaN(CodeGenerator* gen, XMMRegister result)
188 1476 : : OutOfLineCode(gen), result_(result) {}
189 :
190 1475 : void Generate() final {
191 1475 : __ Xorpd(result_, result_);
192 1475 : __ Divsd(result_, result_);
193 1476 : }
194 :
195 : private:
196 : XMMRegister const result_;
197 : };
198 :
199 0 : class OutOfLineTruncateDoubleToI final : public OutOfLineCode {
200 : public:
201 45401 : OutOfLineTruncateDoubleToI(CodeGenerator* gen, Register result,
202 : XMMRegister input,
203 : UnwindingInfoWriter* unwinding_info_writer)
204 : : OutOfLineCode(gen),
205 : result_(result),
206 : input_(input),
207 : unwinding_info_writer_(unwinding_info_writer),
208 90806 : zone_(gen->zone()) {}
209 :
210 45404 : void Generate() final {
211 227047 : __ subp(rsp, Immediate(kDoubleSize));
212 : unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
213 90814 : kDoubleSize);
214 90815 : __ Movsd(MemOperand(rsp, 0), input_);
215 90816 : __ SlowTruncateToIDelayed(zone_, result_);
216 45412 : __ addp(rsp, Immediate(kDoubleSize));
217 : unwinding_info_writer_->MaybeIncreaseBaseOffsetAt(__ pc_offset(),
218 90828 : -kDoubleSize);
219 45414 : }
220 :
221 : private:
222 : Register const result_;
223 : XMMRegister const input_;
224 : UnwindingInfoWriter* const unwinding_info_writer_;
225 : Zone* zone_;
226 : };
227 :
228 :
229 0 : class OutOfLineRecordWrite final : public OutOfLineCode {
230 : public:
231 320882 : OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand operand,
232 : Register value, Register scratch0, Register scratch1,
233 : RecordWriteMode mode)
234 : : OutOfLineCode(gen),
235 : object_(object),
236 : operand_(operand),
237 : value_(value),
238 : scratch0_(scratch0),
239 : scratch1_(scratch1),
240 : mode_(mode),
241 641764 : zone_(gen->zone()) {}
242 :
243 320882 : void Generate() final {
244 320882 : if (mode_ > RecordWriteMode::kValueIsPointer) {
245 1555508 : __ JumpIfSmi(value_, exit());
246 : }
247 : __ CheckPageFlag(value_, scratch0_,
248 : MemoryChunk::kPointersToHereAreInterestingMask, zero,
249 641764 : exit());
250 320882 : __ leap(scratch1_, operand_);
251 :
252 : RememberedSetAction const remembered_set_action =
253 : mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
254 320882 : : OMIT_REMEMBERED_SET;
255 : SaveFPRegsMode const save_fp_mode =
256 641764 : frame()->DidAllocateDoubleRegisters() ? kSaveFPRegs : kDontSaveFPRegs;
257 :
258 : #ifdef V8_CSA_WRITE_BARRIER
259 : __ CallRecordWriteStub(object_, scratch1_, remembered_set_action,
260 320882 : save_fp_mode);
261 : #else
262 : __ CallStubDelayed(
263 : new (zone_) RecordWriteStub(nullptr, object_, scratch0_, scratch1_,
264 : remembered_set_action, save_fp_mode));
265 : #endif
266 320882 : }
267 :
268 : private:
269 : Register const object_;
270 : Operand const operand_;
271 : Register const value_;
272 : Register const scratch0_;
273 : Register const scratch1_;
274 : RecordWriteMode const mode_;
275 : Zone* zone_;
276 : };
277 :
278 0 : class WasmOutOfLineTrap final : public OutOfLineCode {
279 : public:
280 : WasmOutOfLineTrap(CodeGenerator* gen, int pc, bool frame_elided,
281 : Instruction* instr)
282 : : OutOfLineCode(gen),
283 : gen_(gen),
284 : pc_(pc),
285 : frame_elided_(frame_elided),
286 5292 : instr_(instr) {}
287 :
288 : // TODO(eholk): Refactor this method to take the code generator as a
289 : // parameter.
290 5287 : void Generate() final {
291 22024 : __ RecordProtectedInstructionLanding(pc_);
292 :
293 5292 : if (frame_elided_) {
294 865 : __ EnterFrame(StackFrame::WASM_COMPILED);
295 : }
296 :
297 10578 : gen_->AssembleSourcePosition(instr_);
298 : __ Call(__ isolate()->builtins()->builtin_handle(
299 : Builtins::kThrowWasmTrapMemOutOfBounds),
300 10580 : RelocInfo::CODE_TARGET);
301 5289 : ReferenceMap* reference_map = new (gen_->zone()) ReferenceMap(gen_->zone());
302 : gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
303 5285 : Safepoint::kNoLazyDeopt);
304 5292 : __ AssertUnreachable(kUnexpectedReturnFromWasmTrap);
305 5291 : }
306 :
307 : private:
308 : CodeGenerator* gen_;
309 : int pc_;
310 : bool frame_elided_;
311 : Instruction* instr_;
312 : };
313 :
314 5483671 : void EmitOOLTrapIfNeeded(Zone* zone, CodeGenerator* codegen,
315 : InstructionCode opcode, Instruction* instr,
316 : X64OperandConverter& i, int pc) {
317 : const X64MemoryProtection protection =
318 5478379 : static_cast<X64MemoryProtection>(MiscField::decode(opcode));
319 5478379 : if (protection == X64MemoryProtection::kProtected) {
320 5292 : const bool frame_elided = !codegen->frame_access_state()->has_frame();
321 : new (zone) WasmOutOfLineTrap(codegen, pc, frame_elided, instr);
322 : }
323 5478372 : }
324 : } // namespace
325 :
326 :
327 : #define ASSEMBLE_UNOP(asm_instr) \
328 : do { \
329 : if (instr->Output()->IsRegister()) { \
330 : __ asm_instr(i.OutputRegister()); \
331 : } else { \
332 : __ asm_instr(i.OutputOperand()); \
333 : } \
334 : } while (0)
335 :
336 : #define ASSEMBLE_BINOP(asm_instr) \
337 : do { \
338 : if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
339 : size_t index = 1; \
340 : Operand right = i.MemoryOperand(&index); \
341 : __ asm_instr(i.InputRegister(0), right); \
342 : } else { \
343 : if (HasImmediateInput(instr, 1)) { \
344 : if (instr->InputAt(0)->IsRegister()) { \
345 : __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
346 : } else { \
347 : __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
348 : } \
349 : } else { \
350 : if (instr->InputAt(1)->IsRegister()) { \
351 : __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
352 : } else { \
353 : __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
354 : } \
355 : } \
356 : } \
357 : } while (0)
358 :
359 : #define ASSEMBLE_COMPARE(asm_instr) \
360 : do { \
361 : if (AddressingModeField::decode(instr->opcode()) != kMode_None) { \
362 : size_t index = 0; \
363 : Operand left = i.MemoryOperand(&index); \
364 : if (HasImmediateInput(instr, index)) { \
365 : __ asm_instr(left, i.InputImmediate(index)); \
366 : } else { \
367 : __ asm_instr(left, i.InputRegister(index)); \
368 : } \
369 : } else { \
370 : if (HasImmediateInput(instr, 1)) { \
371 : if (instr->InputAt(0)->IsRegister()) { \
372 : __ asm_instr(i.InputRegister(0), i.InputImmediate(1)); \
373 : } else { \
374 : __ asm_instr(i.InputOperand(0), i.InputImmediate(1)); \
375 : } \
376 : } else { \
377 : if (instr->InputAt(1)->IsRegister()) { \
378 : __ asm_instr(i.InputRegister(0), i.InputRegister(1)); \
379 : } else { \
380 : __ asm_instr(i.InputRegister(0), i.InputOperand(1)); \
381 : } \
382 : } \
383 : } \
384 : } while (0)
385 :
386 : #define ASSEMBLE_MULT(asm_instr) \
387 : do { \
388 : if (HasImmediateInput(instr, 1)) { \
389 : if (instr->InputAt(0)->IsRegister()) { \
390 : __ asm_instr(i.OutputRegister(), i.InputRegister(0), \
391 : i.InputImmediate(1)); \
392 : } else { \
393 : __ asm_instr(i.OutputRegister(), i.InputOperand(0), \
394 : i.InputImmediate(1)); \
395 : } \
396 : } else { \
397 : if (instr->InputAt(1)->IsRegister()) { \
398 : __ asm_instr(i.OutputRegister(), i.InputRegister(1)); \
399 : } else { \
400 : __ asm_instr(i.OutputRegister(), i.InputOperand(1)); \
401 : } \
402 : } \
403 : } while (0)
404 :
405 :
406 : #define ASSEMBLE_SHIFT(asm_instr, width) \
407 : do { \
408 : if (HasImmediateInput(instr, 1)) { \
409 : if (instr->Output()->IsRegister()) { \
410 : __ asm_instr(i.OutputRegister(), Immediate(i.InputInt##width(1))); \
411 : } else { \
412 : __ asm_instr(i.OutputOperand(), Immediate(i.InputInt##width(1))); \
413 : } \
414 : } else { \
415 : if (instr->Output()->IsRegister()) { \
416 : __ asm_instr##_cl(i.OutputRegister()); \
417 : } else { \
418 : __ asm_instr##_cl(i.OutputOperand()); \
419 : } \
420 : } \
421 : } while (0)
422 :
423 :
424 : #define ASSEMBLE_MOVX(asm_instr) \
425 : do { \
426 : if (instr->addressing_mode() != kMode_None) { \
427 : __ asm_instr(i.OutputRegister(), i.MemoryOperand()); \
428 : } else if (instr->InputAt(0)->IsRegister()) { \
429 : __ asm_instr(i.OutputRegister(), i.InputRegister(0)); \
430 : } else { \
431 : __ asm_instr(i.OutputRegister(), i.InputOperand(0)); \
432 : } \
433 : } while (0)
434 :
435 : #define ASSEMBLE_SSE_BINOP(asm_instr) \
436 : do { \
437 : if (instr->InputAt(1)->IsFPRegister()) { \
438 : __ asm_instr(i.InputDoubleRegister(0), i.InputDoubleRegister(1)); \
439 : } else { \
440 : __ asm_instr(i.InputDoubleRegister(0), i.InputOperand(1)); \
441 : } \
442 : } while (0)
443 :
444 : #define ASSEMBLE_SSE_UNOP(asm_instr) \
445 : do { \
446 : if (instr->InputAt(0)->IsFPRegister()) { \
447 : __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0)); \
448 : } else { \
449 : __ asm_instr(i.OutputDoubleRegister(), i.InputOperand(0)); \
450 : } \
451 : } while (0)
452 :
453 : #define ASSEMBLE_AVX_BINOP(asm_instr) \
454 : do { \
455 : CpuFeatureScope avx_scope(tasm(), AVX); \
456 : if (instr->InputAt(1)->IsFPRegister()) { \
457 : __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
458 : i.InputDoubleRegister(1)); \
459 : } else { \
460 : __ asm_instr(i.OutputDoubleRegister(), i.InputDoubleRegister(0), \
461 : i.InputOperand(1)); \
462 : } \
463 : } while (0)
464 :
465 : #define ASSEMBLE_CHECKED_LOAD_FLOAT(asm_instr, OutOfLineLoadNaN) \
466 : do { \
467 : auto result = i.OutputDoubleRegister(); \
468 : auto buffer = i.InputRegister(0); \
469 : auto index1 = i.InputRegister(1); \
470 : auto index2 = i.InputUint32(2); \
471 : OutOfLineCode* ool; \
472 : if (instr->InputAt(3)->IsRegister()) { \
473 : auto length = i.InputRegister(3); \
474 : DCHECK_EQ(0u, index2); \
475 : __ cmpl(index1, length); \
476 : ool = new (zone()) OutOfLineLoadNaN(this, result); \
477 : } else { \
478 : auto length = i.InputUint32(3); \
479 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
480 : DCHECK_LE(index2, length); \
481 : __ cmpl(index1, Immediate(length - index2, rmode)); \
482 : class OutOfLineLoadFloat final : public OutOfLineCode { \
483 : public: \
484 : OutOfLineLoadFloat(CodeGenerator* gen, XMMRegister result, \
485 : Register buffer, Register index1, int32_t index2, \
486 : int32_t length, RelocInfo::Mode rmode) \
487 : : OutOfLineCode(gen), \
488 : result_(result), \
489 : buffer_(buffer), \
490 : index1_(index1), \
491 : index2_(index2), \
492 : length_(length), \
493 : rmode_(rmode) {} \
494 : \
495 : void Generate() final { \
496 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
497 : __ Pcmpeqd(result_, result_); \
498 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
499 : __ j(above_equal, exit()); \
500 : __ asm_instr(result_, \
501 : Operand(buffer_, kScratchRegister, times_1, 0)); \
502 : } \
503 : \
504 : private: \
505 : XMMRegister const result_; \
506 : Register const buffer_; \
507 : Register const index1_; \
508 : int32_t const index2_; \
509 : int32_t const length_; \
510 : RelocInfo::Mode rmode_; \
511 : }; \
512 : ool = new (zone()) OutOfLineLoadFloat(this, result, buffer, index1, \
513 : index2, length, rmode); \
514 : } \
515 : __ j(above_equal, ool->entry()); \
516 : __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
517 : __ bind(ool->exit()); \
518 : } while (false)
519 :
520 : #define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
521 : do { \
522 : auto result = i.OutputRegister(); \
523 : auto buffer = i.InputRegister(0); \
524 : auto index1 = i.InputRegister(1); \
525 : auto index2 = i.InputUint32(2); \
526 : OutOfLineCode* ool; \
527 : if (instr->InputAt(3)->IsRegister()) { \
528 : auto length = i.InputRegister(3); \
529 : DCHECK_EQ(0u, index2); \
530 : __ cmpl(index1, length); \
531 : ool = new (zone()) OutOfLineLoadZero(this, result); \
532 : } else { \
533 : auto length = i.InputUint32(3); \
534 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
535 : DCHECK_LE(index2, length); \
536 : __ cmpl(index1, Immediate(length - index2, rmode)); \
537 : class OutOfLineLoadInteger final : public OutOfLineCode { \
538 : public: \
539 : OutOfLineLoadInteger(CodeGenerator* gen, Register result, \
540 : Register buffer, Register index1, int32_t index2, \
541 : int32_t length, RelocInfo::Mode rmode) \
542 : : OutOfLineCode(gen), \
543 : result_(result), \
544 : buffer_(buffer), \
545 : index1_(index1), \
546 : index2_(index2), \
547 : length_(length), \
548 : rmode_(rmode) {} \
549 : \
550 : void Generate() final { \
551 : Label oob; \
552 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
553 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
554 : __ j(above_equal, &oob, Label::kNear); \
555 : __ asm_instr(result_, \
556 : Operand(buffer_, kScratchRegister, times_1, 0)); \
557 : __ jmp(exit()); \
558 : __ bind(&oob); \
559 : __ xorl(result_, result_); \
560 : } \
561 : \
562 : private: \
563 : Register const result_; \
564 : Register const buffer_; \
565 : Register const index1_; \
566 : int32_t const index2_; \
567 : int32_t const length_; \
568 : RelocInfo::Mode const rmode_; \
569 : }; \
570 : ool = new (zone()) OutOfLineLoadInteger(this, result, buffer, index1, \
571 : index2, length, rmode); \
572 : } \
573 : __ j(above_equal, ool->entry()); \
574 : __ asm_instr(result, Operand(buffer, index1, times_1, index2)); \
575 : __ bind(ool->exit()); \
576 : } while (false)
577 :
578 : #define ASSEMBLE_CHECKED_STORE_FLOAT(asm_instr) \
579 : do { \
580 : auto buffer = i.InputRegister(0); \
581 : auto index1 = i.InputRegister(1); \
582 : auto index2 = i.InputUint32(2); \
583 : auto value = i.InputDoubleRegister(4); \
584 : if (instr->InputAt(3)->IsRegister()) { \
585 : auto length = i.InputRegister(3); \
586 : DCHECK_EQ(0u, index2); \
587 : Label done; \
588 : __ cmpl(index1, length); \
589 : __ j(above_equal, &done, Label::kNear); \
590 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
591 : __ bind(&done); \
592 : } else { \
593 : auto length = i.InputUint32(3); \
594 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
595 : DCHECK_LE(index2, length); \
596 : __ cmpl(index1, Immediate(length - index2, rmode)); \
597 : class OutOfLineStoreFloat final : public OutOfLineCode { \
598 : public: \
599 : OutOfLineStoreFloat(CodeGenerator* gen, Register buffer, \
600 : Register index1, int32_t index2, int32_t length, \
601 : XMMRegister value, RelocInfo::Mode rmode) \
602 : : OutOfLineCode(gen), \
603 : buffer_(buffer), \
604 : index1_(index1), \
605 : index2_(index2), \
606 : length_(length), \
607 : value_(value), \
608 : rmode_(rmode) {} \
609 : \
610 : void Generate() final { \
611 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
612 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
613 : __ j(above_equal, exit()); \
614 : __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
615 : value_); \
616 : } \
617 : \
618 : private: \
619 : Register const buffer_; \
620 : Register const index1_; \
621 : int32_t const index2_; \
622 : int32_t const length_; \
623 : XMMRegister const value_; \
624 : RelocInfo::Mode rmode_; \
625 : }; \
626 : auto ool = new (zone()) OutOfLineStoreFloat( \
627 : this, buffer, index1, index2, length, value, rmode); \
628 : __ j(above_equal, ool->entry()); \
629 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
630 : __ bind(ool->exit()); \
631 : } \
632 : } while (false)
633 :
634 : #define ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Value) \
635 : do { \
636 : auto buffer = i.InputRegister(0); \
637 : auto index1 = i.InputRegister(1); \
638 : auto index2 = i.InputUint32(2); \
639 : if (instr->InputAt(3)->IsRegister()) { \
640 : auto length = i.InputRegister(3); \
641 : DCHECK_EQ(0u, index2); \
642 : Label done; \
643 : __ cmpl(index1, length); \
644 : __ j(above_equal, &done, Label::kNear); \
645 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
646 : __ bind(&done); \
647 : } else { \
648 : auto length = i.InputUint32(3); \
649 : RelocInfo::Mode rmode = i.ToConstant(instr->InputAt(3)).rmode(); \
650 : DCHECK_LE(index2, length); \
651 : __ cmpl(index1, Immediate(length - index2, rmode)); \
652 : class OutOfLineStoreInteger final : public OutOfLineCode { \
653 : public: \
654 : OutOfLineStoreInteger(CodeGenerator* gen, Register buffer, \
655 : Register index1, int32_t index2, int32_t length, \
656 : Value value, RelocInfo::Mode rmode) \
657 : : OutOfLineCode(gen), \
658 : buffer_(buffer), \
659 : index1_(index1), \
660 : index2_(index2), \
661 : length_(length), \
662 : value_(value), \
663 : rmode_(rmode) {} \
664 : \
665 : void Generate() final { \
666 : __ leal(kScratchRegister, Operand(index1_, index2_)); \
667 : __ cmpl(kScratchRegister, Immediate(length_, rmode_)); \
668 : __ j(above_equal, exit()); \
669 : __ asm_instr(Operand(buffer_, kScratchRegister, times_1, 0), \
670 : value_); \
671 : } \
672 : \
673 : private: \
674 : Register const buffer_; \
675 : Register const index1_; \
676 : int32_t const index2_; \
677 : int32_t const length_; \
678 : Value const value_; \
679 : RelocInfo::Mode rmode_; \
680 : }; \
681 : auto ool = new (zone()) OutOfLineStoreInteger( \
682 : this, buffer, index1, index2, length, value, rmode); \
683 : __ j(above_equal, ool->entry()); \
684 : __ asm_instr(Operand(buffer, index1, times_1, index2), value); \
685 : __ bind(ool->exit()); \
686 : } \
687 : } while (false)
688 :
689 : #define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
690 : do { \
691 : if (instr->InputAt(4)->IsRegister()) { \
692 : Register value = i.InputRegister(4); \
693 : ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Register); \
694 : } else { \
695 : Immediate value = i.InputImmediate(4); \
696 : ASSEMBLE_CHECKED_STORE_INTEGER_IMPL(asm_instr, Immediate); \
697 : } \
698 : } while (false)
699 :
700 : #define ASSEMBLE_IEEE754_BINOP(name) \
701 : do { \
702 : __ PrepareCallCFunction(2); \
703 : __ CallCFunction( \
704 : ExternalReference::ieee754_##name##_function(__ isolate()), 2); \
705 : } while (false)
706 :
707 : #define ASSEMBLE_IEEE754_UNOP(name) \
708 : do { \
709 : __ PrepareCallCFunction(1); \
710 : __ CallCFunction( \
711 : ExternalReference::ieee754_##name##_function(__ isolate()), 1); \
712 : } while (false)
713 :
714 : #define ASSEMBLE_ATOMIC_BINOP(bin_inst, mov_inst, cmpxchg_inst) \
715 : do { \
716 : Label binop; \
717 : __ bind(&binop); \
718 : __ mov_inst(rax, i.MemoryOperand(1)); \
719 : __ movl(i.TempRegister(0), rax); \
720 : __ bin_inst(i.TempRegister(0), i.InputRegister(0)); \
721 : __ lock(); \
722 : __ cmpxchg_inst(i.MemoryOperand(1), i.TempRegister(0)); \
723 : __ j(not_equal, &binop); \
724 : } while (false)
725 :
726 1077568 : void CodeGenerator::AssembleDeconstructFrame() {
727 2155136 : unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
728 1077589 : __ movq(rsp, rbp);
729 1077596 : __ popq(rbp);
730 1077592 : }
731 :
732 373178 : void CodeGenerator::AssemblePrepareTailCall() {
733 186589 : if (frame_access_state()->has_frame()) {
734 46994 : __ movq(rbp, MemOperand(rbp, 0));
735 : }
736 : frame_access_state()->SetFrameAccessToSP();
737 186589 : }
738 :
739 930 : void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
740 : Register scratch1,
741 : Register scratch2,
742 : Register scratch3) {
743 : DCHECK(!AreAliased(args_reg, scratch1, scratch2, scratch3));
744 : Label done;
745 :
746 : // Check if current frame is an arguments adaptor frame.
747 : __ cmpp(Operand(rbp, CommonFrameConstants::kContextOrFrameTypeOffset),
748 1860 : Immediate(StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR)));
749 930 : __ j(not_equal, &done, Label::kNear);
750 :
751 : // Load arguments count from current arguments adaptor frame (note, it
752 : // does not include receiver).
753 930 : Register caller_args_count_reg = scratch1;
754 : __ SmiToInteger32(
755 : caller_args_count_reg,
756 930 : Operand(rbp, ArgumentsAdaptorFrameConstants::kLengthOffset));
757 :
758 : ParameterCount callee_args_count(args_reg);
759 : __ PrepareForTailCall(callee_args_count, caller_args_count_reg, scratch2,
760 930 : scratch3);
761 930 : __ bind(&done);
762 930 : }
763 :
764 : namespace {
765 :
766 413304 : void AdjustStackPointerForTailCall(Assembler* assembler,
767 : FrameAccessState* state,
768 : int new_slot_above_sp,
769 : bool allow_shrinkage = true) {
770 : int current_sp_offset = state->GetSPToFPSlotCount() +
771 413304 : StandardFrameConstants::kFixedSlotCountAboveFp;
772 413304 : int stack_slot_delta = new_slot_above_sp - current_sp_offset;
773 413304 : if (stack_slot_delta > 0) {
774 260 : assembler->subq(rsp, Immediate(stack_slot_delta * kPointerSize));
775 : state->IncreaseSPDelta(stack_slot_delta);
776 413174 : } else if (allow_shrinkage && stack_slot_delta < 0) {
777 46038 : assembler->addq(rsp, Immediate(-stack_slot_delta * kPointerSize));
778 : state->IncreaseSPDelta(stack_slot_delta);
779 : }
780 413304 : }
781 :
782 : } // namespace
783 :
784 186607 : void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
785 453394 : int first_unused_stack_slot) {
786 : CodeGenerator::PushTypeFlags flags(kImmediatePush | kScalarPush);
787 : ZoneVector<MoveOperands*> pushes(zone());
788 186607 : GetPushCompatibleMoves(instr, flags, &pushes);
789 :
790 197175 : if (!pushes.empty() &&
791 21136 : (LocationOperand::cast(pushes.back()->destination()).index() + 1 ==
792 : first_unused_stack_slot)) {
793 : X64OperandConverter g(this, instr);
794 61226 : for (auto move : pushes) {
795 : LocationOperand destination_location(
796 : LocationOperand::cast(move->destination()));
797 40090 : InstructionOperand source(move->source());
798 40090 : AdjustStackPointerForTailCall(tasm(), frame_access_state(),
799 40090 : destination_location.index());
800 40090 : if (source.IsStackSlot()) {
801 : LocationOperand source_location(LocationOperand::cast(source));
802 10586 : __ Push(g.SlotToOperand(source_location.index()));
803 29504 : } else if (source.IsRegister()) {
804 : LocationOperand source_location(LocationOperand::cast(source));
805 29504 : __ Push(source_location.GetRegister());
806 0 : } else if (source.IsImmediate()) {
807 0 : __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
808 : } else {
809 : // Pushes of non-scalar data types is not supported.
810 0 : UNIMPLEMENTED();
811 : }
812 : frame_access_state()->IncreaseSPDelta(1);
813 : move->Eliminate();
814 : }
815 : }
816 186607 : AdjustStackPointerForTailCall(tasm(), frame_access_state(),
817 186607 : first_unused_stack_slot, false);
818 186607 : }
819 :
820 186607 : void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
821 186607 : int first_unused_stack_slot) {
822 186607 : AdjustStackPointerForTailCall(tasm(), frame_access_state(),
823 186607 : first_unused_stack_slot);
824 186607 : }
825 :
826 : // Check if the code object is marked for deoptimization. If it is, then it
827 : // jumps to CompileLazyDeoptimizedCode builtin. In order to do this we need to:
828 : // 1. load the address of the current instruction;
829 : // 2. read from memory the word that contains that bit, which can be found in
830 : // the first set of flags ({kKindSpecificFlags1Offset});
831 : // 3. test kMarkedForDeoptimizationBit in those flags; and
832 : // 4. if it is not zero then it jumps to the builtin.
833 443344 : void CodeGenerator::BailoutIfDeoptimized() {
834 : Label current;
835 : // Load effective address to get the address of the current instruction into
836 : // rcx.
837 886693 : __ leaq(rcx, Operand(¤t));
838 443349 : __ bind(¤t);
839 443346 : int pc = __ pc_offset();
840 443346 : int offset = Code::kKindSpecificFlags1Offset - (Code::kHeaderSize + pc);
841 : __ testl(Operand(rcx, offset),
842 886695 : Immediate(1 << Code::kMarkedForDeoptimizationBit));
843 : Handle<Code> code = isolate()->builtins()->builtin_handle(
844 443349 : Builtins::kCompileLazyDeoptimizedCode);
845 443345 : __ j(not_zero, code, RelocInfo::CODE_TARGET);
846 443347 : }
847 :
848 : // Assembles an instruction after register allocation, producing machine code.
849 44716154 : CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
850 59445190 : Instruction* instr) {
851 : X64OperandConverter i(this, instr);
852 : InstructionCode opcode = instr->opcode();
853 44716154 : ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
854 44716154 : switch (arch_opcode) {
855 : case kArchCallCodeObject: {
856 4389782 : if (HasImmediateInput(instr, 0)) {
857 4385516 : Handle<Code> code = i.InputCode(0);
858 4385516 : __ Call(code, RelocInfo::CODE_TARGET);
859 : } else {
860 4299 : Register reg = i.InputRegister(0);
861 4299 : __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
862 4299 : __ call(reg);
863 : }
864 4389826 : RecordCallPosition(instr);
865 : frame_access_state()->ClearSPDelta();
866 : break;
867 : }
868 : case kArchTailCallCodeObjectFromJSFunction:
869 : case kArchTailCallCodeObject: {
870 156302 : if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
871 : AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
872 : i.TempRegister(0), i.TempRegister(1),
873 930 : i.TempRegister(2));
874 : }
875 156302 : if (HasImmediateInput(instr, 0)) {
876 155398 : Handle<Code> code = i.InputCode(0);
877 155398 : __ jmp(code, RelocInfo::CODE_TARGET);
878 : } else {
879 1808 : Register reg = i.InputRegister(0);
880 904 : __ addp(reg, Immediate(Code::kHeaderSize - kHeapObjectTag));
881 904 : __ jmp(reg);
882 : }
883 : unwinding_info_writer_.MarkBlockWillExit();
884 : frame_access_state()->ClearSPDelta();
885 156302 : frame_access_state()->SetFrameAccessToDefault();
886 156302 : break;
887 : }
888 : case kArchTailCallAddress: {
889 30287 : CHECK(!HasImmediateInput(instr, 0));
890 30287 : Register reg = i.InputRegister(0);
891 30287 : __ jmp(reg);
892 : unwinding_info_writer_.MarkBlockWillExit();
893 : frame_access_state()->ClearSPDelta();
894 30287 : frame_access_state()->SetFrameAccessToDefault();
895 : break;
896 : }
897 : case kArchCallJSFunction: {
898 : Register func = i.InputRegister(0);
899 152973 : if (FLAG_debug_code) {
900 : // Check the function's context matches the context argument.
901 8 : __ cmpp(rsi, FieldOperand(func, JSFunction::kContextOffset));
902 8 : __ Assert(equal, kWrongFunctionContext);
903 : }
904 152973 : __ movp(rcx, FieldOperand(func, JSFunction::kCodeOffset));
905 152973 : __ addp(rcx, Immediate(Code::kHeaderSize - kHeapObjectTag));
906 152973 : __ call(rcx);
907 : frame_access_state()->ClearSPDelta();
908 152973 : RecordCallPosition(instr);
909 : break;
910 : }
911 : case kArchPrepareCallCFunction: {
912 : // Frame alignment requires using FP-relative frame addressing.
913 : frame_access_state()->SetFrameAccessToFP();
914 41300 : int const num_parameters = MiscField::decode(instr->opcode());
915 41300 : __ PrepareCallCFunction(num_parameters);
916 41300 : break;
917 : }
918 : case kArchSaveCallerRegisters: {
919 : fp_mode_ =
920 378 : static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode()));
921 : DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
922 : // kReturnRegister0 should have been saved before entering the stub.
923 378 : int bytes = __ PushCallerSaved(fp_mode_, kReturnRegister0);
924 : DCHECK_EQ(0, bytes % kPointerSize);
925 : DCHECK_EQ(0, frame_access_state()->sp_delta());
926 378 : frame_access_state()->IncreaseSPDelta(bytes / kPointerSize);
927 : DCHECK(!caller_registers_saved_);
928 378 : caller_registers_saved_ = true;
929 378 : break;
930 : }
931 : case kArchRestoreCallerRegisters: {
932 : DCHECK(fp_mode_ ==
933 : static_cast<SaveFPRegsMode>(MiscField::decode(instr->opcode())));
934 : DCHECK(fp_mode_ == kDontSaveFPRegs || fp_mode_ == kSaveFPRegs);
935 : // Don't overwrite the returned value.
936 378 : int bytes = __ PopCallerSaved(fp_mode_, kReturnRegister0);
937 378 : frame_access_state()->IncreaseSPDelta(-(bytes / kPointerSize));
938 : DCHECK_EQ(0, frame_access_state()->sp_delta());
939 : DCHECK(caller_registers_saved_);
940 378 : caller_registers_saved_ = false;
941 378 : break;
942 : }
943 : case kArchPrepareTailCall:
944 186589 : AssemblePrepareTailCall();
945 186589 : break;
946 : case kArchCallCFunction: {
947 : int const num_parameters = MiscField::decode(instr->opcode());
948 41300 : if (HasImmediateInput(instr, 0)) {
949 40655 : ExternalReference ref = i.InputExternalReference(0);
950 40655 : __ CallCFunction(ref, num_parameters);
951 : } else {
952 645 : Register func = i.InputRegister(0);
953 645 : __ CallCFunction(func, num_parameters);
954 : }
955 41300 : frame_access_state()->SetFrameAccessToDefault();
956 : // Ideally, we should decrement SP delta to match the change of stack
957 : // pointer in CallCFunction. However, for certain architectures (e.g.
958 : // ARM), there may be more strict alignment requirement, causing old SP
959 : // to be saved on the stack. In those cases, we can not calculate the SP
960 : // delta statically.
961 : frame_access_state()->ClearSPDelta();
962 41300 : if (caller_registers_saved_) {
963 : // Need to re-sync SP delta introduced in kArchSaveCallerRegisters.
964 : // Here, we assume the sequence to be:
965 : // kArchSaveCallerRegisters;
966 : // kArchCallCFunction;
967 : // kArchRestoreCallerRegisters;
968 : int bytes =
969 378 : __ RequiredStackSizeForCallerSaved(fp_mode_, kReturnRegister0);
970 378 : frame_access_state()->IncreaseSPDelta(bytes / kPointerSize);
971 : }
972 : break;
973 : }
974 : case kArchJmp:
975 3391140 : AssembleArchJump(i.InputRpo(0));
976 3391130 : break;
977 : case kArchLookupSwitch:
978 20780 : AssembleArchLookupSwitch(instr);
979 20780 : break;
980 : case kArchTableSwitch:
981 7807 : AssembleArchTableSwitch(instr);
982 7807 : break;
983 : case kArchComment: {
984 : Address comment_string = i.InputExternalReference(0).address();
985 0 : __ RecordComment(reinterpret_cast<const char*>(comment_string));
986 0 : break;
987 : }
988 : case kArchDebugAbort:
989 : DCHECK(i.InputRegister(0) == rdx);
990 0 : if (!frame_access_state()->has_frame()) {
991 : // We don't actually want to generate a pile of code for this, so just
992 : // claim there is a stack frame, without generating one.
993 0 : FrameScope scope(tasm(), StackFrame::NONE);
994 : __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS),
995 0 : RelocInfo::CODE_TARGET);
996 : } else {
997 : __ Call(isolate()->builtins()->builtin_handle(Builtins::kAbortJS),
998 0 : RelocInfo::CODE_TARGET);
999 : }
1000 0 : __ int3();
1001 0 : break;
1002 : case kArchDebugBreak:
1003 22395 : __ int3();
1004 22395 : break;
1005 : case kArchNop:
1006 : case kArchThrowTerminator:
1007 : // don't emit code for nops.
1008 : break;
1009 : case kArchDeoptimize: {
1010 : int deopt_state_id =
1011 51672 : BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
1012 : CodeGenResult result =
1013 51672 : AssembleDeoptimizerCall(deopt_state_id, current_source_position_);
1014 51672 : if (result != kSuccess) return result;
1015 : break;
1016 : }
1017 : case kArchRet:
1018 1510596 : AssembleReturn(instr->InputAt(0));
1019 1510605 : break;
1020 : case kArchStackPointer:
1021 62 : __ movq(i.OutputRegister(), rsp);
1022 : break;
1023 : case kArchFramePointer:
1024 19989 : __ movq(i.OutputRegister(), rbp);
1025 : break;
1026 : case kArchParentFramePointer:
1027 24681 : if (frame_access_state()->has_frame()) {
1028 33960 : __ movq(i.OutputRegister(), Operand(rbp, 0));
1029 : } else {
1030 13361 : __ movq(i.OutputRegister(), rbp);
1031 : }
1032 : break;
1033 : case kArchTruncateDoubleToI: {
1034 : auto result = i.OutputRegister();
1035 : auto input = i.InputDoubleRegister(0);
1036 : auto ool = new (zone()) OutOfLineTruncateDoubleToI(
1037 45405 : this, result, input, &unwinding_info_writer_);
1038 : // We use Cvttsd2siq instead of Cvttsd2si due to performance reasons. The
1039 : // use of Cvttsd2siq requires the movl below to avoid sign extension.
1040 45401 : __ Cvttsd2siq(result, input);
1041 45408 : __ cmpq(result, Immediate(1));
1042 45408 : __ j(overflow, ool->entry());
1043 45406 : __ bind(ool->exit());
1044 : __ movl(result, result);
1045 : break;
1046 : }
1047 : case kArchStoreWithWriteBarrier: {
1048 : RecordWriteMode mode =
1049 : static_cast<RecordWriteMode>(MiscField::decode(instr->opcode()));
1050 : Register object = i.InputRegister(0);
1051 320882 : size_t index = 0;
1052 320882 : Operand operand = i.MemoryOperand(&index);
1053 320882 : Register value = i.InputRegister(index);
1054 : Register scratch0 = i.TempRegister(0);
1055 : Register scratch1 = i.TempRegister(1);
1056 : auto ool = new (zone()) OutOfLineRecordWrite(this, object, operand, value,
1057 641764 : scratch0, scratch1, mode);
1058 320882 : __ movp(operand, value);
1059 : __ CheckPageFlag(object, scratch0,
1060 : MemoryChunk::kPointersFromHereAreInterestingMask,
1061 320882 : not_zero, ool->entry());
1062 320882 : __ bind(ool->exit());
1063 : break;
1064 : }
1065 : case kArchStackSlot: {
1066 : FrameOffset offset =
1067 2354 : frame_access_state()->GetFrameOffset(i.InputInt32(0));
1068 1177 : Register base = offset.from_stack_pointer() ? rsp : rbp;
1069 3531 : __ leaq(i.OutputRegister(), Operand(base, offset.offset()));
1070 : break;
1071 : }
1072 : case kIeee754Float64Acos:
1073 68 : ASSEMBLE_IEEE754_UNOP(acos);
1074 68 : break;
1075 : case kIeee754Float64Acosh:
1076 67 : ASSEMBLE_IEEE754_UNOP(acosh);
1077 67 : break;
1078 : case kIeee754Float64Asin:
1079 68 : ASSEMBLE_IEEE754_UNOP(asin);
1080 68 : break;
1081 : case kIeee754Float64Asinh:
1082 67 : ASSEMBLE_IEEE754_UNOP(asinh);
1083 67 : break;
1084 : case kIeee754Float64Atan:
1085 81 : ASSEMBLE_IEEE754_UNOP(atan);
1086 81 : break;
1087 : case kIeee754Float64Atanh:
1088 67 : ASSEMBLE_IEEE754_UNOP(atanh);
1089 67 : break;
1090 : case kIeee754Float64Atan2:
1091 91 : ASSEMBLE_IEEE754_BINOP(atan2);
1092 91 : break;
1093 : case kIeee754Float64Cbrt:
1094 67 : ASSEMBLE_IEEE754_UNOP(cbrt);
1095 67 : break;
1096 : case kIeee754Float64Cos:
1097 256 : ASSEMBLE_IEEE754_UNOP(cos);
1098 256 : break;
1099 : case kIeee754Float64Cosh:
1100 75 : ASSEMBLE_IEEE754_UNOP(cosh);
1101 75 : break;
1102 : case kIeee754Float64Exp:
1103 81 : ASSEMBLE_IEEE754_UNOP(exp);
1104 81 : break;
1105 : case kIeee754Float64Expm1:
1106 67 : ASSEMBLE_IEEE754_UNOP(expm1);
1107 67 : break;
1108 : case kIeee754Float64Log:
1109 232 : ASSEMBLE_IEEE754_UNOP(log);
1110 232 : break;
1111 : case kIeee754Float64Log1p:
1112 67 : ASSEMBLE_IEEE754_UNOP(log1p);
1113 67 : break;
1114 : case kIeee754Float64Log2:
1115 67 : ASSEMBLE_IEEE754_UNOP(log2);
1116 67 : break;
1117 : case kIeee754Float64Log10:
1118 67 : ASSEMBLE_IEEE754_UNOP(log10);
1119 67 : break;
1120 : case kIeee754Float64Pow: {
1121 : // TODO(bmeurer): Improve integration of the stub.
1122 140 : __ Movsd(xmm2, xmm0);
1123 : __ CallStubDelayed(new (zone())
1124 140 : MathPowStub(nullptr, MathPowStub::DOUBLE));
1125 140 : __ Movsd(xmm0, xmm3);
1126 140 : break;
1127 : }
1128 : case kIeee754Float64Sin:
1129 256 : ASSEMBLE_IEEE754_UNOP(sin);
1130 256 : break;
1131 : case kIeee754Float64Sinh:
1132 75 : ASSEMBLE_IEEE754_UNOP(sinh);
1133 75 : break;
1134 : case kIeee754Float64Tan:
1135 81 : ASSEMBLE_IEEE754_UNOP(tan);
1136 81 : break;
1137 : case kIeee754Float64Tanh:
1138 75 : ASSEMBLE_IEEE754_UNOP(tanh);
1139 75 : break;
1140 : case kX64Add32:
1141 289938 : ASSEMBLE_BINOP(addl);
1142 : break;
1143 : case kX64Add:
1144 187232 : ASSEMBLE_BINOP(addq);
1145 : break;
1146 : case kX64Sub32:
1147 347690 : ASSEMBLE_BINOP(subl);
1148 : break;
1149 : case kX64Sub:
1150 194412 : ASSEMBLE_BINOP(subq);
1151 : break;
1152 : case kX64And32:
1153 540291 : ASSEMBLE_BINOP(andl);
1154 : break;
1155 : case kX64And:
1156 868395 : ASSEMBLE_BINOP(andq);
1157 : break;
1158 : case kX64Cmp8:
1159 750757 : ASSEMBLE_COMPARE(cmpb);
1160 : break;
1161 : case kX64Cmp16:
1162 17496 : ASSEMBLE_COMPARE(cmpw);
1163 : break;
1164 : case kX64Cmp32:
1165 4257283 : ASSEMBLE_COMPARE(cmpl);
1166 : break;
1167 : case kX64Cmp:
1168 6588705 : ASSEMBLE_COMPARE(cmpq);
1169 : break;
1170 : case kX64Test8:
1171 312536 : ASSEMBLE_COMPARE(testb);
1172 : break;
1173 : case kX64Test16:
1174 7152 : ASSEMBLE_COMPARE(testw);
1175 : break;
1176 : case kX64Test32:
1177 187799 : ASSEMBLE_COMPARE(testl);
1178 : break;
1179 : case kX64Test:
1180 1998878 : ASSEMBLE_COMPARE(testq);
1181 : break;
1182 : case kX64Imul32:
1183 182313 : ASSEMBLE_MULT(imull);
1184 : break;
1185 : case kX64Imul:
1186 9243 : ASSEMBLE_MULT(imulq);
1187 : break;
1188 : case kX64ImulHigh32:
1189 9698 : if (instr->InputAt(1)->IsRegister()) {
1190 4849 : __ imull(i.InputRegister(1));
1191 : } else {
1192 0 : __ imull(i.InputOperand(1));
1193 : }
1194 : break;
1195 : case kX64UmulHigh32:
1196 1258 : if (instr->InputAt(1)->IsRegister()) {
1197 629 : __ mull(i.InputRegister(1));
1198 : } else {
1199 0 : __ mull(i.InputOperand(1));
1200 : }
1201 : break;
1202 : case kX64Idiv32:
1203 5486 : __ cdq();
1204 5486 : __ idivl(i.InputRegister(1));
1205 : break;
1206 : case kX64Idiv:
1207 1028 : __ cqo();
1208 1028 : __ idivq(i.InputRegister(1));
1209 : break;
1210 : case kX64Udiv32:
1211 1164 : __ xorl(rdx, rdx);
1212 1164 : __ divl(i.InputRegister(1));
1213 : break;
1214 : case kX64Udiv:
1215 866 : __ xorq(rdx, rdx);
1216 866 : __ divq(i.InputRegister(1));
1217 : break;
1218 : case kX64Not:
1219 110 : ASSEMBLE_UNOP(notq);
1220 : break;
1221 : case kX64Not32:
1222 6938 : ASSEMBLE_UNOP(notl);
1223 : break;
1224 : case kX64Neg:
1225 9340 : ASSEMBLE_UNOP(negq);
1226 : break;
1227 : case kX64Neg32:
1228 10336 : ASSEMBLE_UNOP(negl);
1229 : break;
1230 : case kX64Or32:
1231 80518 : ASSEMBLE_BINOP(orl);
1232 : break;
1233 : case kX64Or:
1234 147880 : ASSEMBLE_BINOP(orq);
1235 : break;
1236 : case kX64Xor32:
1237 39512 : ASSEMBLE_BINOP(xorl);
1238 : break;
1239 : case kX64Xor:
1240 1647 : ASSEMBLE_BINOP(xorq);
1241 : break;
1242 : case kX64Shl32:
1243 41342 : ASSEMBLE_SHIFT(shll, 5);
1244 : break;
1245 : case kX64Shl:
1246 1080654 : ASSEMBLE_SHIFT(shlq, 6);
1247 : break;
1248 : case kX64Shr32:
1249 169721 : ASSEMBLE_SHIFT(shrl, 5);
1250 : break;
1251 : case kX64Shr:
1252 1175984 : ASSEMBLE_SHIFT(shrq, 6);
1253 : break;
1254 : case kX64Sar32:
1255 46519 : ASSEMBLE_SHIFT(sarl, 5);
1256 : break;
1257 : case kX64Sar:
1258 127634 : ASSEMBLE_SHIFT(sarq, 6);
1259 : break;
1260 : case kX64Ror32:
1261 3741 : ASSEMBLE_SHIFT(rorl, 5);
1262 : break;
1263 : case kX64Ror:
1264 468 : ASSEMBLE_SHIFT(rorq, 6);
1265 : break;
1266 : case kX64Lzcnt:
1267 74 : if (instr->InputAt(0)->IsRegister()) {
1268 37 : __ Lzcntq(i.OutputRegister(), i.InputRegister(0));
1269 : } else {
1270 0 : __ Lzcntq(i.OutputRegister(), i.InputOperand(0));
1271 : }
1272 : break;
1273 : case kX64Lzcnt32:
1274 3710 : if (instr->InputAt(0)->IsRegister()) {
1275 1695 : __ Lzcntl(i.OutputRegister(), i.InputRegister(0));
1276 : } else {
1277 320 : __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
1278 : }
1279 : break;
1280 : case kX64Tzcnt:
1281 54 : if (instr->InputAt(0)->IsRegister()) {
1282 27 : __ Tzcntq(i.OutputRegister(), i.InputRegister(0));
1283 : } else {
1284 0 : __ Tzcntq(i.OutputRegister(), i.InputOperand(0));
1285 : }
1286 : break;
1287 : case kX64Tzcnt32:
1288 1714 : if (instr->InputAt(0)->IsRegister()) {
1289 857 : __ Tzcntl(i.OutputRegister(), i.InputRegister(0));
1290 : } else {
1291 0 : __ Tzcntl(i.OutputRegister(), i.InputOperand(0));
1292 : }
1293 : break;
1294 : case kX64Popcnt:
1295 98 : if (instr->InputAt(0)->IsRegister()) {
1296 49 : __ Popcntq(i.OutputRegister(), i.InputRegister(0));
1297 : } else {
1298 0 : __ Popcntq(i.OutputRegister(), i.InputOperand(0));
1299 : }
1300 : break;
1301 : case kX64Popcnt32:
1302 314 : if (instr->InputAt(0)->IsRegister()) {
1303 157 : __ Popcntl(i.OutputRegister(), i.InputRegister(0));
1304 : } else {
1305 0 : __ Popcntl(i.OutputRegister(), i.InputOperand(0));
1306 : }
1307 : break;
1308 : case kSSEFloat32Cmp:
1309 0 : ASSEMBLE_SSE_BINOP(Ucomiss);
1310 : break;
1311 : case kSSEFloat32Add:
1312 0 : ASSEMBLE_SSE_BINOP(addss);
1313 : break;
1314 : case kSSEFloat32Sub:
1315 0 : ASSEMBLE_SSE_BINOP(subss);
1316 : break;
1317 : case kSSEFloat32Mul:
1318 0 : ASSEMBLE_SSE_BINOP(mulss);
1319 : break;
1320 : case kSSEFloat32Div:
1321 0 : ASSEMBLE_SSE_BINOP(divss);
1322 : // Don't delete this mov. It may improve performance on some CPUs,
1323 : // when there is a (v)mulss depending on the result.
1324 0 : __ movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1325 0 : break;
1326 : case kSSEFloat32Abs: {
1327 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1328 0 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1329 0 : __ psrlq(kScratchDoubleReg, 33);
1330 0 : __ andps(i.OutputDoubleRegister(), kScratchDoubleReg);
1331 0 : break;
1332 : }
1333 : case kSSEFloat32Neg: {
1334 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1335 0 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1336 0 : __ psllq(kScratchDoubleReg, 31);
1337 0 : __ xorps(i.OutputDoubleRegister(), kScratchDoubleReg);
1338 0 : break;
1339 : }
1340 : case kSSEFloat32Sqrt:
1341 588 : ASSEMBLE_SSE_UNOP(sqrtss);
1342 : break;
1343 : case kSSEFloat32ToFloat64:
1344 73524 : ASSEMBLE_SSE_UNOP(Cvtss2sd);
1345 : break;
1346 : case kSSEFloat32Round: {
1347 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
1348 : RoundingMode const mode =
1349 : static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1350 512 : __ Roundss(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1351 : break;
1352 : }
1353 : case kSSEFloat32ToInt32:
1354 542 : if (instr->InputAt(0)->IsFPRegister()) {
1355 271 : __ Cvttss2si(i.OutputRegister(), i.InputDoubleRegister(0));
1356 : } else {
1357 0 : __ Cvttss2si(i.OutputRegister(), i.InputOperand(0));
1358 : }
1359 : break;
1360 : case kSSEFloat32ToUint32: {
1361 134 : if (instr->InputAt(0)->IsFPRegister()) {
1362 67 : __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1363 : } else {
1364 0 : __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1365 : }
1366 : break;
1367 : }
1368 : case kSSEFloat64Cmp:
1369 4074 : ASSEMBLE_SSE_BINOP(Ucomisd);
1370 : break;
1371 : case kSSEFloat64Add:
1372 768 : ASSEMBLE_SSE_BINOP(addsd);
1373 : break;
1374 : case kSSEFloat64Sub:
1375 1158 : ASSEMBLE_SSE_BINOP(subsd);
1376 : break;
1377 : case kSSEFloat64Mul:
1378 108 : ASSEMBLE_SSE_BINOP(mulsd);
1379 : break;
1380 : case kSSEFloat64Div:
1381 78 : ASSEMBLE_SSE_BINOP(divsd);
1382 : // Don't delete this mov. It may improve performance on some CPUs,
1383 : // when there is a (v)mulsd depending on the result.
1384 52 : __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1385 26 : break;
1386 : case kSSEFloat64Mod: {
1387 1365 : __ subq(rsp, Immediate(kDoubleSize));
1388 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1389 2730 : kDoubleSize);
1390 : // Move values to st(0) and st(1).
1391 2730 : __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(1));
1392 1365 : __ fld_d(Operand(rsp, 0));
1393 2730 : __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
1394 1365 : __ fld_d(Operand(rsp, 0));
1395 : // Loop while fprem isn't done.
1396 : Label mod_loop;
1397 1365 : __ bind(&mod_loop);
1398 : // This instructions traps on all kinds inputs, but we are assuming the
1399 : // floating point control word is set to ignore them all.
1400 1365 : __ fprem();
1401 : // The following 2 instruction implicitly use rax.
1402 1365 : __ fnstsw_ax();
1403 1365 : if (CpuFeatures::IsSupported(SAHF)) {
1404 : CpuFeatureScope sahf_scope(tasm(), SAHF);
1405 1341 : __ sahf();
1406 : } else {
1407 : __ shrl(rax, Immediate(8));
1408 24 : __ andl(rax, Immediate(0xFF));
1409 24 : __ pushq(rax);
1410 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1411 48 : kPointerSize);
1412 24 : __ popfq();
1413 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1414 48 : -kPointerSize);
1415 : }
1416 1365 : __ j(parity_even, &mod_loop);
1417 : // Move output to stack and clean up.
1418 1365 : __ fstp(1);
1419 1365 : __ fstp_d(Operand(rsp, 0));
1420 2730 : __ Movsd(i.OutputDoubleRegister(), Operand(rsp, 0));
1421 1365 : __ addq(rsp, Immediate(kDoubleSize));
1422 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
1423 2730 : -kDoubleSize);
1424 : break;
1425 : }
1426 : case kSSEFloat32Max: {
1427 : Label compare_nan, compare_swap, done_compare;
1428 122 : if (instr->InputAt(1)->IsFPRegister()) {
1429 61 : __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1430 : } else {
1431 0 : __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1432 : }
1433 : auto ool =
1434 61 : new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1435 61 : __ j(parity_even, ool->entry());
1436 61 : __ j(above, &done_compare, Label::kNear);
1437 61 : __ j(below, &compare_swap, Label::kNear);
1438 122 : __ Movmskps(kScratchRegister, i.InputDoubleRegister(0));
1439 : __ testl(kScratchRegister, Immediate(1));
1440 61 : __ j(zero, &done_compare, Label::kNear);
1441 61 : __ bind(&compare_swap);
1442 122 : if (instr->InputAt(1)->IsFPRegister()) {
1443 122 : __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1444 : } else {
1445 0 : __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1446 : }
1447 61 : __ bind(&done_compare);
1448 61 : __ bind(ool->exit());
1449 : break;
1450 : }
1451 : case kSSEFloat32Min: {
1452 : Label compare_swap, done_compare;
1453 122 : if (instr->InputAt(1)->IsFPRegister()) {
1454 61 : __ Ucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1455 : } else {
1456 0 : __ Ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1457 : }
1458 : auto ool =
1459 61 : new (zone()) OutOfLineLoadFloat32NaN(this, i.OutputDoubleRegister());
1460 61 : __ j(parity_even, ool->entry());
1461 61 : __ j(below, &done_compare, Label::kNear);
1462 61 : __ j(above, &compare_swap, Label::kNear);
1463 122 : if (instr->InputAt(1)->IsFPRegister()) {
1464 122 : __ Movmskps(kScratchRegister, i.InputDoubleRegister(1));
1465 : } else {
1466 0 : __ Movss(kScratchDoubleReg, i.InputOperand(1));
1467 0 : __ Movmskps(kScratchRegister, kScratchDoubleReg);
1468 : }
1469 : __ testl(kScratchRegister, Immediate(1));
1470 61 : __ j(zero, &done_compare, Label::kNear);
1471 61 : __ bind(&compare_swap);
1472 122 : if (instr->InputAt(1)->IsFPRegister()) {
1473 122 : __ Movss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1474 : } else {
1475 0 : __ Movss(i.InputDoubleRegister(0), i.InputOperand(1));
1476 : }
1477 61 : __ bind(&done_compare);
1478 61 : __ bind(ool->exit());
1479 : break;
1480 : }
1481 : case kSSEFloat64Max: {
1482 : Label compare_nan, compare_swap, done_compare;
1483 284 : if (instr->InputAt(1)->IsFPRegister()) {
1484 142 : __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1485 : } else {
1486 0 : __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1487 : }
1488 : auto ool =
1489 142 : new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1490 142 : __ j(parity_even, ool->entry());
1491 142 : __ j(above, &done_compare, Label::kNear);
1492 142 : __ j(below, &compare_swap, Label::kNear);
1493 284 : __ Movmskpd(kScratchRegister, i.InputDoubleRegister(0));
1494 : __ testl(kScratchRegister, Immediate(1));
1495 142 : __ j(zero, &done_compare, Label::kNear);
1496 142 : __ bind(&compare_swap);
1497 284 : if (instr->InputAt(1)->IsFPRegister()) {
1498 284 : __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1499 : } else {
1500 0 : __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1501 : }
1502 142 : __ bind(&done_compare);
1503 142 : __ bind(ool->exit());
1504 : break;
1505 : }
1506 : case kSSEFloat64Min: {
1507 : Label compare_swap, done_compare;
1508 602 : if (instr->InputAt(1)->IsFPRegister()) {
1509 293 : __ Ucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1510 : } else {
1511 16 : __ Ucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1512 : }
1513 : auto ool =
1514 301 : new (zone()) OutOfLineLoadFloat64NaN(this, i.OutputDoubleRegister());
1515 301 : __ j(parity_even, ool->entry());
1516 301 : __ j(below, &done_compare, Label::kNear);
1517 301 : __ j(above, &compare_swap, Label::kNear);
1518 602 : if (instr->InputAt(1)->IsFPRegister()) {
1519 586 : __ Movmskpd(kScratchRegister, i.InputDoubleRegister(1));
1520 : } else {
1521 8 : __ Movsd(kScratchDoubleReg, i.InputOperand(1));
1522 8 : __ Movmskpd(kScratchRegister, kScratchDoubleReg);
1523 : }
1524 : __ testl(kScratchRegister, Immediate(1));
1525 301 : __ j(zero, &done_compare, Label::kNear);
1526 301 : __ bind(&compare_swap);
1527 602 : if (instr->InputAt(1)->IsFPRegister()) {
1528 586 : __ Movsd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1529 : } else {
1530 16 : __ Movsd(i.InputDoubleRegister(0), i.InputOperand(1));
1531 : }
1532 301 : __ bind(&done_compare);
1533 301 : __ bind(ool->exit());
1534 : break;
1535 : }
1536 : case kSSEFloat64Abs: {
1537 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1538 6 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1539 6 : __ psrlq(kScratchDoubleReg, 1);
1540 12 : __ andpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1541 6 : break;
1542 : }
1543 : case kSSEFloat64Neg: {
1544 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1545 182 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
1546 182 : __ psllq(kScratchDoubleReg, 63);
1547 364 : __ xorpd(i.OutputDoubleRegister(), kScratchDoubleReg);
1548 182 : break;
1549 : }
1550 : case kSSEFloat64Sqrt:
1551 1263 : ASSEMBLE_SSE_UNOP(Sqrtsd);
1552 : break;
1553 : case kSSEFloat64Round: {
1554 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
1555 : RoundingMode const mode =
1556 : static_cast<RoundingMode>(MiscField::decode(instr->opcode()));
1557 18797 : __ Roundsd(i.OutputDoubleRegister(), i.InputDoubleRegister(0), mode);
1558 : break;
1559 : }
1560 : case kSSEFloat64ToFloat32:
1561 61902 : ASSEMBLE_SSE_UNOP(Cvtsd2ss);
1562 : break;
1563 : case kSSEFloat64ToInt32:
1564 217294 : if (instr->InputAt(0)->IsFPRegister()) {
1565 86211 : __ Cvttsd2si(i.OutputRegister(), i.InputDoubleRegister(0));
1566 : } else {
1567 44872 : __ Cvttsd2si(i.OutputRegister(), i.InputOperand(0));
1568 : }
1569 : break;
1570 : case kSSEFloat64ToUint32: {
1571 638 : if (instr->InputAt(0)->IsFPRegister()) {
1572 319 : __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1573 : } else {
1574 0 : __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1575 : }
1576 638 : if (MiscField::decode(instr->opcode())) {
1577 534 : __ AssertZeroExtended(i.OutputRegister());
1578 : }
1579 : break;
1580 : }
1581 : case kSSEFloat32ToInt64:
1582 148 : if (instr->InputAt(0)->IsFPRegister()) {
1583 74 : __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1584 : } else {
1585 0 : __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1586 : }
1587 74 : if (instr->OutputCount() > 1) {
1588 138 : __ Set(i.OutputRegister(1), 1);
1589 : Label done;
1590 : Label fail;
1591 : __ Move(kScratchDoubleReg, static_cast<float>(INT64_MIN));
1592 138 : if (instr->InputAt(0)->IsFPRegister()) {
1593 138 : __ Ucomiss(kScratchDoubleReg, i.InputDoubleRegister(0));
1594 : } else {
1595 0 : __ Ucomiss(kScratchDoubleReg, i.InputOperand(0));
1596 : }
1597 : // If the input is NaN, then the conversion fails.
1598 69 : __ j(parity_even, &fail);
1599 : // If the input is INT64_MIN, then the conversion succeeds.
1600 69 : __ j(equal, &done);
1601 138 : __ cmpq(i.OutputRegister(0), Immediate(1));
1602 : // If the conversion results in INT64_MIN, but the input was not
1603 : // INT64_MIN, then the conversion fails.
1604 69 : __ j(no_overflow, &done);
1605 69 : __ bind(&fail);
1606 138 : __ Set(i.OutputRegister(1), 0);
1607 69 : __ bind(&done);
1608 : }
1609 : break;
1610 : case kSSEFloat64ToInt64:
1611 252 : if (instr->InputAt(0)->IsFPRegister()) {
1612 126 : __ Cvttsd2siq(i.OutputRegister(0), i.InputDoubleRegister(0));
1613 : } else {
1614 0 : __ Cvttsd2siq(i.OutputRegister(0), i.InputOperand(0));
1615 : }
1616 126 : if (instr->OutputCount() > 1) {
1617 242 : __ Set(i.OutputRegister(1), 1);
1618 : Label done;
1619 : Label fail;
1620 : __ Move(kScratchDoubleReg, static_cast<double>(INT64_MIN));
1621 242 : if (instr->InputAt(0)->IsFPRegister()) {
1622 242 : __ Ucomisd(kScratchDoubleReg, i.InputDoubleRegister(0));
1623 : } else {
1624 0 : __ Ucomisd(kScratchDoubleReg, i.InputOperand(0));
1625 : }
1626 : // If the input is NaN, then the conversion fails.
1627 121 : __ j(parity_even, &fail);
1628 : // If the input is INT64_MIN, then the conversion succeeds.
1629 121 : __ j(equal, &done);
1630 242 : __ cmpq(i.OutputRegister(0), Immediate(1));
1631 : // If the conversion results in INT64_MIN, but the input was not
1632 : // INT64_MIN, then the conversion fails.
1633 121 : __ j(no_overflow, &done);
1634 121 : __ bind(&fail);
1635 242 : __ Set(i.OutputRegister(1), 0);
1636 121 : __ bind(&done);
1637 : }
1638 : break;
1639 : case kSSEFloat32ToUint64: {
1640 : Label done;
1641 : Label success;
1642 74 : if (instr->OutputCount() > 1) {
1643 69 : __ Set(i.OutputRegister(1), 0);
1644 : }
1645 : // There does not exist a Float32ToUint64 instruction, so we have to use
1646 : // the Float32ToInt64 instruction.
1647 148 : if (instr->InputAt(0)->IsFPRegister()) {
1648 148 : __ Cvttss2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1649 : } else {
1650 0 : __ Cvttss2siq(i.OutputRegister(), i.InputOperand(0));
1651 : }
1652 : // Check if the result of the Float32ToInt64 conversion is positive, we
1653 : // are already done.
1654 148 : __ testq(i.OutputRegister(), i.OutputRegister());
1655 74 : __ j(positive, &success);
1656 : // The result of the first conversion was negative, which means that the
1657 : // input value was not within the positive int64 range. We subtract 2^64
1658 : // and convert it again to see if it is within the uint64 range.
1659 74 : __ Move(kScratchDoubleReg, -9223372036854775808.0f);
1660 148 : if (instr->InputAt(0)->IsFPRegister()) {
1661 148 : __ addss(kScratchDoubleReg, i.InputDoubleRegister(0));
1662 : } else {
1663 0 : __ addss(kScratchDoubleReg, i.InputOperand(0));
1664 : }
1665 148 : __ Cvttss2siq(i.OutputRegister(), kScratchDoubleReg);
1666 74 : __ testq(i.OutputRegister(), i.OutputRegister());
1667 : // The only possible negative value here is 0x80000000000000000, which is
1668 : // used on x64 to indicate an integer overflow.
1669 74 : __ j(negative, &done);
1670 : // The input value is within uint64 range and the second conversion worked
1671 : // successfully, but we still have to undo the subtraction we did
1672 : // earlier.
1673 74 : __ Set(kScratchRegister, 0x8000000000000000);
1674 148 : __ orq(i.OutputRegister(), kScratchRegister);
1675 74 : __ bind(&success);
1676 74 : if (instr->OutputCount() > 1) {
1677 138 : __ Set(i.OutputRegister(1), 1);
1678 : }
1679 74 : __ bind(&done);
1680 : break;
1681 : }
1682 : case kSSEFloat64ToUint64: {
1683 : Label done;
1684 : Label success;
1685 270 : if (instr->OutputCount() > 1) {
1686 79 : __ Set(i.OutputRegister(1), 0);
1687 : }
1688 : // There does not exist a Float64ToUint64 instruction, so we have to use
1689 : // the Float64ToInt64 instruction.
1690 540 : if (instr->InputAt(0)->IsFPRegister()) {
1691 540 : __ Cvttsd2siq(i.OutputRegister(), i.InputDoubleRegister(0));
1692 : } else {
1693 0 : __ Cvttsd2siq(i.OutputRegister(), i.InputOperand(0));
1694 : }
1695 : // Check if the result of the Float64ToInt64 conversion is positive, we
1696 : // are already done.
1697 540 : __ testq(i.OutputRegister(), i.OutputRegister());
1698 270 : __ j(positive, &success);
1699 : // The result of the first conversion was negative, which means that the
1700 : // input value was not within the positive int64 range. We subtract 2^64
1701 : // and convert it again to see if it is within the uint64 range.
1702 270 : __ Move(kScratchDoubleReg, -9223372036854775808.0);
1703 540 : if (instr->InputAt(0)->IsFPRegister()) {
1704 540 : __ addsd(kScratchDoubleReg, i.InputDoubleRegister(0));
1705 : } else {
1706 0 : __ addsd(kScratchDoubleReg, i.InputOperand(0));
1707 : }
1708 540 : __ Cvttsd2siq(i.OutputRegister(), kScratchDoubleReg);
1709 270 : __ testq(i.OutputRegister(), i.OutputRegister());
1710 : // The only possible negative value here is 0x80000000000000000, which is
1711 : // used on x64 to indicate an integer overflow.
1712 270 : __ j(negative, &done);
1713 : // The input value is within uint64 range and the second conversion worked
1714 : // successfully, but we still have to undo the subtraction we did
1715 : // earlier.
1716 270 : __ Set(kScratchRegister, 0x8000000000000000);
1717 540 : __ orq(i.OutputRegister(), kScratchRegister);
1718 270 : __ bind(&success);
1719 270 : if (instr->OutputCount() > 1) {
1720 158 : __ Set(i.OutputRegister(1), 1);
1721 : }
1722 270 : __ bind(&done);
1723 : break;
1724 : }
1725 : case kSSEInt32ToFloat64:
1726 598756 : if (instr->InputAt(0)->IsRegister()) {
1727 295316 : __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1728 : } else {
1729 8124 : __ Cvtlsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1730 : }
1731 : break;
1732 : case kSSEInt32ToFloat32:
1733 1130 : if (instr->InputAt(0)->IsRegister()) {
1734 555 : __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1735 : } else {
1736 20 : __ Cvtlsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1737 : }
1738 : break;
1739 : case kSSEInt64ToFloat32:
1740 90 : if (instr->InputAt(0)->IsRegister()) {
1741 45 : __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputRegister(0));
1742 : } else {
1743 0 : __ Cvtqsi2ss(i.OutputDoubleRegister(), i.InputOperand(0));
1744 : }
1745 : break;
1746 : case kSSEInt64ToFloat64:
1747 2110 : if (instr->InputAt(0)->IsRegister()) {
1748 1055 : __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputRegister(0));
1749 : } else {
1750 0 : __ Cvtqsi2sd(i.OutputDoubleRegister(), i.InputOperand(0));
1751 : }
1752 : break;
1753 : case kSSEUint64ToFloat32:
1754 94 : if (instr->InputAt(0)->IsRegister()) {
1755 47 : __ movq(kScratchRegister, i.InputRegister(0));
1756 : } else {
1757 0 : __ movq(kScratchRegister, i.InputOperand(0));
1758 : }
1759 : __ Cvtqui2ss(i.OutputDoubleRegister(), kScratchRegister,
1760 94 : i.TempRegister(0));
1761 47 : break;
1762 : case kSSEUint64ToFloat64:
1763 354 : if (instr->InputAt(0)->IsRegister()) {
1764 177 : __ movq(kScratchRegister, i.InputRegister(0));
1765 : } else {
1766 0 : __ movq(kScratchRegister, i.InputOperand(0));
1767 : }
1768 : __ Cvtqui2sd(i.OutputDoubleRegister(), kScratchRegister,
1769 348 : i.TempRegister(0));
1770 177 : break;
1771 : case kSSEUint32ToFloat64:
1772 5102 : if (instr->InputAt(0)->IsRegister()) {
1773 2071 : __ movl(kScratchRegister, i.InputRegister(0));
1774 : } else {
1775 480 : __ movl(kScratchRegister, i.InputOperand(0));
1776 : }
1777 5104 : __ Cvtqsi2sd(i.OutputDoubleRegister(), kScratchRegister);
1778 2552 : break;
1779 : case kSSEUint32ToFloat32:
1780 246 : if (instr->InputAt(0)->IsRegister()) {
1781 123 : __ movl(kScratchRegister, i.InputRegister(0));
1782 : } else {
1783 0 : __ movl(kScratchRegister, i.InputOperand(0));
1784 : }
1785 246 : __ Cvtqsi2ss(i.OutputDoubleRegister(), kScratchRegister);
1786 123 : break;
1787 : case kSSEFloat64ExtractLowWord32:
1788 10 : if (instr->InputAt(0)->IsFPStackSlot()) {
1789 0 : __ movl(i.OutputRegister(), i.InputOperand(0));
1790 : } else {
1791 5 : __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
1792 : }
1793 : break;
1794 : case kSSEFloat64ExtractHighWord32:
1795 145066 : if (instr->InputAt(0)->IsFPStackSlot()) {
1796 85540 : __ movl(i.OutputRegister(), i.InputOperand(0, kDoubleSize / 2));
1797 : } else {
1798 29763 : __ Pextrd(i.OutputRegister(), i.InputDoubleRegister(0), 1);
1799 : }
1800 : break;
1801 : case kSSEFloat64InsertLowWord32:
1802 10 : if (instr->InputAt(1)->IsRegister()) {
1803 5 : __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 0);
1804 : } else {
1805 0 : __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 0);
1806 : }
1807 : break;
1808 : case kSSEFloat64InsertHighWord32:
1809 10 : if (instr->InputAt(1)->IsRegister()) {
1810 5 : __ Pinsrd(i.OutputDoubleRegister(), i.InputRegister(1), 1);
1811 : } else {
1812 0 : __ Pinsrd(i.OutputDoubleRegister(), i.InputOperand(1), 1);
1813 : }
1814 : break;
1815 : case kSSEFloat64LoadLowWord32:
1816 0 : if (instr->InputAt(0)->IsRegister()) {
1817 0 : __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
1818 : } else {
1819 0 : __ Movd(i.OutputDoubleRegister(), i.InputOperand(0));
1820 : }
1821 : break;
1822 : case kAVXFloat32Cmp: {
1823 : CpuFeatureScope avx_scope(tasm(), AVX);
1824 3886 : if (instr->InputAt(1)->IsFPRegister()) {
1825 1921 : __ vucomiss(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1826 : } else {
1827 44 : __ vucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
1828 : }
1829 : break;
1830 : }
1831 : case kAVXFloat32Add:
1832 1878 : ASSEMBLE_AVX_BINOP(vaddss);
1833 : break;
1834 : case kAVXFloat32Sub:
1835 4815 : ASSEMBLE_AVX_BINOP(vsubss);
1836 : break;
1837 : case kAVXFloat32Mul:
1838 1713 : ASSEMBLE_AVX_BINOP(vmulss);
1839 : break;
1840 : case kAVXFloat32Div:
1841 1509 : ASSEMBLE_AVX_BINOP(vdivss);
1842 : // Don't delete this mov. It may improve performance on some CPUs,
1843 : // when there is a (v)mulss depending on the result.
1844 1006 : __ Movaps(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1845 503 : break;
1846 : case kAVXFloat64Cmp: {
1847 : CpuFeatureScope avx_scope(tasm(), AVX);
1848 422794 : if (instr->InputAt(1)->IsFPRegister()) {
1849 184568 : __ vucomisd(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
1850 : } else {
1851 80487 : __ vucomisd(i.InputDoubleRegister(0), i.InputOperand(1));
1852 : }
1853 : break;
1854 : }
1855 : case kAVXFloat64Add:
1856 181590 : ASSEMBLE_AVX_BINOP(vaddsd);
1857 : break;
1858 : case kAVXFloat64Sub:
1859 66729 : ASSEMBLE_AVX_BINOP(vsubsd);
1860 : break;
1861 : case kAVXFloat64Mul:
1862 41472 : ASSEMBLE_AVX_BINOP(vmulsd);
1863 : break;
1864 : case kAVXFloat64Div:
1865 35010 : ASSEMBLE_AVX_BINOP(vdivsd);
1866 : // Don't delete this mov. It may improve performance on some CPUs,
1867 : // when there is a (v)mulsd depending on the result.
1868 23340 : __ Movapd(i.OutputDoubleRegister(), i.OutputDoubleRegister());
1869 11671 : break;
1870 : case kAVXFloat32Abs: {
1871 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1872 : CpuFeatureScope avx_scope(tasm(), AVX);
1873 85 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1874 : __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 33);
1875 170 : if (instr->InputAt(0)->IsFPRegister()) {
1876 : __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1877 85 : i.InputDoubleRegister(0));
1878 : } else {
1879 : __ vandps(i.OutputDoubleRegister(), kScratchDoubleReg,
1880 0 : i.InputOperand(0));
1881 : }
1882 : break;
1883 : }
1884 : case kAVXFloat32Neg: {
1885 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1886 : CpuFeatureScope avx_scope(tasm(), AVX);
1887 225 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1888 : __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 31);
1889 456 : if (instr->InputAt(0)->IsFPRegister()) {
1890 : __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1891 228 : i.InputDoubleRegister(0));
1892 : } else {
1893 : __ vxorps(i.OutputDoubleRegister(), kScratchDoubleReg,
1894 0 : i.InputOperand(0));
1895 : }
1896 : break;
1897 : }
1898 : case kAVXFloat64Abs: {
1899 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1900 : CpuFeatureScope avx_scope(tasm(), AVX);
1901 367 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1902 : __ vpsrlq(kScratchDoubleReg, kScratchDoubleReg, 1);
1903 734 : if (instr->InputAt(0)->IsFPRegister()) {
1904 : __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1905 367 : i.InputDoubleRegister(0));
1906 : } else {
1907 : __ vandpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1908 0 : i.InputOperand(0));
1909 : }
1910 : break;
1911 : }
1912 : case kAVXFloat64Neg: {
1913 : // TODO(bmeurer): Use RIP relative 128-bit constants.
1914 : CpuFeatureScope avx_scope(tasm(), AVX);
1915 9913 : __ vpcmpeqd(kScratchDoubleReg, kScratchDoubleReg, kScratchDoubleReg);
1916 : __ vpsllq(kScratchDoubleReg, kScratchDoubleReg, 63);
1917 19826 : if (instr->InputAt(0)->IsFPRegister()) {
1918 : __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1919 9817 : i.InputDoubleRegister(0));
1920 : } else {
1921 : __ vxorpd(i.OutputDoubleRegister(), kScratchDoubleReg,
1922 192 : i.InputOperand(0));
1923 : }
1924 : break;
1925 : }
1926 : case kSSEFloat64SilenceNaN:
1927 1825 : __ Xorpd(kScratchDoubleReg, kScratchDoubleReg);
1928 3650 : __ Subsd(i.InputDoubleRegister(0), kScratchDoubleReg);
1929 1825 : break;
1930 : case kX64Movsxbl:
1931 13844 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1932 21733 : ASSEMBLE_MOVX(movsxbl);
1933 13844 : __ AssertZeroExtended(i.OutputRegister());
1934 6922 : break;
1935 : case kX64Movzxbl:
1936 290874 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1937 438376 : ASSEMBLE_MOVX(movzxbl);
1938 290874 : __ AssertZeroExtended(i.OutputRegister());
1939 145437 : break;
1940 : case kX64Movsxbq:
1941 12724 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1942 19086 : ASSEMBLE_MOVX(movsxbq);
1943 : break;
1944 : case kX64Movzxbq:
1945 13336 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1946 20004 : ASSEMBLE_MOVX(movzxbq);
1947 13336 : __ AssertZeroExtended(i.OutputRegister());
1948 6668 : break;
1949 : case kX64Movb: {
1950 24062 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1951 12031 : size_t index = 0;
1952 12031 : Operand operand = i.MemoryOperand(&index);
1953 24062 : if (HasImmediateInput(instr, index)) {
1954 4332 : __ movb(operand, Immediate(i.InputInt8(index)));
1955 : } else {
1956 19730 : __ movb(operand, i.InputRegister(index));
1957 : }
1958 : break;
1959 : }
1960 : case kX64Movsxwl:
1961 6180 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1962 9956 : ASSEMBLE_MOVX(movsxwl);
1963 6180 : __ AssertZeroExtended(i.OutputRegister());
1964 3090 : break;
1965 : case kX64Movzxwl:
1966 44088 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1967 71248 : ASSEMBLE_MOVX(movzxwl);
1968 44090 : __ AssertZeroExtended(i.OutputRegister());
1969 22045 : break;
1970 : case kX64Movsxwq:
1971 9952 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1972 14928 : ASSEMBLE_MOVX(movsxwq);
1973 : break;
1974 : case kX64Movzxwq:
1975 0 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1976 0 : ASSEMBLE_MOVX(movzxwq);
1977 0 : __ AssertZeroExtended(i.OutputRegister());
1978 0 : break;
1979 : case kX64Movw: {
1980 16090 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1981 8045 : size_t index = 0;
1982 8045 : Operand operand = i.MemoryOperand(&index);
1983 16090 : if (HasImmediateInput(instr, index)) {
1984 702 : __ movw(operand, Immediate(i.InputInt16(index)));
1985 : } else {
1986 15388 : __ movw(operand, i.InputRegister(index));
1987 : }
1988 : break;
1989 : }
1990 : case kX64Movl:
1991 1276930 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
1992 638466 : if (instr->HasOutput()) {
1993 389331 : if (instr->addressing_mode() == kMode_None) {
1994 156242 : if (instr->InputAt(0)->IsRegister()) {
1995 153346 : __ movl(i.OutputRegister(), i.InputRegister(0));
1996 : } else {
1997 2896 : __ movl(i.OutputRegister(), i.InputOperand(0));
1998 : }
1999 : } else {
2000 622410 : __ movl(i.OutputRegister(), i.MemoryOperand());
2001 : }
2002 778598 : __ AssertZeroExtended(i.OutputRegister());
2003 : } else {
2004 249135 : size_t index = 0;
2005 249135 : Operand operand = i.MemoryOperand(&index);
2006 498272 : if (HasImmediateInput(instr, index)) {
2007 68181 : __ movl(operand, i.InputImmediate(index));
2008 : } else {
2009 361910 : __ movl(operand, i.InputRegister(index));
2010 : }
2011 : }
2012 : break;
2013 : case kX64Movsxlq:
2014 160432 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
2015 268441 : ASSEMBLE_MOVX(movsxlq);
2016 : break;
2017 : case kX64Movq:
2018 8219362 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
2019 4109678 : if (instr->HasOutput()) {
2020 5278760 : __ movq(i.OutputRegister(), i.MemoryOperand());
2021 : } else {
2022 1470322 : size_t index = 0;
2023 1470322 : Operand operand = i.MemoryOperand(&index);
2024 2940640 : if (HasImmediateInput(instr, index)) {
2025 53562 : __ movq(operand, i.InputImmediate(index));
2026 : } else {
2027 2833516 : __ movq(operand, i.InputRegister(index));
2028 : }
2029 : }
2030 : break;
2031 : case kX64Movss:
2032 37170 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
2033 18585 : if (instr->HasOutput()) {
2034 22506 : __ movss(i.OutputDoubleRegister(), i.MemoryOperand());
2035 : } else {
2036 7333 : size_t index = 0;
2037 7333 : Operand operand = i.MemoryOperand(&index);
2038 14666 : __ movss(operand, i.InputDoubleRegister(index));
2039 : }
2040 : break;
2041 : case kX64Movsd:
2042 831474 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
2043 415728 : if (instr->HasOutput()) {
2044 577342 : __ Movsd(i.OutputDoubleRegister(), i.MemoryOperand());
2045 : } else {
2046 127073 : size_t index = 0;
2047 127073 : Operand operand = i.MemoryOperand(&index);
2048 254150 : __ Movsd(operand, i.InputDoubleRegister(index));
2049 : }
2050 : break;
2051 : case kX64Movdqu: {
2052 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2053 240 : EmitOOLTrapIfNeeded(zone(), this, opcode, instr, i, __ pc_offset());
2054 120 : if (instr->HasOutput()) {
2055 156 : __ movdqu(i.OutputSimd128Register(), i.MemoryOperand());
2056 : } else {
2057 42 : size_t index = 0;
2058 42 : Operand operand = i.MemoryOperand(&index);
2059 84 : __ movdqu(operand, i.InputSimd128Register(index));
2060 : }
2061 : break;
2062 : }
2063 : case kX64BitcastFI:
2064 125568 : if (instr->InputAt(0)->IsFPStackSlot()) {
2065 0 : __ movl(i.OutputRegister(), i.InputOperand(0));
2066 : } else {
2067 62784 : __ Movd(i.OutputRegister(), i.InputDoubleRegister(0));
2068 : }
2069 : break;
2070 : case kX64BitcastDL:
2071 124780 : if (instr->InputAt(0)->IsFPStackSlot()) {
2072 0 : __ movq(i.OutputRegister(), i.InputOperand(0));
2073 : } else {
2074 62390 : __ Movq(i.OutputRegister(), i.InputDoubleRegister(0));
2075 : }
2076 : break;
2077 : case kX64BitcastIF:
2078 438 : if (instr->InputAt(0)->IsRegister()) {
2079 219 : __ Movd(i.OutputDoubleRegister(), i.InputRegister(0));
2080 : } else {
2081 0 : __ movss(i.OutputDoubleRegister(), i.InputOperand(0));
2082 : }
2083 : break;
2084 : case kX64BitcastLD:
2085 550 : if (instr->InputAt(0)->IsRegister()) {
2086 275 : __ Movq(i.OutputDoubleRegister(), i.InputRegister(0));
2087 : } else {
2088 0 : __ Movsd(i.OutputDoubleRegister(), i.InputOperand(0));
2089 : }
2090 : break;
2091 : case kX64Lea32: {
2092 : AddressingMode mode = AddressingModeField::decode(instr->opcode());
2093 : // Shorten "leal" to "addl", "subl" or "shll" if the register allocation
2094 : // and addressing mode just happens to work out. The "addl"/"subl" forms
2095 : // in these cases are faster based on measurements.
2096 223533 : if (i.InputRegister(0) == i.OutputRegister()) {
2097 94124 : if (mode == kMode_MRI) {
2098 68580 : int32_t constant_summand = i.InputInt32(1);
2099 68584 : if (constant_summand > 0) {
2100 86512 : __ addl(i.OutputRegister(), Immediate(constant_summand));
2101 25328 : } else if (constant_summand < 0) {
2102 75321 : __ subl(i.OutputRegister(), Immediate(-constant_summand));
2103 : }
2104 25544 : } else if (mode == kMode_MR1) {
2105 10745 : if (i.InputRegister(1) == i.OutputRegister()) {
2106 887 : __ shll(i.OutputRegister(), Immediate(1));
2107 : } else {
2108 9858 : __ addl(i.OutputRegister(), i.InputRegister(1));
2109 : }
2110 14799 : } else if (mode == kMode_M2) {
2111 0 : __ shll(i.OutputRegister(), Immediate(1));
2112 14799 : } else if (mode == kMode_M4) {
2113 265 : __ shll(i.OutputRegister(), Immediate(2));
2114 14534 : } else if (mode == kMode_M8) {
2115 117 : __ shll(i.OutputRegister(), Immediate(3));
2116 : } else {
2117 28836 : __ leal(i.OutputRegister(), i.MemoryOperand());
2118 : }
2119 150620 : } else if (mode == kMode_MR1 &&
2120 : i.InputRegister(1) == i.OutputRegister()) {
2121 14193 : __ addl(i.OutputRegister(), i.InputRegister(0));
2122 : } else {
2123 230438 : __ leal(i.OutputRegister(), i.MemoryOperand());
2124 : }
2125 447084 : __ AssertZeroExtended(i.OutputRegister());
2126 223542 : break;
2127 : }
2128 : case kX64Lea: {
2129 : AddressingMode mode = AddressingModeField::decode(instr->opcode());
2130 : // Shorten "leaq" to "addq", "subq" or "shlq" if the register allocation
2131 : // and addressing mode just happens to work out. The "addq"/"subq" forms
2132 : // in these cases are faster based on measurements.
2133 1123591 : if (i.InputRegister(0) == i.OutputRegister()) {
2134 418397 : if (mode == kMode_MRI) {
2135 369448 : int32_t constant_summand = i.InputInt32(1);
2136 369475 : if (constant_summand > 0) {
2137 466914 : __ addq(i.OutputRegister(), Immediate(constant_summand));
2138 136018 : } else if (constant_summand < 0) {
2139 405903 : __ subq(i.OutputRegister(), Immediate(-constant_summand));
2140 : }
2141 48949 : } else if (mode == kMode_MR1) {
2142 21500 : if (i.InputRegister(1) == i.OutputRegister()) {
2143 2246 : __ shlq(i.OutputRegister(), Immediate(1));
2144 : } else {
2145 19254 : __ addq(i.OutputRegister(), i.InputRegister(1));
2146 : }
2147 27449 : } else if (mode == kMode_M2) {
2148 0 : __ shlq(i.OutputRegister(), Immediate(1));
2149 27449 : } else if (mode == kMode_M4) {
2150 1054 : __ shlq(i.OutputRegister(), Immediate(2));
2151 26395 : } else if (mode == kMode_M8) {
2152 6972 : __ shlq(i.OutputRegister(), Immediate(3));
2153 : } else {
2154 38846 : __ leaq(i.OutputRegister(), i.MemoryOperand());
2155 : }
2156 766407 : } else if (mode == kMode_MR1 &&
2157 : i.InputRegister(1) == i.OutputRegister()) {
2158 23957 : __ addq(i.OutputRegister(), i.InputRegister(0));
2159 : } else {
2160 1362496 : __ leaq(i.OutputRegister(), i.MemoryOperand());
2161 : }
2162 : break;
2163 : }
2164 : case kX64Dec32:
2165 0 : __ decl(i.OutputRegister());
2166 : break;
2167 : case kX64Inc32:
2168 0 : __ incl(i.OutputRegister());
2169 : break;
2170 : case kX64Push:
2171 3698046 : if (AddressingModeField::decode(instr->opcode()) != kMode_None) {
2172 135062 : size_t index = 0;
2173 135062 : Operand operand = i.MemoryOperand(&index);
2174 135062 : __ pushq(operand);
2175 : frame_access_state()->IncreaseSPDelta(1);
2176 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2177 270124 : kPointerSize);
2178 3562984 : } else if (HasImmediateInput(instr, 0)) {
2179 194900 : __ pushq(i.InputImmediate(0));
2180 : frame_access_state()->IncreaseSPDelta(1);
2181 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2182 389800 : kPointerSize);
2183 3368084 : } else if (instr->InputAt(0)->IsRegister()) {
2184 2910131 : __ pushq(i.InputRegister(0));
2185 : frame_access_state()->IncreaseSPDelta(1);
2186 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2187 5820266 : kPointerSize);
2188 457953 : } else if (instr->InputAt(0)->IsFPRegister()) {
2189 : // TODO(titzer): use another machine instruction?
2190 28694 : __ subq(rsp, Immediate(kDoubleSize));
2191 : frame_access_state()->IncreaseSPDelta(kDoubleSize / kPointerSize);
2192 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2193 57388 : kDoubleSize);
2194 57388 : __ Movsd(Operand(rsp, 0), i.InputDoubleRegister(0));
2195 : } else {
2196 429259 : __ pushq(i.InputOperand(0));
2197 : frame_access_state()->IncreaseSPDelta(1);
2198 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
2199 858518 : kPointerSize);
2200 : }
2201 : break;
2202 : case kX64Poke: {
2203 : int const slot = MiscField::decode(instr->opcode());
2204 1903 : if (HasImmediateInput(instr, 0)) {
2205 1276 : __ movq(Operand(rsp, slot * kPointerSize), i.InputImmediate(0));
2206 : } else {
2207 2530 : __ movq(Operand(rsp, slot * kPointerSize), i.InputRegister(0));
2208 : }
2209 : break;
2210 : }
2211 : case kX64I32x4Splat: {
2212 354 : XMMRegister dst = i.OutputSimd128Register();
2213 354 : __ movd(dst, i.InputRegister(0));
2214 354 : __ pshufd(dst, dst, 0x0);
2215 : break;
2216 : }
2217 : case kX64I32x4ExtractLane: {
2218 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2219 1680 : __ Pextrd(i.OutputRegister(), i.InputSimd128Register(0), i.InputInt8(1));
2220 : break;
2221 : }
2222 : case kX64I32x4ReplaceLane: {
2223 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2224 156 : if (instr->InputAt(2)->IsRegister()) {
2225 : __ Pinsrd(i.OutputSimd128Register(), i.InputRegister(2),
2226 156 : i.InputInt8(1));
2227 : } else {
2228 0 : __ Pinsrd(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2229 : }
2230 : break;
2231 : }
2232 : case kX64I32x4Neg: {
2233 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2234 : XMMRegister dst = i.OutputSimd128Register();
2235 : XMMRegister src = i.InputSimd128Register(0);
2236 6 : if (dst == src) {
2237 6 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
2238 : __ psignd(dst, kScratchDoubleReg);
2239 : } else {
2240 0 : __ pxor(dst, dst);
2241 : __ psubd(dst, src);
2242 : }
2243 : break;
2244 : }
2245 : case kX64I32x4Shl: {
2246 12 : __ pslld(i.OutputSimd128Register(), i.InputInt8(1));
2247 6 : break;
2248 : }
2249 : case kX64I32x4ShrS: {
2250 12 : __ psrad(i.OutputSimd128Register(), i.InputInt8(1));
2251 6 : break;
2252 : }
2253 : case kX64I32x4Add: {
2254 12 : __ paddd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2255 : break;
2256 : }
2257 : case kX64I32x4AddHoriz: {
2258 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2259 6 : __ phaddd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2260 : break;
2261 : }
2262 : case kX64I32x4Sub: {
2263 6 : __ psubd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2264 : break;
2265 : }
2266 : case kX64I32x4Mul: {
2267 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2268 6 : __ pmulld(i.OutputSimd128Register(), i.InputSimd128Register(1));
2269 : break;
2270 : }
2271 : case kX64I32x4MinS: {
2272 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2273 6 : __ pminsd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2274 : break;
2275 : }
2276 : case kX64I32x4MaxS: {
2277 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2278 6 : __ pmaxsd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2279 : break;
2280 : }
2281 : case kX64I32x4Eq: {
2282 6 : __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2283 : break;
2284 : }
2285 : case kX64I32x4Ne: {
2286 12 : __ pcmpeqd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2287 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
2288 12 : __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
2289 : break;
2290 : }
2291 : case kX64I32x4GtS: {
2292 12 : __ pcmpgtd(i.OutputSimd128Register(), i.InputSimd128Register(1));
2293 : break;
2294 : }
2295 : case kX64I32x4GeS: {
2296 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2297 : XMMRegister dst = i.OutputSimd128Register();
2298 : XMMRegister src = i.InputSimd128Register(1);
2299 12 : __ pminsd(dst, src);
2300 : __ pcmpeqd(dst, src);
2301 : break;
2302 : }
2303 : case kX64I32x4ShrU: {
2304 12 : __ psrld(i.OutputSimd128Register(), i.InputInt8(1));
2305 6 : break;
2306 : }
2307 : case kX64I32x4MinU: {
2308 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2309 6 : __ pminud(i.OutputSimd128Register(), i.InputSimd128Register(1));
2310 : break;
2311 : }
2312 : case kX64I32x4MaxU: {
2313 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2314 6 : __ pmaxud(i.OutputSimd128Register(), i.InputSimd128Register(1));
2315 : break;
2316 : }
2317 : case kX64I32x4GtU: {
2318 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2319 : XMMRegister dst = i.OutputSimd128Register();
2320 : XMMRegister src = i.InputSimd128Register(1);
2321 12 : __ pmaxud(dst, src);
2322 : __ pcmpeqd(dst, src);
2323 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
2324 : __ pxor(dst, kScratchDoubleReg);
2325 : break;
2326 : }
2327 : case kX64I32x4GeU: {
2328 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2329 : XMMRegister dst = i.OutputSimd128Register();
2330 : XMMRegister src = i.InputSimd128Register(1);
2331 12 : __ pminud(dst, src);
2332 : __ pcmpeqd(dst, src);
2333 : break;
2334 : }
2335 : case kX64S128Zero: {
2336 0 : XMMRegister dst = i.OutputSimd128Register();
2337 0 : __ xorps(dst, dst);
2338 : break;
2339 : }
2340 : case kX64I16x8Splat: {
2341 324 : XMMRegister dst = i.OutputSimd128Register();
2342 324 : __ movd(dst, i.InputRegister(0));
2343 324 : __ pshuflw(dst, dst, 0x0);
2344 324 : __ pshufhw(dst, dst, 0x0);
2345 324 : __ pshufd(dst, dst, 0x0);
2346 : break;
2347 : }
2348 : case kX64I16x8ExtractLane: {
2349 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2350 1680 : Register dst = i.OutputRegister();
2351 3360 : __ pextrw(dst, i.InputSimd128Register(0), i.InputInt8(1));
2352 1680 : __ movsxwl(dst, dst);
2353 : break;
2354 : }
2355 : case kX64I16x8ReplaceLane: {
2356 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2357 144 : if (instr->InputAt(2)->IsRegister()) {
2358 : __ pinsrw(i.OutputSimd128Register(), i.InputRegister(2),
2359 144 : i.InputInt8(1));
2360 : } else {
2361 0 : __ pinsrw(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2362 : }
2363 : break;
2364 : }
2365 : case kX64I16x8Neg: {
2366 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2367 : XMMRegister dst = i.OutputSimd128Register();
2368 : XMMRegister src = i.InputSimd128Register(0);
2369 6 : if (dst == src) {
2370 6 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
2371 : __ psignw(dst, kScratchDoubleReg);
2372 : } else {
2373 0 : __ pxor(dst, dst);
2374 : __ psubw(dst, src);
2375 : }
2376 : break;
2377 : }
2378 : case kX64I16x8Shl: {
2379 12 : __ psllw(i.OutputSimd128Register(), i.InputInt8(1));
2380 6 : break;
2381 : }
2382 : case kX64I16x8ShrS: {
2383 12 : __ psraw(i.OutputSimd128Register(), i.InputInt8(1));
2384 6 : break;
2385 : }
2386 : case kX64I16x8Add: {
2387 6 : __ paddw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2388 : break;
2389 : }
2390 : case kX64I16x8AddSaturateS: {
2391 6 : __ paddsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2392 : break;
2393 : }
2394 : case kX64I16x8AddHoriz: {
2395 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2396 6 : __ phaddw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2397 : break;
2398 : }
2399 : case kX64I16x8Sub: {
2400 6 : __ psubw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2401 : break;
2402 : }
2403 : case kX64I16x8SubSaturateS: {
2404 6 : __ psubsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2405 : break;
2406 : }
2407 : case kX64I16x8Mul: {
2408 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2409 6 : __ pmullw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2410 : break;
2411 : }
2412 : case kX64I16x8MinS: {
2413 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2414 6 : __ pminsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2415 : break;
2416 : }
2417 : case kX64I16x8MaxS: {
2418 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2419 6 : __ pmaxsw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2420 : break;
2421 : }
2422 : case kX64I16x8Eq: {
2423 6 : __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2424 : break;
2425 : }
2426 : case kX64I16x8Ne: {
2427 12 : __ pcmpeqw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2428 : __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg);
2429 12 : __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
2430 : break;
2431 : }
2432 : case kX64I16x8GtS: {
2433 12 : __ pcmpgtw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2434 : break;
2435 : }
2436 : case kX64I16x8GeS: {
2437 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2438 : XMMRegister dst = i.OutputSimd128Register();
2439 : XMMRegister src = i.InputSimd128Register(1);
2440 12 : __ pminsw(dst, src);
2441 : __ pcmpeqw(dst, src);
2442 : break;
2443 : }
2444 : case kX64I16x8ShrU: {
2445 12 : __ psrlw(i.OutputSimd128Register(), i.InputInt8(1));
2446 6 : break;
2447 : }
2448 : case kX64I16x8AddSaturateU: {
2449 6 : __ paddusw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2450 : break;
2451 : }
2452 : case kX64I16x8SubSaturateU: {
2453 6 : __ psubusw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2454 : break;
2455 : }
2456 : case kX64I16x8MinU: {
2457 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2458 6 : __ pminuw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2459 : break;
2460 : }
2461 : case kX64I16x8MaxU: {
2462 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2463 6 : __ pmaxuw(i.OutputSimd128Register(), i.InputSimd128Register(1));
2464 : break;
2465 : }
2466 : case kX64I16x8GtU: {
2467 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2468 : XMMRegister dst = i.OutputSimd128Register();
2469 : XMMRegister src = i.InputSimd128Register(1);
2470 12 : __ pmaxuw(dst, src);
2471 : __ pcmpeqw(dst, src);
2472 : __ pcmpeqw(kScratchDoubleReg, kScratchDoubleReg);
2473 : __ pxor(dst, kScratchDoubleReg);
2474 : break;
2475 : }
2476 : case kX64I16x8GeU: {
2477 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2478 : XMMRegister dst = i.OutputSimd128Register();
2479 : XMMRegister src = i.InputSimd128Register(1);
2480 24 : __ pminuw(dst, src);
2481 : __ pcmpeqw(dst, src);
2482 : break;
2483 : }
2484 : case kX64I8x16Splat: {
2485 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2486 : XMMRegister dst = i.OutputSimd128Register();
2487 294 : __ movd(dst, i.InputRegister(0));
2488 294 : __ xorps(kScratchDoubleReg, kScratchDoubleReg);
2489 : __ pshufb(dst, kScratchDoubleReg);
2490 : break;
2491 : }
2492 : case kX64I8x16ExtractLane: {
2493 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2494 3648 : Register dst = i.OutputRegister();
2495 7296 : __ pextrb(dst, i.InputSimd128Register(0), i.InputInt8(1));
2496 3648 : __ movsxbl(dst, dst);
2497 : break;
2498 : }
2499 : case kX64I8x16ReplaceLane: {
2500 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2501 240 : if (instr->InputAt(2)->IsRegister()) {
2502 : __ pinsrb(i.OutputSimd128Register(), i.InputRegister(2),
2503 240 : i.InputInt8(1));
2504 : } else {
2505 0 : __ pinsrb(i.OutputSimd128Register(), i.InputOperand(2), i.InputInt8(1));
2506 : }
2507 : break;
2508 : }
2509 : case kX64I8x16Neg: {
2510 : CpuFeatureScope sse_scope(tasm(), SSSE3);
2511 : XMMRegister dst = i.OutputSimd128Register();
2512 : XMMRegister src = i.InputSimd128Register(0);
2513 6 : if (dst == src) {
2514 6 : __ pcmpeqd(kScratchDoubleReg, kScratchDoubleReg);
2515 : __ psignb(dst, kScratchDoubleReg);
2516 : } else {
2517 0 : __ pxor(dst, dst);
2518 : __ psubb(dst, src);
2519 : }
2520 : break;
2521 : }
2522 : case kX64I8x16Add: {
2523 6 : __ paddb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2524 : break;
2525 : }
2526 : case kX64I8x16AddSaturateS: {
2527 6 : __ paddsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2528 : break;
2529 : }
2530 : case kX64I8x16Sub: {
2531 6 : __ psubb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2532 : break;
2533 : }
2534 : case kX64I8x16SubSaturateS: {
2535 6 : __ psubsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2536 : break;
2537 : }
2538 : case kX64I8x16MinS: {
2539 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2540 6 : __ pminsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2541 : break;
2542 : }
2543 : case kX64I8x16MaxS: {
2544 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2545 6 : __ pmaxsb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2546 : break;
2547 : }
2548 : case kX64I8x16Eq: {
2549 6 : __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2550 : break;
2551 : }
2552 : case kX64I8x16Ne: {
2553 12 : __ pcmpeqb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2554 : __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg);
2555 12 : __ pxor(i.OutputSimd128Register(), kScratchDoubleReg);
2556 : break;
2557 : }
2558 : case kX64I8x16GtS: {
2559 12 : __ pcmpgtb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2560 : break;
2561 : }
2562 : case kX64I8x16GeS: {
2563 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2564 : XMMRegister dst = i.OutputSimd128Register();
2565 : XMMRegister src = i.InputSimd128Register(1);
2566 12 : __ pminsb(dst, src);
2567 : __ pcmpeqb(dst, src);
2568 : break;
2569 : }
2570 : case kX64I8x16AddSaturateU: {
2571 6 : __ paddusb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2572 : break;
2573 : }
2574 : case kX64I8x16SubSaturateU: {
2575 6 : __ psubusb(i.OutputSimd128Register(), i.InputSimd128Register(1));
2576 : break;
2577 : }
2578 : case kX64I8x16MinU: {
2579 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2580 6 : __ pminub(i.OutputSimd128Register(), i.InputSimd128Register(1));
2581 : break;
2582 : }
2583 : case kX64I8x16MaxU: {
2584 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2585 6 : __ pmaxub(i.OutputSimd128Register(), i.InputSimd128Register(1));
2586 : break;
2587 : }
2588 : case kX64I8x16GtU: {
2589 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2590 : XMMRegister dst = i.OutputSimd128Register();
2591 : XMMRegister src = i.InputSimd128Register(1);
2592 12 : __ pmaxub(dst, src);
2593 : __ pcmpeqb(dst, src);
2594 : __ pcmpeqb(kScratchDoubleReg, kScratchDoubleReg);
2595 : __ pxor(dst, kScratchDoubleReg);
2596 : break;
2597 : }
2598 : case kX64I8x16GeU: {
2599 : CpuFeatureScope sse_scope(tasm(), SSE4_1);
2600 : XMMRegister dst = i.OutputSimd128Register();
2601 : XMMRegister src = i.InputSimd128Register(1);
2602 0 : __ pminub(dst, src);
2603 : __ pcmpeqb(dst, src);
2604 : break;
2605 : }
2606 : case kX64S128And: {
2607 6 : __ pand(i.OutputSimd128Register(), i.InputSimd128Register(1));
2608 : break;
2609 : }
2610 : case kX64S128Or: {
2611 6 : __ por(i.OutputSimd128Register(), i.InputSimd128Register(1));
2612 : break;
2613 : }
2614 : case kX64S128Xor: {
2615 6 : __ pxor(i.OutputSimd128Register(), i.InputSimd128Register(1));
2616 : break;
2617 : }
2618 : case kX64S128Not: {
2619 : XMMRegister dst = i.OutputSimd128Register();
2620 : XMMRegister src = i.InputSimd128Register(0);
2621 6 : if (dst == src) {
2622 6 : __ movaps(kScratchDoubleReg, dst);
2623 : __ pcmpeqd(dst, dst);
2624 : __ pxor(dst, kScratchDoubleReg);
2625 : } else {
2626 0 : __ pcmpeqd(dst, dst);
2627 : __ pxor(dst, src);
2628 : }
2629 :
2630 : break;
2631 : }
2632 : case kX64S128Select: {
2633 : // Mask used here is stored in dst.
2634 36 : XMMRegister dst = i.OutputSimd128Register();
2635 36 : __ movaps(kScratchDoubleReg, i.InputSimd128Register(1));
2636 72 : __ xorps(kScratchDoubleReg, i.InputSimd128Register(2));
2637 36 : __ andps(dst, kScratchDoubleReg);
2638 72 : __ xorps(dst, i.InputSimd128Register(2));
2639 : break;
2640 : }
2641 : case kCheckedLoadInt8:
2642 11310 : ASSEMBLE_CHECKED_LOAD_INTEGER(movsxbl);
2643 3770 : break;
2644 : case kCheckedLoadUint8:
2645 11344 : ASSEMBLE_CHECKED_LOAD_INTEGER(movzxbl);
2646 2836 : break;
2647 : case kCheckedLoadInt16:
2648 3810 : ASSEMBLE_CHECKED_LOAD_INTEGER(movsxwl);
2649 1270 : break;
2650 : case kCheckedLoadUint16:
2651 2868 : ASSEMBLE_CHECKED_LOAD_INTEGER(movzxwl);
2652 717 : break;
2653 : case kCheckedLoadWord32:
2654 236360 : ASSEMBLE_CHECKED_LOAD_INTEGER(movl);
2655 58862 : break;
2656 : case kCheckedLoadWord64:
2657 72 : ASSEMBLE_CHECKED_LOAD_INTEGER(movq);
2658 6 : break;
2659 : case kCheckedLoadFloat32:
2660 23268 : ASSEMBLE_CHECKED_LOAD_FLOAT(Movss, OutOfLineLoadFloat32NaN);
2661 7666 : break;
2662 : case kCheckedLoadFloat64:
2663 3531 : ASSEMBLE_CHECKED_LOAD_FLOAT(Movsd, OutOfLineLoadFloat64NaN);
2664 1087 : break;
2665 : case kCheckedStoreWord8:
2666 26015 : ASSEMBLE_CHECKED_STORE_INTEGER(movb);
2667 : break;
2668 : case kCheckedStoreWord16:
2669 11988 : ASSEMBLE_CHECKED_STORE_INTEGER(movw);
2670 : break;
2671 : case kCheckedStoreWord32:
2672 307337 : ASSEMBLE_CHECKED_STORE_INTEGER(movl);
2673 : break;
2674 : case kCheckedStoreWord64:
2675 72 : ASSEMBLE_CHECKED_STORE_INTEGER(movq);
2676 : break;
2677 : case kCheckedStoreFloat32:
2678 23075 : ASSEMBLE_CHECKED_STORE_FLOAT(Movss);
2679 : break;
2680 : case kCheckedStoreFloat64:
2681 4475 : ASSEMBLE_CHECKED_STORE_FLOAT(Movsd);
2682 : break;
2683 : case kX64StackCheck:
2684 666206 : __ CompareRoot(rsp, Heap::kStackLimitRootIndex);
2685 666214 : break;
2686 : case kAtomicExchangeInt8: {
2687 280 : __ xchgb(i.InputRegister(0), i.MemoryOperand(1));
2688 280 : __ movsxbl(i.InputRegister(0), i.InputRegister(0));
2689 140 : break;
2690 : }
2691 : case kAtomicExchangeUint8: {
2692 156 : __ xchgb(i.InputRegister(0), i.MemoryOperand(1));
2693 78 : __ movzxbl(i.InputRegister(0), i.InputRegister(0));
2694 : break;
2695 : }
2696 : case kAtomicExchangeInt16: {
2697 280 : __ xchgw(i.InputRegister(0), i.MemoryOperand(1));
2698 280 : __ movsxwl(i.InputRegister(0), i.InputRegister(0));
2699 140 : break;
2700 : }
2701 : case kAtomicExchangeUint16: {
2702 156 : __ xchgw(i.InputRegister(0), i.MemoryOperand(1));
2703 78 : __ movzxwl(i.InputRegister(0), i.InputRegister(0));
2704 : break;
2705 : }
2706 : case kAtomicExchangeWord32: {
2707 508 : __ xchgl(i.InputRegister(0), i.MemoryOperand(1));
2708 : break;
2709 : }
2710 : case kAtomicCompareExchangeInt8: {
2711 62 : __ lock();
2712 124 : __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1));
2713 62 : __ movsxbl(rax, rax);
2714 62 : break;
2715 : }
2716 : case kAtomicCompareExchangeUint8: {
2717 78 : __ lock();
2718 156 : __ cmpxchgb(i.MemoryOperand(2), i.InputRegister(1));
2719 : __ movzxbl(rax, rax);
2720 : break;
2721 : }
2722 : case kAtomicCompareExchangeInt16: {
2723 62 : __ lock();
2724 124 : __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1));
2725 62 : __ movsxwl(rax, rax);
2726 62 : break;
2727 : }
2728 : case kAtomicCompareExchangeUint16: {
2729 78 : __ lock();
2730 156 : __ cmpxchgw(i.MemoryOperand(2), i.InputRegister(1));
2731 : __ movzxwl(rax, rax);
2732 : break;
2733 : }
2734 : case kAtomicCompareExchangeWord32: {
2735 140 : __ lock();
2736 140 : __ cmpxchgl(i.MemoryOperand(2), i.InputRegister(1));
2737 : break;
2738 : }
2739 : #define ATOMIC_BINOP_CASE(op, inst) \
2740 : case kAtomic##op##Int8: \
2741 : ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \
2742 : __ movsxbl(rax, rax); \
2743 : break; \
2744 : case kAtomic##op##Uint8: \
2745 : ASSEMBLE_ATOMIC_BINOP(inst, movb, cmpxchgb); \
2746 : __ movzxbl(rax, rax); \
2747 : break; \
2748 : case kAtomic##op##Int16: \
2749 : ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \
2750 : __ movsxwl(rax, rax); \
2751 : break; \
2752 : case kAtomic##op##Uint16: \
2753 : ASSEMBLE_ATOMIC_BINOP(inst, movw, cmpxchgw); \
2754 : __ movzxwl(rax, rax); \
2755 : break; \
2756 : case kAtomic##op##Word32: \
2757 : ASSEMBLE_ATOMIC_BINOP(inst, movl, cmpxchgl); \
2758 : break;
2759 2100 : ATOMIC_BINOP_CASE(Add, addl)
2760 2100 : ATOMIC_BINOP_CASE(Sub, subl)
2761 2100 : ATOMIC_BINOP_CASE(And, andl)
2762 2100 : ATOMIC_BINOP_CASE(Or, orl)
2763 2100 : ATOMIC_BINOP_CASE(Xor, xorl)
2764 : #undef ATOMIC_BINOP_CASE
2765 : case kAtomicLoadInt8:
2766 : case kAtomicLoadUint8:
2767 : case kAtomicLoadInt16:
2768 : case kAtomicLoadUint16:
2769 : case kAtomicLoadWord32:
2770 : case kAtomicStoreWord8:
2771 : case kAtomicStoreWord16:
2772 : case kAtomicStoreWord32:
2773 0 : UNREACHABLE(); // Won't be generated by instruction selector.
2774 : break;
2775 : }
2776 : return kSuccess;
2777 : } // NOLINT(readability/fn_size)
2778 :
2779 : namespace {
2780 :
2781 4309476 : Condition FlagsConditionToCondition(FlagsCondition condition) {
2782 4309476 : switch (condition) {
2783 : case kUnorderedEqual:
2784 : case kEqual:
2785 : return equal;
2786 : case kUnorderedNotEqual:
2787 : case kNotEqual:
2788 1084299 : return not_equal;
2789 : case kSignedLessThan:
2790 103491 : return less;
2791 : case kSignedGreaterThanOrEqual:
2792 41659 : return greater_equal;
2793 : case kSignedLessThanOrEqual:
2794 31667 : return less_equal;
2795 : case kSignedGreaterThan:
2796 35909 : return greater;
2797 : case kUnsignedLessThan:
2798 67851 : return below;
2799 : case kUnsignedGreaterThanOrEqual:
2800 130522 : return above_equal;
2801 : case kUnsignedLessThanOrEqual:
2802 876306 : return below_equal;
2803 : case kUnsignedGreaterThan:
2804 40091 : return above;
2805 : case kOverflow:
2806 204408 : return overflow;
2807 : case kNotOverflow:
2808 1302 : return no_overflow;
2809 : default:
2810 : break;
2811 : }
2812 0 : UNREACHABLE();
2813 : }
2814 :
2815 : } // namespace
2816 :
2817 : // Assembles branches after this instruction.
2818 3609646 : void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
2819 : Label::Distance flabel_distance =
2820 3609646 : branch->fallthru ? Label::kNear : Label::kFar;
2821 3609646 : Label* tlabel = branch->true_label;
2822 3609646 : Label* flabel = branch->false_label;
2823 3609646 : if (branch->condition == kUnorderedEqual) {
2824 38512 : __ j(parity_even, flabel, flabel_distance);
2825 3571134 : } else if (branch->condition == kUnorderedNotEqual) {
2826 82724 : __ j(parity_even, tlabel);
2827 : }
2828 3609646 : __ j(FlagsConditionToCondition(branch->condition), tlabel);
2829 :
2830 3609655 : if (!branch->fallthru) __ jmp(flabel, flabel_distance);
2831 3609655 : }
2832 :
2833 302299 : void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr,
2834 : BranchInfo* branch) {
2835 : Label::Distance flabel_distance =
2836 302299 : branch->fallthru ? Label::kNear : Label::kFar;
2837 302299 : Label* tlabel = branch->true_label;
2838 302299 : Label* flabel = branch->false_label;
2839 : Label nodeopt;
2840 302299 : if (branch->condition == kUnorderedEqual) {
2841 0 : __ j(parity_even, flabel, flabel_distance);
2842 302299 : } else if (branch->condition == kUnorderedNotEqual) {
2843 6434 : __ j(parity_even, tlabel);
2844 : }
2845 302299 : __ j(FlagsConditionToCondition(branch->condition), tlabel);
2846 :
2847 302306 : if (FLAG_deopt_every_n_times > 0) {
2848 : ExternalReference counter =
2849 268 : ExternalReference::stress_deopt_count(isolate());
2850 :
2851 268 : __ pushfq();
2852 268 : __ pushq(rax);
2853 268 : __ load_rax(counter);
2854 : __ decl(rax);
2855 268 : __ j(not_zero, &nodeopt);
2856 :
2857 268 : __ Set(rax, FLAG_deopt_every_n_times);
2858 268 : __ store_rax(counter);
2859 268 : __ popq(rax);
2860 268 : __ popfq();
2861 268 : __ jmp(tlabel);
2862 :
2863 268 : __ bind(&nodeopt);
2864 268 : __ store_rax(counter);
2865 268 : __ popq(rax);
2866 268 : __ popfq();
2867 : }
2868 :
2869 302306 : if (!branch->fallthru) {
2870 0 : __ jmp(flabel, flabel_distance);
2871 : }
2872 302306 : }
2873 :
2874 5608892 : void CodeGenerator::AssembleArchJump(RpoNumber target) {
2875 5608892 : if (!IsNextInAssemblyOrder(target)) __ jmp(GetLabel(target));
2876 3412136 : }
2877 :
2878 197640 : void CodeGenerator::AssembleArchTrap(Instruction* instr,
2879 395280 : FlagsCondition condition) {
2880 0 : class OutOfLineTrap final : public OutOfLineCode {
2881 : public:
2882 : OutOfLineTrap(CodeGenerator* gen, bool frame_elided, Instruction* instr)
2883 : : OutOfLineCode(gen),
2884 : frame_elided_(frame_elided),
2885 : instr_(instr),
2886 197661 : gen_(gen) {}
2887 :
2888 197648 : void Generate() final {
2889 395296 : X64OperandConverter i(gen_, instr_);
2890 :
2891 : Builtins::Name trap_id =
2892 197648 : static_cast<Builtins::Name>(i.InputInt32(instr_->InputCount() - 1));
2893 238613 : bool old_has_frame = __ has_frame();
2894 197671 : if (frame_elided_) {
2895 : __ set_has_frame(true);
2896 20461 : __ EnterFrame(StackFrame::WASM_COMPILED);
2897 : }
2898 197660 : GenerateCallToTrap(trap_id);
2899 197674 : if (frame_elided_) {
2900 : __ set_has_frame(old_has_frame);
2901 : }
2902 197674 : }
2903 :
2904 : private:
2905 197650 : void GenerateCallToTrap(Builtins::Name trap_id) {
2906 197650 : if (trap_id == Builtins::builtin_count) {
2907 : // We cannot test calls to the runtime in cctest/test-run-wasm.
2908 : // Therefore we emit a call to C here instead of a call to the runtime.
2909 627970 : __ PrepareCallCFunction(0);
2910 : __ CallCFunction(ExternalReference::wasm_call_trap_callback_for_testing(
2911 : __ isolate()),
2912 69856 : 0);
2913 34928 : __ LeaveFrame(StackFrame::WASM_COMPILED);
2914 232600 : CallDescriptor* descriptor = gen_->linkage()->GetIncomingDescriptor();
2915 34928 : size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
2916 : // Use rcx as a scratch register, we return anyways immediately.
2917 69856 : __ Ret(static_cast<int>(pop_size), rcx);
2918 : } else {
2919 162722 : gen_->AssembleSourcePosition(instr_);
2920 : __ Call(__ isolate()->builtins()->builtin_handle(trap_id),
2921 325487 : RelocInfo::CODE_TARGET);
2922 : ReferenceMap* reference_map =
2923 162744 : new (gen_->zone()) ReferenceMap(gen_->zone());
2924 : gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
2925 162738 : Safepoint::kNoLazyDeopt);
2926 162771 : __ AssertUnreachable(kUnexpectedReturnFromWasmTrap);
2927 : }
2928 197672 : }
2929 :
2930 : bool frame_elided_;
2931 : Instruction* instr_;
2932 : CodeGenerator* gen_;
2933 : };
2934 197640 : bool frame_elided = !frame_access_state()->has_frame();
2935 : auto ool = new (zone()) OutOfLineTrap(this, frame_elided, instr);
2936 197617 : Label* tlabel = ool->entry();
2937 : Label end;
2938 197617 : if (condition == kUnorderedEqual) {
2939 0 : __ j(parity_even, &end);
2940 197617 : } else if (condition == kUnorderedNotEqual) {
2941 630 : __ j(parity_even, tlabel);
2942 : }
2943 197619 : __ j(FlagsConditionToCondition(condition), tlabel);
2944 197676 : __ bind(&end);
2945 197692 : }
2946 :
2947 : // Assembles boolean materializations after this instruction.
2948 399828 : void CodeGenerator::AssembleArchBoolean(Instruction* instr,
2949 : FlagsCondition condition) {
2950 : X64OperandConverter i(this, instr);
2951 : Label done;
2952 :
2953 : // Materialize a full 64-bit 1 or 0 value. The result register is always the
2954 : // last output of the instruction.
2955 : Label check;
2956 : DCHECK_NE(0u, instr->OutputCount());
2957 199914 : Register reg = i.OutputRegister(instr->OutputCount() - 1);
2958 199914 : if (condition == kUnorderedEqual) {
2959 4148 : __ j(parity_odd, &check, Label::kNear);
2960 : __ movl(reg, Immediate(0));
2961 4147 : __ jmp(&done, Label::kNear);
2962 195766 : } else if (condition == kUnorderedNotEqual) {
2963 1148 : __ j(parity_odd, &check, Label::kNear);
2964 : __ movl(reg, Immediate(1));
2965 1148 : __ jmp(&done, Label::kNear);
2966 : }
2967 199913 : __ bind(&check);
2968 199916 : __ setcc(FlagsConditionToCondition(condition), reg);
2969 : __ movzxbl(reg, reg);
2970 199917 : __ bind(&done);
2971 199918 : }
2972 :
2973 :
2974 152657 : void CodeGenerator::AssembleArchLookupSwitch(Instruction* instr) {
2975 : X64OperandConverter i(this, instr);
2976 20780 : Register input = i.InputRegister(0);
2977 152656 : for (size_t index = 2; index < instr->InputCount(); index += 2) {
2978 111096 : __ cmpl(input, Immediate(i.InputInt32(index + 0)));
2979 111097 : __ j(equal, GetLabel(i.InputRpo(index + 1)));
2980 : }
2981 20780 : AssembleArchJump(i.InputRpo(1));
2982 20780 : }
2983 :
2984 :
2985 342541 : void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
2986 : X64OperandConverter i(this, instr);
2987 7807 : Register input = i.InputRegister(0);
2988 7807 : int32_t const case_count = static_cast<int32_t>(instr->InputCount() - 2);
2989 7807 : Label** cases = zone()->NewArray<Label*>(case_count);
2990 326927 : for (int32_t index = 0; index < case_count; ++index) {
2991 638240 : cases[index] = GetLabel(i.InputRpo(index + 2));
2992 : }
2993 7807 : Label* const table = AddJumpTable(cases, case_count);
2994 7807 : __ cmpl(input, Immediate(case_count));
2995 15614 : __ j(above_equal, GetLabel(i.InputRpo(1)));
2996 15614 : __ leaq(kScratchRegister, Operand(table));
2997 7807 : __ jmp(Operand(kScratchRegister, input, times_8, 0));
2998 7807 : }
2999 :
3000 : namespace {
3001 :
3002 : static const int kQuadWordSize = 16;
3003 :
3004 : } // namespace
3005 :
3006 1301525 : void CodeGenerator::FinishFrame(Frame* frame) {
3007 2603050 : CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
3008 :
3009 : const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
3010 1301525 : if (saves_fp != 0) {
3011 : frame->AlignSavedCalleeRegisterSlots();
3012 0 : if (saves_fp != 0) { // Save callee-saved XMM registers.
3013 : const uint32_t saves_fp_count = base::bits::CountPopulation(saves_fp);
3014 0 : frame->AllocateSavedCalleeRegisterSlots(saves_fp_count *
3015 0 : (kQuadWordSize / kPointerSize));
3016 : }
3017 : }
3018 : const RegList saves = descriptor->CalleeSavedRegisters();
3019 1301525 : if (saves != 0) { // Save callee-saved registers.
3020 : int count = 0;
3021 3662512 : for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
3022 3662512 : if (((1 << i) & saves)) {
3023 1144535 : ++count;
3024 : }
3025 : }
3026 : frame->AllocateSavedCalleeRegisterSlots(count);
3027 : }
3028 1301525 : }
3029 :
3030 5159584 : void CodeGenerator::AssembleConstructFrame() {
3031 3224113 : CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
3032 1074688 : if (frame_access_state()->has_frame()) {
3033 1074710 : int pc_base = __ pc_offset();
3034 :
3035 1074710 : if (descriptor->IsCFunctionCall()) {
3036 228907 : __ pushq(rbp);
3037 : __ movq(rbp, rsp);
3038 845803 : } else if (descriptor->IsJSFunctionCall()) {
3039 619711 : __ Prologue();
3040 619715 : if (descriptor->PushArgumentCount()) {
3041 15917 : __ pushq(kJavaScriptCallArgCountRegister);
3042 : }
3043 : } else {
3044 226092 : __ StubPrologue(info()->GetOutputStackFrameType());
3045 : }
3046 :
3047 1074698 : unwinding_info_writer_.MarkFrameConstructed(pc_base);
3048 : }
3049 : int shrink_slots =
3050 1074667 : frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
3051 :
3052 1074702 : if (info()->is_osr()) {
3053 : // TurboFan OSR-compiled functions cannot be entered directly.
3054 5809 : __ Abort(kShouldNotDirectlyEnterOsrFunction);
3055 :
3056 : // Unoptimized code jumps directly to this entrypoint while the unoptimized
3057 : // frame is still on the stack. Optimized code uses OSR values directly from
3058 : // the unoptimized frame. Thus, all that needs to be done is to allocate the
3059 : // remaining stack slots.
3060 5809 : if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
3061 11618 : osr_pc_offset_ = __ pc_offset();
3062 5809 : shrink_slots -= static_cast<int>(osr_helper()->UnoptimizedFrameSlots());
3063 : }
3064 :
3065 : const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
3066 1074702 : if (shrink_slots > 0) {
3067 634717 : if (info()->IsWasm() && shrink_slots > 128) {
3068 : // For WebAssembly functions with big frames we have to do the stack
3069 : // overflow check before we construct the frame. Otherwise we may not
3070 : // have enough space on the stack to call the runtime for the stack
3071 : // overflow.
3072 : Label done;
3073 :
3074 : // If the frame is bigger than the stack, we throw the stack overflow
3075 : // exception unconditionally. Thereby we can avoid the integer overflow
3076 : // check in the condition code.
3077 10 : if (shrink_slots * kPointerSize < FLAG_stack_size * 1024) {
3078 : __ Move(kScratchRegister,
3079 10 : ExternalReference::address_of_real_stack_limit(__ isolate()));
3080 20 : __ movq(kScratchRegister, Operand(kScratchRegister, 0));
3081 10 : __ addq(kScratchRegister, Immediate(shrink_slots * kPointerSize));
3082 10 : __ cmpq(rsp, kScratchRegister);
3083 10 : __ j(above_equal, &done);
3084 : }
3085 10 : if (!frame_access_state()->has_frame()) {
3086 : __ set_has_frame(true);
3087 0 : __ EnterFrame(StackFrame::WASM_COMPILED);
3088 : }
3089 10 : __ Move(rsi, Smi::kZero);
3090 10 : __ CallRuntimeDelayed(zone(), Runtime::kThrowWasmStackOverflow);
3091 : ReferenceMap* reference_map = new (zone()) ReferenceMap(zone());
3092 : RecordSafepoint(reference_map, Safepoint::kSimple, 0,
3093 10 : Safepoint::kNoLazyDeopt);
3094 10 : __ AssertUnreachable(kUnexpectedReturnFromWasmTrap);
3095 10 : __ bind(&done);
3096 : }
3097 1269434 : __ subq(rsp, Immediate(shrink_slots * kPointerSize));
3098 : }
3099 :
3100 1074701 : if (saves_fp != 0) { // Save callee-saved XMM registers.
3101 : const uint32_t saves_fp_count = base::bits::CountPopulation(saves_fp);
3102 0 : const int stack_size = saves_fp_count * kQuadWordSize;
3103 : // Adjust the stack pointer.
3104 0 : __ subp(rsp, Immediate(stack_size));
3105 : // Store the registers on the stack.
3106 : int slot_idx = 0;
3107 0 : for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
3108 0 : if (!((1 << i) & saves_fp)) continue;
3109 : __ movdqu(Operand(rsp, kQuadWordSize * slot_idx),
3110 0 : XMMRegister::from_code(i));
3111 0 : slot_idx++;
3112 : }
3113 : }
3114 :
3115 : const RegList saves = descriptor->CalleeSavedRegisters();
3116 1074701 : if (saves != 0) { // Save callee-saved registers.
3117 3662512 : for (int i = Register::kNumRegisters - 1; i >= 0; i--) {
3118 3662512 : if (!((1 << i) & saves)) continue;
3119 1144535 : __ pushq(Register::from_code(i));
3120 : }
3121 : }
3122 1074701 : }
3123 :
3124 2779299 : void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
3125 6042156 : CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
3126 :
3127 : // Restore registers.
3128 : const RegList saves = descriptor->CalleeSavedRegisters();
3129 1510539 : if (saves != 0) {
3130 3868464 : for (int i = 0; i < Register::kNumRegisters; i++) {
3131 3868464 : if (!((1 << i) & saves)) continue;
3132 1208895 : __ popq(Register::from_code(i));
3133 : }
3134 : }
3135 : const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
3136 1510539 : if (saves_fp != 0) {
3137 : const uint32_t saves_fp_count = base::bits::CountPopulation(saves_fp);
3138 0 : const int stack_size = saves_fp_count * kQuadWordSize;
3139 : // Load the registers from the stack.
3140 : int slot_idx = 0;
3141 0 : for (int i = 0; i < XMMRegister::kNumRegisters; i++) {
3142 0 : if (!((1 << i) & saves_fp)) continue;
3143 : __ movdqu(XMMRegister::from_code(i),
3144 0 : Operand(rsp, kQuadWordSize * slot_idx));
3145 0 : slot_idx++;
3146 : }
3147 : // Adjust the stack pointer.
3148 0 : __ addp(rsp, Immediate(stack_size));
3149 : }
3150 :
3151 : unwinding_info_writer_.MarkBlockWillExit();
3152 :
3153 : // Might need rcx for scratch if pop_size is too big or if there is a variable
3154 : // pop count.
3155 : DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rcx.bit());
3156 : DCHECK_EQ(0u, descriptor->CalleeSavedRegisters() & rdx.bit());
3157 1510539 : size_t pop_size = descriptor->StackParameterCount() * kPointerSize;
3158 : X64OperandConverter g(this, nullptr);
3159 1510539 : if (descriptor->IsCFunctionCall()) {
3160 241779 : AssembleDeconstructFrame();
3161 1268760 : } else if (frame_access_state()->has_frame()) {
3162 2168423 : if (pop->IsImmediate() && g.ToConstant(pop).ToInt32() == 0) {
3163 : // Canonicalize JSFunction return sites for now.
3164 1077735 : if (return_label_.is_bound()) {
3165 298808 : __ jmp(&return_label_);
3166 1510638 : return;
3167 : } else {
3168 778927 : __ bind(&return_label_);
3169 778903 : AssembleDeconstructFrame();
3170 : }
3171 : } else {
3172 12970 : AssembleDeconstructFrame();
3173 : }
3174 : }
3175 :
3176 1211717 : if (pop->IsImmediate()) {
3177 : DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
3178 2397566 : pop_size += g.ToConstant(pop).ToInt32() * kPointerSize;
3179 1198813 : CHECK_LT(pop_size, static_cast<size_t>(std::numeric_limits<int>::max()));
3180 1198813 : __ Ret(static_cast<int>(pop_size), rcx);
3181 : } else {
3182 : Register pop_reg = g.ToRegister(pop);
3183 12964 : Register scratch_reg = pop_reg == rcx ? rdx : rcx;
3184 12964 : __ popq(scratch_reg);
3185 25928 : __ leaq(rsp, Operand(rsp, pop_reg, times_8, static_cast<int>(pop_size)));
3186 12964 : __ jmp(scratch_reg);
3187 : }
3188 : }
3189 :
3190 1301268 : void CodeGenerator::FinishCode() {}
3191 :
3192 27149840 : void CodeGenerator::AssembleMove(InstructionOperand* source,
3193 : InstructionOperand* destination) {
3194 : X64OperandConverter g(this, nullptr);
3195 : // Dispatch on the source and destination operand kinds. Not all
3196 : // combinations are possible.
3197 27149840 : if (source->IsRegister()) {
3198 : DCHECK(destination->IsRegister() || destination->IsStackSlot());
3199 : Register src = g.ToRegister(source);
3200 5691234 : if (destination->IsRegister()) {
3201 3381998 : __ movq(g.ToRegister(destination), src);
3202 : } else {
3203 2309251 : __ movq(g.ToOperand(destination), src);
3204 : }
3205 21458606 : } else if (source->IsStackSlot()) {
3206 : DCHECK(destination->IsRegister() || destination->IsStackSlot());
3207 6954973 : Operand src = g.ToOperand(source);
3208 6954973 : if (destination->IsRegister()) {
3209 : Register dst = g.ToRegister(destination);
3210 6876576 : __ movq(dst, src);
3211 : } else {
3212 : // Spill on demand to use a temporary register for memory-to-memory
3213 : // moves.
3214 : Register tmp = kScratchRegister;
3215 78397 : Operand dst = g.ToOperand(destination);
3216 78397 : __ movq(tmp, src);
3217 : __ movq(dst, tmp);
3218 : }
3219 14503640 : } else if (source->IsConstant()) {
3220 : ConstantOperand* constant_source = ConstantOperand::cast(source);
3221 13795034 : Constant src = g.ToConstant(constant_source);
3222 14409604 : if (destination->IsRegister() || destination->IsStackSlot()) {
3223 : Register dst = destination->IsRegister() ? g.ToRegister(destination)
3224 13212279 : : kScratchRegister;
3225 13212279 : switch (src.type()) {
3226 : case Constant::kInt32: {
3227 5033302 : if (RelocInfo::IsWasmPtrReference(src.rmode())) {
3228 0 : __ movq(dst, src.ToInt64(), src.rmode());
3229 : } else {
3230 : // TODO(dcarney): don't need scratch in this case.
3231 2516651 : int32_t value = src.ToInt32();
3232 2516651 : if (RelocInfo::IsWasmSizeReference(src.rmode())) {
3233 5 : __ movl(dst, Immediate(value, src.rmode()));
3234 2516646 : } else if (value == 0) {
3235 897244 : __ xorl(dst, dst);
3236 : } else {
3237 1619402 : __ movl(dst, Immediate(value));
3238 : }
3239 : }
3240 : break;
3241 : }
3242 : case Constant::kInt64:
3243 2051574 : if (RelocInfo::IsWasmPtrReference(src.rmode())) {
3244 183969 : __ movq(dst, src.ToInt64(), src.rmode());
3245 : } else {
3246 : DCHECK(!RelocInfo::IsWasmSizeReference(src.rmode()));
3247 841818 : __ Set(dst, src.ToInt64());
3248 : }
3249 : break;
3250 : case Constant::kFloat32:
3251 504 : __ MoveNumber(dst, src.ToFloat32());
3252 252 : break;
3253 : case Constant::kFloat64:
3254 3112847 : __ MoveNumber(dst, src.ToFloat64().value());
3255 3112845 : break;
3256 : case Constant::kExternalReference:
3257 1255919 : __ Move(dst, src.ToExternalReference());
3258 : break;
3259 : case Constant::kHeapObject: {
3260 5300832 : Handle<HeapObject> src_object = src.ToHeapObject();
3261 : Heap::RootListIndex index;
3262 5300829 : if (IsMaterializableFromRoot(src_object, &index)) {
3263 1741037 : __ LoadRoot(dst, index);
3264 : } else {
3265 3559793 : __ Move(dst, src_object);
3266 : }
3267 : break;
3268 : }
3269 : case Constant::kRpoNumber:
3270 0 : UNREACHABLE(); // TODO(dcarney): load of labels on x64.
3271 : break;
3272 : }
3273 13212311 : if (destination->IsStackSlot()) {
3274 31788 : __ movq(g.ToOperand(destination), kScratchRegister);
3275 : }
3276 582779 : } else if (src.type() == Constant::kFloat32) {
3277 : // TODO(turbofan): Can we do better here?
3278 150700 : uint32_t src_const = bit_cast<uint32_t>(src.ToFloat32());
3279 150700 : if (destination->IsFPRegister()) {
3280 150148 : __ Move(g.ToDoubleRegister(destination), src_const);
3281 : } else {
3282 : DCHECK(destination->IsFPStackSlot());
3283 552 : Operand dst = g.ToOperand(destination);
3284 552 : __ movl(dst, Immediate(src_const));
3285 : }
3286 : } else {
3287 : DCHECK_EQ(Constant::kFloat64, src.type());
3288 : uint64_t src_const = src.ToFloat64().AsUint64();
3289 432079 : if (destination->IsFPRegister()) {
3290 431056 : __ Move(g.ToDoubleRegister(destination), src_const);
3291 : } else {
3292 : DCHECK(destination->IsFPStackSlot());
3293 1023 : __ movq(kScratchRegister, src_const);
3294 : __ movq(g.ToOperand(destination), kScratchRegister);
3295 : }
3296 : }
3297 708606 : } else if (source->IsFPRegister()) {
3298 354682 : XMMRegister src = g.ToDoubleRegister(source);
3299 354682 : if (destination->IsFPRegister()) {
3300 89334 : XMMRegister dst = g.ToDoubleRegister(destination);
3301 89334 : __ Movapd(dst, src);
3302 : } else {
3303 : DCHECK(destination->IsFPStackSlot());
3304 265346 : Operand dst = g.ToOperand(destination);
3305 : MachineRepresentation rep =
3306 : LocationOperand::cast(source)->representation();
3307 265346 : if (rep != MachineRepresentation::kSimd128) {
3308 265346 : __ Movsd(dst, src);
3309 : } else {
3310 0 : __ Movups(dst, src);
3311 : }
3312 : }
3313 353924 : } else if (source->IsFPStackSlot()) {
3314 : DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
3315 353924 : Operand src = g.ToOperand(source);
3316 : MachineRepresentation rep = LocationOperand::cast(source)->representation();
3317 353924 : if (destination->IsFPRegister()) {
3318 328401 : XMMRegister dst = g.ToDoubleRegister(destination);
3319 328401 : if (rep != MachineRepresentation::kSimd128) {
3320 328401 : __ Movsd(dst, src);
3321 : } else {
3322 0 : __ Movups(dst, src);
3323 : }
3324 : } else {
3325 25523 : Operand dst = g.ToOperand(destination);
3326 25523 : if (rep != MachineRepresentation::kSimd128) {
3327 25523 : __ Movsd(kScratchDoubleReg, src);
3328 25523 : __ Movsd(dst, kScratchDoubleReg);
3329 : } else {
3330 0 : __ Movups(kScratchDoubleReg, src);
3331 0 : __ Movups(dst, kScratchDoubleReg);
3332 : }
3333 : }
3334 : } else {
3335 0 : UNREACHABLE();
3336 : }
3337 27149900 : }
3338 :
3339 :
3340 41899 : void CodeGenerator::AssembleSwap(InstructionOperand* source,
3341 3626 : InstructionOperand* destination) {
3342 : X64OperandConverter g(this, nullptr);
3343 : // Dispatch on the source and destination operand kinds. Not all
3344 : // combinations are possible.
3345 72917 : if (source->IsRegister() && destination->IsRegister()) {
3346 : // Register-register.
3347 : Register src = g.ToRegister(source);
3348 : Register dst = g.ToRegister(destination);
3349 29205 : __ movq(kScratchRegister, src);
3350 : __ movq(src, dst);
3351 : __ movq(dst, kScratchRegister);
3352 14507 : } else if (source->IsRegister() && destination->IsStackSlot()) {
3353 : Register src = g.ToRegister(source);
3354 1813 : __ pushq(src);
3355 : frame_access_state()->IncreaseSPDelta(1);
3356 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3357 3626 : kPointerSize);
3358 1813 : Operand dst = g.ToOperand(destination);
3359 : __ movq(src, dst);
3360 : frame_access_state()->IncreaseSPDelta(-1);
3361 1813 : dst = g.ToOperand(destination);
3362 1813 : __ popq(dst);
3363 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3364 3626 : -kPointerSize);
3365 32643 : } else if ((source->IsStackSlot() && destination->IsStackSlot()) ||
3366 1774 : (source->IsFPStackSlot() && destination->IsFPStackSlot())) {
3367 : // Memory-memory.
3368 6101 : Operand src = g.ToOperand(source);
3369 6101 : Operand dst = g.ToOperand(destination);
3370 : MachineRepresentation rep = LocationOperand::cast(source)->representation();
3371 6101 : if (rep != MachineRepresentation::kSimd128) {
3372 : Register tmp = kScratchRegister;
3373 6101 : __ movq(tmp, dst);
3374 6101 : __ pushq(src); // Then use stack to copy src to destination.
3375 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3376 12202 : kPointerSize);
3377 6101 : __ popq(dst);
3378 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3379 12202 : -kPointerSize);
3380 : __ movq(src, tmp);
3381 : } else {
3382 : // Without AVX, misaligned reads and writes will trap. Move using the
3383 : // stack, in two parts.
3384 0 : __ movups(kScratchDoubleReg, dst); // Save dst in scratch register.
3385 0 : __ pushq(src); // Then use stack to copy src to destination.
3386 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3387 0 : kPointerSize);
3388 0 : __ popq(dst);
3389 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3390 0 : -kPointerSize);
3391 0 : __ pushq(g.ToOperand(source, kPointerSize));
3392 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3393 0 : kPointerSize);
3394 0 : __ popq(g.ToOperand(destination, kPointerSize));
3395 : unwinding_info_writer_.MaybeIncreaseBaseOffsetAt(__ pc_offset(),
3396 0 : -kPointerSize);
3397 0 : __ movups(src, kScratchDoubleReg);
3398 : }
3399 9560 : } else if (source->IsFPRegister() && destination->IsFPRegister()) {
3400 : // XMM register-register swap.
3401 1441 : XMMRegister src = g.ToDoubleRegister(source);
3402 1441 : XMMRegister dst = g.ToDoubleRegister(destination);
3403 1441 : __ Movapd(kScratchDoubleReg, src);
3404 1441 : __ Movapd(src, dst);
3405 1441 : __ Movapd(dst, kScratchDoubleReg);
3406 6678 : } else if (source->IsFPRegister() && destination->IsFPStackSlot()) {
3407 : // XMM register-memory swap.
3408 3339 : XMMRegister src = g.ToDoubleRegister(source);
3409 3339 : Operand dst = g.ToOperand(destination);
3410 : MachineRepresentation rep = LocationOperand::cast(source)->representation();
3411 3339 : if (rep != MachineRepresentation::kSimd128) {
3412 3339 : __ Movsd(kScratchDoubleReg, src);
3413 3339 : __ Movsd(src, dst);
3414 3339 : __ Movsd(dst, kScratchDoubleReg);
3415 : } else {
3416 0 : __ Movups(kScratchDoubleReg, src);
3417 0 : __ Movups(src, dst);
3418 0 : __ Movups(dst, kScratchDoubleReg);
3419 : }
3420 : } else {
3421 : // No other combinations are possible.
3422 0 : UNREACHABLE();
3423 : }
3424 41899 : }
3425 :
3426 :
3427 7807 : void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
3428 326927 : for (size_t index = 0; index < target_count; ++index) {
3429 319120 : __ dq(targets[index]);
3430 : }
3431 7807 : }
3432 :
3433 : #undef __
3434 :
3435 : } // namespace compiler
3436 : } // namespace internal
3437 : } // namespace v8
|
