Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include <algorithm>
6 :
7 : #include "src/base/adapters.h"
8 : #include "src/base/overflowing-math.h"
9 : #include "src/compiler/backend/instruction-selector-impl.h"
10 : #include "src/compiler/node-matchers.h"
11 : #include "src/compiler/node-properties.h"
12 : #include "src/roots-inl.h"
13 :
14 : namespace v8 {
15 : namespace internal {
16 : namespace compiler {
17 :
18 : // Adds X64-specific methods for generating operands.
19 : class X64OperandGenerator final : public OperandGenerator {
20 : public:
21 : explicit X64OperandGenerator(InstructionSelector* selector)
22 : : OperandGenerator(selector) {}
23 :
24 28874472 : bool CanBeImmediate(Node* node) {
25 28874472 : switch (node->opcode()) {
26 : case IrOpcode::kInt32Constant:
27 : case IrOpcode::kRelocatableInt32Constant:
28 : return true;
29 : case IrOpcode::kInt64Constant: {
30 12068975 : const int64_t value = OpParameter<int64_t>(node->op());
31 12068975 : return std::numeric_limits<int32_t>::min() < value &&
32 : value <= std::numeric_limits<int32_t>::max();
33 : }
34 : case IrOpcode::kNumberConstant: {
35 46108 : const double value = OpParameter<double>(node->op());
36 46108 : return bit_cast<int64_t>(value) == 0;
37 : }
38 : default:
39 : return false;
40 : }
41 : }
42 :
43 224765 : int32_t GetImmediateIntegerValue(Node* node) {
44 : DCHECK(CanBeImmediate(node));
45 224765 : if (node->opcode() == IrOpcode::kInt32Constant) {
46 171381 : return OpParameter<int32_t>(node->op());
47 : }
48 : DCHECK_EQ(IrOpcode::kInt64Constant, node->opcode());
49 53384 : return static_cast<int32_t>(OpParameter<int64_t>(node->op()));
50 : }
51 :
52 12934927 : bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
53 : int effect_level) {
54 13567468 : if (input->opcode() != IrOpcode::kLoad ||
55 2258076 : !selector()->CanCover(node, input)) {
56 : return false;
57 : }
58 1625624 : if (effect_level != selector()->GetEffectLevel(input)) {
59 : return false;
60 : }
61 : MachineRepresentation rep =
62 1625578 : LoadRepresentationOf(input->op()).representation();
63 1625577 : switch (opcode) {
64 : case kX64And:
65 : case kX64Or:
66 : case kX64Xor:
67 : case kX64Add:
68 : case kX64Sub:
69 : case kX64Push:
70 : case kX64Cmp:
71 : case kX64Test:
72 1688655 : return rep == MachineRepresentation::kWord64 || IsAnyTagged(rep);
73 : case kX64And32:
74 : case kX64Or32:
75 : case kX64Xor32:
76 : case kX64Add32:
77 : case kX64Sub32:
78 : case kX64Cmp32:
79 : case kX64Test32:
80 131629 : return rep == MachineRepresentation::kWord32;
81 : case kX64Cmp16:
82 : case kX64Test16:
83 272314 : return rep == MachineRepresentation::kWord16;
84 : case kX64Cmp8:
85 : case kX64Test8:
86 104485 : return rep == MachineRepresentation::kWord8;
87 : default:
88 : break;
89 : }
90 : return false;
91 : }
92 :
93 13622535 : AddressingMode GenerateMemoryOperandInputs(Node* index, int scale_exponent,
94 11363828 : Node* base, Node* displacement,
95 : DisplacementMode displacement_mode,
96 : InstructionOperand inputs[],
97 : size_t* input_count) {
98 : AddressingMode mode = kMode_MRI;
99 13622535 : if (base != nullptr && (index != nullptr || displacement != nullptr)) {
100 11465940 : if (base->opcode() == IrOpcode::kInt32Constant &&
101 102112 : OpParameter<int32_t>(base->op()) == 0) {
102 : base = nullptr;
103 11511607 : } else if (base->opcode() == IrOpcode::kInt64Constant &&
104 214410 : OpParameter<int64_t>(base->op()) == 0) {
105 : base = nullptr;
106 : }
107 : }
108 13622535 : if (base != nullptr) {
109 13307317 : inputs[(*input_count)++] = UseRegister(base);
110 13307354 : if (index != nullptr) {
111 : DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
112 1492185 : inputs[(*input_count)++] = UseRegister(index);
113 1492155 : if (displacement != nullptr) {
114 : inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
115 : ? UseNegatedImmediate(displacement)
116 536470 : : UseImmediate(displacement);
117 : static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
118 : kMode_MR4I, kMode_MR8I};
119 536468 : mode = kMRnI_modes[scale_exponent];
120 : } else {
121 : static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
122 : kMode_MR4, kMode_MR8};
123 955685 : mode = kMRn_modes[scale_exponent];
124 : }
125 : } else {
126 11815169 : if (displacement == nullptr) {
127 : mode = kMode_MR;
128 : } else {
129 : inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
130 : ? UseNegatedImmediate(displacement)
131 9756591 : : UseImmediate(displacement);
132 : mode = kMode_MRI;
133 : }
134 : }
135 : } else {
136 : DCHECK(scale_exponent >= 0 && scale_exponent <= 3);
137 315218 : if (displacement != nullptr) {
138 261054 : if (index == nullptr) {
139 56636 : inputs[(*input_count)++] = UseRegister(displacement);
140 : mode = kMode_MR;
141 : } else {
142 204418 : inputs[(*input_count)++] = UseRegister(index);
143 : inputs[(*input_count)++] = displacement_mode == kNegativeDisplacement
144 : ? UseNegatedImmediate(displacement)
145 204418 : : UseImmediate(displacement);
146 : static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
147 : kMode_M4I, kMode_M8I};
148 204418 : mode = kMnI_modes[scale_exponent];
149 : }
150 : } else {
151 54164 : inputs[(*input_count)++] = UseRegister(index);
152 : static const AddressingMode kMn_modes[] = {kMode_MR, kMode_MR1,
153 : kMode_M4, kMode_M8};
154 54172 : mode = kMn_modes[scale_exponent];
155 54172 : if (mode == kMode_MR1) {
156 : // [%r1 + %r1*1] has a smaller encoding than [%r1*2+0]
157 12337 : inputs[(*input_count)++] = UseRegister(index);
158 : }
159 : }
160 : }
161 13622563 : return mode;
162 : }
163 :
164 11638332 : AddressingMode GetEffectiveAddressMemoryOperand(Node* operand,
165 : InstructionOperand inputs[],
166 : size_t* input_count) {
167 11638332 : if (selector()->CanAddressRelativeToRootsRegister()) {
168 3242961 : LoadMatcher<ExternalReferenceMatcher> m(operand);
169 3242962 : if (m.index().HasValue() && m.object().HasValue()) {
170 : ptrdiff_t const delta =
171 485515 : m.index().Value() +
172 : TurboAssemblerBase::RootRegisterOffsetForExternalReference(
173 485515 : selector()->isolate(), m.object().Value());
174 485514 : if (is_int32(delta)) {
175 485516 : inputs[(*input_count)++] = TempImmediate(static_cast<int32_t>(delta));
176 485515 : return kMode_Root;
177 : }
178 : }
179 : }
180 : BaseWithIndexAndDisplacement64Matcher m(operand, AddressOption::kAllowAll);
181 : DCHECK(m.matches());
182 11152879 : if (m.displacement() == nullptr || CanBeImmediate(m.displacement())) {
183 : return GenerateMemoryOperandInputs(
184 : m.index(), m.scale(), m.base(), m.displacement(),
185 11149601 : m.displacement_mode(), inputs, input_count);
186 4658 : } else if (m.base() == nullptr &&
187 1402 : m.displacement_mode() == kPositiveDisplacement) {
188 : // The displacement cannot be an immediate, but we can use the
189 : // displacement as base instead and still benefit from addressing
190 : // modes for the scale.
191 : return GenerateMemoryOperandInputs(m.index(), m.scale(), m.displacement(),
192 : nullptr, m.displacement_mode(), inputs,
193 1402 : input_count);
194 : } else {
195 3708 : inputs[(*input_count)++] = UseRegister(operand->InputAt(0));
196 3740 : inputs[(*input_count)++] = UseRegister(operand->InputAt(1));
197 1874 : return kMode_MR1;
198 : }
199 : }
200 :
201 348714 : InstructionOperand GetEffectiveIndexOperand(Node* index,
202 : AddressingMode* mode) {
203 348714 : if (CanBeImmediate(index)) {
204 292531 : *mode = kMode_MRI;
205 292531 : return UseImmediate(index);
206 : } else {
207 56184 : *mode = kMode_MR1;
208 56184 : return UseUniqueRegister(index);
209 : }
210 : }
211 :
212 : bool CanBeBetterLeftOperand(Node* node) const {
213 942863 : return !selector()->IsLive(node);
214 : }
215 : };
216 :
217 : namespace {
218 5929080 : ArchOpcode GetLoadOpcode(LoadRepresentation load_rep) {
219 : ArchOpcode opcode = kArchNop;
220 5929080 : switch (load_rep.representation()) {
221 : case MachineRepresentation::kFloat32:
222 : opcode = kX64Movss;
223 20207 : break;
224 : case MachineRepresentation::kFloat64:
225 : opcode = kX64Movsd;
226 433674 : break;
227 : case MachineRepresentation::kBit: // Fall through.
228 : case MachineRepresentation::kWord8:
229 229731 : opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
230 229731 : break;
231 : case MachineRepresentation::kWord16:
232 158239 : opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
233 158239 : break;
234 : case MachineRepresentation::kWord32:
235 : opcode = kX64Movl;
236 474276 : break;
237 : #ifdef V8_COMPRESS_POINTERS
238 : case MachineRepresentation::kTaggedSigned:
239 : return kX64MovqDecompressTaggedSigned;
240 : case MachineRepresentation::kTaggedPointer:
241 : return kX64MovqDecompressTaggedPointer;
242 : case MachineRepresentation::kTagged:
243 : return kX64MovqDecompressAnyTagged;
244 : #else
245 : case MachineRepresentation::kTaggedSigned: // Fall through.
246 : case MachineRepresentation::kTaggedPointer: // Fall through.
247 : case MachineRepresentation::kTagged: // Fall through.
248 : #endif
249 : case MachineRepresentation::kWord64:
250 : opcode = kX64Movq;
251 4595458 : break;
252 : case MachineRepresentation::kSimd128: // Fall through.
253 : opcode = kX64Movdqu;
254 17465 : break;
255 : case MachineRepresentation::kNone:
256 0 : UNREACHABLE();
257 : break;
258 : }
259 5929080 : return opcode;
260 : }
261 :
262 4347006 : ArchOpcode GetStoreOpcode(StoreRepresentation store_rep) {
263 4347006 : switch (store_rep.representation()) {
264 : case MachineRepresentation::kFloat32:
265 : return kX64Movss;
266 : break;
267 : case MachineRepresentation::kFloat64:
268 : return kX64Movsd;
269 : break;
270 : case MachineRepresentation::kBit: // Fall through.
271 : case MachineRepresentation::kWord8:
272 : return kX64Movb;
273 : break;
274 : case MachineRepresentation::kWord16:
275 : return kX64Movw;
276 : break;
277 : case MachineRepresentation::kWord32:
278 : return kX64Movl;
279 : break;
280 : case MachineRepresentation::kTaggedSigned: // Fall through.
281 : case MachineRepresentation::kTaggedPointer: // Fall through.
282 : case MachineRepresentation::kTagged: // Fall through.
283 : case MachineRepresentation::kWord64:
284 : return kX64Movq;
285 : break;
286 : case MachineRepresentation::kSimd128: // Fall through.
287 : return kX64Movdqu;
288 : break;
289 : case MachineRepresentation::kNone:
290 0 : UNREACHABLE();
291 : }
292 0 : UNREACHABLE();
293 : }
294 :
295 : } // namespace
296 :
297 2509 : void InstructionSelector::VisitStackSlot(Node* node) {
298 1254 : StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
299 1254 : int slot = frame_->AllocateSpillSlot(rep.size());
300 : OperandGenerator g(this);
301 :
302 : Emit(kArchStackSlot, g.DefineAsRegister(node),
303 2509 : sequence()->AddImmediate(Constant(slot)), 0, nullptr);
304 1255 : }
305 :
306 170 : void InstructionSelector::VisitDebugAbort(Node* node) {
307 : X64OperandGenerator g(this);
308 170 : Emit(kArchDebugAbort, g.NoOutput(), g.UseFixed(node->InputAt(0), rdx));
309 170 : }
310 :
311 1 : void InstructionSelector::VisitSpeculationFence(Node* node) {
312 : X64OperandGenerator g(this);
313 1 : Emit(kLFence, g.NoOutput());
314 1 : }
315 :
316 11858183 : void InstructionSelector::VisitLoad(Node* node) {
317 5929044 : LoadRepresentation load_rep = LoadRepresentationOf(node->op());
318 : X64OperandGenerator g(this);
319 :
320 5929084 : ArchOpcode opcode = GetLoadOpcode(load_rep);
321 : size_t temp_count = 0;
322 17787195 : InstructionOperand temps[2];
323 : #ifdef V8_COMPRESS_POINTERS
324 : if (opcode == kX64MovqDecompressAnyTagged) {
325 : temps[temp_count++] = g.TempRegister();
326 : }
327 : #ifdef DEBUG
328 : if (opcode == kX64MovqDecompressTaggedSigned ||
329 : opcode == kX64MovqDecompressTaggedPointer ||
330 : opcode == kX64MovqDecompressAnyTagged) {
331 : temps[temp_count++] = g.TempRegister();
332 : }
333 : #endif // DEBUG
334 : #endif // V8_COMPRESS_POINTERS
335 : DCHECK_LE(temp_count, arraysize(temps));
336 5929065 : InstructionOperand outputs[] = {g.DefineAsRegister(node)};
337 23716316 : InstructionOperand inputs[3];
338 5929112 : size_t input_count = 0;
339 : AddressingMode mode =
340 5929112 : g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
341 5929139 : InstructionCode code = opcode | AddressingModeField::encode(mode);
342 5929139 : if (node->opcode() == IrOpcode::kProtectedLoad) {
343 198467 : code |= MiscField::encode(kMemoryAccessProtected);
344 5730672 : } else if (node->opcode() == IrOpcode::kPoisonedLoad) {
345 0 : CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
346 0 : code |= MiscField::encode(kMemoryAccessPoisoned);
347 : }
348 5929139 : Emit(code, 1, outputs, input_count, inputs, temp_count, temps);
349 5929123 : }
350 :
351 0 : void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }
352 :
353 198465 : void InstructionSelector::VisitProtectedLoad(Node* node) { VisitLoad(node); }
354 :
355 8805298 : void InstructionSelector::VisitStore(Node* node) {
356 : X64OperandGenerator g(this);
357 : Node* base = node->InputAt(0);
358 : Node* index = node->InputAt(1);
359 1800447 : Node* value = node->InputAt(2);
360 :
361 4402649 : StoreRepresentation store_rep = StoreRepresentationOf(node->op());
362 : WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
363 :
364 4402650 : if (write_barrier_kind != kNoWriteBarrier) {
365 : DCHECK(CanBeTaggedPointer(store_rep.representation()));
366 : AddressingMode addressing_mode;
367 : InstructionOperand inputs[] = {
368 : g.UseUniqueRegister(base),
369 : g.GetEffectiveIndexOperand(index, &addressing_mode),
370 313550 : g.UseUniqueRegister(value)};
371 : RecordWriteMode record_write_mode = RecordWriteMode::kValueIsAny;
372 313550 : switch (write_barrier_kind) {
373 : case kNoWriteBarrier:
374 0 : UNREACHABLE();
375 : break;
376 : case kMapWriteBarrier:
377 : record_write_mode = RecordWriteMode::kValueIsMap;
378 33929 : break;
379 : case kPointerWriteBarrier:
380 : record_write_mode = RecordWriteMode::kValueIsPointer;
381 31351 : break;
382 : case kFullWriteBarrier:
383 : record_write_mode = RecordWriteMode::kValueIsAny;
384 : break;
385 : }
386 627100 : InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
387 : InstructionCode code = kArchStoreWithWriteBarrier;
388 627100 : code |= AddressingModeField::encode(addressing_mode);
389 313550 : code |= MiscField::encode(static_cast<int>(record_write_mode));
390 313550 : Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
391 : } else {
392 4089100 : ArchOpcode opcode = GetStoreOpcode(store_rep);
393 20445509 : InstructionOperand inputs[4];
394 4089101 : size_t input_count = 0;
395 : AddressingMode addressing_mode =
396 4089101 : g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
397 : InstructionCode code =
398 4089116 : opcode | AddressingModeField::encode(addressing_mode);
399 8178220 : if ((ElementSizeLog2Of(store_rep.representation()) <
400 1800447 : kSystemPointerSizeLog2) &&
401 4210774 : (value->opcode() == IrOpcode::kTruncateInt64ToInt32) &&
402 121670 : CanCover(node, value)) {
403 : value = value->InputAt(0);
404 : }
405 : InstructionOperand value_operand =
406 4089104 : g.CanBeImmediate(value) ? g.UseImmediate(value) : g.UseRegister(value);
407 4089111 : inputs[input_count++] = value_operand;
408 : Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count,
409 4089111 : inputs);
410 : }
411 4402666 : }
412 :
413 515814 : void InstructionSelector::VisitProtectedStore(Node* node) {
414 : X64OperandGenerator g(this);
415 : Node* value = node->InputAt(2);
416 :
417 257907 : StoreRepresentation store_rep = StoreRepresentationOf(node->op());
418 :
419 257911 : ArchOpcode opcode = GetStoreOpcode(store_rep);
420 1289550 : InstructionOperand inputs[4];
421 257914 : size_t input_count = 0;
422 : AddressingMode addressing_mode =
423 257914 : g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
424 257909 : InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) |
425 257909 : MiscField::encode(kMemoryAccessProtected);
426 : InstructionOperand value_operand =
427 257909 : g.CanBeImmediate(value) ? g.UseImmediate(value) : g.UseRegister(value);
428 257913 : inputs[input_count++] = value_operand;
429 257913 : Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count, inputs);
430 257918 : }
431 :
432 : // Architecture supports unaligned access, therefore VisitLoad is used instead
433 0 : void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }
434 :
435 : // Architecture supports unaligned access, therefore VisitStore is used instead
436 0 : void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
437 :
438 : // Shared routine for multiple binary operations.
439 1414017 : static void VisitBinop(InstructionSelector* selector, Node* node,
440 1727596 : InstructionCode opcode, FlagsContinuation* cont) {
441 : X64OperandGenerator g(selector);
442 1007189 : Int32BinopMatcher m(node);
443 : Node* left = m.left().node();
444 : Node* right = m.right().node();
445 9065201 : InstructionOperand inputs[8];
446 1007265 : size_t input_count = 0;
447 2014515 : InstructionOperand outputs[1];
448 : size_t output_count = 0;
449 :
450 : // TODO(turbofan): match complex addressing modes.
451 1007265 : if (left == right) {
452 : // If both inputs refer to the same operand, enforce allocating a register
453 : // for both of them to ensure that we don't end up generating code like
454 : // this:
455 : //
456 : // mov rax, [rbp-0x10]
457 : // add rax, [rbp-0x10]
458 : // jo label
459 2752 : InstructionOperand const input = g.UseRegister(left);
460 2752 : inputs[input_count++] = input;
461 2752 : inputs[input_count++] = input;
462 1004513 : } else if (g.CanBeImmediate(right)) {
463 597660 : inputs[input_count++] = g.UseRegister(left);
464 597662 : inputs[input_count++] = g.UseImmediate(right);
465 : } else {
466 406847 : int effect_level = selector->GetEffectLevel(node);
467 406839 : if (cont->IsBranch()) {
468 : effect_level = selector->GetEffectLevel(
469 36680 : cont->true_block()->PredecessorAt(0)->control_input());
470 : }
471 739156 : if (node->op()->HasProperty(Operator::kCommutative) &&
472 667748 : g.CanBeBetterLeftOperand(right) &&
473 248609 : (!g.CanBeBetterLeftOperand(left) ||
474 248659 : !g.CanBeMemoryOperand(opcode, node, right, effect_level))) {
475 : std::swap(left, right);
476 : }
477 406831 : if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) {
478 2132 : inputs[input_count++] = g.UseRegister(left);
479 : AddressingMode addressing_mode =
480 2147 : g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
481 2150 : opcode |= AddressingModeField::encode(addressing_mode);
482 : } else {
483 404695 : inputs[input_count++] = g.UseRegister(left);
484 404696 : inputs[input_count++] = g.Use(right);
485 : }
486 : }
487 :
488 1007263 : if (cont->IsBranch()) {
489 276822 : inputs[input_count++] = g.Label(cont->true_block());
490 276806 : inputs[input_count++] = g.Label(cont->false_block());
491 : }
492 :
493 1007257 : outputs[output_count++] = g.DefineSameAsFirst(node);
494 :
495 : DCHECK_NE(0u, input_count);
496 : DCHECK_EQ(1u, output_count);
497 : DCHECK_GE(arraysize(inputs), input_count);
498 : DCHECK_GE(arraysize(outputs), output_count);
499 :
500 : selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
501 1007258 : inputs, cont);
502 1007262 : }
503 :
504 : // Shared routine for multiple binary operations.
505 680121 : static void VisitBinop(InstructionSelector* selector, Node* node,
506 : InstructionCode opcode) {
507 : FlagsContinuation cont;
508 680136 : VisitBinop(selector, node, opcode, &cont);
509 680178 : }
510 :
511 200590 : void InstructionSelector::VisitWord32And(Node* node) {
512 : X64OperandGenerator g(this);
513 200590 : Uint32BinopMatcher m(node);
514 200591 : if (m.right().Is(0xFF)) {
515 2131 : Emit(kX64Movzxbl, g.DefineAsRegister(node), g.Use(m.left().node()));
516 198460 : } else if (m.right().Is(0xFFFF)) {
517 6214 : Emit(kX64Movzxwl, g.DefineAsRegister(node), g.Use(m.left().node()));
518 : } else {
519 192246 : VisitBinop(this, node, kX64And32);
520 : }
521 200591 : }
522 :
523 186195 : void InstructionSelector::VisitWord64And(Node* node) {
524 186195 : VisitBinop(this, node, kX64And);
525 186206 : }
526 :
527 81324 : void InstructionSelector::VisitWord32Or(Node* node) {
528 81324 : VisitBinop(this, node, kX64Or32);
529 81325 : }
530 :
531 89393 : void InstructionSelector::VisitWord64Or(Node* node) {
532 89393 : VisitBinop(this, node, kX64Or);
533 89405 : }
534 :
535 42968 : void InstructionSelector::VisitWord32Xor(Node* node) {
536 : X64OperandGenerator g(this);
537 42968 : Uint32BinopMatcher m(node);
538 42968 : if (m.right().Is(-1)) {
539 3423 : Emit(kX64Not32, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
540 : } else {
541 39545 : VisitBinop(this, node, kX64Xor32);
542 : }
543 42968 : }
544 :
545 425 : void InstructionSelector::VisitWord64Xor(Node* node) {
546 : X64OperandGenerator g(this);
547 425 : Uint64BinopMatcher m(node);
548 425 : if (m.right().Is(-1)) {
549 55 : Emit(kX64Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
550 : } else {
551 370 : VisitBinop(this, node, kX64Xor);
552 : }
553 425 : }
554 :
555 : namespace {
556 :
557 23 : bool TryMergeTruncateInt64ToInt32IntoLoad(InstructionSelector* selector,
558 46 : Node* node, Node* load) {
559 23 : if (load->opcode() == IrOpcode::kLoad && selector->CanCover(node, load)) {
560 23 : LoadRepresentation load_rep = LoadRepresentationOf(load->op());
561 23 : MachineRepresentation rep = load_rep.representation();
562 : InstructionCode opcode = kArchNop;
563 : switch (rep) {
564 : case MachineRepresentation::kBit: // Fall through.
565 : case MachineRepresentation::kWord8:
566 0 : opcode = load_rep.IsSigned() ? kX64Movsxbl : kX64Movzxbl;
567 0 : break;
568 : case MachineRepresentation::kWord16:
569 0 : opcode = load_rep.IsSigned() ? kX64Movsxwl : kX64Movzxwl;
570 0 : break;
571 : case MachineRepresentation::kWord32:
572 : case MachineRepresentation::kWord64:
573 : case MachineRepresentation::kTaggedSigned:
574 : case MachineRepresentation::kTagged:
575 : opcode = kX64Movl;
576 : break;
577 : default:
578 0 : UNREACHABLE();
579 : return false;
580 : }
581 : X64OperandGenerator g(selector);
582 23 : InstructionOperand outputs[] = {g.DefineAsRegister(node)};
583 23 : size_t input_count = 0;
584 92 : InstructionOperand inputs[3];
585 : AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
586 23 : node->InputAt(0), inputs, &input_count);
587 23 : opcode |= AddressingModeField::encode(mode);
588 23 : selector->Emit(opcode, 1, outputs, input_count, inputs);
589 : return true;
590 : }
591 : return false;
592 : }
593 :
594 : // Shared routine for multiple 32-bit shift operations.
595 : // TODO(bmeurer): Merge this with VisitWord64Shift using template magic?
596 284532 : void VisitWord32Shift(InstructionSelector* selector, Node* node,
597 : ArchOpcode opcode) {
598 : X64OperandGenerator g(selector);
599 284532 : Int32BinopMatcher m(node);
600 284533 : Node* left = m.left().node();
601 : Node* right = m.right().node();
602 :
603 294969 : if (left->opcode() == IrOpcode::kTruncateInt64ToInt32 &&
604 10436 : selector->CanCover(node, left)) {
605 : left = left->InputAt(0);
606 : }
607 :
608 284533 : if (g.CanBeImmediate(right)) {
609 : selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
610 270978 : g.UseImmediate(right));
611 : } else {
612 : selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
613 13556 : g.UseFixed(right, rcx));
614 : }
615 284533 : }
616 :
617 : // Shared routine for multiple 64-bit shift operations.
618 : // TODO(bmeurer): Merge this with VisitWord32Shift using template magic?
619 318841 : void VisitWord64Shift(InstructionSelector* selector, Node* node,
620 : ArchOpcode opcode) {
621 : X64OperandGenerator g(selector);
622 318841 : Int64BinopMatcher m(node);
623 : Node* left = m.left().node();
624 : Node* right = m.right().node();
625 :
626 318841 : if (g.CanBeImmediate(right)) {
627 : selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
628 313795 : g.UseImmediate(right));
629 : } else {
630 5045 : if (m.right().IsWord64And()) {
631 112 : Int64BinopMatcher mright(right);
632 112 : if (mright.right().Is(0x3F)) {
633 : right = mright.left().node();
634 : }
635 : }
636 : selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
637 5045 : g.UseFixed(right, rcx));
638 : }
639 318841 : }
640 :
641 : // Shared routine for multiple shift operations with continuation.
642 : template <typename BinopMatcher, int Bits>
643 29390 : bool TryVisitWordShift(InstructionSelector* selector, Node* node,
644 : ArchOpcode opcode, FlagsContinuation* cont) {
645 : X64OperandGenerator g(selector);
646 29390 : BinopMatcher m(node);
647 : Node* left = m.left().node();
648 : Node* right = m.right().node();
649 :
650 : // If the shift count is 0, the flags are not affected.
651 58780 : if (!g.CanBeImmediate(right) ||
652 : (g.GetImmediateIntegerValue(right) & (Bits - 1)) == 0) {
653 : return false;
654 : }
655 29380 : InstructionOperand output = g.DefineSameAsFirst(node);
656 88140 : InstructionOperand inputs[2];
657 29380 : inputs[0] = g.UseRegister(left);
658 29380 : inputs[1] = g.UseImmediate(right);
659 29380 : selector->EmitWithContinuation(opcode, 1, &output, 2, inputs, cont);
660 29380 : return true;
661 : }
662 :
663 2471499 : void EmitLea(InstructionSelector* selector, InstructionCode opcode,
664 : Node* result, Node* index, int scale, Node* base,
665 : Node* displacement, DisplacementMode displacement_mode) {
666 : X64OperandGenerator g(selector);
667 :
668 12357427 : InstructionOperand inputs[4];
669 2471499 : size_t input_count = 0;
670 : AddressingMode mode =
671 : g.GenerateMemoryOperandInputs(index, scale, base, displacement,
672 2471499 : displacement_mode, inputs, &input_count);
673 :
674 : DCHECK_NE(0u, input_count);
675 : DCHECK_GE(arraysize(inputs), input_count);
676 :
677 4943036 : InstructionOperand outputs[1];
678 2471521 : outputs[0] = g.DefineAsRegister(result);
679 :
680 2471522 : opcode = AddressingModeField::encode(mode) | opcode;
681 :
682 2471522 : selector->Emit(opcode, 1, outputs, input_count, inputs);
683 2471538 : }
684 :
685 : } // namespace
686 :
687 65080 : void InstructionSelector::VisitWord32Shl(Node* node) {
688 65080 : Int32ScaleMatcher m(node, true);
689 65083 : if (m.matches()) {
690 : Node* index = node->InputAt(0);
691 14179 : Node* base = m.power_of_two_plus_one() ? index : nullptr;
692 : EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
693 14179 : kPositiveDisplacement);
694 79273 : return;
695 : }
696 50904 : VisitWord32Shift(this, node, kX64Shl32);
697 : }
698 :
699 398013 : void InstructionSelector::VisitWord64Shl(Node* node) {
700 : X64OperandGenerator g(this);
701 398013 : Int64ScaleMatcher m(node, true);
702 398025 : if (m.matches()) {
703 : Node* index = node->InputAt(0);
704 36879 : Node* base = m.power_of_two_plus_one() ? index : nullptr;
705 : EmitLea(this, kX64Lea, node, index, m.scale(), base, nullptr,
706 36879 : kPositiveDisplacement);
707 36879 : return;
708 : } else {
709 361146 : Int64BinopMatcher m(node);
710 497044 : if ((m.left().IsChangeInt32ToInt64() ||
711 629340 : m.left().IsChangeUint32ToUint64()) &&
712 : m.right().IsInRange(32, 63)) {
713 : // There's no need to sign/zero-extend to 64-bit if we shift out the upper
714 : // 32 bits anyway.
715 : Emit(kX64Shl, g.DefineSameAsFirst(node),
716 : g.UseRegister(m.left().node()->InputAt(0)),
717 536397 : g.UseImmediate(m.right().node()));
718 268201 : return;
719 : }
720 : }
721 92950 : VisitWord64Shift(this, node, kX64Shl);
722 : }
723 :
724 119363 : void InstructionSelector::VisitWord32Shr(Node* node) {
725 119363 : VisitWord32Shift(this, node, kX64Shr32);
726 119364 : }
727 :
728 : namespace {
729 :
730 21 : inline AddressingMode AddDisplacementToAddressingMode(AddressingMode mode) {
731 21 : switch (mode) {
732 : case kMode_MR:
733 : return kMode_MRI;
734 : break;
735 : case kMode_MR1:
736 0 : return kMode_MR1I;
737 : break;
738 : case kMode_MR2:
739 0 : return kMode_MR2I;
740 : break;
741 : case kMode_MR4:
742 0 : return kMode_MR4I;
743 : break;
744 : case kMode_MR8:
745 0 : return kMode_MR8I;
746 : break;
747 : case kMode_M1:
748 0 : return kMode_M1I;
749 : break;
750 : case kMode_M2:
751 0 : return kMode_M2I;
752 : break;
753 : case kMode_M4:
754 0 : return kMode_M4I;
755 : break;
756 : case kMode_M8:
757 0 : return kMode_M8I;
758 : break;
759 : case kMode_None:
760 : case kMode_MRI:
761 : case kMode_MR1I:
762 : case kMode_MR2I:
763 : case kMode_MR4I:
764 : case kMode_MR8I:
765 : case kMode_M1I:
766 : case kMode_M2I:
767 : case kMode_M4I:
768 : case kMode_M8I:
769 : case kMode_Root:
770 0 : UNREACHABLE();
771 : }
772 0 : UNREACHABLE();
773 : }
774 :
775 530743 : bool TryMatchLoadWord64AndShiftRight(InstructionSelector* selector, Node* node,
776 : InstructionCode opcode) {
777 : DCHECK(IrOpcode::kWord64Sar == node->opcode() ||
778 : IrOpcode::kWord64Shr == node->opcode());
779 : X64OperandGenerator g(selector);
780 530743 : Int64BinopMatcher m(node);
781 1217677 : if (selector->CanCover(m.node(), m.left().node()) && m.left().IsLoad() &&
782 : m.right().Is(32)) {
783 : DCHECK_EQ(selector->GetEffectLevel(node),
784 : selector->GetEffectLevel(m.left().node()));
785 : // Just load and sign-extend the interesting 4 bytes instead. This happens,
786 : // for example, when we're loading and untagging SMIs.
787 : BaseWithIndexAndDisplacement64Matcher mleft(m.left().node(),
788 : AddressOption::kAllowAll);
789 390771 : if (mleft.matches() && (mleft.displacement() == nullptr ||
790 195375 : g.CanBeImmediate(mleft.displacement()))) {
791 195396 : size_t input_count = 0;
792 781575 : InstructionOperand inputs[3];
793 : AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
794 195396 : m.left().node(), inputs, &input_count);
795 195394 : if (mleft.displacement() == nullptr) {
796 : // Make sure that the addressing mode indicates the presence of an
797 : // immediate displacement. It seems that we never use M1 and M2, but we
798 : // handle them here anyways.
799 21 : mode = AddDisplacementToAddressingMode(mode);
800 21 : inputs[input_count++] = ImmediateOperand(ImmediateOperand::INLINE, 4);
801 : } else {
802 : // In the case that the base address was zero, the displacement will be
803 : // in a register and replacing it with an immediate is not allowed. This
804 : // usually only happens in dead code anyway.
805 390746 : if (!inputs[input_count - 1].IsImmediate()) return false;
806 : int32_t displacement = g.GetImmediateIntegerValue(mleft.displacement());
807 : inputs[input_count - 1] =
808 390732 : ImmediateOperand(ImmediateOperand::INLINE, displacement + 4);
809 : }
810 195387 : InstructionOperand outputs[] = {g.DefineAsRegister(node)};
811 195389 : InstructionCode code = opcode | AddressingModeField::encode(mode);
812 195389 : selector->Emit(code, 1, outputs, input_count, inputs);
813 195389 : return true;
814 : }
815 : }
816 : return false;
817 : }
818 :
819 : } // namespace
820 :
821 44932 : void InstructionSelector::VisitWord64Shr(Node* node) {
822 89864 : if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movl)) return;
823 44931 : VisitWord64Shift(this, node, kX64Shr);
824 : }
825 :
826 47365 : void InstructionSelector::VisitWord32Sar(Node* node) {
827 : X64OperandGenerator g(this);
828 47365 : Int32BinopMatcher m(node);
829 87190 : if (CanCover(m.node(), m.left().node()) && m.left().IsWord32Shl()) {
830 1303 : Int32BinopMatcher mleft(m.left().node());
831 1870 : if (mleft.right().Is(16) && m.right().Is(16)) {
832 567 : Emit(kX64Movsxwl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
833 567 : return;
834 1460 : } else if (mleft.right().Is(24) && m.right().Is(24)) {
835 724 : Emit(kX64Movsxbl, g.DefineAsRegister(node), g.Use(mleft.left().node()));
836 724 : return;
837 : }
838 : }
839 46074 : VisitWord32Shift(this, node, kX64Sar32);
840 : }
841 :
842 198444 : void InstructionSelector::VisitWord64Sar(Node* node) {
843 396889 : if (TryMatchLoadWord64AndShiftRight(this, node, kX64Movsxlq)) return;
844 180774 : VisitWord64Shift(this, node, kX64Sar);
845 : }
846 :
847 68192 : void InstructionSelector::VisitWord32Ror(Node* node) {
848 68192 : VisitWord32Shift(this, node, kX64Ror32);
849 68192 : }
850 :
851 186 : void InstructionSelector::VisitWord64Ror(Node* node) {
852 186 : VisitWord64Shift(this, node, kX64Ror);
853 186 : }
854 :
855 0 : void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
856 :
857 0 : void InstructionSelector::VisitWord64ReverseBits(Node* node) { UNREACHABLE(); }
858 :
859 15 : void InstructionSelector::VisitWord64ReverseBytes(Node* node) {
860 : X64OperandGenerator g(this);
861 15 : Emit(kX64Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
862 15 : }
863 :
864 55 : void InstructionSelector::VisitWord32ReverseBytes(Node* node) {
865 : X64OperandGenerator g(this);
866 55 : Emit(kX64Bswap32, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
867 55 : }
868 :
869 318691 : void InstructionSelector::VisitInt32Add(Node* node) {
870 : X64OperandGenerator g(this);
871 :
872 : // Try to match the Add to a leal pattern
873 318691 : BaseWithIndexAndDisplacement32Matcher m(node);
874 956086 : if (m.matches() &&
875 589352 : (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
876 : EmitLea(this, kX64Lea32, node, m.index(), m.scale(), m.base(),
877 318692 : m.displacement(), m.displacement_mode());
878 637422 : return;
879 : }
880 :
881 : // No leal pattern match, use addl
882 0 : VisitBinop(this, node, kX64Add32);
883 : }
884 :
885 2117326 : void InstructionSelector::VisitInt64Add(Node* node) {
886 : X64OperandGenerator g(this);
887 :
888 : // Try to match the Add to a leaq pattern
889 2117326 : BaseWithIndexAndDisplacement64Matcher m(node);
890 6352127 : if (m.matches() &&
891 3908964 : (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
892 : EmitLea(this, kX64Lea, node, m.index(), m.scale(), m.base(),
893 2074301 : m.displacement(), m.displacement_mode());
894 4191655 : return;
895 : }
896 :
897 : // No leal pattern match, use addq
898 43067 : VisitBinop(this, node, kX64Add);
899 : }
900 :
901 33620 : void InstructionSelector::VisitInt64AddWithOverflow(Node* node) {
902 33620 : if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
903 : FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
904 67240 : return VisitBinop(this, node, kX64Add, &cont);
905 : }
906 : FlagsContinuation cont;
907 0 : VisitBinop(this, node, kX64Add, &cont);
908 : }
909 :
910 77301 : void InstructionSelector::VisitInt32Sub(Node* node) {
911 : X64OperandGenerator g(this);
912 : DCHECK_EQ(node->InputCount(), 2);
913 77301 : Node* input1 = node->InputAt(0);
914 : Node* input2 = node->InputAt(1);
915 81999 : if (input1->opcode() == IrOpcode::kTruncateInt64ToInt32 &&
916 4698 : g.CanBeImmediate(input2)) {
917 : int32_t imm = g.GetImmediateIntegerValue(input2);
918 9 : InstructionOperand int64_input = g.UseRegister(input1->InputAt(0));
919 9 : if (imm == 0) {
920 : // Emit "movl" for subtraction of 0.
921 9 : Emit(kX64Movl, g.DefineAsRegister(node), int64_input);
922 : } else {
923 : // Omit truncation and turn subtractions of constant values into immediate
924 : // "leal" instructions by negating the value.
925 : Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI),
926 0 : g.DefineAsRegister(node), int64_input, g.TempImmediate(-imm));
927 : }
928 77299 : return;
929 : }
930 :
931 77292 : Int32BinopMatcher m(node);
932 77293 : if (m.left().Is(0)) {
933 7045 : Emit(kX64Neg32, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
934 70248 : } else if (m.right().Is(0)) {
935 : // TODO(jarin): We should be able to use {EmitIdentity} here
936 : // (https://crbug.com/v8/7947).
937 861 : Emit(kArchNop, g.DefineSameAsFirst(node), g.Use(m.left().node()));
938 113197 : } else if (m.right().HasValue() && g.CanBeImmediate(m.right().node())) {
939 : // Turn subtractions of constant values into immediate "leal" instructions
940 : // by negating the value.
941 : Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI),
942 : g.DefineAsRegister(node), g.UseRegister(m.left().node()),
943 131426 : g.TempImmediate(base::NegateWithWraparound(m.right().Value())));
944 : } else {
945 25577 : VisitBinop(this, node, kX64Sub32);
946 : }
947 : }
948 :
949 35893 : void InstructionSelector::VisitInt64Sub(Node* node) {
950 : X64OperandGenerator g(this);
951 35893 : Int64BinopMatcher m(node);
952 35901 : if (m.left().Is(0)) {
953 10652 : Emit(kX64Neg, g.DefineSameAsFirst(node), g.UseRegister(m.right().node()));
954 : } else {
955 28093 : if (m.right().HasValue() && g.CanBeImmediate(m.right().node())) {
956 : // Turn subtractions of constant values into immediate "leaq" instructions
957 : // by negating the value.
958 : Emit(kX64Lea | AddressingModeField::encode(kMode_MRI),
959 : g.DefineAsRegister(node), g.UseRegister(m.left().node()),
960 5614 : g.TempImmediate(-static_cast<int32_t>(m.right().Value())));
961 38711 : return;
962 : }
963 22449 : VisitBinop(this, node, kX64Sub);
964 : }
965 : }
966 :
967 33620 : void InstructionSelector::VisitInt64SubWithOverflow(Node* node) {
968 33620 : if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
969 : FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
970 67240 : return VisitBinop(this, node, kX64Sub, &cont);
971 : }
972 : FlagsContinuation cont;
973 0 : VisitBinop(this, node, kX64Sub, &cont);
974 : }
975 :
976 : namespace {
977 :
978 81760 : void VisitMul(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
979 : X64OperandGenerator g(selector);
980 81760 : Int32BinopMatcher m(node);
981 : Node* left = m.left().node();
982 : Node* right = m.right().node();
983 81760 : if (g.CanBeImmediate(right)) {
984 : selector->Emit(opcode, g.DefineAsRegister(node), g.Use(left),
985 72983 : g.UseImmediate(right));
986 : } else {
987 8777 : if (g.CanBeBetterLeftOperand(right)) {
988 : std::swap(left, right);
989 : }
990 : selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
991 8777 : g.Use(right));
992 : }
993 81760 : }
994 :
995 9285 : void VisitMulHigh(InstructionSelector* selector, Node* node,
996 : ArchOpcode opcode) {
997 : X64OperandGenerator g(selector);
998 : Node* left = node->InputAt(0);
999 : Node* right = node->InputAt(1);
1000 9285 : if (selector->IsLive(left) && !selector->IsLive(right)) {
1001 : std::swap(left, right);
1002 : }
1003 9285 : InstructionOperand temps[] = {g.TempRegister(rax)};
1004 : // TODO(turbofan): We use UseUniqueRegister here to improve register
1005 : // allocation.
1006 : selector->Emit(opcode, g.DefineAsFixed(node, rdx), g.UseFixed(left, rax),
1007 9285 : g.UseUniqueRegister(right), arraysize(temps), temps);
1008 9285 : }
1009 :
1010 77186 : void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1011 : X64OperandGenerator g(selector);
1012 77186 : InstructionOperand temps[] = {g.TempRegister(rdx)};
1013 : selector->Emit(
1014 : opcode, g.DefineAsFixed(node, rax), g.UseFixed(node->InputAt(0), rax),
1015 154372 : g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
1016 77186 : }
1017 :
1018 77029 : void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
1019 : X64OperandGenerator g(selector);
1020 77029 : InstructionOperand temps[] = {g.TempRegister(rax)};
1021 : selector->Emit(
1022 : opcode, g.DefineAsFixed(node, rdx), g.UseFixed(node->InputAt(0), rax),
1023 154058 : g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
1024 77029 : }
1025 :
1026 : } // namespace
1027 :
1028 87065 : void InstructionSelector::VisitInt32Mul(Node* node) {
1029 87065 : Int32ScaleMatcher m(node, true);
1030 87065 : if (m.matches()) {
1031 : Node* index = node->InputAt(0);
1032 27447 : Node* base = m.power_of_two_plus_one() ? index : nullptr;
1033 : EmitLea(this, kX64Lea32, node, index, m.scale(), base, nullptr,
1034 27447 : kPositiveDisplacement);
1035 114512 : return;
1036 : }
1037 59618 : VisitMul(this, node, kX64Imul32);
1038 : }
1039 :
1040 17405 : void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
1041 : // TODO(mvstanton): Use Int32ScaleMatcher somehow.
1042 17405 : if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1043 : FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1044 34810 : return VisitBinop(this, node, kX64Imul32, &cont);
1045 : }
1046 : FlagsContinuation cont;
1047 0 : VisitBinop(this, node, kX64Imul32, &cont);
1048 : }
1049 :
1050 22142 : void InstructionSelector::VisitInt64Mul(Node* node) {
1051 22142 : VisitMul(this, node, kX64Imul);
1052 22142 : }
1053 :
1054 7731 : void InstructionSelector::VisitInt32MulHigh(Node* node) {
1055 7731 : VisitMulHigh(this, node, kX64ImulHigh32);
1056 7731 : }
1057 :
1058 36917 : void InstructionSelector::VisitInt32Div(Node* node) {
1059 36917 : VisitDiv(this, node, kX64Idiv32);
1060 36917 : }
1061 :
1062 2856 : void InstructionSelector::VisitInt64Div(Node* node) {
1063 2856 : VisitDiv(this, node, kX64Idiv);
1064 2856 : }
1065 :
1066 35581 : void InstructionSelector::VisitUint32Div(Node* node) {
1067 35581 : VisitDiv(this, node, kX64Udiv32);
1068 35581 : }
1069 :
1070 1832 : void InstructionSelector::VisitUint64Div(Node* node) {
1071 1832 : VisitDiv(this, node, kX64Udiv);
1072 1832 : }
1073 :
1074 37656 : void InstructionSelector::VisitInt32Mod(Node* node) {
1075 37656 : VisitMod(this, node, kX64Idiv32);
1076 37656 : }
1077 :
1078 1814 : void InstructionSelector::VisitInt64Mod(Node* node) {
1079 1814 : VisitMod(this, node, kX64Idiv);
1080 1814 : }
1081 :
1082 35754 : void InstructionSelector::VisitUint32Mod(Node* node) {
1083 35754 : VisitMod(this, node, kX64Udiv32);
1084 35754 : }
1085 :
1086 1805 : void InstructionSelector::VisitUint64Mod(Node* node) {
1087 1805 : VisitMod(this, node, kX64Udiv);
1088 1805 : }
1089 :
1090 1554 : void InstructionSelector::VisitUint32MulHigh(Node* node) {
1091 1554 : VisitMulHigh(this, node, kX64UmulHigh32);
1092 1554 : }
1093 :
1094 71 : void InstructionSelector::VisitTryTruncateFloat32ToInt64(Node* node) {
1095 : X64OperandGenerator g(this);
1096 71 : InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1097 213 : InstructionOperand outputs[2];
1098 : size_t output_count = 0;
1099 71 : outputs[output_count++] = g.DefineAsRegister(node);
1100 :
1101 71 : Node* success_output = NodeProperties::FindProjection(node, 1);
1102 71 : if (success_output) {
1103 66 : outputs[output_count++] = g.DefineAsRegister(success_output);
1104 : }
1105 :
1106 71 : Emit(kSSEFloat32ToInt64, output_count, outputs, 1, inputs);
1107 71 : }
1108 :
1109 684 : void InstructionSelector::VisitTryTruncateFloat64ToInt64(Node* node) {
1110 : X64OperandGenerator g(this);
1111 684 : InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1112 2066 : InstructionOperand outputs[2];
1113 : size_t output_count = 0;
1114 692 : outputs[output_count++] = g.DefineAsRegister(node);
1115 :
1116 699 : Node* success_output = NodeProperties::FindProjection(node, 1);
1117 696 : if (success_output) {
1118 693 : outputs[output_count++] = g.DefineAsRegister(success_output);
1119 : }
1120 :
1121 699 : Emit(kSSEFloat64ToInt64, output_count, outputs, 1, inputs);
1122 700 : }
1123 :
1124 71 : void InstructionSelector::VisitTryTruncateFloat32ToUint64(Node* node) {
1125 : X64OperandGenerator g(this);
1126 71 : InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1127 213 : InstructionOperand outputs[2];
1128 : size_t output_count = 0;
1129 71 : outputs[output_count++] = g.DefineAsRegister(node);
1130 :
1131 71 : Node* success_output = NodeProperties::FindProjection(node, 1);
1132 71 : if (success_output) {
1133 66 : outputs[output_count++] = g.DefineAsRegister(success_output);
1134 : }
1135 :
1136 71 : Emit(kSSEFloat32ToUint64, output_count, outputs, 1, inputs);
1137 71 : }
1138 :
1139 80 : void InstructionSelector::VisitTryTruncateFloat64ToUint64(Node* node) {
1140 : X64OperandGenerator g(this);
1141 80 : InstructionOperand inputs[] = {g.UseRegister(node->InputAt(0))};
1142 240 : InstructionOperand outputs[2];
1143 : size_t output_count = 0;
1144 80 : outputs[output_count++] = g.DefineAsRegister(node);
1145 :
1146 80 : Node* success_output = NodeProperties::FindProjection(node, 1);
1147 80 : if (success_output) {
1148 75 : outputs[output_count++] = g.DefineAsRegister(success_output);
1149 : }
1150 :
1151 80 : Emit(kSSEFloat64ToUint64, output_count, outputs, 1, inputs);
1152 80 : }
1153 :
1154 156684 : void InstructionSelector::VisitChangeInt32ToInt64(Node* node) {
1155 : X64OperandGenerator g(this);
1156 227532 : Node* const value = node->InputAt(0);
1157 156684 : if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
1158 70848 : LoadRepresentation load_rep = LoadRepresentationOf(value->op());
1159 70848 : MachineRepresentation rep = load_rep.representation();
1160 : InstructionCode opcode = kArchNop;
1161 70848 : switch (rep) {
1162 : case MachineRepresentation::kBit: // Fall through.
1163 : case MachineRepresentation::kWord8:
1164 27978 : opcode = load_rep.IsSigned() ? kX64Movsxbq : kX64Movzxbq;
1165 27978 : break;
1166 : case MachineRepresentation::kWord16:
1167 9697 : opcode = load_rep.IsSigned() ? kX64Movsxwq : kX64Movzxwq;
1168 9697 : break;
1169 : case MachineRepresentation::kWord32:
1170 33173 : opcode = load_rep.IsSigned() ? kX64Movsxlq : kX64Movl;
1171 33173 : break;
1172 : default:
1173 0 : UNREACHABLE();
1174 : return;
1175 : }
1176 70848 : InstructionOperand outputs[] = {g.DefineAsRegister(node)};
1177 70848 : size_t input_count = 0;
1178 283392 : InstructionOperand inputs[3];
1179 : AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1180 70848 : node->InputAt(0), inputs, &input_count);
1181 70848 : opcode |= AddressingModeField::encode(mode);
1182 70848 : Emit(opcode, 1, outputs, input_count, inputs);
1183 : } else {
1184 85836 : Emit(kX64Movsxlq, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1185 : }
1186 156684 : }
1187 :
1188 : namespace {
1189 :
1190 289586 : bool ZeroExtendsWord32ToWord64(Node* node) {
1191 289586 : switch (node->opcode()) {
1192 : case IrOpcode::kWord32And:
1193 : case IrOpcode::kWord32Or:
1194 : case IrOpcode::kWord32Xor:
1195 : case IrOpcode::kWord32Shl:
1196 : case IrOpcode::kWord32Shr:
1197 : case IrOpcode::kWord32Sar:
1198 : case IrOpcode::kWord32Ror:
1199 : case IrOpcode::kWord32Equal:
1200 : case IrOpcode::kInt32Add:
1201 : case IrOpcode::kInt32Sub:
1202 : case IrOpcode::kInt32Mul:
1203 : case IrOpcode::kInt32MulHigh:
1204 : case IrOpcode::kInt32Div:
1205 : case IrOpcode::kInt32LessThan:
1206 : case IrOpcode::kInt32LessThanOrEqual:
1207 : case IrOpcode::kInt32Mod:
1208 : case IrOpcode::kUint32Div:
1209 : case IrOpcode::kUint32LessThan:
1210 : case IrOpcode::kUint32LessThanOrEqual:
1211 : case IrOpcode::kUint32Mod:
1212 : case IrOpcode::kUint32MulHigh:
1213 : // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1214 : // zero-extension is a no-op.
1215 : return true;
1216 : case IrOpcode::kProjection: {
1217 504 : Node* const value = node->InputAt(0);
1218 504 : switch (value->opcode()) {
1219 : case IrOpcode::kInt32AddWithOverflow:
1220 : case IrOpcode::kInt32SubWithOverflow:
1221 : case IrOpcode::kInt32MulWithOverflow:
1222 : return true;
1223 : default:
1224 0 : return false;
1225 : }
1226 : }
1227 : case IrOpcode::kLoad:
1228 : case IrOpcode::kPoisonedLoad: {
1229 : // The movzxbl/movsxbl/movzxwl/movsxwl/movl operations implicitly
1230 : // zero-extend to 64-bit on x64, so the zero-extension is a no-op.
1231 157765 : LoadRepresentation load_rep = LoadRepresentationOf(node->op());
1232 157765 : switch (load_rep.representation()) {
1233 : case MachineRepresentation::kWord8:
1234 : case MachineRepresentation::kWord16:
1235 : case MachineRepresentation::kWord32:
1236 : return true;
1237 : default:
1238 126 : return false;
1239 : }
1240 : }
1241 : default:
1242 45896 : return false;
1243 : }
1244 : }
1245 :
1246 : } // namespace
1247 :
1248 289585 : void InstructionSelector::VisitChangeUint32ToUint64(Node* node) {
1249 : X64OperandGenerator g(this);
1250 : Node* value = node->InputAt(0);
1251 289585 : if (ZeroExtendsWord32ToWord64(value)) {
1252 : // These 32-bit operations implicitly zero-extend to 64-bit on x64, so the
1253 : // zero-extension is a no-op.
1254 533272 : return EmitIdentity(node);
1255 : }
1256 46046 : Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1257 : }
1258 :
1259 : namespace {
1260 :
1261 706173 : void VisitRO(InstructionSelector* selector, Node* node,
1262 : InstructionCode opcode) {
1263 : X64OperandGenerator g(selector);
1264 706173 : selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
1265 706202 : }
1266 :
1267 100301 : void VisitRR(InstructionSelector* selector, Node* node,
1268 : InstructionCode opcode) {
1269 : X64OperandGenerator g(selector);
1270 : selector->Emit(opcode, g.DefineAsRegister(node),
1271 100301 : g.UseRegister(node->InputAt(0)));
1272 100307 : }
1273 :
1274 830 : void VisitRRO(InstructionSelector* selector, Node* node,
1275 : InstructionCode opcode) {
1276 : X64OperandGenerator g(selector);
1277 : selector->Emit(opcode, g.DefineSameAsFirst(node),
1278 1660 : g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)));
1279 830 : }
1280 :
1281 130040 : void VisitFloatBinop(InstructionSelector* selector, Node* node,
1282 : ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
1283 : X64OperandGenerator g(selector);
1284 130040 : InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
1285 130041 : InstructionOperand operand1 = g.Use(node->InputAt(1));
1286 130041 : if (selector->IsSupported(AVX)) {
1287 129429 : selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
1288 : } else {
1289 612 : selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
1290 : }
1291 130039 : }
1292 :
1293 10942 : void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
1294 : ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
1295 : X64OperandGenerator g(selector);
1296 10942 : if (selector->IsSupported(AVX)) {
1297 10858 : selector->Emit(avx_opcode, g.DefineAsRegister(node), g.Use(input));
1298 : } else {
1299 84 : selector->Emit(sse_opcode, g.DefineSameAsFirst(node), g.UseRegister(input));
1300 : }
1301 10943 : }
1302 :
1303 : } // namespace
1304 :
1305 : #define RO_OP_LIST(V) \
1306 : V(Word64Clz, kX64Lzcnt) \
1307 : V(Word32Clz, kX64Lzcnt32) \
1308 : V(Word64Ctz, kX64Tzcnt) \
1309 : V(Word32Ctz, kX64Tzcnt32) \
1310 : V(Word64Popcnt, kX64Popcnt) \
1311 : V(Word32Popcnt, kX64Popcnt32) \
1312 : V(Float64Sqrt, kSSEFloat64Sqrt) \
1313 : V(Float32Sqrt, kSSEFloat32Sqrt) \
1314 : V(ChangeFloat64ToInt32, kSSEFloat64ToInt32) \
1315 : V(ChangeFloat64ToInt64, kSSEFloat64ToInt64) \
1316 : V(ChangeFloat64ToUint32, kSSEFloat64ToUint32 | MiscField::encode(1)) \
1317 : V(TruncateFloat64ToInt64, kSSEFloat64ToInt64) \
1318 : V(TruncateFloat64ToUint32, kSSEFloat64ToUint32 | MiscField::encode(0)) \
1319 : V(ChangeFloat64ToUint64, kSSEFloat64ToUint64) \
1320 : V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32) \
1321 : V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64) \
1322 : V(TruncateFloat32ToInt32, kSSEFloat32ToInt32) \
1323 : V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \
1324 : V(ChangeInt32ToFloat64, kSSEInt32ToFloat64) \
1325 : V(ChangeInt64ToFloat64, kSSEInt64ToFloat64) \
1326 : V(ChangeUint32ToFloat64, kSSEUint32ToFloat64) \
1327 : V(RoundFloat64ToInt32, kSSEFloat64ToInt32) \
1328 : V(RoundInt32ToFloat32, kSSEInt32ToFloat32) \
1329 : V(RoundInt64ToFloat32, kSSEInt64ToFloat32) \
1330 : V(RoundUint64ToFloat32, kSSEUint64ToFloat32) \
1331 : V(RoundInt64ToFloat64, kSSEInt64ToFloat64) \
1332 : V(RoundUint64ToFloat64, kSSEUint64ToFloat64) \
1333 : V(RoundUint32ToFloat32, kSSEUint32ToFloat32) \
1334 : V(BitcastFloat32ToInt32, kX64BitcastFI) \
1335 : V(BitcastFloat64ToInt64, kX64BitcastDL) \
1336 : V(BitcastInt32ToFloat32, kX64BitcastIF) \
1337 : V(BitcastInt64ToFloat64, kX64BitcastLD) \
1338 : V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32) \
1339 : V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \
1340 : V(SignExtendWord8ToInt32, kX64Movsxbl) \
1341 : V(SignExtendWord16ToInt32, kX64Movsxwl) \
1342 : V(SignExtendWord8ToInt64, kX64Movsxbq) \
1343 : V(SignExtendWord16ToInt64, kX64Movsxwq) \
1344 : V(SignExtendWord32ToInt64, kX64Movsxlq)
1345 :
1346 : #define RR_OP_LIST(V) \
1347 : V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown)) \
1348 : V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown)) \
1349 : V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp)) \
1350 : V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp)) \
1351 : V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \
1352 : V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \
1353 : V(Float32RoundTiesEven, \
1354 : kSSEFloat32Round | MiscField::encode(kRoundToNearest)) \
1355 : V(Float64RoundTiesEven, kSSEFloat64Round | MiscField::encode(kRoundToNearest))
1356 :
1357 : #define RO_VISITOR(Name, opcode) \
1358 : void InstructionSelector::Visit##Name(Node* node) { \
1359 : VisitRO(this, node, opcode); \
1360 : }
1361 706180 : RO_OP_LIST(RO_VISITOR)
1362 : #undef RO_VISITOR
1363 : #undef RO_OP_LIST
1364 :
1365 : #define RR_VISITOR(Name, opcode) \
1366 : void InstructionSelector::Visit##Name(Node* node) { \
1367 : VisitRR(this, node, opcode); \
1368 : }
1369 44413 : RR_OP_LIST(RR_VISITOR)
1370 : #undef RR_VISITOR
1371 : #undef RR_OP_LIST
1372 :
1373 55890 : void InstructionSelector::VisitTruncateFloat64ToWord32(Node* node) {
1374 55890 : VisitRR(this, node, kArchTruncateDoubleToI);
1375 55896 : }
1376 :
1377 536321 : void InstructionSelector::VisitTruncateInt64ToInt32(Node* node) {
1378 : X64OperandGenerator g(this);
1379 513872 : Node* value = node->InputAt(0);
1380 536321 : if (CanCover(node, value)) {
1381 513872 : switch (value->opcode()) {
1382 : case IrOpcode::kWord64Sar:
1383 : case IrOpcode::kWord64Shr: {
1384 509388 : Int64BinopMatcher m(value);
1385 509380 : if (m.right().Is(32)) {
1386 796216 : if (CanCoverTransitively(node, value, value->InputAt(0)) &&
1387 287381 : TryMatchLoadWord64AndShiftRight(this, value, kX64Movl)) {
1388 686561 : return EmitIdentity(node);
1389 : }
1390 : Emit(kX64Shr, g.DefineSameAsFirst(node),
1391 662262 : g.UseRegister(m.left().node()), g.TempImmediate(32));
1392 331132 : return;
1393 : }
1394 546 : break;
1395 : }
1396 : case IrOpcode::kLoad: {
1397 23 : if (TryMergeTruncateInt64ToInt32IntoLoad(this, node, value)) {
1398 : return;
1399 : }
1400 : break;
1401 : }
1402 : default:
1403 : break;
1404 : }
1405 : }
1406 27478 : Emit(kX64Movl, g.DefineAsRegister(node), g.Use(value));
1407 : }
1408 :
1409 2166 : void InstructionSelector::VisitFloat32Add(Node* node) {
1410 2166 : VisitFloatBinop(this, node, kAVXFloat32Add, kSSEFloat32Add);
1411 2166 : }
1412 :
1413 3177 : void InstructionSelector::VisitFloat32Sub(Node* node) {
1414 3177 : VisitFloatBinop(this, node, kAVXFloat32Sub, kSSEFloat32Sub);
1415 3177 : }
1416 :
1417 1052 : void InstructionSelector::VisitFloat32Mul(Node* node) {
1418 1052 : VisitFloatBinop(this, node, kAVXFloat32Mul, kSSEFloat32Mul);
1419 1052 : }
1420 :
1421 416 : void InstructionSelector::VisitFloat32Div(Node* node) {
1422 416 : VisitFloatBinop(this, node, kAVXFloat32Div, kSSEFloat32Div);
1423 416 : }
1424 :
1425 101 : void InstructionSelector::VisitFloat32Abs(Node* node) {
1426 101 : VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat32Abs, kSSEFloat32Abs);
1427 101 : }
1428 :
1429 89 : void InstructionSelector::VisitFloat32Max(Node* node) {
1430 89 : VisitRRO(this, node, kSSEFloat32Max);
1431 89 : }
1432 :
1433 94 : void InstructionSelector::VisitFloat32Min(Node* node) {
1434 94 : VisitRRO(this, node, kSSEFloat32Min);
1435 94 : }
1436 :
1437 80725 : void InstructionSelector::VisitFloat64Add(Node* node) {
1438 80725 : VisitFloatBinop(this, node, kAVXFloat64Add, kSSEFloat64Add);
1439 80725 : }
1440 :
1441 16079 : void InstructionSelector::VisitFloat64Sub(Node* node) {
1442 16079 : VisitFloatBinop(this, node, kAVXFloat64Sub, kSSEFloat64Sub);
1443 16079 : }
1444 :
1445 14297 : void InstructionSelector::VisitFloat64Mul(Node* node) {
1446 14297 : VisitFloatBinop(this, node, kAVXFloat64Mul, kSSEFloat64Mul);
1447 14297 : }
1448 :
1449 12147 : void InstructionSelector::VisitFloat64Div(Node* node) {
1450 12147 : VisitFloatBinop(this, node, kAVXFloat64Div, kSSEFloat64Div);
1451 12147 : }
1452 :
1453 1628 : void InstructionSelector::VisitFloat64Mod(Node* node) {
1454 : X64OperandGenerator g(this);
1455 1628 : InstructionOperand temps[] = {g.TempRegister(rax)};
1456 : Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
1457 : g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)), 1,
1458 3256 : temps);
1459 1628 : }
1460 :
1461 282 : void InstructionSelector::VisitFloat64Max(Node* node) {
1462 282 : VisitRRO(this, node, kSSEFloat64Max);
1463 282 : }
1464 :
1465 365 : void InstructionSelector::VisitFloat64Min(Node* node) {
1466 365 : VisitRRO(this, node, kSSEFloat64Min);
1467 365 : }
1468 :
1469 665 : void InstructionSelector::VisitFloat64Abs(Node* node) {
1470 665 : VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat64Abs, kSSEFloat64Abs);
1471 665 : }
1472 :
1473 0 : void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
1474 0 : UNREACHABLE();
1475 : }
1476 :
1477 207 : void InstructionSelector::VisitFloat32Neg(Node* node) {
1478 207 : VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat32Neg, kSSEFloat32Neg);
1479 207 : }
1480 :
1481 9970 : void InstructionSelector::VisitFloat64Neg(Node* node) {
1482 9970 : VisitFloatUnop(this, node, node->InputAt(0), kAVXFloat64Neg, kSSEFloat64Neg);
1483 9971 : }
1484 :
1485 476 : void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
1486 : InstructionCode opcode) {
1487 : X64OperandGenerator g(this);
1488 : Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0),
1489 : g.UseFixed(node->InputAt(1), xmm1))
1490 952 : ->MarkAsCall();
1491 476 : }
1492 :
1493 2892 : void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
1494 : InstructionCode opcode) {
1495 : X64OperandGenerator g(this);
1496 : Emit(opcode, g.DefineAsFixed(node, xmm0), g.UseFixed(node->InputAt(0), xmm0))
1497 2892 : ->MarkAsCall();
1498 2892 : }
1499 :
1500 5355805 : void InstructionSelector::EmitPrepareArguments(
1501 5382092 : ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
1502 2857424 : Node* node) {
1503 : X64OperandGenerator g(this);
1504 :
1505 : // Prepare for C function call.
1506 5355805 : if (call_descriptor->IsCFunctionCall()) {
1507 26287 : Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>(
1508 : call_descriptor->ParameterCount())),
1509 26287 : 0, nullptr, 0, nullptr);
1510 :
1511 : // Poke any stack arguments.
1512 59374 : for (size_t n = 0; n < arguments->size(); ++n) {
1513 33087 : PushParameter input = (*arguments)[n];
1514 3400 : if (input.node) {
1515 : int slot = static_cast<int>(n);
1516 3400 : InstructionOperand value = g.CanBeImmediate(input.node)
1517 : ? g.UseImmediate(input.node)
1518 3400 : : g.UseRegister(input.node);
1519 3400 : Emit(kX64Poke | MiscField::encode(slot), g.NoOutput(), value);
1520 : }
1521 : }
1522 : } else {
1523 : // Push any stack arguments.
1524 5329518 : int effect_level = GetEffectLevel(node);
1525 11448144 : for (PushParameter input : base::Reversed(*arguments)) {
1526 : // Skip any alignment holes in pushed nodes. We may have one in case of a
1527 : // Simd128 stack argument.
1528 3059302 : if (input.node == nullptr) continue;
1529 3058829 : if (g.CanBeImmediate(input.node)) {
1530 201412 : Emit(kX64Push, g.NoOutput(), g.UseImmediate(input.node));
1531 5714845 : } else if (IsSupported(ATOM) ||
1532 2857418 : sequence()->IsFP(GetVirtualRegister(input.node))) {
1533 : // TODO(titzer): X64Push cannot handle stack->stack double moves
1534 : // because there is no way to encode fixed double slots.
1535 0 : Emit(kX64Push, g.NoOutput(), g.UseRegister(input.node));
1536 2857427 : } else if (g.CanBeMemoryOperand(kX64Push, node, input.node,
1537 : effect_level)) {
1538 25845 : InstructionOperand outputs[1];
1539 103380 : InstructionOperand inputs[4];
1540 25845 : size_t input_count = 0;
1541 : InstructionCode opcode = kX64Push;
1542 : AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1543 25845 : input.node, inputs, &input_count);
1544 25845 : opcode |= AddressingModeField::encode(mode);
1545 25845 : Emit(opcode, 0, outputs, input_count, inputs);
1546 : } else {
1547 2831571 : Emit(kX64Push, g.NoOutput(), g.UseAny(input.node));
1548 : }
1549 : }
1550 : }
1551 5355827 : }
1552 :
1553 5355823 : void InstructionSelector::EmitPrepareResults(
1554 : ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
1555 : Node* node) {
1556 : X64OperandGenerator g(this);
1557 :
1558 : int reverse_slot = 0;
1559 15942042 : for (PushParameter output : *results) {
1560 10454426 : if (!output.location.IsCallerFrameSlot()) continue;
1561 18378 : reverse_slot += output.location.GetSizeInPointers();
1562 : // Skip any alignment holes in nodes.
1563 18378 : if (output.node == nullptr) continue;
1564 : DCHECK(!call_descriptor->IsCFunctionCall());
1565 6338 : if (output.location.GetType() == MachineType::Float32()) {
1566 : MarkAsFloat32(output.node);
1567 4758 : } else if (output.location.GetType() == MachineType::Float64()) {
1568 : MarkAsFloat64(output.node);
1569 : }
1570 6338 : InstructionOperand result = g.DefineAsRegister(output.node);
1571 6338 : InstructionOperand slot = g.UseImmediate(reverse_slot);
1572 6338 : Emit(kX64Peek, 1, &result, 1, &slot);
1573 : }
1574 5355840 : }
1575 :
1576 118262 : bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
1577 :
1578 1344 : int InstructionSelector::GetTempsCountForTailCallFromJSFunction() { return 3; }
1579 :
1580 : namespace {
1581 :
1582 1068004 : void VisitCompareWithMemoryOperand(InstructionSelector* selector,
1583 : InstructionCode opcode, Node* left,
1584 : InstructionOperand right,
1585 : FlagsContinuation* cont) {
1586 : DCHECK_EQ(IrOpcode::kLoad, left->opcode());
1587 : X64OperandGenerator g(selector);
1588 1068004 : size_t input_count = 0;
1589 5340016 : InstructionOperand inputs[4];
1590 : AddressingMode addressing_mode =
1591 1068004 : g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
1592 1068029 : opcode |= AddressingModeField::encode(addressing_mode);
1593 1068029 : inputs[input_count++] = right;
1594 :
1595 1068029 : selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
1596 1068043 : }
1597 :
1598 : // Shared routine for multiple compare operations.
1599 : void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1600 : InstructionOperand left, InstructionOperand right,
1601 : FlagsContinuation* cont) {
1602 4208922 : selector->EmitWithContinuation(opcode, left, right, cont);
1603 : }
1604 :
1605 : // Shared routine for multiple compare operations.
1606 883620 : void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1607 : Node* left, Node* right, FlagsContinuation* cont,
1608 : bool commutative) {
1609 : X64OperandGenerator g(selector);
1610 1224514 : if (commutative && g.CanBeBetterLeftOperand(right)) {
1611 : std::swap(left, right);
1612 : }
1613 883620 : VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
1614 883637 : }
1615 :
1616 15217717 : MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
1617 7182315 : if (hint_node->opcode() == IrOpcode::kLoad) {
1618 1530111 : MachineType hint = LoadRepresentationOf(hint_node->op());
1619 1530106 : if (node->opcode() == IrOpcode::kInt32Constant ||
1620 : node->opcode() == IrOpcode::kInt64Constant) {
1621 : int64_t constant = node->opcode() == IrOpcode::kInt32Constant
1622 : ? OpParameter<int32_t>(node->op())
1623 756021 : : OpParameter<int64_t>(node->op());
1624 756021 : if (hint == MachineType::Int8()) {
1625 0 : if (constant >= std::numeric_limits<int8_t>::min() &&
1626 : constant <= std::numeric_limits<int8_t>::max()) {
1627 0 : return hint;
1628 : }
1629 756021 : } else if (hint == MachineType::Uint8()) {
1630 106350 : if (constant >= std::numeric_limits<uint8_t>::min() &&
1631 : constant <= std::numeric_limits<uint8_t>::max()) {
1632 102318 : return hint;
1633 : }
1634 649671 : } else if (hint == MachineType::Int16()) {
1635 112 : if (constant >= std::numeric_limits<int16_t>::min() &&
1636 : constant <= std::numeric_limits<int16_t>::max()) {
1637 112 : return hint;
1638 : }
1639 649559 : } else if (hint == MachineType::Uint16()) {
1640 449270 : if (constant >= std::numeric_limits<uint16_t>::min() &&
1641 : constant <= std::numeric_limits<uint16_t>::max()) {
1642 449270 : return hint;
1643 : }
1644 200289 : } else if (hint == MachineType::Int32()) {
1645 24145 : return hint;
1646 176144 : } else if (hint == MachineType::Uint32()) {
1647 107228 : if (constant >= 0) return hint;
1648 : }
1649 : }
1650 : }
1651 : return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op())
1652 6505296 : : MachineType::None();
1653 : }
1654 :
1655 : // Tries to match the size of the given opcode to that of the operands, if
1656 : // possible.
1657 3591157 : InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
1658 : Node* right, FlagsContinuation* cont) {
1659 : // TODO(epertoso): we can probably get some size information out phi nodes.
1660 : // If the load representations don't match, both operands will be
1661 : // zero/sign-extended to 32bit.
1662 3591157 : MachineType left_type = MachineTypeForNarrow(left, right);
1663 3591160 : MachineType right_type = MachineTypeForNarrow(right, left);
1664 3591183 : if (left_type == right_type) {
1665 2891413 : switch (left_type.representation()) {
1666 : case MachineRepresentation::kBit:
1667 : case MachineRepresentation::kWord8: {
1668 104739 : if (opcode == kX64Test32) return kX64Test8;
1669 3765 : if (opcode == kX64Cmp32) {
1670 3765 : if (left_type.semantic() == MachineSemantic::kUint32) {
1671 : cont->OverwriteUnsignedIfSigned();
1672 : } else {
1673 112 : CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1674 : }
1675 : return kX64Cmp8;
1676 : }
1677 : break;
1678 : }
1679 : case MachineRepresentation::kWord16:
1680 451735 : if (opcode == kX64Test32) return kX64Test16;
1681 426149 : if (opcode == kX64Cmp32) {
1682 426149 : if (left_type.semantic() == MachineSemantic::kUint32) {
1683 : cont->OverwriteUnsignedIfSigned();
1684 : } else {
1685 112 : CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
1686 : }
1687 : return kX64Cmp16;
1688 : }
1689 : break;
1690 : default:
1691 : break;
1692 : }
1693 : }
1694 : return opcode;
1695 : }
1696 :
1697 : // Shared routine for multiple word compare operations.
1698 4798526 : void VisitWordCompare(InstructionSelector* selector, Node* node,
1699 6484245 : InstructionCode opcode, FlagsContinuation* cont) {
1700 : X64OperandGenerator g(selector);
1701 : Node* left = node->InputAt(0);
1702 : Node* right = node->InputAt(1);
1703 :
1704 : // The 32-bit comparisons automatically truncate Word64
1705 : // values to Word32 range, no need to do that explicitly.
1706 3591154 : if (opcode == kX64Cmp32 || opcode == kX64Test32) {
1707 3371868 : if (left->opcode() == IrOpcode::kTruncateInt64ToInt32 &&
1708 34864 : selector->CanCover(node, left)) {
1709 : left = left->InputAt(0);
1710 : }
1711 :
1712 3378473 : if (right->opcode() == IrOpcode::kTruncateInt64ToInt32 &&
1713 41469 : selector->CanCover(node, right)) {
1714 : right = right->InputAt(0);
1715 : }
1716 : }
1717 :
1718 3591154 : opcode = TryNarrowOpcodeSize(opcode, left, right, cont);
1719 :
1720 : // If one of the two inputs is an immediate, make sure it's on the right, or
1721 : // if one of the two inputs is a memory operand, make sure it's on the left.
1722 3591162 : int effect_level = selector->GetEffectLevel(node);
1723 3591173 : if (cont->IsBranch()) {
1724 : effect_level = selector->GetEffectLevel(
1725 2893072 : cont->true_block()->PredecessorAt(0)->control_input());
1726 : }
1727 :
1728 10558390 : if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
1729 3773674 : (g.CanBeMemoryOperand(opcode, node, right, effect_level) &&
1730 397645 : !g.CanBeMemoryOperand(opcode, node, left, effect_level))) {
1731 592470 : if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1732 : std::swap(left, right);
1733 : }
1734 :
1735 : // Match immediates on right side of comparison.
1736 3591153 : if (g.CanBeImmediate(right)) {
1737 2438110 : if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
1738 : return VisitCompareWithMemoryOperand(selector, opcode, left,
1739 491638 : g.UseImmediate(right), cont);
1740 : }
1741 : return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
1742 1946481 : cont);
1743 : }
1744 :
1745 : // Match memory operands on left side of comparison.
1746 1153053 : if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
1747 : return VisitCompareWithMemoryOperand(selector, opcode, left,
1748 538147 : g.UseRegister(right), cont);
1749 : }
1750 :
1751 : return VisitCompare(selector, opcode, left, right, cont,
1752 614902 : node->op()->HasProperty(Operator::kCommutative));
1753 : }
1754 :
1755 : // Shared routine for 64-bit word comparison operations.
1756 4337757 : void VisitWord64Compare(InstructionSelector* selector, Node* node,
1757 554416 : FlagsContinuation* cont) {
1758 : X64OperandGenerator g(selector);
1759 2819746 : if (selector->CanUseRootsRegister()) {
1760 : const RootsTable& roots_table = selector->isolate()->roots_table();
1761 : RootIndex root_index;
1762 2632055 : HeapObjectBinopMatcher m(node);
1763 3674977 : if (m.right().HasValue() &&
1764 : roots_table.IsRootHandle(m.right().Value(), &root_index)) {
1765 963602 : if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1766 : InstructionCode opcode =
1767 : kX64Cmp | AddressingModeField::encode(kMode_Root);
1768 : return VisitCompare(
1769 : selector, opcode,
1770 : g.TempImmediate(
1771 : TurboAssemblerBase::RootRegisterOffsetForRootIndex(root_index)),
1772 1927180 : g.UseRegister(m.left().node()), cont);
1773 1668448 : } else if (m.left().HasValue() &&
1774 : roots_table.IsRootHandle(m.left().Value(), &root_index)) {
1775 : InstructionCode opcode =
1776 : kX64Cmp | AddressingModeField::encode(kMode_Root);
1777 : return VisitCompare(
1778 : selector, opcode,
1779 : g.TempImmediate(
1780 : TurboAssemblerBase::RootRegisterOffsetForRootIndex(root_index)),
1781 0 : g.UseRegister(m.right().node()), cont);
1782 : }
1783 : }
1784 1856187 : if (selector->isolate() != nullptr) {
1785 : StackCheckMatcher<Int64BinopMatcher, IrOpcode::kUint64LessThan> m(
1786 : selector->isolate(), node);
1787 1761163 : if (m.Matched()) {
1788 : // Compare(Load(js_stack_limit), LoadStackPointer)
1789 554409 : if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
1790 : InstructionCode opcode = cont->Encode(kX64StackCheck);
1791 554416 : CHECK(cont->IsBranch());
1792 554416 : selector->EmitWithContinuation(opcode, cont);
1793 554428 : return;
1794 : }
1795 : }
1796 : WasmStackCheckMatcher<Int64BinopMatcher, IrOpcode::kUint64LessThan> wasm_m(
1797 : node);
1798 1301773 : if (wasm_m.Matched()) {
1799 : // This is a wasm stack check. By structure, we know that we can use the
1800 : // stack pointer directly, as wasm code does not modify the stack at points
1801 : // where stack checks are performed.
1802 : Node* left = node->InputAt(0);
1803 : LocationOperand rsp(InstructionOperand::EXPLICIT, LocationOperand::REGISTER,
1804 : InstructionSequence::DefaultRepresentation(),
1805 : RegisterCode::kRegCode_rsp);
1806 21746 : return VisitCompareWithMemoryOperand(selector, kX64Cmp, left, rsp, cont);
1807 : }
1808 1280042 : VisitWordCompare(selector, node, kX64Cmp, cont);
1809 : }
1810 :
1811 : // Shared routine for comparison with zero.
1812 1550629 : void VisitCompareZero(InstructionSelector* selector, Node* user, Node* node,
1813 1935534 : InstructionCode opcode, FlagsContinuation* cont) {
1814 : X64OperandGenerator g(selector);
1815 1701788 : if (cont->IsBranch() &&
1816 225263 : (cont->condition() == kNotEqual || cont->condition() == kEqual)) {
1817 536071 : switch (node->opcode()) {
1818 : #define FLAGS_SET_BINOP_LIST(V) \
1819 : V(kInt32Add, VisitBinop, kX64Add32) \
1820 : V(kInt32Sub, VisitBinop, kX64Sub32) \
1821 : V(kWord32And, VisitBinop, kX64And32) \
1822 : V(kWord32Or, VisitBinop, kX64Or32) \
1823 : V(kInt64Add, VisitBinop, kX64Add) \
1824 : V(kInt64Sub, VisitBinop, kX64Sub) \
1825 : V(kWord64And, VisitBinop, kX64And) \
1826 : V(kWord64Or, VisitBinop, kX64Or)
1827 : #define FLAGS_SET_BINOP(opcode, Visit, archOpcode) \
1828 : case IrOpcode::opcode: \
1829 : if (selector->IsOnlyUserOfNodeInSameBlock(user, node)) { \
1830 : return Visit(selector, node, archOpcode, cont); \
1831 : } \
1832 : break;
1833 157104 : FLAGS_SET_BINOP_LIST(FLAGS_SET_BINOP)
1834 : #undef FLAGS_SET_BINOP_LIST
1835 : #undef FLAGS_SET_BINOP
1836 :
1837 : #define TRY_VISIT_WORD32_SHIFT TryVisitWordShift<Int32BinopMatcher, 32>
1838 : #define TRY_VISIT_WORD64_SHIFT TryVisitWordShift<Int64BinopMatcher, 64>
1839 : // Skip Word64Sar/Word32Sar since no instruction reduction in most cases.
1840 : #define FLAGS_SET_SHIFT_LIST(V) \
1841 : V(kWord32Shl, TRY_VISIT_WORD32_SHIFT, kX64Shl32) \
1842 : V(kWord32Shr, TRY_VISIT_WORD32_SHIFT, kX64Shr32) \
1843 : V(kWord64Shl, TRY_VISIT_WORD64_SHIFT, kX64Shl) \
1844 : V(kWord64Shr, TRY_VISIT_WORD64_SHIFT, kX64Shr)
1845 : #define FLAGS_SET_SHIFT(opcode, TryVisit, archOpcode) \
1846 : case IrOpcode::opcode: \
1847 : if (selector->IsOnlyUserOfNodeInSameBlock(user, node)) { \
1848 : if (TryVisit(selector, node, archOpcode, cont)) return; \
1849 : } \
1850 : break;
1851 165 : FLAGS_SET_SHIFT_LIST(FLAGS_SET_SHIFT)
1852 : #undef TRY_VISIT_WORD32_SHIFT
1853 : #undef TRY_VISIT_WORD64_SHIFT
1854 : #undef FLAGS_SET_SHIFT_LIST
1855 : #undef FLAGS_SET_SHIFT
1856 : default:
1857 : break;
1858 : }
1859 : }
1860 431693 : int effect_level = selector->GetEffectLevel(node);
1861 431699 : if (cont->IsBranch()) {
1862 : effect_level = selector->GetEffectLevel(
1863 384905 : cont->true_block()->PredecessorAt(0)->control_input());
1864 : }
1865 431700 : if (node->opcode() == IrOpcode::kLoad) {
1866 31382 : switch (LoadRepresentationOf(node->op()).representation()) {
1867 : case MachineRepresentation::kWord8:
1868 6831 : if (opcode == kX64Cmp32) {
1869 : opcode = kX64Cmp8;
1870 0 : } else if (opcode == kX64Test32) {
1871 : opcode = kX64Test8;
1872 : }
1873 : break;
1874 : case MachineRepresentation::kWord16:
1875 843 : if (opcode == kX64Cmp32) {
1876 : opcode = kX64Cmp16;
1877 0 : } else if (opcode == kX64Test32) {
1878 : opcode = kX64Test16;
1879 : }
1880 : break;
1881 : default:
1882 : break;
1883 : }
1884 : }
1885 431700 : if (g.CanBeMemoryOperand(opcode, user, node, effect_level)) {
1886 : VisitCompareWithMemoryOperand(selector, opcode, node, g.TempImmediate(0),
1887 16489 : cont);
1888 : } else {
1889 415202 : VisitCompare(selector, opcode, g.Use(node), g.TempImmediate(0), cont);
1890 : }
1891 : }
1892 :
1893 : // Shared routine for multiple float32 compare operations (inputs commuted).
1894 2844 : void VisitFloat32Compare(InstructionSelector* selector, Node* node,
1895 : FlagsContinuation* cont) {
1896 : Node* const left = node->InputAt(0);
1897 : Node* const right = node->InputAt(1);
1898 : InstructionCode const opcode =
1899 2844 : selector->IsSupported(AVX) ? kAVXFloat32Cmp : kSSEFloat32Cmp;
1900 2844 : VisitCompare(selector, opcode, right, left, cont, false);
1901 2844 : }
1902 :
1903 : // Shared routine for multiple float64 compare operations (inputs commuted).
1904 209514 : void VisitFloat64Compare(InstructionSelector* selector, Node* node,
1905 : FlagsContinuation* cont) {
1906 : Node* const left = node->InputAt(0);
1907 : Node* const right = node->InputAt(1);
1908 : InstructionCode const opcode =
1909 209514 : selector->IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
1910 209514 : VisitCompare(selector, opcode, right, left, cont, false);
1911 209513 : }
1912 :
1913 : // Shared routine for Word32/Word64 Atomic Binops
1914 24652 : void VisitAtomicBinop(InstructionSelector* selector, Node* node,
1915 : ArchOpcode opcode) {
1916 : X64OperandGenerator g(selector);
1917 : Node* base = node->InputAt(0);
1918 : Node* index = node->InputAt(1);
1919 : Node* value = node->InputAt(2);
1920 : AddressingMode addressing_mode;
1921 : InstructionOperand inputs[] = {
1922 : g.UseUniqueRegister(value), g.UseUniqueRegister(base),
1923 24652 : g.GetEffectiveIndexOperand(index, &addressing_mode)};
1924 24649 : InstructionOperand outputs[] = {g.DefineAsFixed(node, rax)};
1925 24650 : InstructionOperand temps[] = {g.TempRegister()};
1926 49300 : InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1927 : selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
1928 24650 : arraysize(temps), temps);
1929 24653 : }
1930 :
1931 : // Shared routine for Word32/Word64 Atomic CmpExchg
1932 1110 : void VisitAtomicCompareExchange(InstructionSelector* selector, Node* node,
1933 : ArchOpcode opcode) {
1934 : X64OperandGenerator g(selector);
1935 : Node* base = node->InputAt(0);
1936 : Node* index = node->InputAt(1);
1937 : Node* old_value = node->InputAt(2);
1938 : Node* new_value = node->InputAt(3);
1939 : AddressingMode addressing_mode;
1940 : InstructionOperand inputs[] = {
1941 : g.UseFixed(old_value, rax), g.UseUniqueRegister(new_value),
1942 : g.UseUniqueRegister(base),
1943 1110 : g.GetEffectiveIndexOperand(index, &addressing_mode)};
1944 1110 : InstructionOperand outputs[] = {g.DefineAsFixed(node, rax)};
1945 2220 : InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1946 1110 : selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
1947 1110 : }
1948 :
1949 : // Shared routine for Word32/Word64 Atomic Exchange
1950 9406 : void VisitAtomicExchange(InstructionSelector* selector, Node* node,
1951 : ArchOpcode opcode) {
1952 : X64OperandGenerator g(selector);
1953 : Node* base = node->InputAt(0);
1954 : Node* index = node->InputAt(1);
1955 : Node* value = node->InputAt(2);
1956 : AddressingMode addressing_mode;
1957 : InstructionOperand inputs[] = {
1958 : g.UseUniqueRegister(value), g.UseUniqueRegister(base),
1959 9406 : g.GetEffectiveIndexOperand(index, &addressing_mode)};
1960 9406 : InstructionOperand outputs[] = {g.DefineSameAsFirst(node)};
1961 18812 : InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1962 9406 : selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
1963 9406 : }
1964 :
1965 : } // namespace
1966 :
1967 : // Shared routine for word comparison against zero.
1968 17634022 : void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
1969 : FlagsContinuation* cont) {
1970 : // Try to combine with comparisons against 0 by simply inverting the branch.
1971 12239725 : while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
1972 1497195 : Int32BinopMatcher m(value);
1973 1497197 : if (!m.right().Is(0)) break;
1974 :
1975 : user = value;
1976 : value = m.left().node();
1977 : cont->Negate();
1978 : }
1979 :
1980 5651296 : if (CanCover(user, value)) {
1981 5394297 : switch (value->opcode()) {
1982 : case IrOpcode::kWord32Equal:
1983 : cont->OverwriteAndNegateIfEqual(kEqual);
1984 560040 : return VisitWordCompare(this, value, kX64Cmp32, cont);
1985 : case IrOpcode::kInt32LessThan:
1986 : cont->OverwriteAndNegateIfEqual(kSignedLessThan);
1987 230001 : return VisitWordCompare(this, value, kX64Cmp32, cont);
1988 : case IrOpcode::kInt32LessThanOrEqual:
1989 : cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
1990 53123 : return VisitWordCompare(this, value, kX64Cmp32, cont);
1991 : case IrOpcode::kUint32LessThan:
1992 : cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
1993 142568 : return VisitWordCompare(this, value, kX64Cmp32, cont);
1994 : case IrOpcode::kUint32LessThanOrEqual:
1995 : cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
1996 57296 : return VisitWordCompare(this, value, kX64Cmp32, cont);
1997 : case IrOpcode::kWord64Equal: {
1998 : cont->OverwriteAndNegateIfEqual(kEqual);
1999 2396590 : Int64BinopMatcher m(value);
2000 2396599 : if (m.right().Is(0)) {
2001 : // Try to combine the branch with a comparison.
2002 : Node* const user = m.node();
2003 674242 : Node* const value = m.left().node();
2004 760866 : if (CanCover(user, value)) {
2005 674242 : switch (value->opcode()) {
2006 : case IrOpcode::kInt64Sub:
2007 10 : return VisitWord64Compare(this, value, cont);
2008 : case IrOpcode::kWord64And:
2009 634009 : return VisitWordCompare(this, value, kX64Test, cont);
2010 : default:
2011 : break;
2012 : }
2013 : }
2014 126857 : return VisitCompareZero(this, user, value, kX64Cmp, cont);
2015 : }
2016 1635733 : return VisitWord64Compare(this, value, cont);
2017 : }
2018 : case IrOpcode::kInt64LessThan:
2019 : cont->OverwriteAndNegateIfEqual(kSignedLessThan);
2020 98858 : return VisitWord64Compare(this, value, cont);
2021 : case IrOpcode::kInt64LessThanOrEqual:
2022 : cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
2023 32256 : return VisitWord64Compare(this, value, cont);
2024 : case IrOpcode::kUint64LessThan:
2025 : cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
2026 930252 : return VisitWord64Compare(this, value, cont);
2027 : case IrOpcode::kUint64LessThanOrEqual:
2028 : cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
2029 82281 : return VisitWord64Compare(this, value, cont);
2030 : case IrOpcode::kFloat32Equal:
2031 : cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
2032 1204 : return VisitFloat32Compare(this, value, cont);
2033 : case IrOpcode::kFloat32LessThan:
2034 : cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
2035 976 : return VisitFloat32Compare(this, value, cont);
2036 : case IrOpcode::kFloat32LessThanOrEqual:
2037 : cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
2038 207 : return VisitFloat32Compare(this, value, cont);
2039 : case IrOpcode::kFloat64Equal:
2040 : cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
2041 172772 : return VisitFloat64Compare(this, value, cont);
2042 : case IrOpcode::kFloat64LessThan: {
2043 72866 : Float64BinopMatcher m(value);
2044 126574 : if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2045 : // This matches the pattern
2046 : //
2047 : // Float64LessThan(#0.0, Float64Abs(x))
2048 : //
2049 : // which TurboFan generates for NumberToBoolean in the general case,
2050 : // and which evaluates to false if x is 0, -0 or NaN. We can compile
2051 : // this to a simple (v)ucomisd using not_equal flags condition, which
2052 : // avoids the costly Float64Abs.
2053 : cont->OverwriteAndNegateIfEqual(kNotEqual);
2054 : InstructionCode const opcode =
2055 53080 : IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2056 : return VisitCompare(this, opcode, m.left().node(),
2057 53080 : m.right().InputAt(0), cont, false);
2058 : }
2059 : cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
2060 19787 : return VisitFloat64Compare(this, value, cont);
2061 : }
2062 : case IrOpcode::kFloat64LessThanOrEqual:
2063 : cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
2064 11252 : return VisitFloat64Compare(this, value, cont);
2065 : case IrOpcode::kProjection:
2066 : // Check if this is the overflow output projection of an
2067 : // <Operation>WithOverflow node.
2068 85862 : if (ProjectionIndexOf(value->op()) == 1u) {
2069 : // We cannot combine the <Operation>WithOverflow with this branch
2070 : // unless the 0th projection (the use of the actual value of the
2071 : // <Operation> is either nullptr, which means there's no use of the
2072 : // actual value, or was already defined, which means it is scheduled
2073 : // *AFTER* this branch).
2074 85831 : Node* const node = value->InputAt(0);
2075 85830 : Node* const result = NodeProperties::FindProjection(node, 0);
2076 85830 : if (result == nullptr || IsDefined(result)) {
2077 85831 : switch (node->opcode()) {
2078 : case IrOpcode::kInt32AddWithOverflow:
2079 : cont->OverwriteAndNegateIfEqual(kOverflow);
2080 56455 : return VisitBinop(this, node, kX64Add32, cont);
2081 : case IrOpcode::kInt32SubWithOverflow:
2082 : cont->OverwriteAndNegateIfEqual(kOverflow);
2083 10705 : return VisitBinop(this, node, kX64Sub32, cont);
2084 : case IrOpcode::kInt32MulWithOverflow:
2085 : cont->OverwriteAndNegateIfEqual(kOverflow);
2086 3753 : return VisitBinop(this, node, kX64Imul32, cont);
2087 : case IrOpcode::kInt64AddWithOverflow:
2088 : cont->OverwriteAndNegateIfEqual(kOverflow);
2089 9530 : return VisitBinop(this, node, kX64Add, cont);
2090 : case IrOpcode::kInt64SubWithOverflow:
2091 : cont->OverwriteAndNegateIfEqual(kOverflow);
2092 5386 : return VisitBinop(this, node, kX64Sub, cont);
2093 : default:
2094 : break;
2095 : }
2096 : }
2097 : }
2098 : break;
2099 : case IrOpcode::kInt32Sub:
2100 1210 : return VisitWordCompare(this, value, kX64Cmp32, cont);
2101 : case IrOpcode::kWord32And:
2102 265678 : return VisitWordCompare(this, value, kX64Test32, cont);
2103 : default:
2104 : break;
2105 : }
2106 : }
2107 :
2108 : // Branch could not be combined with a compare, emit compare against 0.
2109 456067 : VisitCompareZero(this, user, value, kX64Cmp32, cont);
2110 : }
2111 :
2112 60366 : void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
2113 : X64OperandGenerator g(this);
2114 39811 : InstructionOperand value_operand = g.UseRegister(node->InputAt(0));
2115 :
2116 : // Emit either ArchTableSwitch or ArchLookupSwitch.
2117 39811 : if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
2118 : static const size_t kMaxTableSwitchValueRange = 2 << 16;
2119 : size_t table_space_cost = 4 + sw.value_range();
2120 : size_t table_time_cost = 3;
2121 19947 : size_t lookup_space_cost = 3 + 2 * sw.case_count();
2122 : size_t lookup_time_cost = sw.case_count();
2123 20258 : if (sw.case_count() > 4 &&
2124 311 : table_space_cost + 3 * table_time_cost <=
2125 615 : lookup_space_cost + 3 * lookup_time_cost &&
2126 20251 : sw.min_value() > std::numeric_limits<int32_t>::min() &&
2127 : sw.value_range() <= kMaxTableSwitchValueRange) {
2128 304 : InstructionOperand index_operand = g.TempRegister();
2129 304 : if (sw.min_value()) {
2130 : // The leal automatically zero extends, so result is a valid 64-bit
2131 : // index.
2132 : Emit(kX64Lea32 | AddressingModeField::encode(kMode_MRI), index_operand,
2133 3 : value_operand, g.TempImmediate(-sw.min_value()));
2134 : } else {
2135 : // Zero extend, because we use it as 64-bit index into the jump table.
2136 301 : Emit(kX64Movl, index_operand, value_operand);
2137 : }
2138 : // Generate a table lookup.
2139 304 : return EmitTableSwitch(sw, index_operand);
2140 : }
2141 : }
2142 :
2143 : // Generate a tree of conditional jumps.
2144 39507 : return EmitBinarySearchSwitch(sw, value_operand);
2145 : }
2146 :
2147 150348 : void InstructionSelector::VisitWord32Equal(Node* const node) {
2148 : Node* user = node;
2149 : FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2150 150348 : Int32BinopMatcher m(user);
2151 150348 : if (m.right().Is(0)) {
2152 221757 : return VisitWordCompareZero(m.node(), m.left().node(), &cont);
2153 : }
2154 78939 : VisitWordCompare(this, node, kX64Cmp32, &cont);
2155 : }
2156 :
2157 68750 : void InstructionSelector::VisitInt32LessThan(Node* node) {
2158 : FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2159 68750 : VisitWordCompare(this, node, kX64Cmp32, &cont);
2160 68750 : }
2161 :
2162 68319 : void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
2163 : FlagsContinuation cont =
2164 : FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2165 68319 : VisitWordCompare(this, node, kX64Cmp32, &cont);
2166 68319 : }
2167 :
2168 74096 : void InstructionSelector::VisitUint32LessThan(Node* node) {
2169 : FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2170 74096 : VisitWordCompare(this, node, kX64Cmp32, &cont);
2171 74096 : }
2172 :
2173 68487 : void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
2174 : FlagsContinuation cont =
2175 : FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2176 68487 : VisitWordCompare(this, node, kX64Cmp32, &cont);
2177 68487 : }
2178 :
2179 39247 : void InstructionSelector::VisitWord64Equal(Node* const node) {
2180 : FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
2181 39247 : Int64BinopMatcher m(node);
2182 39248 : if (m.right().Is(0)) {
2183 : // Try to combine the equality check with a comparison.
2184 : Node* const user = m.node();
2185 8699 : Node* const value = m.left().node();
2186 9199 : if (CanCover(user, value)) {
2187 8699 : switch (value->opcode()) {
2188 : case IrOpcode::kInt64Sub:
2189 8616 : return VisitWord64Compare(this, value, &cont);
2190 : case IrOpcode::kWord64And:
2191 8616 : return VisitWordCompare(this, value, kX64Test, &cont);
2192 : default:
2193 : break;
2194 : }
2195 : }
2196 : }
2197 30632 : VisitWord64Compare(this, node, &cont);
2198 : }
2199 :
2200 17405 : void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
2201 17405 : if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2202 : FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2203 34810 : return VisitBinop(this, node, kX64Add32, &cont);
2204 : }
2205 : FlagsContinuation cont;
2206 0 : VisitBinop(this, node, kX64Add32, &cont);
2207 : }
2208 :
2209 17405 : void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
2210 17405 : if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
2211 : FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
2212 34810 : return VisitBinop(this, node, kX64Sub32, &cont);
2213 : }
2214 : FlagsContinuation cont;
2215 0 : VisitBinop(this, node, kX64Sub32, &cont);
2216 : }
2217 :
2218 1476 : void InstructionSelector::VisitInt64LessThan(Node* node) {
2219 : FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
2220 1476 : VisitWord64Compare(this, node, &cont);
2221 1476 : }
2222 :
2223 943 : void InstructionSelector::VisitInt64LessThanOrEqual(Node* node) {
2224 : FlagsContinuation cont =
2225 : FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
2226 943 : VisitWord64Compare(this, node, &cont);
2227 943 : }
2228 :
2229 6818 : void InstructionSelector::VisitUint64LessThan(Node* node) {
2230 : FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
2231 6818 : VisitWord64Compare(this, node, &cont);
2232 6818 : }
2233 :
2234 510 : void InstructionSelector::VisitUint64LessThanOrEqual(Node* node) {
2235 : FlagsContinuation cont =
2236 : FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
2237 510 : VisitWord64Compare(this, node, &cont);
2238 510 : }
2239 :
2240 136 : void InstructionSelector::VisitFloat32Equal(Node* node) {
2241 : FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2242 136 : VisitFloat32Compare(this, node, &cont);
2243 136 : }
2244 :
2245 182 : void InstructionSelector::VisitFloat32LessThan(Node* node) {
2246 : FlagsContinuation cont =
2247 : FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2248 182 : VisitFloat32Compare(this, node, &cont);
2249 182 : }
2250 :
2251 139 : void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
2252 : FlagsContinuation cont =
2253 : FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2254 139 : VisitFloat32Compare(this, node, &cont);
2255 139 : }
2256 :
2257 2732 : void InstructionSelector::VisitFloat64Equal(Node* node) {
2258 : FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
2259 2732 : VisitFloat64Compare(this, node, &cont);
2260 2732 : }
2261 :
2262 4852 : void InstructionSelector::VisitFloat64LessThan(Node* node) {
2263 4852 : Float64BinopMatcher m(node);
2264 8252 : if (m.left().Is(0.0) && m.right().IsFloat64Abs()) {
2265 : // This matches the pattern
2266 : //
2267 : // Float64LessThan(#0.0, Float64Abs(x))
2268 : //
2269 : // which TurboFan generates for NumberToBoolean in the general case,
2270 : // and which evaluates to false if x is 0, -0 or NaN. We can compile
2271 : // this to a simple (v)ucomisd using not_equal flags condition, which
2272 : // avoids the costly Float64Abs.
2273 : FlagsContinuation cont = FlagsContinuation::ForSet(kNotEqual, node);
2274 : InstructionCode const opcode =
2275 3288 : IsSupported(AVX) ? kAVXFloat64Cmp : kSSEFloat64Cmp;
2276 : return VisitCompare(this, opcode, m.left().node(), m.right().InputAt(0),
2277 8140 : &cont, false);
2278 : }
2279 : FlagsContinuation cont =
2280 : FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
2281 1564 : VisitFloat64Compare(this, node, &cont);
2282 : }
2283 :
2284 1409 : void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
2285 : FlagsContinuation cont =
2286 : FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
2287 1409 : VisitFloat64Compare(this, node, &cont);
2288 1409 : }
2289 :
2290 117 : void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
2291 : X64OperandGenerator g(this);
2292 : Node* left = node->InputAt(0);
2293 : Node* right = node->InputAt(1);
2294 : Float64Matcher mleft(left);
2295 117 : if (mleft.HasValue() && (bit_cast<uint64_t>(mleft.Value()) >> 32) == 0u) {
2296 112 : Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
2297 229 : return;
2298 : }
2299 : Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
2300 5 : g.UseRegister(left), g.Use(right));
2301 : }
2302 :
2303 117 : void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
2304 : X64OperandGenerator g(this);
2305 : Node* left = node->InputAt(0);
2306 : Node* right = node->InputAt(1);
2307 : Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
2308 117 : g.UseRegister(left), g.Use(right));
2309 117 : }
2310 :
2311 5818 : void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
2312 : X64OperandGenerator g(this);
2313 : Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
2314 5818 : g.UseRegister(node->InputAt(0)));
2315 5818 : }
2316 :
2317 1098 : void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
2318 1098 : LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2319 : DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
2320 : load_rep.representation() == MachineRepresentation::kWord16 ||
2321 : load_rep.representation() == MachineRepresentation::kWord32);
2322 : USE(load_rep);
2323 1098 : VisitLoad(node);
2324 1098 : }
2325 :
2326 613 : void InstructionSelector::VisitWord64AtomicLoad(Node* node) {
2327 613 : LoadRepresentation load_rep = LoadRepresentationOf(node->op());
2328 : USE(load_rep);
2329 613 : VisitLoad(node);
2330 613 : }
2331 :
2332 2130 : void InstructionSelector::VisitWord32AtomicStore(Node* node) {
2333 2130 : MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2334 : ArchOpcode opcode = kArchNop;
2335 2130 : switch (rep) {
2336 : case MachineRepresentation::kWord8:
2337 : opcode = kWord32AtomicExchangeInt8;
2338 : break;
2339 : case MachineRepresentation::kWord16:
2340 : opcode = kWord32AtomicExchangeInt16;
2341 658 : break;
2342 : case MachineRepresentation::kWord32:
2343 : opcode = kWord32AtomicExchangeWord32;
2344 648 : break;
2345 : default:
2346 0 : UNREACHABLE();
2347 : return;
2348 : }
2349 2130 : VisitAtomicExchange(this, node, opcode);
2350 2130 : }
2351 :
2352 2627 : void InstructionSelector::VisitWord64AtomicStore(Node* node) {
2353 2627 : MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
2354 : ArchOpcode opcode = kArchNop;
2355 2627 : switch (rep) {
2356 : case MachineRepresentation::kWord8:
2357 : opcode = kX64Word64AtomicExchangeUint8;
2358 : break;
2359 : case MachineRepresentation::kWord16:
2360 : opcode = kX64Word64AtomicExchangeUint16;
2361 647 : break;
2362 : case MachineRepresentation::kWord32:
2363 : opcode = kX64Word64AtomicExchangeUint32;
2364 443 : break;
2365 : case MachineRepresentation::kWord64:
2366 : opcode = kX64Word64AtomicExchangeUint64;
2367 438 : break;
2368 : default:
2369 0 : UNREACHABLE();
2370 : return;
2371 : }
2372 2627 : VisitAtomicExchange(this, node, opcode);
2373 2627 : }
2374 :
2375 2412 : void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
2376 2412 : MachineType type = AtomicOpType(node->op());
2377 : ArchOpcode opcode = kArchNop;
2378 2412 : if (type == MachineType::Int8()) {
2379 : opcode = kWord32AtomicExchangeInt8;
2380 2300 : } else if (type == MachineType::Uint8()) {
2381 : opcode = kWord32AtomicExchangeUint8;
2382 1310 : } else if (type == MachineType::Int16()) {
2383 : opcode = kWord32AtomicExchangeInt16;
2384 1198 : } else if (type == MachineType::Uint16()) {
2385 : opcode = kWord32AtomicExchangeUint16;
2386 968 : } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2387 : opcode = kWord32AtomicExchangeWord32;
2388 : } else {
2389 0 : UNREACHABLE();
2390 : return;
2391 : }
2392 2412 : VisitAtomicExchange(this, node, opcode);
2393 2412 : }
2394 :
2395 2237 : void InstructionSelector::VisitWord64AtomicExchange(Node* node) {
2396 2237 : MachineType type = AtomicOpType(node->op());
2397 : ArchOpcode opcode = kArchNop;
2398 2237 : if (type == MachineType::Uint8()) {
2399 : opcode = kX64Word64AtomicExchangeUint8;
2400 1274 : } else if (type == MachineType::Uint16()) {
2401 : opcode = kX64Word64AtomicExchangeUint16;
2402 943 : } else if (type == MachineType::Uint32()) {
2403 : opcode = kX64Word64AtomicExchangeUint32;
2404 536 : } else if (type == MachineType::Uint64()) {
2405 : opcode = kX64Word64AtomicExchangeUint64;
2406 : } else {
2407 0 : UNREACHABLE();
2408 : return;
2409 : }
2410 2237 : VisitAtomicExchange(this, node, opcode);
2411 2236 : }
2412 :
2413 756 : void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
2414 756 : MachineType type = AtomicOpType(node->op());
2415 : ArchOpcode opcode = kArchNop;
2416 756 : if (type == MachineType::Int8()) {
2417 : opcode = kWord32AtomicCompareExchangeInt8;
2418 644 : } else if (type == MachineType::Uint8()) {
2419 : opcode = kWord32AtomicCompareExchangeUint8;
2420 507 : } else if (type == MachineType::Int16()) {
2421 : opcode = kWord32AtomicCompareExchangeInt16;
2422 395 : } else if (type == MachineType::Uint16()) {
2423 : opcode = kWord32AtomicCompareExchangeUint16;
2424 404 : } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2425 : opcode = kWord32AtomicCompareExchangeWord32;
2426 : } else {
2427 0 : UNREACHABLE();
2428 : return;
2429 : }
2430 756 : VisitAtomicCompareExchange(this, node, opcode);
2431 756 : }
2432 :
2433 354 : void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) {
2434 354 : MachineType type = AtomicOpType(node->op());
2435 : ArchOpcode opcode = kArchNop;
2436 354 : if (type == MachineType::Uint8()) {
2437 : opcode = kX64Word64AtomicCompareExchangeUint8;
2438 329 : } else if (type == MachineType::Uint16()) {
2439 : opcode = kX64Word64AtomicCompareExchangeUint16;
2440 304 : } else if (type == MachineType::Uint32()) {
2441 : opcode = kX64Word64AtomicCompareExchangeUint32;
2442 279 : } else if (type == MachineType::Uint64()) {
2443 : opcode = kX64Word64AtomicCompareExchangeUint64;
2444 : } else {
2445 0 : UNREACHABLE();
2446 : return;
2447 : }
2448 354 : VisitAtomicCompareExchange(this, node, opcode);
2449 354 : }
2450 :
2451 13146 : void InstructionSelector::VisitWord32AtomicBinaryOperation(
2452 13146 : Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
2453 : ArchOpcode uint16_op, ArchOpcode word32_op) {
2454 13146 : MachineType type = AtomicOpType(node->op());
2455 : ArchOpcode opcode = kArchNop;
2456 13146 : if (type == MachineType::Int8()) {
2457 : opcode = int8_op;
2458 12586 : } else if (type == MachineType::Uint8()) {
2459 : opcode = uint8_op;
2460 7504 : } else if (type == MachineType::Int16()) {
2461 : opcode = int16_op;
2462 6944 : } else if (type == MachineType::Uint16()) {
2463 : opcode = uint16_op;
2464 5880 : } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2465 : opcode = word32_op;
2466 : } else {
2467 0 : UNREACHABLE();
2468 : return;
2469 : }
2470 13146 : VisitAtomicBinop(this, node, opcode);
2471 13147 : }
2472 :
2473 : #define VISIT_ATOMIC_BINOP(op) \
2474 : void InstructionSelector::VisitWord32Atomic##op(Node* node) { \
2475 : VisitWord32AtomicBinaryOperation( \
2476 : node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \
2477 : kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \
2478 : kWord32Atomic##op##Word32); \
2479 : }
2480 2660 : VISIT_ATOMIC_BINOP(Add)
2481 2744 : VISIT_ATOMIC_BINOP(Sub)
2482 2406 : VISIT_ATOMIC_BINOP(And)
2483 2454 : VISIT_ATOMIC_BINOP(Or)
2484 2883 : VISIT_ATOMIC_BINOP(Xor)
2485 : #undef VISIT_ATOMIC_BINOP
2486 :
2487 11504 : void InstructionSelector::VisitWord64AtomicBinaryOperation(
2488 11504 : Node* node, ArchOpcode uint8_op, ArchOpcode uint16_op, ArchOpcode uint32_op,
2489 : ArchOpcode word64_op) {
2490 11504 : MachineType type = AtomicOpType(node->op());
2491 : ArchOpcode opcode = kArchNop;
2492 11504 : if (type == MachineType::Uint8()) {
2493 : opcode = uint8_op;
2494 7871 : } else if (type == MachineType::Uint16()) {
2495 : opcode = uint16_op;
2496 4723 : } else if (type == MachineType::Uint32()) {
2497 : opcode = uint32_op;
2498 2717 : } else if (type == MachineType::Uint64()) {
2499 : opcode = word64_op;
2500 : } else {
2501 0 : UNREACHABLE();
2502 : return;
2503 : }
2504 11504 : VisitAtomicBinop(this, node, opcode);
2505 11506 : }
2506 :
2507 : #define VISIT_ATOMIC_BINOP(op) \
2508 : void InstructionSelector::VisitWord64Atomic##op(Node* node) { \
2509 : VisitWord64AtomicBinaryOperation( \
2510 : node, kX64Word64Atomic##op##Uint8, kX64Word64Atomic##op##Uint16, \
2511 : kX64Word64Atomic##op##Uint32, kX64Word64Atomic##op##Uint64); \
2512 : }
2513 2164 : VISIT_ATOMIC_BINOP(Add)
2514 2298 : VISIT_ATOMIC_BINOP(Sub)
2515 2295 : VISIT_ATOMIC_BINOP(And)
2516 2343 : VISIT_ATOMIC_BINOP(Or)
2517 2406 : VISIT_ATOMIC_BINOP(Xor)
2518 : #undef VISIT_ATOMIC_BINOP
2519 :
2520 : #define SIMD_TYPES(V) \
2521 : V(F32x4) \
2522 : V(I32x4) \
2523 : V(I16x8) \
2524 : V(I8x16)
2525 :
2526 : #define SIMD_BINOP_LIST(V) \
2527 : V(F32x4Add) \
2528 : V(F32x4AddHoriz) \
2529 : V(F32x4Sub) \
2530 : V(F32x4Mul) \
2531 : V(F32x4Min) \
2532 : V(F32x4Max) \
2533 : V(F32x4Eq) \
2534 : V(F32x4Ne) \
2535 : V(F32x4Lt) \
2536 : V(F32x4Le) \
2537 : V(I32x4Add) \
2538 : V(I32x4AddHoriz) \
2539 : V(I32x4Sub) \
2540 : V(I32x4Mul) \
2541 : V(I32x4MinS) \
2542 : V(I32x4MaxS) \
2543 : V(I32x4Eq) \
2544 : V(I32x4Ne) \
2545 : V(I32x4GtS) \
2546 : V(I32x4GeS) \
2547 : V(I32x4MinU) \
2548 : V(I32x4MaxU) \
2549 : V(I32x4GtU) \
2550 : V(I32x4GeU) \
2551 : V(I16x8SConvertI32x4) \
2552 : V(I16x8Add) \
2553 : V(I16x8AddSaturateS) \
2554 : V(I16x8AddHoriz) \
2555 : V(I16x8Sub) \
2556 : V(I16x8SubSaturateS) \
2557 : V(I16x8Mul) \
2558 : V(I16x8MinS) \
2559 : V(I16x8MaxS) \
2560 : V(I16x8Eq) \
2561 : V(I16x8Ne) \
2562 : V(I16x8GtS) \
2563 : V(I16x8GeS) \
2564 : V(I16x8AddSaturateU) \
2565 : V(I16x8SubSaturateU) \
2566 : V(I16x8MinU) \
2567 : V(I16x8MaxU) \
2568 : V(I16x8GtU) \
2569 : V(I16x8GeU) \
2570 : V(I8x16SConvertI16x8) \
2571 : V(I8x16Add) \
2572 : V(I8x16AddSaturateS) \
2573 : V(I8x16Sub) \
2574 : V(I8x16SubSaturateS) \
2575 : V(I8x16MinS) \
2576 : V(I8x16MaxS) \
2577 : V(I8x16Eq) \
2578 : V(I8x16Ne) \
2579 : V(I8x16GtS) \
2580 : V(I8x16GeS) \
2581 : V(I8x16AddSaturateU) \
2582 : V(I8x16SubSaturateU) \
2583 : V(I8x16MinU) \
2584 : V(I8x16MaxU) \
2585 : V(I8x16GtU) \
2586 : V(I8x16GeU) \
2587 : V(S128And) \
2588 : V(S128Or) \
2589 : V(S128Xor)
2590 :
2591 : #define SIMD_UNOP_LIST(V) \
2592 : V(F32x4SConvertI32x4) \
2593 : V(F32x4Abs) \
2594 : V(F32x4Neg) \
2595 : V(F32x4RecipApprox) \
2596 : V(F32x4RecipSqrtApprox) \
2597 : V(I32x4SConvertI16x8Low) \
2598 : V(I32x4SConvertI16x8High) \
2599 : V(I32x4Neg) \
2600 : V(I32x4UConvertI16x8Low) \
2601 : V(I32x4UConvertI16x8High) \
2602 : V(I16x8SConvertI8x16Low) \
2603 : V(I16x8SConvertI8x16High) \
2604 : V(I16x8Neg) \
2605 : V(I16x8UConvertI8x16Low) \
2606 : V(I16x8UConvertI8x16High) \
2607 : V(I8x16Neg) \
2608 : V(S128Not)
2609 :
2610 : #define SIMD_SHIFT_OPCODES(V) \
2611 : V(I32x4Shl) \
2612 : V(I32x4ShrS) \
2613 : V(I32x4ShrU) \
2614 : V(I16x8Shl) \
2615 : V(I16x8ShrS) \
2616 : V(I16x8ShrU) \
2617 : V(I8x16Shl) \
2618 : V(I8x16ShrS) \
2619 : V(I8x16ShrU)
2620 :
2621 : #define SIMD_ANYTRUE_LIST(V) \
2622 : V(S1x4AnyTrue) \
2623 : V(S1x8AnyTrue) \
2624 : V(S1x16AnyTrue)
2625 :
2626 : #define SIMD_ALLTRUE_LIST(V) \
2627 : V(S1x4AllTrue) \
2628 : V(S1x8AllTrue) \
2629 : V(S1x16AllTrue)
2630 :
2631 18 : void InstructionSelector::VisitS128Zero(Node* node) {
2632 : X64OperandGenerator g(this);
2633 18 : Emit(kX64S128Zero, g.DefineAsRegister(node));
2634 18 : }
2635 :
2636 : #define VISIT_SIMD_SPLAT(Type) \
2637 : void InstructionSelector::Visit##Type##Splat(Node* node) { \
2638 : X64OperandGenerator g(this); \
2639 : Emit(kX64##Type##Splat, g.DefineAsRegister(node), \
2640 : g.Use(node->InputAt(0))); \
2641 : }
2642 8280 : SIMD_TYPES(VISIT_SIMD_SPLAT)
2643 : #undef VISIT_SIMD_SPLAT
2644 :
2645 : #define VISIT_SIMD_EXTRACT_LANE(Type) \
2646 : void InstructionSelector::Visit##Type##ExtractLane(Node* node) { \
2647 : X64OperandGenerator g(this); \
2648 : int32_t lane = OpParameter<int32_t>(node->op()); \
2649 : Emit(kX64##Type##ExtractLane, g.DefineAsRegister(node), \
2650 : g.UseRegister(node->InputAt(0)), g.UseImmediate(lane)); \
2651 : }
2652 74556 : SIMD_TYPES(VISIT_SIMD_EXTRACT_LANE)
2653 : #undef VISIT_SIMD_EXTRACT_LANE
2654 :
2655 : #define VISIT_SIMD_REPLACE_LANE(Type) \
2656 : void InstructionSelector::Visit##Type##ReplaceLane(Node* node) { \
2657 : X64OperandGenerator g(this); \
2658 : int32_t lane = OpParameter<int32_t>(node->op()); \
2659 : Emit(kX64##Type##ReplaceLane, g.DefineSameAsFirst(node), \
2660 : g.UseRegister(node->InputAt(0)), g.UseImmediate(lane), \
2661 : g.Use(node->InputAt(1))); \
2662 : }
2663 10720 : SIMD_TYPES(VISIT_SIMD_REPLACE_LANE)
2664 : #undef VISIT_SIMD_REPLACE_LANE
2665 :
2666 : #define VISIT_SIMD_SHIFT(Opcode) \
2667 : void InstructionSelector::Visit##Opcode(Node* node) { \
2668 : X64OperandGenerator g(this); \
2669 : int32_t value = OpParameter<int32_t>(node->op()); \
2670 : Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
2671 : g.UseRegister(node->InputAt(0)), g.UseImmediate(value)); \
2672 : }
2673 4770 : SIMD_SHIFT_OPCODES(VISIT_SIMD_SHIFT)
2674 : #undef VISIT_SIMD_SHIFT
2675 : #undef SIMD_SHIFT_OPCODES
2676 :
2677 : #define VISIT_SIMD_UNOP(Opcode) \
2678 : void InstructionSelector::Visit##Opcode(Node* node) { \
2679 : X64OperandGenerator g(this); \
2680 : Emit(kX64##Opcode, g.DefineAsRegister(node), \
2681 : g.UseRegister(node->InputAt(0))); \
2682 : }
2683 330 : SIMD_UNOP_LIST(VISIT_SIMD_UNOP)
2684 : #undef VISIT_SIMD_UNOP
2685 : #undef SIMD_UNOP_LIST
2686 :
2687 : #define VISIT_SIMD_BINOP(Opcode) \
2688 : void InstructionSelector::Visit##Opcode(Node* node) { \
2689 : X64OperandGenerator g(this); \
2690 : Emit(kX64##Opcode, g.DefineSameAsFirst(node), \
2691 : g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))); \
2692 : }
2693 2835 : SIMD_BINOP_LIST(VISIT_SIMD_BINOP)
2694 : #undef VISIT_SIMD_BINOP
2695 : #undef SIMD_BINOP_LIST
2696 :
2697 : #define VISIT_SIMD_ANYTRUE(Opcode) \
2698 : void InstructionSelector::Visit##Opcode(Node* node) { \
2699 : X64OperandGenerator g(this); \
2700 : InstructionOperand temps[] = {g.TempRegister()}; \
2701 : Emit(kX64##Opcode, g.DefineAsRegister(node), \
2702 : g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps); \
2703 : }
2704 405 : SIMD_ANYTRUE_LIST(VISIT_SIMD_ANYTRUE)
2705 : #undef VISIT_SIMD_ANYTRUE
2706 : #undef SIMD_ANYTRUE_LIST
2707 :
2708 : #define VISIT_SIMD_ALLTRUE(Opcode) \
2709 : void InstructionSelector::Visit##Opcode(Node* node) { \
2710 : X64OperandGenerator g(this); \
2711 : InstructionOperand temps[] = {g.TempRegister()}; \
2712 : Emit(kX64##Opcode, g.DefineAsRegister(node), \
2713 : g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps); \
2714 : }
2715 405 : SIMD_ALLTRUE_LIST(VISIT_SIMD_ALLTRUE)
2716 : #undef VISIT_SIMD_ALLTRUE
2717 : #undef SIMD_ALLTRUE_LIST
2718 : #undef SIMD_TYPES
2719 :
2720 35 : void InstructionSelector::VisitS128Select(Node* node) {
2721 : X64OperandGenerator g(this);
2722 : Emit(kX64S128Select, g.DefineSameAsFirst(node),
2723 : g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
2724 105 : g.UseRegister(node->InputAt(2)));
2725 35 : }
2726 :
2727 5 : void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
2728 : X64OperandGenerator g(this);
2729 : Emit(kX64F32x4UConvertI32x4, g.DefineSameAsFirst(node),
2730 5 : g.UseRegister(node->InputAt(0)));
2731 5 : }
2732 :
2733 10 : void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
2734 : X64OperandGenerator g(this);
2735 : Emit(kX64I32x4SConvertF32x4, g.DefineSameAsFirst(node),
2736 10 : g.UseRegister(node->InputAt(0)));
2737 10 : }
2738 :
2739 10 : void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
2740 : X64OperandGenerator g(this);
2741 10 : InstructionOperand temps[] = {g.TempSimd128Register()};
2742 : Emit(kX64I32x4UConvertF32x4, g.DefineSameAsFirst(node),
2743 10 : g.UseRegister(node->InputAt(0)), arraysize(temps), temps);
2744 10 : }
2745 :
2746 10 : void InstructionSelector::VisitI16x8UConvertI32x4(Node* node) {
2747 : X64OperandGenerator g(this);
2748 : Emit(kX64I16x8UConvertI32x4, g.DefineSameAsFirst(node),
2749 20 : g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
2750 10 : }
2751 :
2752 10 : void InstructionSelector::VisitI8x16UConvertI16x8(Node* node) {
2753 : X64OperandGenerator g(this);
2754 : Emit(kX64I8x16UConvertI16x8, g.DefineSameAsFirst(node),
2755 20 : g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
2756 10 : }
2757 :
2758 10 : void InstructionSelector::VisitI8x16Mul(Node* node) {
2759 : X64OperandGenerator g(this);
2760 10 : InstructionOperand temps[] = {g.TempSimd128Register()};
2761 : Emit(kX64I8x16Mul, g.DefineSameAsFirst(node),
2762 : g.UseUniqueRegister(node->InputAt(0)),
2763 20 : g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
2764 10 : }
2765 :
2766 0 : void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
2767 0 : UNREACHABLE();
2768 : }
2769 :
2770 0 : void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
2771 0 : UNREACHABLE();
2772 : }
2773 :
2774 : namespace {
2775 :
2776 : // Packs a 4 lane shuffle into a single imm8 suitable for use by pshufd,
2777 : // pshuflw, and pshufhw.
2778 : uint8_t PackShuffle4(uint8_t* shuffle) {
2779 19845 : return (shuffle[0] & 3) | ((shuffle[1] & 3) << 2) | ((shuffle[2] & 3) << 4) |
2780 19845 : ((shuffle[3] & 3) << 6);
2781 : }
2782 :
2783 : // Gets an 8 bit lane mask suitable for 16x8 pblendw.
2784 : uint8_t PackBlend8(const uint8_t* shuffle16x8) {
2785 : int8_t result = 0;
2786 640 : for (int i = 0; i < 8; ++i) {
2787 640 : result |= (shuffle16x8[i] >= 8 ? 1 : 0) << i;
2788 : }
2789 80 : return result;
2790 : }
2791 :
2792 : // Gets an 8 bit lane mask suitable for 32x4 pblendw.
2793 : uint8_t PackBlend4(const uint8_t* shuffle32x4) {
2794 : int8_t result = 0;
2795 7820 : for (int i = 0; i < 4; ++i) {
2796 7820 : result |= (shuffle32x4[i] >= 4 ? 0x3 : 0) << (i * 2);
2797 : }
2798 60 : return result;
2799 : }
2800 :
2801 : // Returns true if shuffle can be decomposed into two 16x4 half shuffles
2802 : // followed by a 16x8 blend.
2803 : // E.g. [3 2 1 0 15 14 13 12].
2804 : bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) {
2805 : *blend_mask = 0;
2806 14880 : for (int i = 0; i < 8; i++) {
2807 15970 : if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false;
2808 14880 : *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i;
2809 : }
2810 : return true;
2811 : }
2812 :
2813 : struct ShuffleEntry {
2814 : uint8_t shuffle[kSimd128Size];
2815 : ArchOpcode opcode;
2816 : bool src0_needs_reg;
2817 : bool src1_needs_reg;
2818 : };
2819 :
2820 : // Shuffles that map to architecture-specific instruction sequences. These are
2821 : // matched very early, so we shouldn't include shuffles that match better in
2822 : // later tests, like 32x4 and 16x8 shuffles. In general, these patterns should
2823 : // map to either a single instruction, or be finer grained, such as zip/unzip or
2824 : // transpose patterns.
2825 : static const ShuffleEntry arch_shuffles[] = {
2826 : {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23},
2827 : kX64S64x2UnpackLow,
2828 : true,
2829 : false},
2830 : {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31},
2831 : kX64S64x2UnpackHigh,
2832 : true,
2833 : false},
2834 : {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
2835 : kX64S32x4UnpackLow,
2836 : true,
2837 : false},
2838 : {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
2839 : kX64S32x4UnpackHigh,
2840 : true,
2841 : false},
2842 : {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
2843 : kX64S16x8UnpackLow,
2844 : true,
2845 : false},
2846 : {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
2847 : kX64S16x8UnpackHigh,
2848 : true,
2849 : false},
2850 : {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
2851 : kX64S8x16UnpackLow,
2852 : true,
2853 : false},
2854 : {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
2855 : kX64S8x16UnpackHigh,
2856 : true,
2857 : false},
2858 :
2859 : {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
2860 : kX64S16x8UnzipLow,
2861 : true,
2862 : false},
2863 : {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
2864 : kX64S16x8UnzipHigh,
2865 : true,
2866 : true},
2867 : {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
2868 : kX64S8x16UnzipLow,
2869 : true,
2870 : true},
2871 : {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
2872 : kX64S8x16UnzipHigh,
2873 : true,
2874 : true},
2875 : {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
2876 : kX64S8x16TransposeLow,
2877 : true,
2878 : true},
2879 : {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
2880 : kX64S8x16TransposeHigh,
2881 : true,
2882 : true},
2883 : {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8},
2884 : kX64S8x8Reverse,
2885 : false,
2886 : false},
2887 : {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12},
2888 : kX64S8x4Reverse,
2889 : false,
2890 : false},
2891 : {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
2892 : kX64S8x2Reverse,
2893 : true,
2894 : true}};
2895 :
2896 15710 : bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
2897 : size_t num_entries, bool is_swizzle,
2898 : const ShuffleEntry** arch_shuffle) {
2899 15710 : uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1;
2900 236190 : for (size_t i = 0; i < num_entries; ++i) {
2901 226110 : const ShuffleEntry& entry = table[i];
2902 : int j = 0;
2903 408220 : for (; j < kSimd128Size; ++j) {
2904 402590 : if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
2905 : break;
2906 : }
2907 : }
2908 226110 : if (j == kSimd128Size) {
2909 5630 : *arch_shuffle = &entry;
2910 5630 : return true;
2911 : }
2912 : }
2913 : return false;
2914 : }
2915 :
2916 : } // namespace
2917 :
2918 16310 : void InstructionSelector::VisitS8x16Shuffle(Node* node) {
2919 : uint8_t shuffle[kSimd128Size];
2920 : bool is_swizzle;
2921 16310 : CanonicalizeShuffle(node, shuffle, &is_swizzle);
2922 :
2923 : int imm_count = 0;
2924 : static const int kMaxImms = 6;
2925 : uint32_t imms[kMaxImms];
2926 : int temp_count = 0;
2927 : static const int kMaxTemps = 2;
2928 48930 : InstructionOperand temps[kMaxTemps];
2929 :
2930 : X64OperandGenerator g(this);
2931 : // Swizzles don't generally need DefineSameAsFirst to avoid a move.
2932 16310 : bool no_same_as_first = is_swizzle;
2933 : // We generally need UseRegister for input0, Use for input1.
2934 : bool src0_needs_reg = true;
2935 : bool src1_needs_reg = false;
2936 : ArchOpcode opcode = kX64S8x16Shuffle; // general shuffle is the default
2937 :
2938 : uint8_t offset;
2939 : uint8_t shuffle32x4[4];
2940 : uint8_t shuffle16x8[8];
2941 : int index;
2942 : const ShuffleEntry* arch_shuffle;
2943 16310 : if (TryMatchConcat(shuffle, &offset)) {
2944 : // Swap inputs from the normal order for (v)palignr.
2945 600 : SwapShuffleInputs(node);
2946 600 : is_swizzle = false; // It's simpler to just handle the general case.
2947 : no_same_as_first = false; // SSE requires same-as-first.
2948 : opcode = kX64S8x16Alignr;
2949 : // palignr takes a single imm8 offset.
2950 600 : imms[imm_count++] = offset;
2951 15710 : } else if (TryMatchArchShuffle(shuffle, arch_shuffles,
2952 : arraysize(arch_shuffles), is_swizzle,
2953 15710 : &arch_shuffle)) {
2954 5630 : opcode = arch_shuffle->opcode;
2955 5630 : src0_needs_reg = arch_shuffle->src0_needs_reg;
2956 : // SSE can't take advantage of both operands in registers and needs
2957 : // same-as-first.
2958 : src1_needs_reg = false;
2959 : no_same_as_first = false;
2960 10080 : } else if (TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
2961 : uint8_t shuffle_mask = PackShuffle4(shuffle32x4);
2962 3555 : if (is_swizzle) {
2963 1600 : if (TryMatchIdentity(shuffle)) {
2964 : // Bypass normal shuffle code generation in this case.
2965 635 : EmitIdentity(node);
2966 16945 : return;
2967 : } else {
2968 : // pshufd takes a single imm8 shuffle mask.
2969 : opcode = kX64S32x4Swizzle;
2970 : no_same_as_first = true;
2971 : src0_needs_reg = false;
2972 965 : imms[imm_count++] = shuffle_mask;
2973 : }
2974 : } else {
2975 : // 2 operand shuffle
2976 : // A blend is more efficient than a general 32x4 shuffle; try it first.
2977 1955 : if (TryMatchBlend(shuffle)) {
2978 : opcode = kX64S16x8Blend;
2979 : uint8_t blend_mask = PackBlend4(shuffle32x4);
2980 60 : imms[imm_count++] = blend_mask;
2981 : } else {
2982 : opcode = kX64S32x4Shuffle;
2983 : no_same_as_first = true;
2984 : src0_needs_reg = false;
2985 1895 : imms[imm_count++] = shuffle_mask;
2986 : int8_t blend_mask = PackBlend4(shuffle32x4);
2987 1895 : imms[imm_count++] = blend_mask;
2988 : }
2989 : }
2990 6525 : } else if (TryMatch16x8Shuffle(shuffle, shuffle16x8)) {
2991 : uint8_t blend_mask;
2992 3175 : if (TryMatchBlend(shuffle)) {
2993 : opcode = kX64S16x8Blend;
2994 : blend_mask = PackBlend8(shuffle16x8);
2995 80 : imms[imm_count++] = blend_mask;
2996 3095 : } else if (TryMatchDup<8>(shuffle, &index)) {
2997 : opcode = kX64S16x8Dup;
2998 : src0_needs_reg = false;
2999 475 : imms[imm_count++] = index;
3000 2620 : } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) {
3001 1530 : opcode = is_swizzle ? kX64S16x8HalfShuffle1 : kX64S16x8HalfShuffle2;
3002 : // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0).
3003 : no_same_as_first = true;
3004 : src0_needs_reg = false;
3005 : uint8_t mask_lo = PackShuffle4(shuffle16x8);
3006 : uint8_t mask_hi = PackShuffle4(shuffle16x8 + 4);
3007 1530 : imms[imm_count++] = mask_lo;
3008 1530 : imms[imm_count++] = mask_hi;
3009 1530 : if (!is_swizzle) imms[imm_count++] = blend_mask;
3010 : }
3011 3350 : } else if (TryMatchDup<16>(shuffle, &index)) {
3012 : opcode = kX64S8x16Dup;
3013 : no_same_as_first = false;
3014 : src0_needs_reg = true;
3015 610 : imms[imm_count++] = index;
3016 : }
3017 15675 : if (opcode == kX64S8x16Shuffle) {
3018 : // Use same-as-first for general swizzle, but not shuffle.
3019 3830 : no_same_as_first = !is_swizzle;
3020 : src0_needs_reg = !no_same_as_first;
3021 3830 : imms[imm_count++] = Pack4Lanes(shuffle);
3022 3830 : imms[imm_count++] = Pack4Lanes(shuffle + 4);
3023 3830 : imms[imm_count++] = Pack4Lanes(shuffle + 8);
3024 3830 : imms[imm_count++] = Pack4Lanes(shuffle + 12);
3025 3830 : temps[temp_count++] = g.TempRegister();
3026 : }
3027 :
3028 : // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra
3029 : // move instruction in the CodeGenerator.
3030 : Node* input0 = node->InputAt(0);
3031 : InstructionOperand dst =
3032 15675 : no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
3033 : InstructionOperand src0 =
3034 15675 : src0_needs_reg ? g.UseRegister(input0) : g.Use(input0);
3035 :
3036 : int input_count = 0;
3037 156750 : InstructionOperand inputs[2 + kMaxImms + kMaxTemps];
3038 15675 : inputs[input_count++] = src0;
3039 15675 : if (!is_swizzle) {
3040 : Node* input1 = node->InputAt(1);
3041 : inputs[input_count++] =
3042 9615 : src1_needs_reg ? g.UseRegister(input1) : g.Use(input1);
3043 : }
3044 25570 : for (int i = 0; i < imm_count; ++i) {
3045 25570 : inputs[input_count++] = g.UseImmediate(imms[i]);
3046 : }
3047 15675 : Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps);
3048 : }
3049 :
3050 : // static
3051 : MachineOperatorBuilder::Flags
3052 6175471 : InstructionSelector::SupportedMachineOperatorFlags() {
3053 : MachineOperatorBuilder::Flags flags =
3054 : MachineOperatorBuilder::kWord32ShiftIsSafe |
3055 : MachineOperatorBuilder::kWord32Ctz | MachineOperatorBuilder::kWord64Ctz |
3056 : MachineOperatorBuilder::kSpeculationFence;
3057 6175471 : if (CpuFeatures::IsSupported(POPCNT)) {
3058 : flags |= MachineOperatorBuilder::kWord32Popcnt |
3059 : MachineOperatorBuilder::kWord64Popcnt;
3060 : }
3061 6175471 : if (CpuFeatures::IsSupported(SSE4_1)) {
3062 : flags |= MachineOperatorBuilder::kFloat32RoundDown |
3063 : MachineOperatorBuilder::kFloat64RoundDown |
3064 : MachineOperatorBuilder::kFloat32RoundUp |
3065 : MachineOperatorBuilder::kFloat64RoundUp |
3066 : MachineOperatorBuilder::kFloat32RoundTruncate |
3067 : MachineOperatorBuilder::kFloat64RoundTruncate |
3068 : MachineOperatorBuilder::kFloat32RoundTiesEven |
3069 : MachineOperatorBuilder::kFloat64RoundTiesEven;
3070 : }
3071 6175471 : return flags;
3072 : }
3073 :
3074 : // static
3075 : MachineOperatorBuilder::AlignmentRequirements
3076 6174648 : InstructionSelector::AlignmentRequirements() {
3077 : return MachineOperatorBuilder::AlignmentRequirements::
3078 6174648 : FullUnalignedAccessSupport();
3079 : }
3080 :
3081 : } // namespace compiler
3082 : } // namespace internal
3083 183867 : } // namespace v8
|
