Line data Source code
1 : // Copyright 2016 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/simd-scalar-lowering.h"
6 :
7 : #include "src/compiler/diamond.h"
8 : #include "src/compiler/linkage.h"
9 : #include "src/compiler/node-matchers.h"
10 : #include "src/compiler/node-properties.h"
11 : #include "src/compiler/node.h"
12 : #include "src/compiler/wasm-compiler.h"
13 :
14 : namespace v8 {
15 : namespace internal {
16 : namespace compiler {
17 :
18 : namespace {
19 : static const int kNumLanes32 = 4;
20 : static const int kNumLanes16 = 8;
21 : static const int kNumLanes8 = 16;
22 : static const int32_t kMask16 = 0xffff;
23 : static const int32_t kMask8 = 0xff;
24 : static const int32_t kShift16 = 16;
25 : static const int32_t kShift8 = 24;
26 : } // anonymous
27 :
28 492 : SimdScalarLowering::SimdScalarLowering(
29 2460 : JSGraph* jsgraph, Signature<MachineRepresentation>* signature)
30 : : jsgraph_(jsgraph),
31 : state_(jsgraph->graph(), 3),
32 : stack_(jsgraph_->zone()),
33 : replacements_(nullptr),
34 : signature_(signature),
35 : placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
36 984 : graph()->start())),
37 1968 : parameter_count_after_lowering_(-1) {
38 : DCHECK_NOT_NULL(graph());
39 : DCHECK_NOT_NULL(graph()->end());
40 984 : replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
41 984 : memset(replacements_, 0, sizeof(Replacement) * graph()->NodeCount());
42 492 : }
43 :
44 1476 : void SimdScalarLowering::LowerGraph() {
45 984 : stack_.push_back({graph()->end(), 0});
46 492 : state_.Set(graph()->end(), State::kOnStack);
47 1476 : replacements_[graph()->end()->id()].type = SimdType::kInt32x4;
48 :
49 145398 : while (!stack_.empty()) {
50 : NodeState& top = stack_.back();
51 288828 : if (top.input_index == top.node->InputCount()) {
52 : // All inputs of top have already been lowered, now lower top.
53 : stack_.pop_back();
54 49560 : state_.Set(top.node, State::kVisited);
55 49560 : LowerNode(top.node);
56 : } else {
57 : // Push the next input onto the stack.
58 143922 : Node* input = top.node->InputAt(top.input_index++);
59 94854 : if (state_.Get(input) == State::kUnvisited) {
60 49068 : SetLoweredType(input, top.node);
61 49068 : if (input->opcode() == IrOpcode::kPhi) {
62 : // To break cycles with phi nodes we push phis on a separate stack so
63 : // that they are processed after all other nodes.
64 60 : PreparePhiReplacement(input);
65 120 : stack_.push_front({input, 0});
66 49008 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
67 : input->opcode() == IrOpcode::kLoop) {
68 24 : stack_.push_front({input, 0});
69 : } else {
70 97992 : stack_.push_back({input, 0});
71 : }
72 : state_.Set(input, State::kOnStack);
73 : }
74 : }
75 : }
76 492 : }
77 :
78 : #define FOREACH_INT32X4_OPCODE(V) \
79 : V(I32x4Splat) \
80 : V(I32x4ExtractLane) \
81 : V(I32x4ReplaceLane) \
82 : V(I32x4SConvertF32x4) \
83 : V(I32x4UConvertF32x4) \
84 : V(I32x4Neg) \
85 : V(I32x4Shl) \
86 : V(I32x4ShrS) \
87 : V(I32x4Add) \
88 : V(I32x4Sub) \
89 : V(I32x4Mul) \
90 : V(I32x4MinS) \
91 : V(I32x4MaxS) \
92 : V(I32x4ShrU) \
93 : V(I32x4MinU) \
94 : V(I32x4MaxU) \
95 : V(I32x4Eq) \
96 : V(I32x4Ne) \
97 : V(I32x4LtS) \
98 : V(I32x4LeS) \
99 : V(I32x4GtS) \
100 : V(I32x4GeS) \
101 : V(I32x4LtU) \
102 : V(I32x4LeU) \
103 : V(I32x4GtU) \
104 : V(I32x4GeU) \
105 : V(S128And) \
106 : V(S128Or) \
107 : V(S128Xor) \
108 : V(S128Not)
109 :
110 : #define FOREACH_FLOAT32X4_OPCODE(V) \
111 : V(F32x4Splat) \
112 : V(F32x4ExtractLane) \
113 : V(F32x4ReplaceLane) \
114 : V(F32x4SConvertI32x4) \
115 : V(F32x4UConvertI32x4) \
116 : V(F32x4Abs) \
117 : V(F32x4Neg) \
118 : V(F32x4Add) \
119 : V(F32x4Sub) \
120 : V(F32x4Mul) \
121 : V(F32x4Min) \
122 : V(F32x4Max)
123 :
124 : #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \
125 : V(F32x4Eq) \
126 : V(F32x4Ne) \
127 : V(F32x4Lt) \
128 : V(F32x4Le) \
129 : V(F32x4Gt) \
130 : V(F32x4Ge)
131 :
132 : #define FOREACH_INT16X8_OPCODE(V) \
133 : V(I16x8Splat) \
134 : V(I16x8ExtractLane) \
135 : V(I16x8ReplaceLane) \
136 : V(I16x8Neg) \
137 : V(I16x8Shl) \
138 : V(I16x8ShrS) \
139 : V(I16x8Add) \
140 : V(I16x8AddSaturateS) \
141 : V(I16x8Sub) \
142 : V(I16x8SubSaturateS) \
143 : V(I16x8Mul) \
144 : V(I16x8MinS) \
145 : V(I16x8MaxS) \
146 : V(I16x8ShrU) \
147 : V(I16x8AddSaturateU) \
148 : V(I16x8SubSaturateU) \
149 : V(I16x8MinU) \
150 : V(I16x8MaxU) \
151 : V(I16x8Eq) \
152 : V(I16x8Ne) \
153 : V(I16x8LtS) \
154 : V(I16x8LeS) \
155 : V(I16x8LtU) \
156 : V(I16x8LeU)
157 :
158 : #define FOREACH_INT8X16_OPCODE(V) \
159 : V(I8x16Splat) \
160 : V(I8x16ExtractLane) \
161 : V(I8x16ReplaceLane) \
162 : V(I8x16Neg) \
163 : V(I8x16Shl) \
164 : V(I8x16ShrS) \
165 : V(I8x16Add) \
166 : V(I8x16AddSaturateS) \
167 : V(I8x16Sub) \
168 : V(I8x16SubSaturateS) \
169 : V(I8x16Mul) \
170 : V(I8x16MinS) \
171 : V(I8x16MaxS) \
172 : V(I8x16ShrU) \
173 : V(I8x16AddSaturateU) \
174 : V(I8x16SubSaturateU) \
175 : V(I8x16MinU) \
176 : V(I8x16MaxU) \
177 : V(I8x16Eq) \
178 : V(I8x16Ne) \
179 : V(I8x16LtS) \
180 : V(I8x16LeS) \
181 : V(I8x16LtU) \
182 : V(I8x16LeU)
183 :
184 2766 : MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) {
185 2766 : switch (simdType) {
186 : case SimdType::kFloat32x4:
187 : return MachineType::Float32();
188 : case SimdType::kInt32x4:
189 : return MachineType::Int32();
190 : case SimdType::kInt16x8:
191 : return MachineType::Int16();
192 : case SimdType::kInt8x16:
193 : return MachineType::Int8();
194 : }
195 : return MachineType::None();
196 : }
197 :
198 165024 : void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
199 49068 : switch (node->opcode()) {
200 : #define CASE_STMT(name) case IrOpcode::k##name:
201 : FOREACH_INT32X4_OPCODE(CASE_STMT)
202 : case IrOpcode::kReturn:
203 : case IrOpcode::kParameter:
204 : case IrOpcode::kCall: {
205 19044 : replacements_[node->id()].type = SimdType::kInt32x4;
206 9522 : break;
207 : }
208 : FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
209 0 : replacements_[node->id()].type = SimdType::kFloat32x4;
210 0 : break;
211 : }
212 : FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) {
213 0 : replacements_[node->id()].type = SimdType::kInt32x4;
214 0 : break;
215 : }
216 : FOREACH_INT16X8_OPCODE(CASE_STMT) {
217 4140 : replacements_[node->id()].type = SimdType::kInt16x8;
218 2070 : break;
219 : }
220 : FOREACH_INT8X16_OPCODE(CASE_STMT) {
221 8064 : replacements_[node->id()].type = SimdType::kInt8x16;
222 4032 : break;
223 : }
224 : default: {
225 33444 : switch (output->opcode()) {
226 : case IrOpcode::kF32x4SConvertI32x4:
227 : case IrOpcode::kF32x4UConvertI32x4: {
228 0 : replacements_[node->id()].type = SimdType::kInt32x4;
229 0 : break;
230 : }
231 : FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT)
232 : case IrOpcode::kI32x4SConvertF32x4:
233 : case IrOpcode::kI32x4UConvertF32x4: {
234 0 : replacements_[node->id()].type = SimdType::kFloat32x4;
235 0 : break;
236 : }
237 : case IrOpcode::kS128Select: {
238 0 : replacements_[node->id()].type = SimdType::kInt32x4;
239 0 : break;
240 : }
241 : default: {
242 100332 : replacements_[node->id()].type = replacements_[output->id()].type;
243 : }
244 : }
245 : }
246 : #undef CASE_STMT
247 : }
248 49068 : }
249 :
250 : static int GetParameterIndexAfterLoweringSimd128(
251 1284 : Signature<MachineRepresentation>* signature, int old_index) {
252 : // In function calls, the simd128 types are passed as 4 Int32 types. The
253 : // parameters are typecast to the types as needed for various operations.
254 : int result = old_index;
255 1284 : for (int i = 0; i < old_index; ++i) {
256 2568 : if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
257 0 : result += 3;
258 : }
259 : }
260 : return result;
261 : }
262 :
263 2748 : int SimdScalarLowering::GetParameterCountAfterLowering() {
264 2256 : if (parameter_count_after_lowering_ == -1) {
265 : // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the
266 : // parameter count after lowering.
267 : parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128(
268 984 : signature(), static_cast<int>(signature()->parameter_count()));
269 : }
270 2256 : return parameter_count_after_lowering_;
271 : }
272 :
273 : static int GetReturnCountAfterLoweringSimd128(
274 12888 : Signature<MachineRepresentation>* signature) {
275 6444 : int result = static_cast<int>(signature->return_count());
276 6444 : for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
277 12888 : if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
278 0 : result += 3;
279 : }
280 : }
281 : return result;
282 : }
283 :
284 50124 : int SimdScalarLowering::NumLanes(SimdType type) {
285 : int num_lanes = 0;
286 50124 : if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) {
287 : num_lanes = kNumLanes32;
288 6474 : } else if (type == SimdType::kInt16x8) {
289 : num_lanes = kNumLanes16;
290 4206 : } else if (type == SimdType::kInt8x16) {
291 : num_lanes = kNumLanes8;
292 : } else {
293 0 : UNREACHABLE();
294 : }
295 50124 : return num_lanes;
296 : }
297 :
298 : constexpr int SimdScalarLowering::kLaneOffsets[];
299 :
300 66 : void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices,
301 792 : SimdType type) {
302 66 : int num_lanes = NumLanes(type);
303 66 : int lane_width = kSimd128Size / num_lanes;
304 66 : int laneIndex = kLaneOffsets[0] / lane_width;
305 66 : new_indices[laneIndex] = index;
306 264 : for (int i = 1; i < num_lanes; ++i) {
307 198 : laneIndex = kLaneOffsets[i * lane_width] / lane_width;
308 198 : new_indices[laneIndex] = graph()->NewNode(
309 : machine()->Int32Add(), index,
310 : graph()->NewNode(
311 594 : common()->Int32Constant(static_cast<int>(i) * lane_width)));
312 : }
313 66 : }
314 :
315 1038 : void SimdScalarLowering::LowerLoadOp(MachineRepresentation rep, Node* node,
316 210 : const Operator* load_op, SimdType type) {
317 1038 : if (rep == MachineRepresentation::kSimd128) {
318 : Node* base = node->InputAt(0);
319 : Node* index = node->InputAt(1);
320 42 : int num_lanes = NumLanes(type);
321 42 : Node** indices = zone()->NewArray<Node*>(num_lanes);
322 42 : GetIndexNodes(index, indices, type);
323 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
324 42 : rep_nodes[0] = node;
325 42 : rep_nodes[0]->ReplaceInput(1, indices[0]);
326 42 : NodeProperties::ChangeOp(rep_nodes[0], load_op);
327 42 : if (node->InputCount() > 2) {
328 : DCHECK_LT(3, node->InputCount());
329 : Node* effect_input = node->InputAt(2);
330 : Node* control_input = node->InputAt(3);
331 168 : for (int i = num_lanes - 1; i > 0; --i) {
332 126 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
333 252 : control_input);
334 : effect_input = rep_nodes[i];
335 : }
336 42 : rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
337 : } else {
338 0 : for (int i = 1; i < num_lanes; ++i) {
339 0 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
340 : }
341 : }
342 42 : ReplaceNode(node, rep_nodes, num_lanes);
343 : } else {
344 996 : DefaultLowering(node);
345 : }
346 1038 : }
347 :
348 24 : void SimdScalarLowering::LowerStoreOp(MachineRepresentation rep, Node* node,
349 : const Operator* store_op,
350 120 : SimdType rep_type) {
351 24 : if (rep == MachineRepresentation::kSimd128) {
352 : Node* base = node->InputAt(0);
353 : Node* index = node->InputAt(1);
354 24 : int num_lanes = NumLanes(rep_type);
355 24 : Node** indices = zone()->NewArray<Node*>(num_lanes);
356 24 : GetIndexNodes(index, indices, rep_type);
357 : DCHECK_LT(2, node->InputCount());
358 : Node* value = node->InputAt(2);
359 : DCHECK(HasReplacement(1, value));
360 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
361 24 : rep_nodes[0] = node;
362 24 : Node** rep_inputs = GetReplacementsWithType(value, rep_type);
363 24 : rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
364 24 : rep_nodes[0]->ReplaceInput(1, indices[0]);
365 24 : NodeProperties::ChangeOp(node, store_op);
366 24 : if (node->InputCount() > 3) {
367 : DCHECK_LT(4, node->InputCount());
368 : Node* effect_input = node->InputAt(3);
369 : Node* control_input = node->InputAt(4);
370 96 : for (int i = num_lanes - 1; i > 0; --i) {
371 72 : rep_nodes[i] =
372 144 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i],
373 216 : effect_input, control_input);
374 : effect_input = rep_nodes[i];
375 : }
376 24 : rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
377 : } else {
378 0 : for (int i = 1; i < num_lanes; ++i) {
379 0 : rep_nodes[i] =
380 0 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
381 : }
382 : }
383 24 : ReplaceNode(node, rep_nodes, num_lanes);
384 : } else {
385 0 : DefaultLowering(node);
386 : }
387 24 : }
388 :
389 42 : void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
390 210 : const Operator* op) {
391 : DCHECK_EQ(2, node->InputCount());
392 42 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
393 42 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
394 42 : int num_lanes = NumLanes(input_rep_type);
395 42 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
396 210 : for (int i = 0; i < num_lanes; ++i) {
397 336 : rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
398 : }
399 42 : ReplaceNode(node, rep_node, num_lanes);
400 42 : }
401 :
402 162 : void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type,
403 : const Operator* op,
404 6210 : bool invert_inputs) {
405 : DCHECK_EQ(2, node->InputCount());
406 162 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
407 162 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
408 162 : int num_lanes = NumLanes(input_rep_type);
409 162 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
410 1674 : for (int i = 0; i < num_lanes; ++i) {
411 : Node* cmp_result = nullptr;
412 1512 : if (invert_inputs) {
413 1344 : cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]);
414 : } else {
415 168 : cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]);
416 : }
417 : Diamond d_cmp(graph(), common(),
418 : graph()->NewNode(machine()->Word32Equal(), cmp_result,
419 4536 : jsgraph_->Int32Constant(0)));
420 : MachineRepresentation rep =
421 : (input_rep_type == SimdType::kFloat32x4)
422 : ? MachineRepresentation::kWord32
423 1512 : : MachineTypeFrom(input_rep_type).representation();
424 1512 : rep_node[i] =
425 1512 : d_cmp.Phi(rep, jsgraph_->Int32Constant(0), jsgraph_->Int32Constant(-1));
426 : }
427 162 : ReplaceNode(node, rep_node, num_lanes);
428 162 : }
429 :
430 4080 : Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) {
431 : return graph()->NewNode(machine()->Word32Sar(),
432 : graph()->NewNode(machine()->Word32Shl(), input,
433 : jsgraph_->Int32Constant(shift)),
434 3264 : jsgraph_->Int32Constant(shift));
435 : }
436 :
437 30 : void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node,
438 : SimdType input_rep_type,
439 366 : const Operator* op) {
440 : DCHECK_EQ(2, node->InputCount());
441 : DCHECK(input_rep_type == SimdType::kInt16x8 ||
442 : input_rep_type == SimdType::kInt8x16);
443 30 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
444 30 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
445 30 : int num_lanes = NumLanes(input_rep_type);
446 30 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
447 : int32_t shift_val =
448 30 : (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8;
449 366 : for (int i = 0; i < num_lanes; ++i) {
450 672 : rep_node[i] = FixUpperBits(graph()->NewNode(op, rep_left[i], rep_right[i]),
451 1008 : shift_val);
452 : }
453 30 : ReplaceNode(node, rep_node, num_lanes);
454 30 : }
455 :
456 1872 : Node* SimdScalarLowering::Mask(Node* input, int32_t mask) {
457 : return graph()->NewNode(machine()->Word32And(), input,
458 1872 : jsgraph_->Int32Constant(mask));
459 : }
460 :
461 48 : void SimdScalarLowering::LowerSaturateBinaryOp(Node* node,
462 : SimdType input_rep_type,
463 : const Operator* op,
464 4080 : bool is_signed) {
465 : DCHECK_EQ(2, node->InputCount());
466 : DCHECK(input_rep_type == SimdType::kInt16x8 ||
467 : input_rep_type == SimdType::kInt8x16);
468 48 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
469 48 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
470 : int32_t min = 0;
471 : int32_t max = 0;
472 : int32_t mask = 0;
473 : int32_t shift_val = 0;
474 : MachineRepresentation phi_rep;
475 48 : if (input_rep_type == SimdType::kInt16x8) {
476 24 : if (is_signed) {
477 : min = std::numeric_limits<int16_t>::min();
478 : max = std::numeric_limits<int16_t>::max();
479 : } else {
480 : min = std::numeric_limits<uint16_t>::min();
481 : max = std::numeric_limits<uint16_t>::max();
482 : }
483 : mask = kMask16;
484 : shift_val = kShift16;
485 : phi_rep = MachineRepresentation::kWord16;
486 : } else {
487 24 : if (is_signed) {
488 : min = std::numeric_limits<int8_t>::min();
489 : max = std::numeric_limits<int8_t>::max();
490 : } else {
491 : min = std::numeric_limits<uint8_t>::min();
492 : max = std::numeric_limits<uint8_t>::max();
493 : }
494 : mask = kMask8;
495 : shift_val = kShift8;
496 : phi_rep = MachineRepresentation::kWord8;
497 : }
498 48 : int num_lanes = NumLanes(input_rep_type);
499 48 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
500 624 : for (int i = 0; i < num_lanes; ++i) {
501 : Node* op_result = nullptr;
502 576 : Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask);
503 576 : Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask);
504 : op_result = graph()->NewNode(op, left, right);
505 : Diamond d_min(graph(), common(),
506 : graph()->NewNode(machine()->Int32LessThan(), op_result,
507 1728 : jsgraph_->Int32Constant(min)));
508 576 : rep_node[i] = d_min.Phi(phi_rep, jsgraph_->Int32Constant(min), op_result);
509 : Diamond d_max(graph(), common(),
510 : graph()->NewNode(machine()->Int32LessThan(),
511 1728 : jsgraph_->Int32Constant(max), rep_node[i]));
512 576 : rep_node[i] = d_max.Phi(phi_rep, jsgraph_->Int32Constant(max), rep_node[i]);
513 : rep_node[i] =
514 576 : is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
515 : }
516 48 : ReplaceNode(node, rep_node, num_lanes);
517 48 : }
518 :
519 0 : void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
520 0 : const Operator* op) {
521 : DCHECK_EQ(1, node->InputCount());
522 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
523 0 : int num_lanes = NumLanes(input_rep_type);
524 0 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
525 0 : for (int i = 0; i < num_lanes; ++i) {
526 0 : rep_node[i] = graph()->NewNode(op, rep[i]);
527 : }
528 0 : ReplaceNode(node, rep_node, num_lanes);
529 0 : }
530 :
531 72 : void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
532 1416 : bool is_max, SimdType type) {
533 : DCHECK_EQ(2, node->InputCount());
534 72 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), type);
535 72 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), type);
536 72 : int num_lanes = NumLanes(type);
537 72 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
538 : MachineRepresentation rep = MachineRepresentation::kNone;
539 72 : if (type == SimdType::kInt32x4) {
540 : rep = MachineRepresentation::kWord32;
541 48 : } else if (type == SimdType::kInt16x8) {
542 : rep = MachineRepresentation::kWord16;
543 24 : } else if (type == SimdType::kInt8x16) {
544 : rep = MachineRepresentation::kWord8;
545 : } else {
546 0 : UNREACHABLE();
547 : }
548 744 : for (int i = 0; i < num_lanes; ++i) {
549 : Diamond d(graph(), common(),
550 1344 : graph()->NewNode(op, rep_left[i], rep_right[i]));
551 672 : if (is_max) {
552 336 : rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]);
553 : } else {
554 336 : rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]);
555 : }
556 : }
557 72 : ReplaceNode(node, rep_node, num_lanes);
558 72 : }
559 :
560 0 : Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
561 0 : if (machine()->Float64RoundTruncate().IsSupported()) {
562 0 : return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
563 : } else {
564 : ExternalReference ref =
565 0 : ExternalReference::wasm_f64_trunc(jsgraph_->isolate());
566 : Node* stack_slot =
567 0 : graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
568 : const Operator* store_op = machine()->Store(
569 0 : StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
570 : Node* effect =
571 : graph()->NewNode(store_op, stack_slot, jsgraph_->Int32Constant(0),
572 0 : input, graph()->start(), graph()->start());
573 0 : Node* function = graph()->NewNode(common()->ExternalConstant(ref));
574 : Node** args = zone()->NewArray<Node*>(4);
575 0 : args[0] = function;
576 0 : args[1] = stack_slot;
577 0 : args[2] = effect;
578 0 : args[3] = graph()->start();
579 : Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
580 : sig_builder.AddParam(MachineType::Pointer());
581 : CallDescriptor* desc =
582 0 : Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
583 0 : Node* call = graph()->NewNode(common()->Call(desc), 4, args);
584 : return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
585 : stack_slot, jsgraph_->Int32Constant(0), call,
586 0 : graph()->start());
587 : }
588 : }
589 :
590 0 : void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
591 : DCHECK_EQ(1, node->InputCount());
592 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4);
593 : Node* rep_node[kNumLanes32];
594 0 : Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
595 : Node* min = graph()->NewNode(
596 0 : common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
597 : Node* max = graph()->NewNode(common()->Float64Constant(
598 0 : static_cast<double>(is_signed ? kMaxInt : 0xffffffffu)));
599 0 : for (int i = 0; i < kNumLanes32; ++i) {
600 : Node* double_rep =
601 0 : graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
602 : Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
603 0 : double_rep, double_rep));
604 : Node* temp =
605 0 : nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
606 : Diamond min_d(graph(), common(),
607 0 : graph()->NewNode(machine()->Float64LessThan(), temp, min));
608 0 : temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
609 : Diamond max_d(graph(), common(),
610 0 : graph()->NewNode(machine()->Float64LessThan(), max, temp));
611 0 : temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
612 0 : Node* trunc = BuildF64Trunc(temp);
613 0 : if (is_signed) {
614 0 : rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
615 : } else {
616 : rep_node[i] =
617 0 : graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
618 : }
619 : }
620 0 : ReplaceNode(node, rep_node, kNumLanes32);
621 0 : }
622 :
623 792 : void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) {
624 : DCHECK_EQ(1, node->InputCount());
625 36 : int32_t shift_amount = OpParameter<int32_t>(node);
626 36 : Node* shift_node = graph()->NewNode(common()->Int32Constant(shift_amount));
627 36 : Node** rep = GetReplacementsWithType(node->InputAt(0), type);
628 36 : int num_lanes = NumLanes(type);
629 36 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
630 252 : for (int i = 0; i < num_lanes; ++i) {
631 216 : rep_node[i] = rep[i];
632 216 : switch (node->opcode()) {
633 : case IrOpcode::kI8x16ShrU:
634 0 : rep_node[i] = Mask(rep_node[i], kMask8);
635 : rep_node[i] =
636 0 : graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
637 0 : break;
638 : case IrOpcode::kI16x8ShrU:
639 48 : rep_node[i] = Mask(rep_node[i], kMask16); // Fall through.
640 : case IrOpcode::kI32x4ShrU:
641 : rep_node[i] =
642 216 : graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
643 72 : break;
644 : case IrOpcode::kI32x4Shl:
645 : rep_node[i] =
646 48 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
647 24 : break;
648 : case IrOpcode::kI16x8Shl:
649 : rep_node[i] =
650 96 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
651 48 : rep_node[i] = FixUpperBits(rep_node[i], kShift16);
652 48 : break;
653 : case IrOpcode::kI8x16Shl:
654 : rep_node[i] =
655 0 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
656 0 : rep_node[i] = FixUpperBits(rep_node[i], kShift8);
657 0 : break;
658 : case IrOpcode::kI32x4ShrS:
659 : case IrOpcode::kI16x8ShrS:
660 : case IrOpcode::kI8x16ShrS:
661 : rep_node[i] =
662 144 : graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node);
663 72 : break;
664 : default:
665 0 : UNREACHABLE();
666 : }
667 : }
668 36 : ReplaceNode(node, rep_node, num_lanes);
669 36 : }
670 :
671 18 : void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
672 354 : const Operator* op) {
673 : DCHECK_EQ(2, node->InputCount());
674 18 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
675 18 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
676 18 : int num_lanes = NumLanes(input_rep_type);
677 18 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
678 186 : for (int i = 0; i < num_lanes; ++i) {
679 : Diamond d(graph(), common(),
680 336 : graph()->NewNode(op, rep_left[i], rep_right[i]));
681 : MachineRepresentation rep =
682 : (input_rep_type == SimdType::kFloat32x4)
683 : ? MachineRepresentation::kWord32
684 168 : : MachineTypeFrom(input_rep_type).representation();
685 168 : rep_node[i] =
686 168 : d.Phi(rep, jsgraph_->Int32Constant(0), jsgraph_->Int32Constant(-1));
687 : }
688 18 : ReplaceNode(node, rep_node, num_lanes);
689 18 : }
690 :
691 116796 : void SimdScalarLowering::LowerNode(Node* node) {
692 : SimdType rep_type = ReplacementType(node);
693 49560 : int num_lanes = NumLanes(rep_type);
694 49560 : switch (node->opcode()) {
695 : case IrOpcode::kStart: {
696 492 : int parameter_count = GetParameterCountAfterLowering();
697 : // Only exchange the node if the parameter count actually changed.
698 492 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
699 : int delta =
700 0 : parameter_count - static_cast<int>(signature()->parameter_count());
701 0 : int new_output_count = node->op()->ValueOutputCount() + delta;
702 0 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
703 : }
704 : break;
705 : }
706 : case IrOpcode::kParameter: {
707 : DCHECK_EQ(1, node->InputCount());
708 : // Only exchange the node if the parameter count actually changed. We do
709 : // not even have to do the default lowering because the the start node,
710 : // the only input of a parameter node, only changes if the parameter count
711 : // changes.
712 3528 : if (GetParameterCountAfterLowering() !=
713 1764 : static_cast<int>(signature()->parameter_count())) {
714 0 : int old_index = ParameterIndexOf(node->op());
715 : int new_index =
716 : GetParameterIndexAfterLoweringSimd128(signature(), old_index);
717 0 : if (old_index == new_index) {
718 0 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
719 :
720 : Node* new_node[kNumLanes32];
721 0 : for (int i = 0; i < kNumLanes32; ++i) {
722 0 : new_node[i] = nullptr;
723 : }
724 0 : new_node[0] = node;
725 0 : if (signature()->GetParam(old_index) ==
726 : MachineRepresentation::kSimd128) {
727 0 : for (int i = 1; i < kNumLanes32; ++i) {
728 : new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
729 0 : graph()->start());
730 : }
731 : }
732 0 : ReplaceNode(node, new_node, kNumLanes32);
733 : }
734 : }
735 : break;
736 : }
737 : case IrOpcode::kLoad: {
738 : MachineRepresentation rep =
739 1038 : LoadRepresentationOf(node->op()).representation();
740 : const Operator* load_op;
741 2076 : load_op = machine()->Load(MachineTypeFrom(rep_type));
742 1038 : LowerLoadOp(rep, node, load_op, rep_type);
743 1038 : break;
744 : }
745 : case IrOpcode::kUnalignedLoad: {
746 : MachineRepresentation rep =
747 0 : UnalignedLoadRepresentationOf(node->op()).representation();
748 : const Operator* load_op;
749 0 : load_op = machine()->UnalignedLoad(MachineTypeFrom(rep_type));
750 0 : LowerLoadOp(rep, node, load_op, rep_type);
751 0 : break;
752 : }
753 : case IrOpcode::kStore: {
754 : MachineRepresentation rep =
755 24 : StoreRepresentationOf(node->op()).representation();
756 : WriteBarrierKind write_barrier_kind =
757 24 : StoreRepresentationOf(node->op()).write_barrier_kind();
758 : const Operator* store_op;
759 : store_op = machine()->Store(StoreRepresentation(
760 48 : MachineTypeFrom(rep_type).representation(), write_barrier_kind));
761 24 : LowerStoreOp(rep, node, store_op, rep_type);
762 24 : break;
763 : }
764 : case IrOpcode::kUnalignedStore: {
765 0 : MachineRepresentation rep = UnalignedStoreRepresentationOf(node->op());
766 : const Operator* store_op;
767 : store_op =
768 0 : machine()->UnalignedStore(MachineTypeFrom(rep_type).representation());
769 0 : LowerStoreOp(rep, node, store_op, rep_type);
770 0 : break;
771 : }
772 : case IrOpcode::kReturn: {
773 6444 : DefaultLowering(node);
774 : int new_return_count = GetReturnCountAfterLoweringSimd128(signature());
775 6444 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
776 0 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
777 : }
778 : break;
779 : }
780 : case IrOpcode::kCall: {
781 : // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor.
782 0 : CallDescriptor* descriptor =
783 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
784 0 : if (DefaultLowering(node) ||
785 0 : (descriptor->ReturnCount() == 1 &&
786 : descriptor->GetReturnType(0) == MachineType::Simd128())) {
787 : // We have to adjust the call descriptor.
788 : const Operator* op =
789 0 : common()->Call(GetI32WasmCallDescriptorForSimd(zone(), descriptor));
790 0 : NodeProperties::ChangeOp(node, op);
791 : }
792 0 : if (descriptor->ReturnCount() == 1 &&
793 : descriptor->GetReturnType(0) == MachineType::Simd128()) {
794 : // We access the additional return values through projections.
795 : Node* rep_node[kNumLanes32];
796 0 : for (int i = 0; i < kNumLanes32; ++i) {
797 : rep_node[i] =
798 0 : graph()->NewNode(common()->Projection(i), node, graph()->start());
799 : }
800 0 : ReplaceNode(node, rep_node, kNumLanes32);
801 : }
802 : break;
803 : }
804 : case IrOpcode::kPhi: {
805 60 : MachineRepresentation rep = PhiRepresentationOf(node->op());
806 60 : if (rep == MachineRepresentation::kSimd128) {
807 : // The replacement nodes have already been created, we only have to
808 : // replace placeholder nodes.
809 : Node** rep_node = GetReplacements(node);
810 54 : for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
811 : Node** rep_input =
812 12 : GetReplacementsWithType(node->InputAt(i), rep_type);
813 60 : for (int j = 0; j < num_lanes; j++) {
814 48 : rep_node[j]->ReplaceInput(i, rep_input[j]);
815 : }
816 : }
817 : } else {
818 54 : DefaultLowering(node);
819 : }
820 : break;
821 : }
822 : #define I32X4_BINOP_CASE(opcode, instruction) \
823 : case IrOpcode::opcode: { \
824 : LowerBinaryOp(node, rep_type, machine()->instruction()); \
825 : break; \
826 : }
827 12 : I32X4_BINOP_CASE(kI32x4Add, Int32Add)
828 6 : I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
829 6 : I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
830 6 : I32X4_BINOP_CASE(kS128And, Word32And)
831 6 : I32X4_BINOP_CASE(kS128Or, Word32Or)
832 6 : I32X4_BINOP_CASE(kS128Xor, Word32Xor)
833 : #undef I32X4_BINOP_CASE
834 : case IrOpcode::kI16x8Add:
835 : case IrOpcode::kI8x16Add: {
836 12 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add());
837 12 : break;
838 : }
839 : case IrOpcode::kI16x8Sub:
840 : case IrOpcode::kI8x16Sub: {
841 12 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub());
842 12 : break;
843 : }
844 : case IrOpcode::kI16x8Mul:
845 : case IrOpcode::kI8x16Mul: {
846 6 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul());
847 6 : break;
848 : }
849 : case IrOpcode::kI16x8AddSaturateS:
850 : case IrOpcode::kI8x16AddSaturateS: {
851 12 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true);
852 12 : break;
853 : }
854 : case IrOpcode::kI16x8SubSaturateS:
855 : case IrOpcode::kI8x16SubSaturateS: {
856 12 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true);
857 12 : break;
858 : }
859 : case IrOpcode::kI16x8AddSaturateU:
860 : case IrOpcode::kI8x16AddSaturateU: {
861 12 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false);
862 12 : break;
863 : }
864 : case IrOpcode::kI16x8SubSaturateU:
865 : case IrOpcode::kI8x16SubSaturateU: {
866 12 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false);
867 12 : break;
868 : }
869 : case IrOpcode::kI32x4MaxS:
870 : case IrOpcode::kI16x8MaxS:
871 : case IrOpcode::kI8x16MaxS: {
872 18 : LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type);
873 18 : break;
874 : }
875 : case IrOpcode::kI32x4MinS:
876 : case IrOpcode::kI16x8MinS:
877 : case IrOpcode::kI8x16MinS: {
878 18 : LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type);
879 18 : break;
880 : }
881 : case IrOpcode::kI32x4MaxU:
882 : case IrOpcode::kI16x8MaxU:
883 : case IrOpcode::kI8x16MaxU: {
884 18 : LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type);
885 18 : break;
886 : }
887 : case IrOpcode::kI32x4MinU:
888 : case IrOpcode::kI16x8MinU:
889 : case IrOpcode::kI8x16MinU: {
890 18 : LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type);
891 18 : break;
892 : }
893 : case IrOpcode::kI32x4Neg:
894 : case IrOpcode::kI16x8Neg:
895 : case IrOpcode::kI8x16Neg: {
896 : DCHECK_EQ(1, node->InputCount());
897 18 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
898 18 : int num_lanes = NumLanes(rep_type);
899 18 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
900 18 : Node* zero = graph()->NewNode(common()->Int32Constant(0));
901 186 : for (int i = 0; i < num_lanes; ++i) {
902 504 : rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
903 168 : if (node->opcode() == IrOpcode::kI16x8Neg) {
904 48 : rep_node[i] = FixUpperBits(rep_node[i], kShift16);
905 120 : } else if (node->opcode() == IrOpcode::kI8x16Neg) {
906 96 : rep_node[i] = FixUpperBits(rep_node[i], kShift8);
907 : }
908 : }
909 18 : ReplaceNode(node, rep_node, num_lanes);
910 18 : break;
911 : }
912 : case IrOpcode::kS128Not: {
913 : DCHECK_EQ(1, node->InputCount());
914 6 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
915 : Node* rep_node[kNumLanes32];
916 6 : Node* mask = graph()->NewNode(common()->Int32Constant(0xffffffff));
917 30 : for (int i = 0; i < kNumLanes32; ++i) {
918 72 : rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
919 : }
920 6 : ReplaceNode(node, rep_node, kNumLanes32);
921 : break;
922 : }
923 : case IrOpcode::kI32x4SConvertF32x4: {
924 0 : LowerConvertFromFloat(node, true);
925 0 : break;
926 : }
927 : case IrOpcode::kI32x4UConvertF32x4: {
928 0 : LowerConvertFromFloat(node, false);
929 0 : break;
930 : }
931 : case IrOpcode::kI32x4Shl:
932 : case IrOpcode::kI16x8Shl:
933 : case IrOpcode::kI8x16Shl:
934 : case IrOpcode::kI32x4ShrS:
935 : case IrOpcode::kI16x8ShrS:
936 : case IrOpcode::kI8x16ShrS:
937 : case IrOpcode::kI32x4ShrU:
938 : case IrOpcode::kI16x8ShrU:
939 : case IrOpcode::kI8x16ShrU: {
940 36 : LowerShiftOp(node, rep_type);
941 36 : break;
942 : }
943 : #define F32X4_BINOP_CASE(name) \
944 : case IrOpcode::kF32x4##name: { \
945 : LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
946 : break; \
947 : }
948 0 : F32X4_BINOP_CASE(Add)
949 0 : F32X4_BINOP_CASE(Sub)
950 0 : F32X4_BINOP_CASE(Mul)
951 0 : F32X4_BINOP_CASE(Min)
952 0 : F32X4_BINOP_CASE(Max)
953 : #undef F32X4_BINOP_CASE
954 : #define F32X4_UNOP_CASE(name) \
955 : case IrOpcode::kF32x4##name: { \
956 : LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
957 : break; \
958 : }
959 0 : F32X4_UNOP_CASE(Abs)
960 0 : F32X4_UNOP_CASE(Neg)
961 : #undef F32x4_UNOP_CASE
962 : case IrOpcode::kF32x4SConvertI32x4: {
963 0 : LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32());
964 0 : break;
965 : }
966 : case IrOpcode::kF32x4UConvertI32x4: {
967 0 : LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32());
968 0 : break;
969 : }
970 : case IrOpcode::kI32x4Splat:
971 : case IrOpcode::kF32x4Splat:
972 : case IrOpcode::kI16x8Splat:
973 : case IrOpcode::kI8x16Splat: {
974 864 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
975 8568 : for (int i = 0; i < num_lanes; ++i) {
976 7704 : if (HasReplacement(0, node->InputAt(0))) {
977 48 : rep_node[i] = GetReplacements(node->InputAt(0))[0];
978 : } else {
979 15360 : rep_node[i] = node->InputAt(0);
980 : }
981 : }
982 864 : ReplaceNode(node, rep_node, num_lanes);
983 864 : break;
984 : }
985 : case IrOpcode::kI32x4ExtractLane:
986 : case IrOpcode::kF32x4ExtractLane:
987 : case IrOpcode::kI16x8ExtractLane:
988 : case IrOpcode::kI8x16ExtractLane: {
989 6018 : int32_t lane = OpParameter<int32_t>(node);
990 6018 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
991 6018 : rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane];
992 73800 : for (int i = 1; i < num_lanes; ++i) {
993 67782 : rep_node[i] = nullptr;
994 : }
995 6018 : ReplaceNode(node, rep_node, num_lanes);
996 6018 : break;
997 : }
998 : case IrOpcode::kI32x4ReplaceLane:
999 : case IrOpcode::kF32x4ReplaceLane:
1000 : case IrOpcode::kI16x8ReplaceLane:
1001 : case IrOpcode::kI8x16ReplaceLane: {
1002 : DCHECK_EQ(2, node->InputCount());
1003 : Node* repNode = node->InputAt(1);
1004 198 : int32_t lane = OpParameter<int32_t>(node);
1005 198 : Node** rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
1006 198 : if (HasReplacement(0, repNode)) {
1007 0 : rep_node[lane] = GetReplacements(repNode)[0];
1008 : } else {
1009 198 : rep_node[lane] = repNode;
1010 : }
1011 198 : ReplaceNode(node, rep_node, num_lanes);
1012 198 : break;
1013 : }
1014 : #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \
1015 : case IrOpcode::simd_op: { \
1016 : LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
1017 : break; \
1018 : }
1019 0 : COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false)
1020 0 : COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false)
1021 0 : COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false)
1022 0 : COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true)
1023 0 : COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true)
1024 6 : COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false)
1025 0 : COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false)
1026 0 : COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false)
1027 12 : COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true)
1028 12 : COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true)
1029 0 : COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false)
1030 0 : COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false)
1031 12 : COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true)
1032 12 : COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true)
1033 6 : COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false)
1034 0 : COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false)
1035 0 : COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false)
1036 12 : COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true)
1037 12 : COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true)
1038 0 : COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false)
1039 0 : COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false)
1040 12 : COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true)
1041 12 : COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true)
1042 6 : COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false)
1043 0 : COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false)
1044 0 : COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false)
1045 12 : COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true)
1046 12 : COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true)
1047 0 : COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false)
1048 0 : COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false)
1049 12 : COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true)
1050 12 : COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true)
1051 : #undef COMPARISON_CASE
1052 : case IrOpcode::kF32x4Ne: {
1053 0 : LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal());
1054 0 : break;
1055 : }
1056 : case IrOpcode::kI32x4Ne: {
1057 6 : LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal());
1058 6 : break;
1059 : }
1060 : case IrOpcode::kI16x8Ne: {
1061 6 : LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal());
1062 6 : break;
1063 : }
1064 : case IrOpcode::kI8x16Ne: {
1065 6 : LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal());
1066 6 : break;
1067 : }
1068 : case IrOpcode::kS128Select: {
1069 : DCHECK_EQ(3, node->InputCount());
1070 : DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 ||
1071 : ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 ||
1072 : ReplacementType(node->InputAt(0)) == SimdType::kInt8x16);
1073 : Node** boolean_input = GetReplacements(node->InputAt(0));
1074 0 : Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
1075 0 : Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
1076 0 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1077 0 : for (int i = 0; i < num_lanes; ++i) {
1078 : Diamond d(graph(), common(),
1079 0 : graph()->NewNode(machine()->Word32Equal(), boolean_input[i],
1080 0 : jsgraph_->Int32Constant(0)));
1081 0 : rep_node[i] = d.Phi(MachineTypeFrom(rep_type).representation(),
1082 0 : rep_right[1], rep_left[0]);
1083 : }
1084 0 : ReplaceNode(node, rep_node, num_lanes);
1085 0 : break;
1086 : }
1087 32226 : default: { DefaultLowering(node); }
1088 : }
1089 49560 : }
1090 :
1091 39720 : bool SimdScalarLowering::DefaultLowering(Node* node) {
1092 : bool something_changed = false;
1093 84882 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1094 : Node* input = node->InputAt(i);
1095 45162 : if (HasReplacement(0, input)) {
1096 : something_changed = true;
1097 6012 : node->ReplaceInput(i, GetReplacements(input)[0]);
1098 : }
1099 45162 : if (HasReplacement(1, input)) {
1100 : something_changed = true;
1101 0 : for (int j = 1; j < ReplacementCount(input); ++j) {
1102 0 : node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
1103 : }
1104 : }
1105 : }
1106 39720 : return something_changed;
1107 : }
1108 :
1109 118104 : void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) {
1110 22752 : replacements_[old->id()].node = zone()->NewArray<Node*>(count);
1111 95352 : for (int i = 0; i < count; ++i) {
1112 175536 : replacements_[old->id()].node[i] = new_nodes[i];
1113 : }
1114 15168 : replacements_[old->id()].num_replacements = count;
1115 7584 : }
1116 :
1117 98226 : bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
1118 208500 : return replacements_[node->id()].node != nullptr &&
1119 12048 : replacements_[node->id()].node[index] != nullptr;
1120 : }
1121 :
1122 49566 : SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
1123 106188 : return replacements_[node->id()].type;
1124 : }
1125 :
1126 7086 : Node** SimdScalarLowering::GetReplacements(Node* node) {
1127 14148 : Node** result = replacements_[node->id()].node;
1128 : DCHECK(result);
1129 0 : return result;
1130 : }
1131 :
1132 0 : int SimdScalarLowering::ReplacementCount(Node* node) {
1133 0 : return replacements_[node->id()].num_replacements;
1134 : }
1135 :
1136 0 : void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) {
1137 0 : for (int i = 0; i < kNumLanes32; ++i) {
1138 0 : if (replacements[i] != nullptr) {
1139 0 : result[i] =
1140 0 : graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]);
1141 : } else {
1142 0 : result[i] = nullptr;
1143 : }
1144 : }
1145 0 : }
1146 :
1147 0 : void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) {
1148 0 : for (int i = 0; i < kNumLanes32; ++i) {
1149 0 : if (replacements[i] != nullptr) {
1150 0 : result[i] =
1151 0 : graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]);
1152 : } else {
1153 0 : result[i] = nullptr;
1154 : }
1155 : }
1156 0 : }
1157 :
1158 7056 : Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
1159 : Node** replacements = GetReplacements(node);
1160 7056 : if (ReplacementType(node) == type) {
1161 : return GetReplacements(node);
1162 : }
1163 0 : int num_lanes = NumLanes(type);
1164 0 : Node** result = zone()->NewArray<Node*>(num_lanes);
1165 0 : if (type == SimdType::kInt32x4) {
1166 0 : if (ReplacementType(node) == SimdType::kFloat32x4) {
1167 0 : Float32ToInt32(replacements, result);
1168 0 : } else if (ReplacementType(node) == SimdType::kInt16x8) {
1169 0 : UNIMPLEMENTED();
1170 : } else {
1171 0 : UNREACHABLE();
1172 : }
1173 0 : } else if (type == SimdType::kFloat32x4) {
1174 0 : if (ReplacementType(node) == SimdType::kInt32x4) {
1175 0 : Int32ToFloat32(replacements, result);
1176 0 : } else if (ReplacementType(node) == SimdType::kInt16x8) {
1177 0 : UNIMPLEMENTED();
1178 : } else {
1179 0 : UNREACHABLE();
1180 : }
1181 0 : } else if (type == SimdType::kInt16x8) {
1182 0 : if (ReplacementType(node) == SimdType::kInt32x4 ||
1183 : ReplacementType(node) == SimdType::kFloat32x4) {
1184 0 : UNIMPLEMENTED();
1185 : } else {
1186 0 : UNREACHABLE();
1187 : }
1188 : } else {
1189 0 : UNREACHABLE();
1190 : }
1191 0 : return result;
1192 : }
1193 :
1194 150 : void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
1195 60 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
1196 60 : if (rep == MachineRepresentation::kSimd128) {
1197 : // We have to create the replacements for a phi node before we actually
1198 : // lower the phi to break potential cycles in the graph. The replacements of
1199 : // input nodes do not exist yet, so we use a placeholder node to pass the
1200 : // graph verifier.
1201 6 : int value_count = phi->op()->ValueInputCount();
1202 : SimdType type = ReplacementType(phi);
1203 6 : int num_lanes = NumLanes(type);
1204 6 : Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes);
1205 30 : for (int i = 0; i < num_lanes; ++i) {
1206 48 : inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
1207 24 : inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
1208 : }
1209 12 : for (int i = 0; i < value_count; ++i) {
1210 48 : for (int j = 0; j < num_lanes; ++j) {
1211 48 : inputs_rep[j][i] = placeholder_;
1212 : }
1213 : }
1214 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
1215 30 : for (int i = 0; i < num_lanes; ++i) {
1216 24 : rep_nodes[i] = graph()->NewNode(
1217 : common()->Phi(MachineTypeFrom(type).representation(), value_count),
1218 72 : value_count + 1, inputs_rep[i], false);
1219 : }
1220 6 : ReplaceNode(phi, rep_nodes, num_lanes);
1221 : }
1222 60 : }
1223 : } // namespace compiler
1224 : } // namespace internal
1225 : } // namespace v8
|
