Line data Source code
1 : // Copyright 2016 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/simd-scalar-lowering.h"
6 :
7 : #include "src/compiler/diamond.h"
8 : #include "src/compiler/linkage.h"
9 : #include "src/compiler/node-matchers.h"
10 : #include "src/compiler/node-properties.h"
11 : #include "src/compiler/node.h"
12 : #include "src/compiler/wasm-compiler.h"
13 :
14 : namespace v8 {
15 : namespace internal {
16 : namespace compiler {
17 :
18 : namespace {
19 : static const int kNumLanes32 = 4;
20 : static const int kNumLanes16 = 8;
21 : static const int kNumLanes8 = 16;
22 : static const int32_t kMask16 = 0xFFFF;
23 : static const int32_t kMask8 = 0xFF;
24 : static const int32_t kShift16 = 16;
25 : static const int32_t kShift8 = 24;
26 : } // anonymous
27 :
28 5085 : SimdScalarLowering::SimdScalarLowering(
29 25425 : MachineGraph* mcgraph, Signature<MachineRepresentation>* signature)
30 : : mcgraph_(mcgraph),
31 : state_(mcgraph->graph(), 3),
32 : stack_(mcgraph_->zone()),
33 : replacements_(nullptr),
34 : signature_(signature),
35 : placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
36 10170 : graph()->start())),
37 20340 : parameter_count_after_lowering_(-1) {
38 : DCHECK_NOT_NULL(graph());
39 : DCHECK_NOT_NULL(graph()->end());
40 10170 : replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
41 : memset(static_cast<void*>(replacements_), 0,
42 10170 : sizeof(Replacement) * graph()->NodeCount());
43 5085 : }
44 :
45 15255 : void SimdScalarLowering::LowerGraph() {
46 10170 : stack_.push_back({graph()->end(), 0});
47 5085 : state_.Set(graph()->end(), State::kOnStack);
48 15255 : replacements_[graph()->end()->id()].type = SimdType::kInt32x4;
49 :
50 547965 : while (!stack_.empty()) {
51 : NodeState& top = stack_.back();
52 1075590 : if (top.input_index == top.node->InputCount()) {
53 : // All inputs of top have already been lowered, now lower top.
54 : stack_.pop_back();
55 181480 : state_.Set(top.node, State::kVisited);
56 181480 : LowerNode(top.node);
57 : } else {
58 : // Push the next input onto the stack.
59 532710 : Node* input = top.node->InputAt(top.input_index++);
60 356315 : if (state_.Get(input) == State::kUnvisited) {
61 176395 : SetLoweredType(input, top.node);
62 176395 : if (input->opcode() == IrOpcode::kPhi) {
63 : // To break cycles with phi nodes we push phis on a separate stack so
64 : // that they are processed after all other nodes.
65 110 : PreparePhiReplacement(input);
66 220 : stack_.push_front({input, 0});
67 176285 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
68 : input->opcode() == IrOpcode::kLoop) {
69 40 : stack_.push_front({input, 0});
70 : } else {
71 352530 : stack_.push_back({input, 0});
72 : }
73 : state_.Set(input, State::kOnStack);
74 : }
75 : }
76 : }
77 5085 : }
78 :
79 : #define FOREACH_INT32X4_OPCODE(V) \
80 : V(I32x4Splat) \
81 : V(I32x4ExtractLane) \
82 : V(I32x4ReplaceLane) \
83 : V(I32x4SConvertF32x4) \
84 : V(I32x4UConvertF32x4) \
85 : V(I32x4SConvertI16x8Low) \
86 : V(I32x4SConvertI16x8High) \
87 : V(I32x4Neg) \
88 : V(I32x4Shl) \
89 : V(I32x4ShrS) \
90 : V(I32x4Add) \
91 : V(I32x4AddHoriz) \
92 : V(I32x4Sub) \
93 : V(I32x4Mul) \
94 : V(I32x4MinS) \
95 : V(I32x4MaxS) \
96 : V(I32x4ShrU) \
97 : V(I32x4MinU) \
98 : V(I32x4MaxU) \
99 : V(I32x4Eq) \
100 : V(I32x4Ne) \
101 : V(I32x4LtS) \
102 : V(I32x4LeS) \
103 : V(I32x4GtS) \
104 : V(I32x4GeS) \
105 : V(I32x4UConvertI16x8Low) \
106 : V(I32x4UConvertI16x8High) \
107 : V(I32x4LtU) \
108 : V(I32x4LeU) \
109 : V(I32x4GtU) \
110 : V(I32x4GeU) \
111 : V(S128And) \
112 : V(S128Or) \
113 : V(S128Xor) \
114 : V(S128Not) \
115 : V(S1x4AnyTrue) \
116 : V(S1x4AllTrue) \
117 : V(S1x8AnyTrue) \
118 : V(S1x8AllTrue) \
119 : V(S1x16AnyTrue) \
120 : V(S1x16AllTrue)
121 :
122 : #define FOREACH_FLOAT32X4_OPCODE(V) \
123 : V(F32x4Splat) \
124 : V(F32x4ExtractLane) \
125 : V(F32x4ReplaceLane) \
126 : V(F32x4SConvertI32x4) \
127 : V(F32x4UConvertI32x4) \
128 : V(F32x4Abs) \
129 : V(F32x4Neg) \
130 : V(F32x4RecipApprox) \
131 : V(F32x4RecipSqrtApprox) \
132 : V(F32x4Add) \
133 : V(F32x4AddHoriz) \
134 : V(F32x4Sub) \
135 : V(F32x4Mul) \
136 : V(F32x4Min) \
137 : V(F32x4Max)
138 :
139 : #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \
140 : V(F32x4Eq) \
141 : V(F32x4Ne) \
142 : V(F32x4Lt) \
143 : V(F32x4Le) \
144 : V(F32x4Gt) \
145 : V(F32x4Ge)
146 :
147 : #define FOREACH_INT16X8_OPCODE(V) \
148 : V(I16x8Splat) \
149 : V(I16x8ExtractLane) \
150 : V(I16x8ReplaceLane) \
151 : V(I16x8SConvertI8x16Low) \
152 : V(I16x8SConvertI8x16High) \
153 : V(I16x8Neg) \
154 : V(I16x8Shl) \
155 : V(I16x8ShrS) \
156 : V(I16x8SConvertI32x4) \
157 : V(I16x8Add) \
158 : V(I16x8AddSaturateS) \
159 : V(I16x8AddHoriz) \
160 : V(I16x8Sub) \
161 : V(I16x8SubSaturateS) \
162 : V(I16x8Mul) \
163 : V(I16x8MinS) \
164 : V(I16x8MaxS) \
165 : V(I16x8UConvertI8x16Low) \
166 : V(I16x8UConvertI8x16High) \
167 : V(I16x8ShrU) \
168 : V(I16x8UConvertI32x4) \
169 : V(I16x8AddSaturateU) \
170 : V(I16x8SubSaturateU) \
171 : V(I16x8MinU) \
172 : V(I16x8MaxU) \
173 : V(I16x8Eq) \
174 : V(I16x8Ne) \
175 : V(I16x8LtS) \
176 : V(I16x8LeS) \
177 : V(I16x8LtU) \
178 : V(I16x8LeU)
179 :
180 : #define FOREACH_INT8X16_OPCODE(V) \
181 : V(I8x16Splat) \
182 : V(I8x16ExtractLane) \
183 : V(I8x16ReplaceLane) \
184 : V(I8x16SConvertI16x8) \
185 : V(I8x16Neg) \
186 : V(I8x16Shl) \
187 : V(I8x16ShrS) \
188 : V(I8x16Add) \
189 : V(I8x16AddSaturateS) \
190 : V(I8x16Sub) \
191 : V(I8x16SubSaturateS) \
192 : V(I8x16Mul) \
193 : V(I8x16MinS) \
194 : V(I8x16MaxS) \
195 : V(I8x16ShrU) \
196 : V(I8x16UConvertI16x8) \
197 : V(I8x16AddSaturateU) \
198 : V(I8x16SubSaturateU) \
199 : V(I8x16MinU) \
200 : V(I8x16MaxU) \
201 : V(I8x16Eq) \
202 : V(I8x16Ne) \
203 : V(I8x16LtS) \
204 : V(I8x16LeS) \
205 : V(I8x16LtU) \
206 : V(I8x16LeU) \
207 : V(S8x16Shuffle)
208 :
209 33345 : MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) {
210 33345 : switch (simdType) {
211 : case SimdType::kFloat32x4:
212 : return MachineType::Float32();
213 : case SimdType::kInt32x4:
214 : return MachineType::Int32();
215 : case SimdType::kInt16x8:
216 : return MachineType::Int16();
217 : case SimdType::kInt8x16:
218 : return MachineType::Int8();
219 : }
220 : return MachineType::None();
221 : }
222 :
223 608490 : void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
224 176395 : switch (node->opcode()) {
225 : #define CASE_STMT(name) case IrOpcode::k##name:
226 : FOREACH_INT32X4_OPCODE(CASE_STMT)
227 : case IrOpcode::kReturn:
228 : case IrOpcode::kParameter:
229 : case IrOpcode::kCall: {
230 57590 : replacements_[node->id()].type = SimdType::kInt32x4;
231 28795 : break;
232 : }
233 : FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
234 1450 : replacements_[node->id()].type = SimdType::kFloat32x4;
235 725 : break;
236 : }
237 : FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) {
238 60 : replacements_[node->id()].type = SimdType::kInt32x4;
239 30 : break;
240 : }
241 : FOREACH_INT16X8_OPCODE(CASE_STMT) {
242 8340 : replacements_[node->id()].type = SimdType::kInt16x8;
243 4170 : break;
244 : }
245 : FOREACH_INT8X16_OPCODE(CASE_STMT) {
246 29650 : replacements_[node->id()].type = SimdType::kInt8x16;
247 14825 : break;
248 : }
249 : default: {
250 127850 : switch (output->opcode()) {
251 : case IrOpcode::kF32x4SConvertI32x4:
252 : case IrOpcode::kF32x4UConvertI32x4:
253 : case IrOpcode::kI16x8SConvertI32x4:
254 : case IrOpcode::kI16x8UConvertI32x4: {
255 0 : replacements_[node->id()].type = SimdType::kInt32x4;
256 0 : break;
257 : }
258 : case IrOpcode::kI8x16SConvertI16x8:
259 : case IrOpcode::kI8x16UConvertI16x8:
260 : case IrOpcode::kI32x4SConvertI16x8Low:
261 : case IrOpcode::kI32x4SConvertI16x8High:
262 : case IrOpcode::kI32x4UConvertI16x8Low:
263 : case IrOpcode::kI32x4UConvertI16x8High: {
264 0 : replacements_[node->id()].type = SimdType::kInt16x8;
265 0 : break;
266 : }
267 : case IrOpcode::kI16x8SConvertI8x16Low:
268 : case IrOpcode::kI16x8SConvertI8x16High:
269 : case IrOpcode::kI16x8UConvertI8x16Low:
270 : case IrOpcode::kI16x8UConvertI8x16High: {
271 0 : replacements_[node->id()].type = SimdType::kInt8x16;
272 0 : break;
273 : }
274 : FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT)
275 : case IrOpcode::kI32x4SConvertF32x4:
276 : case IrOpcode::kI32x4UConvertF32x4: {
277 0 : replacements_[node->id()].type = SimdType::kFloat32x4;
278 0 : break;
279 : }
280 : case IrOpcode::kS128Select: {
281 0 : replacements_[node->id()].type = SimdType::kInt32x4;
282 0 : break;
283 : }
284 : default: {
285 383550 : replacements_[node->id()].type = replacements_[output->id()].type;
286 : }
287 : }
288 : }
289 : #undef CASE_STMT
290 : }
291 176395 : }
292 :
293 : static int GetParameterIndexAfterLoweringSimd128(
294 2995 : Signature<MachineRepresentation>* signature, int old_index) {
295 : // In function calls, the simd128 types are passed as 4 Int32 types. The
296 : // parameters are typecast to the types as needed for various operations.
297 : int result = old_index;
298 2995 : for (int i = 0; i < old_index; ++i) {
299 5990 : if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
300 0 : result += 3;
301 : }
302 : }
303 : return result;
304 : }
305 :
306 18230 : int SimdScalarLowering::GetParameterCountAfterLowering() {
307 13145 : if (parameter_count_after_lowering_ == -1) {
308 : // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the
309 : // parameter count after lowering.
310 : parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128(
311 10170 : signature(), static_cast<int>(signature()->parameter_count()));
312 : }
313 13145 : return parameter_count_after_lowering_;
314 : }
315 :
316 : static int GetReturnCountAfterLoweringSimd128(
317 32890 : Signature<MachineRepresentation>* signature) {
318 16445 : int result = static_cast<int>(signature->return_count());
319 16445 : for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
320 32890 : if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
321 0 : result += 3;
322 : }
323 : }
324 : return result;
325 : }
326 :
327 216790 : int SimdScalarLowering::NumLanes(SimdType type) {
328 : int num_lanes = 0;
329 216790 : if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) {
330 : num_lanes = kNumLanes32;
331 91345 : } else if (type == SimdType::kInt16x8) {
332 : num_lanes = kNumLanes16;
333 86610 : } else if (type == SimdType::kInt8x16) {
334 : num_lanes = kNumLanes8;
335 : } else {
336 0 : UNREACHABLE();
337 : }
338 216790 : return num_lanes;
339 : }
340 :
341 : constexpr int SimdScalarLowering::kLaneOffsets[];
342 :
343 16860 : void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices,
344 1003680 : SimdType type) {
345 16860 : int num_lanes = NumLanes(type);
346 16860 : int lane_width = kSimd128Size / num_lanes;
347 16860 : int laneIndex = kLaneOffsets[0] / lane_width;
348 16860 : new_indices[laneIndex] = index;
349 267780 : for (int i = 1; i < num_lanes; ++i) {
350 250920 : laneIndex = kLaneOffsets[i * lane_width] / lane_width;
351 250920 : new_indices[laneIndex] = graph()->NewNode(
352 : machine()->Int32Add(), index,
353 : graph()->NewNode(
354 752760 : common()->Int32Constant(static_cast<int>(i) * lane_width)));
355 : }
356 16860 : }
357 :
358 302095 : void SimdScalarLowering::LowerLoadOp(Node* node, SimdType type) {
359 27005 : MachineRepresentation rep = LoadRepresentationOf(node->op()).representation();
360 : const Operator* load_op;
361 27005 : switch (node->opcode()) {
362 : case IrOpcode::kLoad:
363 53990 : load_op = machine()->Load(MachineTypeFrom(type));
364 26995 : break;
365 : case IrOpcode::kUnalignedLoad:
366 0 : load_op = machine()->UnalignedLoad(MachineTypeFrom(type));
367 0 : break;
368 : case IrOpcode::kProtectedLoad:
369 20 : load_op = machine()->ProtectedLoad(MachineTypeFrom(type));
370 10 : break;
371 : default:
372 0 : UNREACHABLE();
373 : }
374 27005 : if (rep == MachineRepresentation::kSimd128) {
375 : Node* base = node->InputAt(0);
376 : Node* index = node->InputAt(1);
377 13080 : int num_lanes = NumLanes(type);
378 13080 : Node** indices = zone()->NewArray<Node*>(num_lanes);
379 13080 : GetIndexNodes(index, indices, type);
380 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
381 13080 : rep_nodes[0] = node;
382 13080 : rep_nodes[0]->ReplaceInput(1, indices[0]);
383 13080 : NodeProperties::ChangeOp(rep_nodes[0], load_op);
384 13080 : if (node->InputCount() > 2) {
385 : DCHECK_LT(3, node->InputCount());
386 : Node* effect_input = node->InputAt(2);
387 : Node* control_input = node->InputAt(3);
388 208000 : for (int i = num_lanes - 1; i > 0; --i) {
389 194920 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
390 389840 : control_input);
391 : effect_input = rep_nodes[i];
392 : }
393 13080 : rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
394 : } else {
395 0 : for (int i = 1; i < num_lanes; ++i) {
396 0 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
397 : }
398 : }
399 13080 : ReplaceNode(node, rep_nodes, num_lanes);
400 : } else {
401 13925 : DefaultLowering(node);
402 : }
403 27005 : }
404 :
405 78675 : void SimdScalarLowering::LowerStoreOp(Node* node) {
406 : // For store operation, use replacement type of its input instead of the
407 : // one of its effected node.
408 : DCHECK_LT(2, node->InputCount());
409 : SimdType rep_type = ReplacementType(node->InputAt(2));
410 3780 : replacements_[node->id()].type = rep_type;
411 : const Operator* store_op;
412 : MachineRepresentation rep;
413 3780 : switch (node->opcode()) {
414 : case IrOpcode::kStore: {
415 3775 : rep = StoreRepresentationOf(node->op()).representation();
416 : WriteBarrierKind write_barrier_kind =
417 3775 : StoreRepresentationOf(node->op()).write_barrier_kind();
418 : store_op = machine()->Store(StoreRepresentation(
419 7550 : MachineTypeFrom(rep_type).representation(), write_barrier_kind));
420 3775 : break;
421 : }
422 : case IrOpcode::kUnalignedStore: {
423 0 : rep = UnalignedStoreRepresentationOf(node->op());
424 : store_op =
425 0 : machine()->UnalignedStore(MachineTypeFrom(rep_type).representation());
426 0 : break;
427 : }
428 : case IrOpcode::kProtectedStore: {
429 5 : rep = StoreRepresentationOf(node->op()).representation();
430 : store_op =
431 10 : machine()->ProtectedStore(MachineTypeFrom(rep_type).representation());
432 5 : break;
433 : }
434 : default:
435 0 : UNREACHABLE();
436 : }
437 3780 : if (rep == MachineRepresentation::kSimd128) {
438 : Node* base = node->InputAt(0);
439 : Node* index = node->InputAt(1);
440 3780 : int num_lanes = NumLanes(rep_type);
441 3780 : Node** indices = zone()->NewArray<Node*>(num_lanes);
442 3780 : GetIndexNodes(index, indices, rep_type);
443 : Node* value = node->InputAt(2);
444 : DCHECK(HasReplacement(1, value));
445 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
446 3780 : rep_nodes[0] = node;
447 3780 : Node** rep_inputs = GetReplacementsWithType(value, rep_type);
448 3780 : rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
449 3780 : rep_nodes[0]->ReplaceInput(1, indices[0]);
450 3780 : NodeProperties::ChangeOp(node, store_op);
451 3780 : if (node->InputCount() > 3) {
452 : DCHECK_LT(4, node->InputCount());
453 : Node* effect_input = node->InputAt(3);
454 : Node* control_input = node->InputAt(4);
455 59780 : for (int i = num_lanes - 1; i > 0; --i) {
456 56000 : rep_nodes[i] =
457 112000 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i],
458 168000 : effect_input, control_input);
459 : effect_input = rep_nodes[i];
460 : }
461 3780 : rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
462 : } else {
463 0 : for (int i = 1; i < num_lanes; ++i) {
464 0 : rep_nodes[i] =
465 0 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
466 : }
467 : }
468 3780 : ReplaceNode(node, rep_nodes, num_lanes);
469 : } else {
470 0 : DefaultLowering(node);
471 : }
472 3780 : }
473 :
474 85 : void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
475 : const Operator* op,
476 425 : bool not_horizontal) {
477 : DCHECK_EQ(2, node->InputCount());
478 85 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
479 85 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
480 85 : int num_lanes = NumLanes(input_rep_type);
481 85 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
482 85 : if (not_horizontal) {
483 300 : for (int i = 0; i < num_lanes; ++i) {
484 600 : rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
485 : }
486 : } else {
487 20 : for (int i = 0; i < num_lanes / 2; ++i) {
488 40 : rep_node[i] = graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]);
489 20 : rep_node[i + num_lanes / 2] =
490 40 : graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]);
491 : }
492 : }
493 85 : ReplaceNode(node, rep_node, num_lanes);
494 85 : }
495 :
496 220 : void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type,
497 : const Operator* op,
498 7900 : bool invert_inputs) {
499 : DCHECK_EQ(2, node->InputCount());
500 220 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
501 220 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
502 220 : int num_lanes = NumLanes(input_rep_type);
503 220 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
504 2140 : for (int i = 0; i < num_lanes; ++i) {
505 : Node* cmp_result = nullptr;
506 1920 : if (invert_inputs) {
507 1120 : cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]);
508 : } else {
509 800 : cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]);
510 : }
511 : Diamond d_cmp(graph(), common(),
512 : graph()->NewNode(machine()->Word32Equal(), cmp_result,
513 5760 : mcgraph_->Int32Constant(0)));
514 : MachineRepresentation rep =
515 : (input_rep_type == SimdType::kFloat32x4)
516 : ? MachineRepresentation::kWord32
517 1920 : : MachineTypeFrom(input_rep_type).representation();
518 1920 : rep_node[i] =
519 1920 : d_cmp.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
520 : }
521 220 : ReplaceNode(node, rep_node, num_lanes);
522 220 : }
523 :
524 10200 : Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) {
525 : return graph()->NewNode(machine()->Word32Sar(),
526 : graph()->NewNode(machine()->Word32Shl(), input,
527 : mcgraph_->Int32Constant(shift)),
528 8160 : mcgraph_->Int32Constant(shift));
529 : }
530 :
531 35 : void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node,
532 : SimdType input_rep_type,
533 : const Operator* op,
534 435 : bool not_horizontal) {
535 : DCHECK_EQ(2, node->InputCount());
536 : DCHECK(input_rep_type == SimdType::kInt16x8 ||
537 : input_rep_type == SimdType::kInt8x16);
538 35 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
539 35 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
540 35 : int num_lanes = NumLanes(input_rep_type);
541 35 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
542 : int32_t shift_val =
543 35 : (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8;
544 35 : if (not_horizontal) {
545 360 : for (int i = 0; i < num_lanes; ++i) {
546 360 : rep_node[i] = FixUpperBits(
547 720 : graph()->NewNode(op, rep_left[i], rep_right[i]), shift_val);
548 : }
549 : } else {
550 20 : for (int i = 0; i < num_lanes / 2; ++i) {
551 20 : rep_node[i] = FixUpperBits(
552 40 : graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]),
553 60 : shift_val);
554 20 : rep_node[i + num_lanes / 2] = FixUpperBits(
555 40 : graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]),
556 60 : shift_val);
557 : }
558 : }
559 35 : ReplaceNode(node, rep_node, num_lanes);
560 35 : }
561 :
562 5280 : Node* SimdScalarLowering::Mask(Node* input, int32_t mask) {
563 : return graph()->NewNode(machine()->Word32And(), input,
564 5280 : mcgraph_->Int32Constant(mask));
565 : }
566 :
567 40 : void SimdScalarLowering::LowerSaturateBinaryOp(Node* node,
568 : SimdType input_rep_type,
569 : const Operator* op,
570 3400 : bool is_signed) {
571 : DCHECK_EQ(2, node->InputCount());
572 : DCHECK(input_rep_type == SimdType::kInt16x8 ||
573 : input_rep_type == SimdType::kInt8x16);
574 40 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
575 40 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
576 : int32_t min = 0;
577 : int32_t max = 0;
578 : int32_t mask = 0;
579 : int32_t shift_val = 0;
580 : MachineRepresentation phi_rep;
581 40 : if (input_rep_type == SimdType::kInt16x8) {
582 20 : if (is_signed) {
583 : min = std::numeric_limits<int16_t>::min();
584 : max = std::numeric_limits<int16_t>::max();
585 : } else {
586 : min = std::numeric_limits<uint16_t>::min();
587 : max = std::numeric_limits<uint16_t>::max();
588 : }
589 : mask = kMask16;
590 : shift_val = kShift16;
591 : phi_rep = MachineRepresentation::kWord16;
592 : } else {
593 20 : if (is_signed) {
594 : min = std::numeric_limits<int8_t>::min();
595 : max = std::numeric_limits<int8_t>::max();
596 : } else {
597 : min = std::numeric_limits<uint8_t>::min();
598 : max = std::numeric_limits<uint8_t>::max();
599 : }
600 : mask = kMask8;
601 : shift_val = kShift8;
602 : phi_rep = MachineRepresentation::kWord8;
603 : }
604 40 : int num_lanes = NumLanes(input_rep_type);
605 40 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
606 520 : for (int i = 0; i < num_lanes; ++i) {
607 : Node* op_result = nullptr;
608 480 : Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask);
609 480 : Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask);
610 : op_result = graph()->NewNode(op, left, right);
611 : Diamond d_min(graph(), common(),
612 : graph()->NewNode(machine()->Int32LessThan(), op_result,
613 1440 : mcgraph_->Int32Constant(min)));
614 480 : rep_node[i] = d_min.Phi(phi_rep, mcgraph_->Int32Constant(min), op_result);
615 : Diamond d_max(graph(), common(),
616 : graph()->NewNode(machine()->Int32LessThan(),
617 1440 : mcgraph_->Int32Constant(max), rep_node[i]));
618 480 : rep_node[i] = d_max.Phi(phi_rep, mcgraph_->Int32Constant(max), rep_node[i]);
619 : rep_node[i] =
620 480 : is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
621 : }
622 40 : ReplaceNode(node, rep_node, num_lanes);
623 40 : }
624 :
625 20 : void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
626 100 : const Operator* op) {
627 : DCHECK_EQ(1, node->InputCount());
628 20 : Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
629 20 : int num_lanes = NumLanes(input_rep_type);
630 20 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
631 100 : for (int i = 0; i < num_lanes; ++i) {
632 160 : rep_node[i] = graph()->NewNode(op, rep[i]);
633 : }
634 20 : ReplaceNode(node, rep_node, num_lanes);
635 20 : }
636 :
637 60 : void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
638 1180 : bool is_max, SimdType type) {
639 : DCHECK_EQ(2, node->InputCount());
640 60 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), type);
641 60 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), type);
642 60 : int num_lanes = NumLanes(type);
643 60 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
644 : MachineRepresentation rep = MachineRepresentation::kNone;
645 60 : if (type == SimdType::kInt32x4) {
646 : rep = MachineRepresentation::kWord32;
647 40 : } else if (type == SimdType::kInt16x8) {
648 : rep = MachineRepresentation::kWord16;
649 20 : } else if (type == SimdType::kInt8x16) {
650 : rep = MachineRepresentation::kWord8;
651 : } else {
652 0 : UNREACHABLE();
653 : }
654 620 : for (int i = 0; i < num_lanes; ++i) {
655 : Diamond d(graph(), common(),
656 1120 : graph()->NewNode(op, rep_left[i], rep_right[i]));
657 560 : if (is_max) {
658 280 : rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]);
659 : } else {
660 280 : rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]);
661 : }
662 : }
663 60 : ReplaceNode(node, rep_node, num_lanes);
664 60 : }
665 :
666 120 : Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
667 40 : if (machine()->Float64RoundTruncate().IsSupported()) {
668 80 : return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
669 : } else {
670 0 : ExternalReference ref = ExternalReference::wasm_f64_trunc();
671 : Node* stack_slot =
672 0 : graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
673 : const Operator* store_op = machine()->Store(
674 0 : StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
675 : Node* effect =
676 : graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(0),
677 0 : input, graph()->start(), graph()->start());
678 0 : Node* function = graph()->NewNode(common()->ExternalConstant(ref));
679 : Node** args = zone()->NewArray<Node*>(4);
680 0 : args[0] = function;
681 0 : args[1] = stack_slot;
682 0 : args[2] = effect;
683 0 : args[3] = graph()->start();
684 : Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
685 : sig_builder.AddParam(MachineType::Pointer());
686 : auto call_descriptor =
687 0 : Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
688 0 : Node* call = graph()->NewNode(common()->Call(call_descriptor), 4, args);
689 : return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
690 : stack_slot, mcgraph_->Int32Constant(0), call,
691 0 : graph()->start());
692 : }
693 : }
694 :
695 590 : void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
696 : DCHECK_EQ(1, node->InputCount());
697 10 : Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4);
698 : Node* rep_node[kNumLanes32];
699 10 : Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
700 : Node* min = graph()->NewNode(
701 20 : common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
702 : Node* max = graph()->NewNode(common()->Float64Constant(
703 20 : static_cast<double>(is_signed ? kMaxInt : 0xFFFFFFFFu)));
704 50 : for (int i = 0; i < kNumLanes32; ++i) {
705 : Node* double_rep =
706 80 : graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
707 : Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
708 80 : double_rep, double_rep));
709 : Node* temp =
710 40 : nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
711 : Diamond min_d(graph(), common(),
712 80 : graph()->NewNode(machine()->Float64LessThan(), temp, min));
713 40 : temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
714 : Diamond max_d(graph(), common(),
715 80 : graph()->NewNode(machine()->Float64LessThan(), max, temp));
716 40 : temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
717 40 : Node* trunc = BuildF64Trunc(temp);
718 40 : if (is_signed) {
719 40 : rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
720 : } else {
721 : rep_node[i] =
722 40 : graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
723 : }
724 : }
725 10 : ReplaceNode(node, rep_node, kNumLanes32);
726 10 : }
727 :
728 40 : void SimdScalarLowering::LowerConvertFromInt(Node* node,
729 : SimdType input_rep_type,
730 : SimdType output_rep_type,
731 40 : bool is_signed, int start_index) {
732 : DCHECK_EQ(1, node->InputCount());
733 40 : Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
734 :
735 : int32_t mask = 0;
736 40 : if (input_rep_type == SimdType::kInt16x8) {
737 : DCHECK_EQ(output_rep_type, SimdType::kInt32x4);
738 : mask = kMask16;
739 : } else {
740 : DCHECK_EQ(output_rep_type, SimdType::kInt16x8);
741 : DCHECK_EQ(input_rep_type, SimdType::kInt8x16);
742 : mask = kMask8;
743 : }
744 :
745 40 : int num_lanes = NumLanes(output_rep_type);
746 40 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
747 280 : for (int i = 0; i < num_lanes; ++i) {
748 240 : rep_node[i] =
749 240 : is_signed ? rep[i + start_index] : Mask(rep[i + start_index], mask);
750 : }
751 :
752 40 : ReplaceNode(node, rep_node, num_lanes);
753 40 : }
754 :
755 20 : void SimdScalarLowering::LowerPack(Node* node, SimdType input_rep_type,
756 760 : SimdType output_rep_type, bool is_signed) {
757 : DCHECK_EQ(2, node->InputCount());
758 20 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
759 20 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
760 : const Operator* less_op =
761 40 : is_signed ? machine()->Int32LessThan() : machine()->Uint32LessThan();
762 : Node* min = nullptr;
763 : Node* max = nullptr;
764 : int32_t shift_val = 0;
765 : MachineRepresentation phi_rep;
766 20 : if (output_rep_type == SimdType::kInt16x8) {
767 : DCHECK(input_rep_type == SimdType::kInt32x4);
768 10 : if (is_signed) {
769 5 : min = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::min());
770 5 : max = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::max());
771 : } else {
772 5 : max = mcgraph_->Uint32Constant(std::numeric_limits<uint16_t>::max());
773 : shift_val = kShift16;
774 : }
775 : phi_rep = MachineRepresentation::kWord16;
776 : } else {
777 : DCHECK(output_rep_type == SimdType::kInt8x16 &&
778 : input_rep_type == SimdType::kInt16x8);
779 10 : if (is_signed) {
780 5 : min = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::min());
781 5 : max = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::max());
782 : } else {
783 5 : max = mcgraph_->Uint32Constant(std::numeric_limits<uint8_t>::max());
784 : shift_val = kShift8;
785 : }
786 : phi_rep = MachineRepresentation::kWord8;
787 : }
788 20 : int num_lanes = NumLanes(output_rep_type);
789 20 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
790 260 : for (int i = 0; i < num_lanes; ++i) {
791 : Node* input = nullptr;
792 240 : if (i < num_lanes / 2)
793 120 : input = rep_left[i];
794 : else
795 120 : input = rep_right[i - num_lanes / 2];
796 240 : if (is_signed) {
797 120 : Diamond d_min(graph(), common(), graph()->NewNode(less_op, input, min));
798 120 : input = d_min.Phi(phi_rep, min, input);
799 : }
800 240 : Diamond d_max(graph(), common(), graph()->NewNode(less_op, max, input));
801 240 : rep_node[i] = d_max.Phi(phi_rep, max, input);
802 : rep_node[i] =
803 240 : is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
804 : }
805 20 : ReplaceNode(node, rep_node, num_lanes);
806 20 : }
807 :
808 19200 : void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) {
809 : DCHECK_EQ(1, node->InputCount());
810 795 : int32_t shift_amount = OpParameter<int32_t>(node->op());
811 795 : Node* shift_node = graph()->NewNode(common()->Int32Constant(shift_amount));
812 795 : Node** rep = GetReplacementsWithType(node->InputAt(0), type);
813 795 : int num_lanes = NumLanes(type);
814 795 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
815 6135 : for (int i = 0; i < num_lanes; ++i) {
816 5340 : rep_node[i] = rep[i];
817 5340 : switch (node->opcode()) {
818 : case IrOpcode::kI8x16ShrU:
819 560 : rep_node[i] = Mask(rep_node[i], kMask8);
820 : rep_node[i] =
821 1120 : graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
822 560 : break;
823 : case IrOpcode::kI16x8ShrU:
824 600 : rep_node[i] = Mask(rep_node[i], kMask16);
825 : V8_FALLTHROUGH;
826 : case IrOpcode::kI32x4ShrU:
827 : rep_node[i] =
828 3660 : graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
829 1220 : break;
830 : case IrOpcode::kI32x4Shl:
831 : rep_node[i] =
832 1240 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
833 620 : break;
834 : case IrOpcode::kI16x8Shl:
835 : rep_node[i] =
836 1200 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
837 600 : rep_node[i] = FixUpperBits(rep_node[i], kShift16);
838 600 : break;
839 : case IrOpcode::kI8x16Shl:
840 : rep_node[i] =
841 1120 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
842 560 : rep_node[i] = FixUpperBits(rep_node[i], kShift8);
843 560 : break;
844 : case IrOpcode::kI32x4ShrS:
845 : case IrOpcode::kI16x8ShrS:
846 : case IrOpcode::kI8x16ShrS:
847 : rep_node[i] =
848 3560 : graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node);
849 1780 : break;
850 : default:
851 0 : UNREACHABLE();
852 : }
853 : }
854 795 : ReplaceNode(node, rep_node, num_lanes);
855 795 : }
856 :
857 80 : void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
858 1520 : const Operator* op) {
859 : DCHECK_EQ(2, node->InputCount());
860 80 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
861 80 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
862 80 : int num_lanes = NumLanes(input_rep_type);
863 80 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
864 800 : for (int i = 0; i < num_lanes; ++i) {
865 : Diamond d(graph(), common(),
866 1440 : graph()->NewNode(op, rep_left[i], rep_right[i]));
867 : MachineRepresentation rep =
868 : (input_rep_type == SimdType::kFloat32x4)
869 : ? MachineRepresentation::kWord32
870 720 : : MachineTypeFrom(input_rep_type).representation();
871 720 : rep_node[i] =
872 720 : d.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
873 : }
874 80 : ReplaceNode(node, rep_node, num_lanes);
875 80 : }
876 :
877 421110 : void SimdScalarLowering::LowerNode(Node* node) {
878 : SimdType rep_type = ReplacementType(node);
879 181480 : int num_lanes = NumLanes(rep_type);
880 181480 : switch (node->opcode()) {
881 : case IrOpcode::kStart: {
882 5085 : int parameter_count = GetParameterCountAfterLowering();
883 : // Only exchange the node if the parameter count actually changed.
884 5085 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
885 : int delta =
886 0 : parameter_count - static_cast<int>(signature()->parameter_count());
887 0 : int new_output_count = node->op()->ValueOutputCount() + delta;
888 0 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
889 : }
890 : break;
891 : }
892 : case IrOpcode::kParameter: {
893 : DCHECK_EQ(1, node->InputCount());
894 : // Only exchange the node if the parameter count actually changed. We do
895 : // not even have to do the default lowering because the the start node,
896 : // the only input of a parameter node, only changes if the parameter count
897 : // changes.
898 16120 : if (GetParameterCountAfterLowering() !=
899 8060 : static_cast<int>(signature()->parameter_count())) {
900 0 : int old_index = ParameterIndexOf(node->op());
901 : int new_index =
902 : GetParameterIndexAfterLoweringSimd128(signature(), old_index);
903 0 : if (old_index == new_index) {
904 0 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
905 :
906 : Node* new_node[kNumLanes32];
907 0 : for (int i = 0; i < kNumLanes32; ++i) {
908 0 : new_node[i] = nullptr;
909 : }
910 0 : new_node[0] = node;
911 0 : if (signature()->GetParam(old_index) ==
912 : MachineRepresentation::kSimd128) {
913 0 : for (int i = 1; i < kNumLanes32; ++i) {
914 : new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
915 0 : graph()->start());
916 : }
917 : }
918 0 : ReplaceNode(node, new_node, kNumLanes32);
919 : }
920 : }
921 : break;
922 : }
923 : case IrOpcode::kLoad:
924 : case IrOpcode::kUnalignedLoad:
925 : case IrOpcode::kProtectedLoad: {
926 27005 : LowerLoadOp(node, rep_type);
927 27005 : break;
928 : }
929 : case IrOpcode::kStore:
930 : case IrOpcode::kUnalignedStore:
931 : case IrOpcode::kProtectedStore: {
932 3780 : LowerStoreOp(node);
933 3780 : break;
934 : }
935 : case IrOpcode::kReturn: {
936 16445 : DefaultLowering(node);
937 : int new_return_count = GetReturnCountAfterLoweringSimd128(signature());
938 16445 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
939 0 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
940 : }
941 : break;
942 : }
943 : case IrOpcode::kCall: {
944 : // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor.
945 0 : auto call_descriptor =
946 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
947 0 : if (DefaultLowering(node) ||
948 0 : (call_descriptor->ReturnCount() == 1 &&
949 : call_descriptor->GetReturnType(0) == MachineType::Simd128())) {
950 : // We have to adjust the call descriptor.
951 : const Operator* op = common()->Call(
952 0 : GetI32WasmCallDescriptorForSimd(zone(), call_descriptor));
953 0 : NodeProperties::ChangeOp(node, op);
954 : }
955 0 : if (call_descriptor->ReturnCount() == 1 &&
956 : call_descriptor->GetReturnType(0) == MachineType::Simd128()) {
957 : // We access the additional return values through projections.
958 : Node* rep_node[kNumLanes32];
959 0 : for (int i = 0; i < kNumLanes32; ++i) {
960 : rep_node[i] =
961 0 : graph()->NewNode(common()->Projection(i), node, graph()->start());
962 : }
963 0 : ReplaceNode(node, rep_node, kNumLanes32);
964 : }
965 : break;
966 : }
967 : case IrOpcode::kPhi: {
968 110 : MachineRepresentation rep = PhiRepresentationOf(node->op());
969 110 : if (rep == MachineRepresentation::kSimd128) {
970 : // The replacement nodes have already been created, we only have to
971 : // replace placeholder nodes.
972 : Node** rep_node = GetReplacements(node);
973 90 : for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
974 : Node** rep_input =
975 20 : GetReplacementsWithType(node->InputAt(i), rep_type);
976 100 : for (int j = 0; j < num_lanes; j++) {
977 80 : rep_node[j]->ReplaceInput(i, rep_input[j]);
978 : }
979 : }
980 : } else {
981 100 : DefaultLowering(node);
982 : }
983 : break;
984 : }
985 : #define I32X4_BINOP_CASE(opcode, instruction) \
986 : case IrOpcode::opcode: { \
987 : LowerBinaryOp(node, rep_type, machine()->instruction()); \
988 : break; \
989 : }
990 15 : I32X4_BINOP_CASE(kI32x4Add, Int32Add)
991 5 : I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
992 5 : I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
993 5 : I32X4_BINOP_CASE(kS128And, Word32And)
994 5 : I32X4_BINOP_CASE(kS128Or, Word32Or)
995 5 : I32X4_BINOP_CASE(kS128Xor, Word32Xor)
996 : #undef I32X4_BINOP_CASE
997 : case IrOpcode::kI32x4AddHoriz: {
998 5 : LowerBinaryOp(node, rep_type, machine()->Int32Add(), false);
999 5 : break;
1000 : }
1001 : case IrOpcode::kI16x8AddHoriz: {
1002 5 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add(), false);
1003 5 : break;
1004 : }
1005 : case IrOpcode::kI16x8Add:
1006 : case IrOpcode::kI8x16Add: {
1007 10 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add());
1008 10 : break;
1009 : }
1010 : case IrOpcode::kI16x8Sub:
1011 : case IrOpcode::kI8x16Sub: {
1012 10 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub());
1013 10 : break;
1014 : }
1015 : case IrOpcode::kI16x8Mul:
1016 : case IrOpcode::kI8x16Mul: {
1017 10 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul());
1018 10 : break;
1019 : }
1020 : case IrOpcode::kI16x8AddSaturateS:
1021 : case IrOpcode::kI8x16AddSaturateS: {
1022 10 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true);
1023 10 : break;
1024 : }
1025 : case IrOpcode::kI16x8SubSaturateS:
1026 : case IrOpcode::kI8x16SubSaturateS: {
1027 10 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true);
1028 10 : break;
1029 : }
1030 : case IrOpcode::kI16x8AddSaturateU:
1031 : case IrOpcode::kI8x16AddSaturateU: {
1032 10 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false);
1033 10 : break;
1034 : }
1035 : case IrOpcode::kI16x8SubSaturateU:
1036 : case IrOpcode::kI8x16SubSaturateU: {
1037 10 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false);
1038 10 : break;
1039 : }
1040 : case IrOpcode::kI32x4MaxS:
1041 : case IrOpcode::kI16x8MaxS:
1042 : case IrOpcode::kI8x16MaxS: {
1043 15 : LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type);
1044 15 : break;
1045 : }
1046 : case IrOpcode::kI32x4MinS:
1047 : case IrOpcode::kI16x8MinS:
1048 : case IrOpcode::kI8x16MinS: {
1049 15 : LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type);
1050 15 : break;
1051 : }
1052 : case IrOpcode::kI32x4MaxU:
1053 : case IrOpcode::kI16x8MaxU:
1054 : case IrOpcode::kI8x16MaxU: {
1055 15 : LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type);
1056 15 : break;
1057 : }
1058 : case IrOpcode::kI32x4MinU:
1059 : case IrOpcode::kI16x8MinU:
1060 : case IrOpcode::kI8x16MinU: {
1061 15 : LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type);
1062 15 : break;
1063 : }
1064 : case IrOpcode::kI32x4Neg:
1065 : case IrOpcode::kI16x8Neg:
1066 : case IrOpcode::kI8x16Neg: {
1067 : DCHECK_EQ(1, node->InputCount());
1068 15 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1069 15 : int num_lanes = NumLanes(rep_type);
1070 15 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1071 15 : Node* zero = graph()->NewNode(common()->Int32Constant(0));
1072 155 : for (int i = 0; i < num_lanes; ++i) {
1073 420 : rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
1074 140 : if (node->opcode() == IrOpcode::kI16x8Neg) {
1075 40 : rep_node[i] = FixUpperBits(rep_node[i], kShift16);
1076 100 : } else if (node->opcode() == IrOpcode::kI8x16Neg) {
1077 80 : rep_node[i] = FixUpperBits(rep_node[i], kShift8);
1078 : }
1079 : }
1080 15 : ReplaceNode(node, rep_node, num_lanes);
1081 15 : break;
1082 : }
1083 : case IrOpcode::kS128Zero: {
1084 : DCHECK_EQ(0, node->InputCount());
1085 : Node* rep_node[kNumLanes32];
1086 0 : for (int i = 0; i < kNumLanes32; ++i) {
1087 0 : rep_node[i] = mcgraph_->Int32Constant(0);
1088 : }
1089 0 : ReplaceNode(node, rep_node, kNumLanes32);
1090 : break;
1091 : }
1092 : case IrOpcode::kS128Not: {
1093 : DCHECK_EQ(1, node->InputCount());
1094 5 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1095 : Node* rep_node[kNumLanes32];
1096 5 : Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF));
1097 25 : for (int i = 0; i < kNumLanes32; ++i) {
1098 60 : rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
1099 : }
1100 5 : ReplaceNode(node, rep_node, kNumLanes32);
1101 : break;
1102 : }
1103 : case IrOpcode::kI32x4SConvertF32x4: {
1104 5 : LowerConvertFromFloat(node, true);
1105 5 : break;
1106 : }
1107 : case IrOpcode::kI32x4UConvertF32x4: {
1108 5 : LowerConvertFromFloat(node, false);
1109 5 : break;
1110 : }
1111 : case IrOpcode::kI32x4SConvertI16x8Low: {
1112 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1113 5 : 0);
1114 5 : break;
1115 : }
1116 : case IrOpcode::kI32x4SConvertI16x8High: {
1117 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1118 5 : 4);
1119 5 : break;
1120 : }
1121 : case IrOpcode::kI32x4UConvertI16x8Low: {
1122 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1123 5 : 0);
1124 5 : break;
1125 : }
1126 : case IrOpcode::kI32x4UConvertI16x8High: {
1127 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1128 5 : 4);
1129 5 : break;
1130 : }
1131 : case IrOpcode::kI16x8SConvertI8x16Low: {
1132 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1133 5 : 0);
1134 5 : break;
1135 : }
1136 : case IrOpcode::kI16x8SConvertI8x16High: {
1137 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1138 5 : 8);
1139 5 : break;
1140 : }
1141 : case IrOpcode::kI16x8UConvertI8x16Low: {
1142 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1143 5 : 0);
1144 5 : break;
1145 : }
1146 : case IrOpcode::kI16x8UConvertI8x16High: {
1147 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1148 5 : 8);
1149 5 : break;
1150 : }
1151 : case IrOpcode::kI16x8SConvertI32x4: {
1152 5 : LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, true);
1153 5 : break;
1154 : }
1155 : case IrOpcode::kI16x8UConvertI32x4: {
1156 5 : LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, false);
1157 5 : break;
1158 : }
1159 : case IrOpcode::kI8x16SConvertI16x8: {
1160 5 : LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, true);
1161 5 : break;
1162 : }
1163 : case IrOpcode::kI8x16UConvertI16x8: {
1164 5 : LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, false);
1165 5 : break;
1166 : }
1167 : case IrOpcode::kI32x4Shl:
1168 : case IrOpcode::kI16x8Shl:
1169 : case IrOpcode::kI8x16Shl:
1170 : case IrOpcode::kI32x4ShrS:
1171 : case IrOpcode::kI16x8ShrS:
1172 : case IrOpcode::kI8x16ShrS:
1173 : case IrOpcode::kI32x4ShrU:
1174 : case IrOpcode::kI16x8ShrU:
1175 : case IrOpcode::kI8x16ShrU: {
1176 795 : LowerShiftOp(node, rep_type);
1177 795 : break;
1178 : }
1179 : case IrOpcode::kF32x4AddHoriz: {
1180 5 : LowerBinaryOp(node, rep_type, machine()->Float32Add(), false);
1181 5 : break;
1182 : }
1183 : #define F32X4_BINOP_CASE(name) \
1184 : case IrOpcode::kF32x4##name: { \
1185 : LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
1186 : break; \
1187 : }
1188 15 : F32X4_BINOP_CASE(Add)
1189 5 : F32X4_BINOP_CASE(Sub)
1190 5 : F32X4_BINOP_CASE(Mul)
1191 5 : F32X4_BINOP_CASE(Min)
1192 5 : F32X4_BINOP_CASE(Max)
1193 : #undef F32X4_BINOP_CASE
1194 : #define F32X4_UNOP_CASE(name) \
1195 : case IrOpcode::kF32x4##name: { \
1196 : LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
1197 : break; \
1198 : }
1199 5 : F32X4_UNOP_CASE(Abs)
1200 5 : F32X4_UNOP_CASE(Neg)
1201 : #undef F32x4_UNOP_CASE
1202 : case IrOpcode::kF32x4RecipApprox:
1203 : case IrOpcode::kF32x4RecipSqrtApprox: {
1204 : DCHECK_EQ(1, node->InputCount());
1205 10 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1206 10 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1207 10 : Node* float_one = graph()->NewNode(common()->Float32Constant(1.0));
1208 50 : for (int i = 0; i < num_lanes; ++i) {
1209 40 : Node* tmp = rep[i];
1210 40 : if (node->opcode() == IrOpcode::kF32x4RecipSqrtApprox) {
1211 20 : tmp = graph()->NewNode(machine()->Float32Sqrt(), rep[i]);
1212 : }
1213 120 : rep_node[i] = graph()->NewNode(machine()->Float32Div(), float_one, tmp);
1214 : }
1215 10 : ReplaceNode(node, rep_node, num_lanes);
1216 10 : break;
1217 : }
1218 : case IrOpcode::kF32x4SConvertI32x4: {
1219 5 : LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32());
1220 5 : break;
1221 : }
1222 : case IrOpcode::kF32x4UConvertI32x4: {
1223 5 : LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32());
1224 5 : break;
1225 : }
1226 : case IrOpcode::kI32x4Splat:
1227 : case IrOpcode::kF32x4Splat:
1228 : case IrOpcode::kI16x8Splat:
1229 : case IrOpcode::kI8x16Splat: {
1230 1735 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1231 14635 : for (int i = 0; i < num_lanes; ++i) {
1232 12900 : if (HasReplacement(0, node->InputAt(0))) {
1233 40 : rep_node[i] = GetReplacements(node->InputAt(0))[0];
1234 : } else {
1235 25760 : rep_node[i] = node->InputAt(0);
1236 : }
1237 : }
1238 1735 : ReplaceNode(node, rep_node, num_lanes);
1239 1735 : break;
1240 : }
1241 : case IrOpcode::kI32x4ExtractLane:
1242 : case IrOpcode::kF32x4ExtractLane:
1243 : case IrOpcode::kI16x8ExtractLane:
1244 : case IrOpcode::kI8x16ExtractLane: {
1245 11350 : int32_t lane = OpParameter<int32_t>(node->op());
1246 11350 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1247 11350 : rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane];
1248 117560 : for (int i = 1; i < num_lanes; ++i) {
1249 106210 : rep_node[i] = nullptr;
1250 : }
1251 11350 : ReplaceNode(node, rep_node, num_lanes);
1252 11350 : break;
1253 : }
1254 : case IrOpcode::kI32x4ReplaceLane:
1255 : case IrOpcode::kF32x4ReplaceLane:
1256 : case IrOpcode::kI16x8ReplaceLane:
1257 : case IrOpcode::kI8x16ReplaceLane: {
1258 : DCHECK_EQ(2, node->InputCount());
1259 : Node* repNode = node->InputAt(1);
1260 230 : int32_t lane = OpParameter<int32_t>(node->op());
1261 230 : Node** old_rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
1262 230 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1263 2430 : for (int i = 0; i < num_lanes; ++i) {
1264 2200 : rep_node[i] = old_rep_node[i];
1265 : }
1266 230 : if (HasReplacement(0, repNode)) {
1267 0 : rep_node[lane] = GetReplacements(repNode)[0];
1268 : } else {
1269 230 : rep_node[lane] = repNode;
1270 : }
1271 230 : ReplaceNode(node, rep_node, num_lanes);
1272 230 : break;
1273 : }
1274 : #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \
1275 : case IrOpcode::simd_op: { \
1276 : LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
1277 : break; \
1278 : }
1279 5 : COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false)
1280 10 : COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false)
1281 10 : COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false)
1282 0 : COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true)
1283 0 : COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true)
1284 25 : COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false)
1285 0 : COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false)
1286 0 : COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false)
1287 10 : COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true)
1288 10 : COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true)
1289 0 : COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false)
1290 0 : COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false)
1291 10 : COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true)
1292 10 : COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true)
1293 25 : COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false)
1294 0 : COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false)
1295 0 : COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false)
1296 10 : COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true)
1297 10 : COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true)
1298 0 : COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false)
1299 0 : COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false)
1300 10 : COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true)
1301 10 : COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true)
1302 25 : COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false)
1303 0 : COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false)
1304 0 : COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false)
1305 10 : COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true)
1306 10 : COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true)
1307 0 : COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false)
1308 0 : COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false)
1309 10 : COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true)
1310 10 : COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true)
1311 : #undef COMPARISON_CASE
1312 : case IrOpcode::kF32x4Ne: {
1313 5 : LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal());
1314 5 : break;
1315 : }
1316 : case IrOpcode::kI32x4Ne: {
1317 25 : LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal());
1318 25 : break;
1319 : }
1320 : case IrOpcode::kI16x8Ne: {
1321 25 : LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal());
1322 25 : break;
1323 : }
1324 : case IrOpcode::kI8x16Ne: {
1325 25 : LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal());
1326 25 : break;
1327 : }
1328 : case IrOpcode::kS128Select: {
1329 : DCHECK_EQ(3, node->InputCount());
1330 : DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 ||
1331 : ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 ||
1332 : ReplacementType(node->InputAt(0)) == SimdType::kInt8x16);
1333 : Node** boolean_input = GetReplacements(node->InputAt(0));
1334 0 : Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
1335 0 : Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
1336 0 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1337 0 : for (int i = 0; i < num_lanes; ++i) {
1338 : Node* tmp1 =
1339 0 : graph()->NewNode(machine()->Word32Xor(), rep_left[i], rep_right[i]);
1340 : Node* tmp2 =
1341 0 : graph()->NewNode(machine()->Word32And(), boolean_input[i], tmp1);
1342 0 : rep_node[i] =
1343 0 : graph()->NewNode(machine()->Word32Xor(), rep_right[i], tmp2);
1344 : }
1345 0 : ReplaceNode(node, rep_node, num_lanes);
1346 0 : break;
1347 : }
1348 : case IrOpcode::kS8x16Shuffle: {
1349 : DCHECK_EQ(2, node->InputCount());
1350 9250 : const uint8_t* shuffle = OpParameter<uint8_t*>(node->op());
1351 9250 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
1352 9250 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
1353 : Node** rep_node = zone()->NewArray<Node*>(16);
1354 157250 : for (int i = 0; i < 16; i++) {
1355 148000 : int lane = shuffle[i];
1356 148000 : rep_node[i] = lane < 16 ? rep_left[lane] : rep_right[lane - 16];
1357 : }
1358 9250 : ReplaceNode(node, rep_node, 16);
1359 9250 : break;
1360 : }
1361 : case IrOpcode::kS1x4AnyTrue:
1362 : case IrOpcode::kS1x4AllTrue:
1363 : case IrOpcode::kS1x8AnyTrue:
1364 : case IrOpcode::kS1x8AllTrue:
1365 : case IrOpcode::kS1x16AnyTrue:
1366 : case IrOpcode::kS1x16AllTrue: {
1367 : DCHECK_EQ(1, node->InputCount());
1368 : SimdType input_rep_type = ReplacementType(node->InputAt(0));
1369 120 : int input_num_lanes = NumLanes(input_rep_type);
1370 : Node** rep = GetReplacements(node->InputAt(0));
1371 120 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1372 120 : Node* true_node = mcgraph_->Int32Constant(-1);
1373 120 : Node* false_node = mcgraph_->Int32Constant(0);
1374 : Node* tmp_result = false_node;
1375 220 : if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1376 200 : node->opcode() == IrOpcode::kS1x8AllTrue ||
1377 : node->opcode() == IrOpcode::kS1x16AllTrue) {
1378 : tmp_result = true_node;
1379 : }
1380 1120 : for (int i = 0; i < input_num_lanes; ++i) {
1381 : Diamond is_false(
1382 : graph(), common(),
1383 3360 : graph()->NewNode(machine()->Word32Equal(), rep[i], false_node));
1384 2160 : if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1385 2000 : node->opcode() == IrOpcode::kS1x8AllTrue ||
1386 : node->opcode() == IrOpcode::kS1x16AllTrue) {
1387 : tmp_result = is_false.Phi(MachineRepresentation::kWord32, false_node,
1388 560 : tmp_result);
1389 : } else {
1390 : tmp_result = is_false.Phi(MachineRepresentation::kWord32, tmp_result,
1391 560 : true_node);
1392 : }
1393 : }
1394 120 : rep_node[0] = tmp_result;
1395 480 : for (int i = 1; i < num_lanes; ++i) {
1396 360 : rep_node[i] = nullptr;
1397 : }
1398 120 : ReplaceNode(node, rep_node, num_lanes);
1399 120 : break;
1400 : }
1401 96875 : default: { DefaultLowering(node); }
1402 : }
1403 181480 : }
1404 :
1405 127345 : bool SimdScalarLowering::DefaultLowering(Node* node) {
1406 : bool something_changed = false;
1407 258285 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1408 : Node* input = node->InputAt(i);
1409 130940 : if (HasReplacement(0, input)) {
1410 : something_changed = true;
1411 11465 : node->ReplaceInput(i, GetReplacements(input)[0]);
1412 : }
1413 130940 : if (HasReplacement(1, input)) {
1414 : something_changed = true;
1415 0 : for (int j = 1; j < ReplacementCount(input); ++j) {
1416 0 : node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
1417 : }
1418 : }
1419 : }
1420 127345 : return something_changed;
1421 : }
1422 :
1423 723480 : void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) {
1424 122970 : replacements_[old->id()].node = zone()->NewArray<Node*>(count);
1425 600510 : for (int i = 0; i < count; ++i) {
1426 1119040 : replacements_[old->id()].node[i] = new_nodes[i];
1427 : }
1428 81980 : replacements_[old->id()].num_replacements = count;
1429 40990 : }
1430 :
1431 275010 : bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
1432 572970 : return replacements_[node->id()].node != nullptr &&
1433 22950 : replacements_[node->id()].node[index] != nullptr;
1434 : }
1435 :
1436 185440 : SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
1437 406615 : return replacements_[node->id()].type;
1438 : }
1439 :
1440 36005 : Node** SimdScalarLowering::GetReplacements(Node* node) {
1441 71870 : Node** result = replacements_[node->id()].node;
1442 : DCHECK(result);
1443 0 : return result;
1444 : }
1445 :
1446 0 : int SimdScalarLowering::ReplacementCount(Node* node) {
1447 0 : return replacements_[node->id()].num_replacements;
1448 : }
1449 :
1450 315 : void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) {
1451 175 : for (int i = 0; i < kNumLanes32; ++i) {
1452 140 : if (replacements[i] != nullptr) {
1453 140 : result[i] =
1454 280 : graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]);
1455 : } else {
1456 0 : result[i] = nullptr;
1457 : }
1458 : }
1459 35 : }
1460 :
1461 135 : void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) {
1462 75 : for (int i = 0; i < kNumLanes32; ++i) {
1463 60 : if (replacements[i] != nullptr) {
1464 60 : result[i] =
1465 120 : graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]);
1466 : } else {
1467 0 : result[i] = nullptr;
1468 : }
1469 : }
1470 15 : }
1471 :
1472 : template <typename T>
1473 0 : void SimdScalarLowering::Int32ToSmallerInt(Node** replacements, Node** result) {
1474 : const int num_ints = sizeof(int32_t) / sizeof(T);
1475 : const int bit_size = sizeof(T) * 8;
1476 : const Operator* sign_extend;
1477 : switch (sizeof(T)) {
1478 : case 1:
1479 0 : sign_extend = machine()->SignExtendWord8ToInt32();
1480 : break;
1481 : case 2:
1482 0 : sign_extend = machine()->SignExtendWord16ToInt32();
1483 : break;
1484 : default:
1485 : UNREACHABLE();
1486 : }
1487 :
1488 0 : for (int i = 0; i < kNumLanes32; i++) {
1489 0 : if (replacements[i] != nullptr) {
1490 0 : for (int j = 0; j < num_ints; j++) {
1491 0 : result[num_ints * i + j] = graph()->NewNode(
1492 : sign_extend,
1493 : graph()->NewNode(machine()->Word32Sar(), replacements[i],
1494 0 : mcgraph_->Int32Constant(j * bit_size)));
1495 : }
1496 : } else {
1497 0 : for (int j = 0; j < num_ints; j++) {
1498 0 : result[num_ints * i + j] = nullptr;
1499 : }
1500 : }
1501 : }
1502 0 : }
1503 :
1504 : template <typename T>
1505 0 : void SimdScalarLowering::SmallerIntToInt32(Node** replacements, Node** result) {
1506 : const int num_ints = sizeof(int32_t) / sizeof(T);
1507 : const int bit_size = sizeof(T) * 8;
1508 : const int bit_mask = (1 << bit_size) - 1;
1509 :
1510 0 : for (int i = 0; i < kNumLanes32; ++i) {
1511 0 : result[i] = mcgraph_->Int32Constant(0);
1512 0 : for (int j = 0; j < num_ints; j++) {
1513 0 : if (replacements[num_ints * i + j] != nullptr) {
1514 : Node* clean_bits = graph()->NewNode(machine()->Word32And(),
1515 : replacements[num_ints * i + j],
1516 0 : mcgraph_->Int32Constant(bit_mask));
1517 : Node* shift = graph()->NewNode(machine()->Word32Shl(), clean_bits,
1518 0 : mcgraph_->Int32Constant(j * bit_size));
1519 0 : result[i] = graph()->NewNode(machine()->Word32Or(), result[i], shift);
1520 : }
1521 : }
1522 : }
1523 0 : }
1524 :
1525 35905 : Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
1526 : Node** replacements = GetReplacements(node);
1527 35855 : if (ReplacementType(node) == type) {
1528 : return GetReplacements(node);
1529 : }
1530 50 : int num_lanes = NumLanes(type);
1531 50 : Node** result = zone()->NewArray<Node*>(num_lanes);
1532 50 : if (type == SimdType::kInt32x4) {
1533 15 : if (ReplacementType(node) == SimdType::kFloat32x4) {
1534 15 : Float32ToInt32(replacements, result);
1535 0 : } else if (ReplacementType(node) == SimdType::kInt16x8) {
1536 0 : SmallerIntToInt32<int16_t>(replacements, result);
1537 0 : } else if (ReplacementType(node) == SimdType::kInt8x16) {
1538 0 : SmallerIntToInt32<int8_t>(replacements, result);
1539 : } else {
1540 0 : UNREACHABLE();
1541 : }
1542 35 : } else if (type == SimdType::kFloat32x4) {
1543 35 : if (ReplacementType(node) == SimdType::kInt32x4) {
1544 35 : Int32ToFloat32(replacements, result);
1545 0 : } else if (ReplacementType(node) == SimdType::kInt16x8) {
1546 0 : UNIMPLEMENTED();
1547 : } else {
1548 0 : UNREACHABLE();
1549 : }
1550 0 : } else if (type == SimdType::kInt16x8) {
1551 0 : if (ReplacementType(node) == SimdType::kInt32x4) {
1552 0 : Int32ToSmallerInt<int16_t>(replacements, result);
1553 0 : } else if (ReplacementType(node) == SimdType::kFloat32x4) {
1554 0 : UNIMPLEMENTED();
1555 : } else {
1556 0 : UNREACHABLE();
1557 : }
1558 0 : } else if (type == SimdType::kInt8x16) {
1559 0 : if (ReplacementType(node) == SimdType::kInt32x4) {
1560 0 : Int32ToSmallerInt<int8_t>(replacements, result);
1561 : } else {
1562 0 : UNIMPLEMENTED();
1563 : }
1564 : } else {
1565 0 : UNREACHABLE();
1566 : }
1567 50 : return result;
1568 : }
1569 :
1570 260 : void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
1571 110 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
1572 110 : if (rep == MachineRepresentation::kSimd128) {
1573 : // We have to create the replacements for a phi node before we actually
1574 : // lower the phi to break potential cycles in the graph. The replacements of
1575 : // input nodes do not exist yet, so we use a placeholder node to pass the
1576 : // graph verifier.
1577 10 : int value_count = phi->op()->ValueInputCount();
1578 : SimdType type = ReplacementType(phi);
1579 10 : int num_lanes = NumLanes(type);
1580 10 : Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes);
1581 50 : for (int i = 0; i < num_lanes; ++i) {
1582 80 : inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
1583 40 : inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
1584 : }
1585 20 : for (int i = 0; i < value_count; ++i) {
1586 80 : for (int j = 0; j < num_lanes; ++j) {
1587 80 : inputs_rep[j][i] = placeholder_;
1588 : }
1589 : }
1590 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
1591 50 : for (int i = 0; i < num_lanes; ++i) {
1592 40 : rep_nodes[i] = graph()->NewNode(
1593 : common()->Phi(MachineTypeFrom(type).representation(), value_count),
1594 120 : value_count + 1, inputs_rep[i], false);
1595 : }
1596 10 : ReplaceNode(phi, rep_nodes, num_lanes);
1597 : }
1598 110 : }
1599 : } // namespace compiler
1600 : } // namespace internal
1601 183867 : } // namespace v8
|
