Line data Source code
1 : // Copyright 2016 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/simd-scalar-lowering.h"
6 :
7 : #include "src/compiler/diamond.h"
8 : #include "src/compiler/linkage.h"
9 : #include "src/compiler/node-matchers.h"
10 : #include "src/compiler/node-properties.h"
11 : #include "src/compiler/node.h"
12 : #include "src/compiler/wasm-compiler.h"
13 :
14 : namespace v8 {
15 : namespace internal {
16 : namespace compiler {
17 :
18 : namespace {
19 : static const int kNumLanes32 = 4;
20 : static const int kNumLanes16 = 8;
21 : static const int kNumLanes8 = 16;
22 : static const int32_t kMask16 = 0xFFFF;
23 : static const int32_t kMask8 = 0xFF;
24 : static const int32_t kShift16 = 16;
25 : static const int32_t kShift8 = 24;
26 : } // anonymous
27 :
28 4084 : SimdScalarLowering::SimdScalarLowering(
29 : MachineGraph* mcgraph, Signature<MachineRepresentation>* signature)
30 : : mcgraph_(mcgraph),
31 : state_(mcgraph->graph(), 3),
32 4084 : stack_(mcgraph_->zone()),
33 : replacements_(nullptr),
34 : signature_(signature),
35 4084 : placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
36 : graph()->start())),
37 12252 : parameter_count_after_lowering_(-1) {
38 : DCHECK_NOT_NULL(graph());
39 : DCHECK_NOT_NULL(graph()->end());
40 4084 : replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
41 4084 : memset(static_cast<void*>(replacements_), 0,
42 : sizeof(Replacement) * graph()->NodeCount());
43 4084 : }
44 :
45 4084 : void SimdScalarLowering::LowerGraph() {
46 8168 : stack_.push_back({graph()->end(), 0});
47 : state_.Set(graph()->end(), State::kOnStack);
48 8168 : replacements_[graph()->end()->id()].type = SimdType::kInt32x4;
49 :
50 250640 : while (!stack_.empty()) {
51 : NodeState& top = stack_.back();
52 493112 : if (top.input_index == top.node->InputCount()) {
53 : // All inputs of top have already been lowered, now lower top.
54 83216 : stack_.pop_back();
55 83216 : state_.Set(top.node, State::kVisited);
56 83216 : LowerNode(top.node);
57 : } else {
58 : // Push the next input onto the stack.
59 163340 : Node* input = top.node->InputAt(top.input_index++);
60 163340 : if (state_.Get(input) == State::kUnvisited) {
61 79132 : SetLoweredType(input, top.node);
62 79132 : if (input->opcode() == IrOpcode::kPhi) {
63 : // To break cycles with phi nodes we push phis on a separate stack so
64 : // that they are processed after all other nodes.
65 88 : PreparePhiReplacement(input);
66 176 : stack_.push_front({input, 0});
67 79044 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
68 : input->opcode() == IrOpcode::kLoop) {
69 32 : stack_.push_front({input, 0});
70 : } else {
71 158056 : stack_.push_back({input, 0});
72 : }
73 : state_.Set(input, State::kOnStack);
74 : }
75 : }
76 : }
77 4084 : }
78 :
79 : #define FOREACH_INT32X4_OPCODE(V) \
80 : V(I32x4Splat) \
81 : V(I32x4ExtractLane) \
82 : V(I32x4ReplaceLane) \
83 : V(I32x4SConvertF32x4) \
84 : V(I32x4UConvertF32x4) \
85 : V(I32x4SConvertI16x8Low) \
86 : V(I32x4SConvertI16x8High) \
87 : V(I32x4Neg) \
88 : V(I32x4Shl) \
89 : V(I32x4ShrS) \
90 : V(I32x4Add) \
91 : V(I32x4AddHoriz) \
92 : V(I32x4Sub) \
93 : V(I32x4Mul) \
94 : V(I32x4MinS) \
95 : V(I32x4MaxS) \
96 : V(I32x4ShrU) \
97 : V(I32x4MinU) \
98 : V(I32x4MaxU) \
99 : V(I32x4Eq) \
100 : V(I32x4Ne) \
101 : V(I32x4LtS) \
102 : V(I32x4LeS) \
103 : V(I32x4GtS) \
104 : V(I32x4GeS) \
105 : V(I32x4UConvertI16x8Low) \
106 : V(I32x4UConvertI16x8High) \
107 : V(I32x4LtU) \
108 : V(I32x4LeU) \
109 : V(I32x4GtU) \
110 : V(I32x4GeU) \
111 : V(S128And) \
112 : V(S128Or) \
113 : V(S128Xor) \
114 : V(S128Not) \
115 : V(S1x4AnyTrue) \
116 : V(S1x4AllTrue) \
117 : V(S1x8AnyTrue) \
118 : V(S1x8AllTrue) \
119 : V(S1x16AnyTrue) \
120 : V(S1x16AllTrue)
121 :
122 : #define FOREACH_FLOAT32X4_OPCODE(V) \
123 : V(F32x4Splat) \
124 : V(F32x4ExtractLane) \
125 : V(F32x4ReplaceLane) \
126 : V(F32x4SConvertI32x4) \
127 : V(F32x4UConvertI32x4) \
128 : V(F32x4Abs) \
129 : V(F32x4Neg) \
130 : V(F32x4RecipApprox) \
131 : V(F32x4RecipSqrtApprox) \
132 : V(F32x4Add) \
133 : V(F32x4AddHoriz) \
134 : V(F32x4Sub) \
135 : V(F32x4Mul) \
136 : V(F32x4Min) \
137 : V(F32x4Max)
138 :
139 : #define FOREACH_FLOAT32X4_TO_INT32X4OPCODE(V) \
140 : V(F32x4Eq) \
141 : V(F32x4Ne) \
142 : V(F32x4Lt) \
143 : V(F32x4Le) \
144 : V(F32x4Gt) \
145 : V(F32x4Ge)
146 :
147 : #define FOREACH_INT16X8_OPCODE(V) \
148 : V(I16x8Splat) \
149 : V(I16x8ExtractLane) \
150 : V(I16x8ReplaceLane) \
151 : V(I16x8SConvertI8x16Low) \
152 : V(I16x8SConvertI8x16High) \
153 : V(I16x8Neg) \
154 : V(I16x8Shl) \
155 : V(I16x8ShrS) \
156 : V(I16x8SConvertI32x4) \
157 : V(I16x8Add) \
158 : V(I16x8AddSaturateS) \
159 : V(I16x8AddHoriz) \
160 : V(I16x8Sub) \
161 : V(I16x8SubSaturateS) \
162 : V(I16x8Mul) \
163 : V(I16x8MinS) \
164 : V(I16x8MaxS) \
165 : V(I16x8UConvertI8x16Low) \
166 : V(I16x8UConvertI8x16High) \
167 : V(I16x8ShrU) \
168 : V(I16x8UConvertI32x4) \
169 : V(I16x8AddSaturateU) \
170 : V(I16x8SubSaturateU) \
171 : V(I16x8MinU) \
172 : V(I16x8MaxU) \
173 : V(I16x8Eq) \
174 : V(I16x8Ne) \
175 : V(I16x8LtS) \
176 : V(I16x8LeS) \
177 : V(I16x8LtU) \
178 : V(I16x8LeU)
179 :
180 : #define FOREACH_INT8X16_OPCODE(V) \
181 : V(I8x16Splat) \
182 : V(I8x16ExtractLane) \
183 : V(I8x16ReplaceLane) \
184 : V(I8x16SConvertI16x8) \
185 : V(I8x16Neg) \
186 : V(I8x16Shl) \
187 : V(I8x16ShrS) \
188 : V(I8x16Add) \
189 : V(I8x16AddSaturateS) \
190 : V(I8x16Sub) \
191 : V(I8x16SubSaturateS) \
192 : V(I8x16Mul) \
193 : V(I8x16MinS) \
194 : V(I8x16MaxS) \
195 : V(I8x16ShrU) \
196 : V(I8x16UConvertI16x8) \
197 : V(I8x16AddSaturateU) \
198 : V(I8x16SubSaturateU) \
199 : V(I8x16MinU) \
200 : V(I8x16MaxU) \
201 : V(I8x16Eq) \
202 : V(I8x16Ne) \
203 : V(I8x16LtS) \
204 : V(I8x16LeS) \
205 : V(I8x16LtU) \
206 : V(I8x16LeU) \
207 : V(S8x16Shuffle)
208 :
209 28108 : MachineType SimdScalarLowering::MachineTypeFrom(SimdType simdType) {
210 28108 : switch (simdType) {
211 : case SimdType::kFloat32x4:
212 : return MachineType::Float32();
213 : case SimdType::kInt32x4:
214 : return MachineType::Int32();
215 : case SimdType::kInt16x8:
216 : return MachineType::Int16();
217 : case SimdType::kInt8x16:
218 : return MachineType::Int8();
219 : }
220 : return MachineType::None();
221 : }
222 :
223 79132 : void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
224 79132 : switch (node->opcode()) {
225 : #define CASE_STMT(name) case IrOpcode::k##name:
226 : FOREACH_INT32X4_OPCODE(CASE_STMT)
227 : case IrOpcode::kReturn:
228 : case IrOpcode::kParameter:
229 : case IrOpcode::kCall: {
230 22008 : replacements_[node->id()].type = SimdType::kInt32x4;
231 11004 : break;
232 : }
233 : FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
234 520 : replacements_[node->id()].type = SimdType::kFloat32x4;
235 260 : break;
236 : }
237 : FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT) {
238 48 : replacements_[node->id()].type = SimdType::kInt32x4;
239 24 : break;
240 : }
241 : FOREACH_INT16X8_OPCODE(CASE_STMT) {
242 1488 : replacements_[node->id()].type = SimdType::kInt16x8;
243 744 : break;
244 : }
245 : FOREACH_INT8X16_OPCODE(CASE_STMT) {
246 14352 : replacements_[node->id()].type = SimdType::kInt8x16;
247 7176 : break;
248 : }
249 : default: {
250 59924 : switch (output->opcode()) {
251 : case IrOpcode::kF32x4SConvertI32x4:
252 : case IrOpcode::kF32x4UConvertI32x4:
253 : case IrOpcode::kI16x8SConvertI32x4:
254 : case IrOpcode::kI16x8UConvertI32x4: {
255 0 : replacements_[node->id()].type = SimdType::kInt32x4;
256 0 : break;
257 : }
258 : case IrOpcode::kI8x16SConvertI16x8:
259 : case IrOpcode::kI8x16UConvertI16x8:
260 : case IrOpcode::kI32x4SConvertI16x8Low:
261 : case IrOpcode::kI32x4SConvertI16x8High:
262 : case IrOpcode::kI32x4UConvertI16x8Low:
263 : case IrOpcode::kI32x4UConvertI16x8High: {
264 0 : replacements_[node->id()].type = SimdType::kInt16x8;
265 0 : break;
266 : }
267 : case IrOpcode::kI16x8SConvertI8x16Low:
268 : case IrOpcode::kI16x8SConvertI8x16High:
269 : case IrOpcode::kI16x8UConvertI8x16Low:
270 : case IrOpcode::kI16x8UConvertI8x16High: {
271 0 : replacements_[node->id()].type = SimdType::kInt8x16;
272 0 : break;
273 : }
274 : FOREACH_FLOAT32X4_TO_INT32X4OPCODE(CASE_STMT)
275 : case IrOpcode::kI32x4SConvertF32x4:
276 : case IrOpcode::kI32x4UConvertF32x4: {
277 0 : replacements_[node->id()].type = SimdType::kFloat32x4;
278 0 : break;
279 : }
280 : case IrOpcode::kS128Select: {
281 0 : replacements_[node->id()].type = SimdType::kInt32x4;
282 0 : break;
283 : }
284 : default: {
285 179772 : replacements_[node->id()].type = replacements_[output->id()].type;
286 : }
287 : }
288 : }
289 : #undef CASE_STMT
290 : }
291 79132 : }
292 :
293 : static int GetParameterIndexAfterLoweringSimd128(
294 : Signature<MachineRepresentation>* signature, int old_index) {
295 : // In function calls, the simd128 types are passed as 4 Int32 types. The
296 : // parameters are typecast to the types as needed for various operations.
297 : int result = old_index;
298 6756 : for (int i = 0; i < old_index; ++i) {
299 2672 : if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
300 0 : result += 3;
301 : }
302 : }
303 : return result;
304 : }
305 :
306 0 : int SimdScalarLowering::GetParameterCountAfterLowering() {
307 9488 : if (parameter_count_after_lowering_ == -1) {
308 : // GetParameterIndexAfterLoweringSimd128(parameter_count) returns the
309 : // parameter count after lowering.
310 4084 : parameter_count_after_lowering_ = GetParameterIndexAfterLoweringSimd128(
311 4084 : signature(), static_cast<int>(signature()->parameter_count()));
312 : }
313 9488 : return parameter_count_after_lowering_;
314 : }
315 :
316 : static int GetReturnCountAfterLoweringSimd128(
317 : Signature<MachineRepresentation>* signature) {
318 4228 : int result = static_cast<int>(signature->return_count());
319 12684 : for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
320 8456 : if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
321 0 : result += 3;
322 : }
323 : }
324 : return result;
325 : }
326 :
327 0 : int SimdScalarLowering::NumLanes(SimdType type) {
328 : int num_lanes = 0;
329 112020 : if (type == SimdType::kFloat32x4 || type == SimdType::kInt32x4) {
330 : num_lanes = kNumLanes32;
331 63952 : } else if (type == SimdType::kInt16x8) {
332 : num_lanes = kNumLanes16;
333 62212 : } else if (type == SimdType::kInt8x16) {
334 : num_lanes = kNumLanes8;
335 : } else {
336 0 : UNREACHABLE();
337 : }
338 0 : return num_lanes;
339 : }
340 :
341 : constexpr int SimdScalarLowering::kLaneOffsets[];
342 :
343 13760 : void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices,
344 : SimdType type) {
345 : int num_lanes = NumLanes(type);
346 13760 : int lane_width = kSimd128Size / num_lanes;
347 13760 : int laneIndex = kLaneOffsets[0] / lane_width;
348 13760 : new_indices[laneIndex] = index;
349 403936 : for (int i = 1; i < num_lanes; ++i) {
350 195088 : laneIndex = kLaneOffsets[i * lane_width] / lane_width;
351 390176 : new_indices[laneIndex] = graph()->NewNode(
352 : machine()->Int32Add(), index,
353 : graph()->NewNode(
354 195088 : common()->Int32Constant(static_cast<int>(i) * lane_width)));
355 : }
356 13760 : }
357 :
358 21868 : void SimdScalarLowering::LowerLoadOp(Node* node, SimdType type) {
359 21868 : MachineRepresentation rep = LoadRepresentationOf(node->op()).representation();
360 : const Operator* load_op;
361 21868 : switch (node->opcode()) {
362 : case IrOpcode::kLoad:
363 21860 : load_op = machine()->Load(MachineTypeFrom(type));
364 21860 : break;
365 : case IrOpcode::kUnalignedLoad:
366 0 : load_op = machine()->UnalignedLoad(MachineTypeFrom(type));
367 0 : break;
368 : case IrOpcode::kProtectedLoad:
369 8 : load_op = machine()->ProtectedLoad(MachineTypeFrom(type));
370 8 : break;
371 : default:
372 0 : UNREACHABLE();
373 : }
374 21868 : if (rep == MachineRepresentation::kSimd128) {
375 : Node* base = node->InputAt(0);
376 : Node* index = node->InputAt(1);
377 : int num_lanes = NumLanes(type);
378 9680 : Node** indices = zone()->NewArray<Node*>(num_lanes);
379 9680 : GetIndexNodes(index, indices, type);
380 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
381 9680 : rep_nodes[0] = node;
382 9680 : rep_nodes[0]->ReplaceInput(1, indices[0]);
383 9680 : NodeProperties::ChangeOp(rep_nodes[0], load_op);
384 9680 : if (node->InputCount() > 2) {
385 : DCHECK_LT(3, node->InputCount());
386 : Node* effect_input = node->InputAt(2);
387 : Node* control_input = node->InputAt(3);
388 153856 : for (int i = num_lanes - 1; i > 0; --i) {
389 144176 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i], effect_input,
390 144176 : control_input);
391 : effect_input = rep_nodes[i];
392 : }
393 9680 : rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
394 : } else {
395 0 : for (int i = 1; i < num_lanes; ++i) {
396 0 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
397 : }
398 : }
399 9680 : ReplaceNode(node, rep_nodes, num_lanes);
400 : } else {
401 12188 : DefaultLowering(node);
402 : }
403 21868 : }
404 :
405 4080 : void SimdScalarLowering::LowerStoreOp(Node* node) {
406 : // For store operation, use replacement type of its input instead of the
407 : // one of its effected node.
408 : DCHECK_LT(2, node->InputCount());
409 : SimdType rep_type = ReplacementType(node->InputAt(2));
410 4080 : replacements_[node->id()].type = rep_type;
411 : const Operator* store_op;
412 : MachineRepresentation rep;
413 4080 : switch (node->opcode()) {
414 : case IrOpcode::kStore: {
415 4076 : rep = StoreRepresentationOf(node->op()).representation();
416 : WriteBarrierKind write_barrier_kind =
417 4076 : StoreRepresentationOf(node->op()).write_barrier_kind();
418 4076 : store_op = machine()->Store(StoreRepresentation(
419 12228 : MachineTypeFrom(rep_type).representation(), write_barrier_kind));
420 4076 : break;
421 : }
422 : case IrOpcode::kUnalignedStore: {
423 0 : rep = UnalignedStoreRepresentationOf(node->op());
424 : store_op =
425 0 : machine()->UnalignedStore(MachineTypeFrom(rep_type).representation());
426 0 : break;
427 : }
428 : case IrOpcode::kProtectedStore: {
429 4 : rep = StoreRepresentationOf(node->op()).representation();
430 : store_op =
431 4 : machine()->ProtectedStore(MachineTypeFrom(rep_type).representation());
432 4 : break;
433 : }
434 : default:
435 0 : UNREACHABLE();
436 : }
437 4080 : if (rep == MachineRepresentation::kSimd128) {
438 : Node* base = node->InputAt(0);
439 : Node* index = node->InputAt(1);
440 : int num_lanes = NumLanes(rep_type);
441 4080 : Node** indices = zone()->NewArray<Node*>(num_lanes);
442 4080 : GetIndexNodes(index, indices, rep_type);
443 : Node* value = node->InputAt(2);
444 : DCHECK(HasReplacement(1, value));
445 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
446 4080 : rep_nodes[0] = node;
447 4080 : Node** rep_inputs = GetReplacementsWithType(value, rep_type);
448 4080 : rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
449 4080 : rep_nodes[0]->ReplaceInput(1, indices[0]);
450 4080 : NodeProperties::ChangeOp(node, store_op);
451 4080 : if (node->InputCount() > 3) {
452 : DCHECK_LT(4, node->InputCount());
453 : Node* effect_input = node->InputAt(3);
454 : Node* control_input = node->InputAt(4);
455 54992 : for (int i = num_lanes - 1; i > 0; --i) {
456 50912 : rep_nodes[i] =
457 50912 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i],
458 50912 : effect_input, control_input);
459 : effect_input = rep_nodes[i];
460 : }
461 4080 : rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
462 : } else {
463 0 : for (int i = 1; i < num_lanes; ++i) {
464 0 : rep_nodes[i] =
465 0 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
466 : }
467 : }
468 4080 : ReplaceNode(node, rep_nodes, num_lanes);
469 : } else {
470 0 : DefaultLowering(node);
471 : }
472 4080 : }
473 :
474 68 : void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
475 : const Operator* op,
476 : bool not_horizontal) {
477 : DCHECK_EQ(2, node->InputCount());
478 68 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
479 68 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
480 : int num_lanes = NumLanes(input_rep_type);
481 68 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
482 68 : if (not_horizontal) {
483 540 : for (int i = 0; i < num_lanes; ++i) {
484 480 : rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
485 : }
486 : } else {
487 40 : for (int i = 0; i < num_lanes / 2; ++i) {
488 32 : rep_node[i] = graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]);
489 16 : rep_node[i + num_lanes / 2] =
490 32 : graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]);
491 : }
492 : }
493 68 : ReplaceNode(node, rep_node, num_lanes);
494 68 : }
495 :
496 176 : void SimdScalarLowering::LowerCompareOp(Node* node, SimdType input_rep_type,
497 : const Operator* op,
498 : bool invert_inputs) {
499 : DCHECK_EQ(2, node->InputCount());
500 176 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
501 176 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
502 : int num_lanes = NumLanes(input_rep_type);
503 176 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
504 3248 : for (int i = 0; i < num_lanes; ++i) {
505 : Node* cmp_result = nullptr;
506 1536 : if (invert_inputs) {
507 896 : cmp_result = graph()->NewNode(op, rep_right[i], rep_left[i]);
508 : } else {
509 640 : cmp_result = graph()->NewNode(op, rep_left[i], rep_right[i]);
510 : }
511 : Diamond d_cmp(graph(), common(),
512 : graph()->NewNode(machine()->Word32Equal(), cmp_result,
513 3072 : mcgraph_->Int32Constant(0)));
514 : MachineRepresentation rep =
515 : (input_rep_type == SimdType::kFloat32x4)
516 : ? MachineRepresentation::kWord32
517 1536 : : MachineTypeFrom(input_rep_type).representation();
518 1536 : rep_node[i] =
519 1536 : d_cmp.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
520 : }
521 176 : ReplaceNode(node, rep_node, num_lanes);
522 176 : }
523 :
524 1632 : Node* SimdScalarLowering::FixUpperBits(Node* input, int32_t shift) {
525 4896 : return graph()->NewNode(machine()->Word32Sar(),
526 : graph()->NewNode(machine()->Word32Shl(), input,
527 1632 : mcgraph_->Int32Constant(shift)),
528 3264 : mcgraph_->Int32Constant(shift));
529 : }
530 :
531 28 : void SimdScalarLowering::LowerBinaryOpForSmallInt(Node* node,
532 : SimdType input_rep_type,
533 : const Operator* op,
534 : bool not_horizontal) {
535 : DCHECK_EQ(2, node->InputCount());
536 : DCHECK(input_rep_type == SimdType::kInt16x8 ||
537 : input_rep_type == SimdType::kInt8x16);
538 28 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
539 28 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
540 : int num_lanes = NumLanes(input_rep_type);
541 28 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
542 : int32_t shift_val =
543 28 : (input_rep_type == SimdType::kInt16x8) ? kShift16 : kShift8;
544 28 : if (not_horizontal) {
545 600 : for (int i = 0; i < num_lanes; ++i) {
546 1152 : rep_node[i] = FixUpperBits(
547 864 : graph()->NewNode(op, rep_left[i], rep_right[i]), shift_val);
548 : }
549 : } else {
550 36 : for (int i = 0; i < num_lanes / 2; ++i) {
551 64 : rep_node[i] = FixUpperBits(
552 32 : graph()->NewNode(op, rep_left[i * 2], rep_left[i * 2 + 1]),
553 16 : shift_val);
554 64 : rep_node[i + num_lanes / 2] = FixUpperBits(
555 32 : graph()->NewNode(op, rep_right[i * 2], rep_right[i * 2 + 1]),
556 16 : shift_val);
557 : }
558 : }
559 28 : ReplaceNode(node, rep_node, num_lanes);
560 28 : }
561 :
562 1408 : Node* SimdScalarLowering::Mask(Node* input, int32_t mask) {
563 1408 : return graph()->NewNode(machine()->Word32And(), input,
564 2816 : mcgraph_->Int32Constant(mask));
565 : }
566 :
567 32 : void SimdScalarLowering::LowerSaturateBinaryOp(Node* node,
568 : SimdType input_rep_type,
569 : const Operator* op,
570 : bool is_signed) {
571 : DCHECK_EQ(2, node->InputCount());
572 : DCHECK(input_rep_type == SimdType::kInt16x8 ||
573 : input_rep_type == SimdType::kInt8x16);
574 32 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
575 32 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
576 : int32_t min = 0;
577 : int32_t max = 0;
578 : int32_t mask = 0;
579 : int32_t shift_val = 0;
580 : MachineRepresentation phi_rep;
581 32 : if (input_rep_type == SimdType::kInt16x8) {
582 16 : if (is_signed) {
583 : min = std::numeric_limits<int16_t>::min();
584 : max = std::numeric_limits<int16_t>::max();
585 : } else {
586 : min = std::numeric_limits<uint16_t>::min();
587 : max = std::numeric_limits<uint16_t>::max();
588 : }
589 : mask = kMask16;
590 : shift_val = kShift16;
591 : phi_rep = MachineRepresentation::kWord16;
592 : } else {
593 16 : if (is_signed) {
594 : min = std::numeric_limits<int8_t>::min();
595 : max = std::numeric_limits<int8_t>::max();
596 : } else {
597 : min = std::numeric_limits<uint8_t>::min();
598 : max = std::numeric_limits<uint8_t>::max();
599 : }
600 : mask = kMask8;
601 : shift_val = kShift8;
602 : phi_rep = MachineRepresentation::kWord8;
603 : }
604 : int num_lanes = NumLanes(input_rep_type);
605 32 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
606 800 : for (int i = 0; i < num_lanes; ++i) {
607 : Node* op_result = nullptr;
608 384 : Node* left = is_signed ? rep_left[i] : Mask(rep_left[i], mask);
609 384 : Node* right = is_signed ? rep_right[i] : Mask(rep_right[i], mask);
610 : op_result = graph()->NewNode(op, left, right);
611 : Diamond d_min(graph(), common(),
612 : graph()->NewNode(machine()->Int32LessThan(), op_result,
613 768 : mcgraph_->Int32Constant(min)));
614 384 : rep_node[i] = d_min.Phi(phi_rep, mcgraph_->Int32Constant(min), op_result);
615 : Diamond d_max(graph(), common(),
616 : graph()->NewNode(machine()->Int32LessThan(),
617 768 : mcgraph_->Int32Constant(max), rep_node[i]));
618 384 : rep_node[i] = d_max.Phi(phi_rep, mcgraph_->Int32Constant(max), rep_node[i]);
619 : rep_node[i] =
620 384 : is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
621 : }
622 32 : ReplaceNode(node, rep_node, num_lanes);
623 32 : }
624 :
625 16 : void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
626 : const Operator* op) {
627 : DCHECK_EQ(1, node->InputCount());
628 16 : Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
629 : int num_lanes = NumLanes(input_rep_type);
630 16 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
631 144 : for (int i = 0; i < num_lanes; ++i) {
632 128 : rep_node[i] = graph()->NewNode(op, rep[i]);
633 : }
634 16 : ReplaceNode(node, rep_node, num_lanes);
635 16 : }
636 :
637 48 : void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
638 : bool is_max, SimdType type) {
639 : DCHECK_EQ(2, node->InputCount());
640 48 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), type);
641 48 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), type);
642 : int num_lanes = NumLanes(type);
643 48 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
644 : MachineRepresentation rep = MachineRepresentation::kNone;
645 48 : if (type == SimdType::kInt32x4) {
646 : rep = MachineRepresentation::kWord32;
647 32 : } else if (type == SimdType::kInt16x8) {
648 : rep = MachineRepresentation::kWord16;
649 16 : } else if (type == SimdType::kInt8x16) {
650 : rep = MachineRepresentation::kWord8;
651 : } else {
652 0 : UNREACHABLE();
653 : }
654 944 : for (int i = 0; i < num_lanes; ++i) {
655 : Diamond d(graph(), common(),
656 896 : graph()->NewNode(op, rep_left[i], rep_right[i]));
657 448 : if (is_max) {
658 224 : rep_node[i] = d.Phi(rep, rep_right[i], rep_left[i]);
659 : } else {
660 224 : rep_node[i] = d.Phi(rep, rep_left[i], rep_right[i]);
661 : }
662 : }
663 48 : ReplaceNode(node, rep_node, num_lanes);
664 48 : }
665 :
666 32 : Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
667 32 : if (machine()->Float64RoundTruncate().IsSupported()) {
668 64 : return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
669 : } else {
670 0 : ExternalReference ref = ExternalReference::wasm_f64_trunc();
671 : Node* stack_slot =
672 0 : graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
673 0 : const Operator* store_op = machine()->Store(
674 0 : StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
675 : Node* effect =
676 0 : graph()->NewNode(store_op, stack_slot, mcgraph_->Int32Constant(0),
677 : input, graph()->start(), graph()->start());
678 0 : Node* function = graph()->NewNode(common()->ExternalConstant(ref));
679 : Node** args = zone()->NewArray<Node*>(4);
680 0 : args[0] = function;
681 0 : args[1] = stack_slot;
682 0 : args[2] = effect;
683 0 : args[3] = graph()->start();
684 : Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
685 : sig_builder.AddParam(MachineType::Pointer());
686 : auto call_descriptor =
687 0 : Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
688 0 : Node* call = graph()->NewNode(common()->Call(call_descriptor), 4, args);
689 0 : return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
690 : stack_slot, mcgraph_->Int32Constant(0), call,
691 : graph()->start());
692 : }
693 : }
694 :
695 8 : void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
696 : DCHECK_EQ(1, node->InputCount());
697 8 : Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32x4);
698 : Node* rep_node[kNumLanes32];
699 8 : Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
700 8 : Node* min = graph()->NewNode(
701 : common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
702 8 : Node* max = graph()->NewNode(common()->Float64Constant(
703 : static_cast<double>(is_signed ? kMaxInt : 0xFFFFFFFFu)));
704 72 : for (int i = 0; i < kNumLanes32; ++i) {
705 : Node* double_rep =
706 32 : graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
707 : Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
708 64 : double_rep, double_rep));
709 : Node* temp =
710 32 : nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
711 : Diamond min_d(graph(), common(),
712 64 : graph()->NewNode(machine()->Float64LessThan(), temp, min));
713 32 : temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
714 : Diamond max_d(graph(), common(),
715 64 : graph()->NewNode(machine()->Float64LessThan(), max, temp));
716 32 : temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
717 32 : Node* trunc = BuildF64Trunc(temp);
718 32 : if (is_signed) {
719 32 : rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
720 : } else {
721 : rep_node[i] =
722 32 : graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
723 : }
724 : }
725 8 : ReplaceNode(node, rep_node, kNumLanes32);
726 8 : }
727 :
728 32 : void SimdScalarLowering::LowerConvertFromInt(Node* node,
729 : SimdType input_rep_type,
730 : SimdType output_rep_type,
731 : bool is_signed, int start_index) {
732 : DCHECK_EQ(1, node->InputCount());
733 32 : Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
734 :
735 : int32_t mask = 0;
736 32 : if (input_rep_type == SimdType::kInt16x8) {
737 : DCHECK_EQ(output_rep_type, SimdType::kInt32x4);
738 : mask = kMask16;
739 : } else {
740 : DCHECK_EQ(output_rep_type, SimdType::kInt16x8);
741 : DCHECK_EQ(input_rep_type, SimdType::kInt8x16);
742 : mask = kMask8;
743 : }
744 :
745 : int num_lanes = NumLanes(output_rep_type);
746 32 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
747 416 : for (int i = 0; i < num_lanes; ++i) {
748 192 : rep_node[i] =
749 192 : is_signed ? rep[i + start_index] : Mask(rep[i + start_index], mask);
750 : }
751 :
752 32 : ReplaceNode(node, rep_node, num_lanes);
753 32 : }
754 :
755 16 : void SimdScalarLowering::LowerPack(Node* node, SimdType input_rep_type,
756 : SimdType output_rep_type, bool is_signed) {
757 : DCHECK_EQ(2, node->InputCount());
758 16 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
759 16 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
760 : const Operator* less_op =
761 16 : is_signed ? machine()->Int32LessThan() : machine()->Uint32LessThan();
762 : Node* min = nullptr;
763 : Node* max = nullptr;
764 : int32_t shift_val = 0;
765 : MachineRepresentation phi_rep;
766 16 : if (output_rep_type == SimdType::kInt16x8) {
767 : DCHECK(input_rep_type == SimdType::kInt32x4);
768 8 : if (is_signed) {
769 4 : min = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::min());
770 4 : max = mcgraph_->Int32Constant(std::numeric_limits<int16_t>::max());
771 : } else {
772 4 : max = mcgraph_->Uint32Constant(std::numeric_limits<uint16_t>::max());
773 : shift_val = kShift16;
774 : }
775 : phi_rep = MachineRepresentation::kWord16;
776 : } else {
777 : DCHECK(output_rep_type == SimdType::kInt8x16 &&
778 : input_rep_type == SimdType::kInt16x8);
779 8 : if (is_signed) {
780 4 : min = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::min());
781 4 : max = mcgraph_->Int32Constant(std::numeric_limits<int8_t>::max());
782 : } else {
783 4 : max = mcgraph_->Uint32Constant(std::numeric_limits<uint8_t>::max());
784 : shift_val = kShift8;
785 : }
786 : phi_rep = MachineRepresentation::kWord8;
787 : }
788 : int num_lanes = NumLanes(output_rep_type);
789 16 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
790 400 : for (int i = 0; i < num_lanes; ++i) {
791 : Node* input = nullptr;
792 192 : if (i < num_lanes / 2)
793 96 : input = rep_left[i];
794 : else
795 96 : input = rep_right[i - num_lanes / 2];
796 192 : if (is_signed) {
797 96 : Diamond d_min(graph(), common(), graph()->NewNode(less_op, input, min));
798 96 : input = d_min.Phi(phi_rep, min, input);
799 : }
800 192 : Diamond d_max(graph(), common(), graph()->NewNode(less_op, max, input));
801 192 : rep_node[i] = d_max.Phi(phi_rep, max, input);
802 : rep_node[i] =
803 192 : is_signed ? rep_node[i] : FixUpperBits(rep_node[i], shift_val);
804 : }
805 16 : ReplaceNode(node, rep_node, num_lanes);
806 16 : }
807 :
808 636 : void SimdScalarLowering::LowerShiftOp(Node* node, SimdType type) {
809 : DCHECK_EQ(1, node->InputCount());
810 636 : int32_t shift_amount = OpParameter<int32_t>(node->op());
811 636 : Node* shift_node = graph()->NewNode(common()->Int32Constant(shift_amount));
812 636 : Node** rep = GetReplacementsWithType(node->InputAt(0), type);
813 : int num_lanes = NumLanes(type);
814 636 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
815 9180 : for (int i = 0; i < num_lanes; ++i) {
816 4272 : rep_node[i] = rep[i];
817 4272 : switch (node->opcode()) {
818 : case IrOpcode::kI8x16ShrU:
819 448 : rep_node[i] = Mask(rep_node[i], kMask8);
820 : rep_node[i] =
821 896 : graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
822 448 : break;
823 : case IrOpcode::kI16x8ShrU:
824 480 : rep_node[i] = Mask(rep_node[i], kMask16);
825 : V8_FALLTHROUGH;
826 : case IrOpcode::kI32x4ShrU:
827 : rep_node[i] =
828 1952 : graph()->NewNode(machine()->Word32Shr(), rep_node[i], shift_node);
829 976 : break;
830 : case IrOpcode::kI32x4Shl:
831 : rep_node[i] =
832 992 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
833 496 : break;
834 : case IrOpcode::kI16x8Shl:
835 : rep_node[i] =
836 960 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
837 480 : rep_node[i] = FixUpperBits(rep_node[i], kShift16);
838 480 : break;
839 : case IrOpcode::kI8x16Shl:
840 : rep_node[i] =
841 896 : graph()->NewNode(machine()->Word32Shl(), rep_node[i], shift_node);
842 448 : rep_node[i] = FixUpperBits(rep_node[i], kShift8);
843 448 : break;
844 : case IrOpcode::kI32x4ShrS:
845 : case IrOpcode::kI16x8ShrS:
846 : case IrOpcode::kI8x16ShrS:
847 : rep_node[i] =
848 2848 : graph()->NewNode(machine()->Word32Sar(), rep_node[i], shift_node);
849 1424 : break;
850 : default:
851 0 : UNREACHABLE();
852 : }
853 : }
854 636 : ReplaceNode(node, rep_node, num_lanes);
855 636 : }
856 :
857 76 : void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
858 : const Operator* op) {
859 : DCHECK_EQ(2, node->InputCount());
860 76 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
861 76 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
862 : int num_lanes = NumLanes(input_rep_type);
863 76 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
864 1452 : for (int i = 0; i < num_lanes; ++i) {
865 : Diamond d(graph(), common(),
866 1376 : graph()->NewNode(op, rep_left[i], rep_right[i]));
867 : MachineRepresentation rep =
868 : (input_rep_type == SimdType::kFloat32x4)
869 : ? MachineRepresentation::kWord32
870 688 : : MachineTypeFrom(input_rep_type).representation();
871 688 : rep_node[i] =
872 688 : d.Phi(rep, mcgraph_->Int32Constant(0), mcgraph_->Int32Constant(-1));
873 : }
874 76 : ReplaceNode(node, rep_node, num_lanes);
875 76 : }
876 :
877 83216 : void SimdScalarLowering::LowerNode(Node* node) {
878 : SimdType rep_type = ReplacementType(node);
879 : int num_lanes = NumLanes(rep_type);
880 83216 : switch (node->opcode()) {
881 : case IrOpcode::kStart: {
882 : int parameter_count = GetParameterCountAfterLowering();
883 : // Only exchange the node if the parameter count actually changed.
884 4084 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
885 : int delta =
886 0 : parameter_count - static_cast<int>(signature()->parameter_count());
887 0 : int new_output_count = node->op()->ValueOutputCount() + delta;
888 0 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
889 : }
890 : break;
891 : }
892 : case IrOpcode::kParameter: {
893 : DCHECK_EQ(1, node->InputCount());
894 : // Only exchange the node if the parameter count actually changed. We do
895 : // not even have to do the default lowering because the the start node,
896 : // the only input of a parameter node, only changes if the parameter count
897 : // changes.
898 5404 : if (GetParameterCountAfterLowering() !=
899 : static_cast<int>(signature()->parameter_count())) {
900 0 : int old_index = ParameterIndexOf(node->op());
901 : int new_index =
902 : GetParameterIndexAfterLoweringSimd128(signature(), old_index);
903 0 : if (old_index == new_index) {
904 0 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
905 :
906 : Node* new_node[kNumLanes32];
907 0 : for (int i = 0; i < kNumLanes32; ++i) {
908 0 : new_node[i] = nullptr;
909 : }
910 0 : new_node[0] = node;
911 0 : if (signature()->GetParam(old_index) ==
912 : MachineRepresentation::kSimd128) {
913 0 : for (int i = 1; i < kNumLanes32; ++i) {
914 0 : new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
915 0 : graph()->start());
916 : }
917 : }
918 0 : ReplaceNode(node, new_node, kNumLanes32);
919 : }
920 : }
921 : break;
922 : }
923 : case IrOpcode::kLoad:
924 : case IrOpcode::kUnalignedLoad:
925 : case IrOpcode::kProtectedLoad: {
926 21868 : LowerLoadOp(node, rep_type);
927 21868 : break;
928 : }
929 : case IrOpcode::kStore:
930 : case IrOpcode::kUnalignedStore:
931 : case IrOpcode::kProtectedStore: {
932 4080 : LowerStoreOp(node);
933 4080 : break;
934 : }
935 : case IrOpcode::kReturn: {
936 4228 : DefaultLowering(node);
937 : int new_return_count = GetReturnCountAfterLoweringSimd128(signature());
938 4228 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
939 0 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
940 : }
941 : break;
942 : }
943 : case IrOpcode::kCall: {
944 : // TODO(turbofan): Make wasm code const-correct wrt. CallDescriptor.
945 : auto call_descriptor =
946 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
947 0 : if (DefaultLowering(node) ||
948 0 : (call_descriptor->ReturnCount() == 1 &&
949 : call_descriptor->GetReturnType(0) == MachineType::Simd128())) {
950 : // We have to adjust the call descriptor.
951 : const Operator* op = common()->Call(
952 0 : GetI32WasmCallDescriptorForSimd(zone(), call_descriptor));
953 0 : NodeProperties::ChangeOp(node, op);
954 : }
955 0 : if (call_descriptor->ReturnCount() == 1 &&
956 : call_descriptor->GetReturnType(0) == MachineType::Simd128()) {
957 : // We access the additional return values through projections.
958 : Node* rep_node[kNumLanes32];
959 0 : for (int i = 0; i < kNumLanes32; ++i) {
960 : rep_node[i] =
961 0 : graph()->NewNode(common()->Projection(i), node, graph()->start());
962 : }
963 0 : ReplaceNode(node, rep_node, kNumLanes32);
964 : }
965 : break;
966 : }
967 : case IrOpcode::kPhi: {
968 88 : MachineRepresentation rep = PhiRepresentationOf(node->op());
969 88 : if (rep == MachineRepresentation::kSimd128) {
970 : // The replacement nodes have already been created, we only have to
971 : // replace placeholder nodes.
972 : Node** rep_node = GetReplacements(node);
973 40 : for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
974 : Node** rep_input =
975 16 : GetReplacementsWithType(node->InputAt(i), rep_type);
976 144 : for (int j = 0; j < num_lanes; j++) {
977 64 : rep_node[j]->ReplaceInput(i, rep_input[j]);
978 : }
979 : }
980 : } else {
981 80 : DefaultLowering(node);
982 : }
983 : break;
984 : }
985 : #define I32X4_BINOP_CASE(opcode, instruction) \
986 : case IrOpcode::opcode: { \
987 : LowerBinaryOp(node, rep_type, machine()->instruction()); \
988 : break; \
989 : }
990 12 : I32X4_BINOP_CASE(kI32x4Add, Int32Add)
991 4 : I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
992 4 : I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
993 4 : I32X4_BINOP_CASE(kS128And, Word32And)
994 4 : I32X4_BINOP_CASE(kS128Or, Word32Or)
995 4 : I32X4_BINOP_CASE(kS128Xor, Word32Xor)
996 : #undef I32X4_BINOP_CASE
997 : case IrOpcode::kI32x4AddHoriz: {
998 4 : LowerBinaryOp(node, rep_type, machine()->Int32Add(), false);
999 4 : break;
1000 : }
1001 : case IrOpcode::kI16x8AddHoriz: {
1002 4 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add(), false);
1003 4 : break;
1004 : }
1005 : case IrOpcode::kI16x8Add:
1006 : case IrOpcode::kI8x16Add: {
1007 8 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Add());
1008 8 : break;
1009 : }
1010 : case IrOpcode::kI16x8Sub:
1011 : case IrOpcode::kI8x16Sub: {
1012 8 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Sub());
1013 8 : break;
1014 : }
1015 : case IrOpcode::kI16x8Mul:
1016 : case IrOpcode::kI8x16Mul: {
1017 8 : LowerBinaryOpForSmallInt(node, rep_type, machine()->Int32Mul());
1018 8 : break;
1019 : }
1020 : case IrOpcode::kI16x8AddSaturateS:
1021 : case IrOpcode::kI8x16AddSaturateS: {
1022 8 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), true);
1023 8 : break;
1024 : }
1025 : case IrOpcode::kI16x8SubSaturateS:
1026 : case IrOpcode::kI8x16SubSaturateS: {
1027 8 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), true);
1028 8 : break;
1029 : }
1030 : case IrOpcode::kI16x8AddSaturateU:
1031 : case IrOpcode::kI8x16AddSaturateU: {
1032 8 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Add(), false);
1033 8 : break;
1034 : }
1035 : case IrOpcode::kI16x8SubSaturateU:
1036 : case IrOpcode::kI8x16SubSaturateU: {
1037 8 : LowerSaturateBinaryOp(node, rep_type, machine()->Int32Sub(), false);
1038 8 : break;
1039 : }
1040 : case IrOpcode::kI32x4MaxS:
1041 : case IrOpcode::kI16x8MaxS:
1042 : case IrOpcode::kI8x16MaxS: {
1043 12 : LowerIntMinMax(node, machine()->Int32LessThan(), true, rep_type);
1044 12 : break;
1045 : }
1046 : case IrOpcode::kI32x4MinS:
1047 : case IrOpcode::kI16x8MinS:
1048 : case IrOpcode::kI8x16MinS: {
1049 12 : LowerIntMinMax(node, machine()->Int32LessThan(), false, rep_type);
1050 12 : break;
1051 : }
1052 : case IrOpcode::kI32x4MaxU:
1053 : case IrOpcode::kI16x8MaxU:
1054 : case IrOpcode::kI8x16MaxU: {
1055 12 : LowerIntMinMax(node, machine()->Uint32LessThan(), true, rep_type);
1056 12 : break;
1057 : }
1058 : case IrOpcode::kI32x4MinU:
1059 : case IrOpcode::kI16x8MinU:
1060 : case IrOpcode::kI8x16MinU: {
1061 12 : LowerIntMinMax(node, machine()->Uint32LessThan(), false, rep_type);
1062 12 : break;
1063 : }
1064 : case IrOpcode::kI32x4Neg:
1065 : case IrOpcode::kI16x8Neg:
1066 : case IrOpcode::kI8x16Neg: {
1067 : DCHECK_EQ(1, node->InputCount());
1068 12 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1069 : int num_lanes = NumLanes(rep_type);
1070 12 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1071 12 : Node* zero = graph()->NewNode(common()->Int32Constant(0));
1072 236 : for (int i = 0; i < num_lanes; ++i) {
1073 224 : rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
1074 112 : if (node->opcode() == IrOpcode::kI16x8Neg) {
1075 32 : rep_node[i] = FixUpperBits(rep_node[i], kShift16);
1076 80 : } else if (node->opcode() == IrOpcode::kI8x16Neg) {
1077 64 : rep_node[i] = FixUpperBits(rep_node[i], kShift8);
1078 : }
1079 : }
1080 12 : ReplaceNode(node, rep_node, num_lanes);
1081 12 : break;
1082 : }
1083 : case IrOpcode::kS128Zero: {
1084 : DCHECK_EQ(0, node->InputCount());
1085 : Node* rep_node[kNumLanes32];
1086 0 : for (int i = 0; i < kNumLanes32; ++i) {
1087 0 : rep_node[i] = mcgraph_->Int32Constant(0);
1088 : }
1089 0 : ReplaceNode(node, rep_node, kNumLanes32);
1090 : break;
1091 : }
1092 : case IrOpcode::kS128Not: {
1093 : DCHECK_EQ(1, node->InputCount());
1094 4 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1095 : Node* rep_node[kNumLanes32];
1096 4 : Node* mask = graph()->NewNode(common()->Int32Constant(0xFFFFFFFF));
1097 36 : for (int i = 0; i < kNumLanes32; ++i) {
1098 32 : rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
1099 : }
1100 4 : ReplaceNode(node, rep_node, kNumLanes32);
1101 : break;
1102 : }
1103 : case IrOpcode::kI32x4SConvertF32x4: {
1104 4 : LowerConvertFromFloat(node, true);
1105 4 : break;
1106 : }
1107 : case IrOpcode::kI32x4UConvertF32x4: {
1108 4 : LowerConvertFromFloat(node, false);
1109 4 : break;
1110 : }
1111 : case IrOpcode::kI32x4SConvertI16x8Low: {
1112 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1113 4 : 0);
1114 4 : break;
1115 : }
1116 : case IrOpcode::kI32x4SConvertI16x8High: {
1117 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, true,
1118 4 : 4);
1119 4 : break;
1120 : }
1121 : case IrOpcode::kI32x4UConvertI16x8Low: {
1122 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1123 4 : 0);
1124 4 : break;
1125 : }
1126 : case IrOpcode::kI32x4UConvertI16x8High: {
1127 : LowerConvertFromInt(node, SimdType::kInt16x8, SimdType::kInt32x4, false,
1128 4 : 4);
1129 4 : break;
1130 : }
1131 : case IrOpcode::kI16x8SConvertI8x16Low: {
1132 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1133 4 : 0);
1134 4 : break;
1135 : }
1136 : case IrOpcode::kI16x8SConvertI8x16High: {
1137 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, true,
1138 4 : 8);
1139 4 : break;
1140 : }
1141 : case IrOpcode::kI16x8UConvertI8x16Low: {
1142 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1143 4 : 0);
1144 4 : break;
1145 : }
1146 : case IrOpcode::kI16x8UConvertI8x16High: {
1147 : LowerConvertFromInt(node, SimdType::kInt8x16, SimdType::kInt16x8, false,
1148 4 : 8);
1149 4 : break;
1150 : }
1151 : case IrOpcode::kI16x8SConvertI32x4: {
1152 4 : LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, true);
1153 4 : break;
1154 : }
1155 : case IrOpcode::kI16x8UConvertI32x4: {
1156 4 : LowerPack(node, SimdType::kInt32x4, SimdType::kInt16x8, false);
1157 4 : break;
1158 : }
1159 : case IrOpcode::kI8x16SConvertI16x8: {
1160 4 : LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, true);
1161 4 : break;
1162 : }
1163 : case IrOpcode::kI8x16UConvertI16x8: {
1164 4 : LowerPack(node, SimdType::kInt16x8, SimdType::kInt8x16, false);
1165 4 : break;
1166 : }
1167 : case IrOpcode::kI32x4Shl:
1168 : case IrOpcode::kI16x8Shl:
1169 : case IrOpcode::kI8x16Shl:
1170 : case IrOpcode::kI32x4ShrS:
1171 : case IrOpcode::kI16x8ShrS:
1172 : case IrOpcode::kI8x16ShrS:
1173 : case IrOpcode::kI32x4ShrU:
1174 : case IrOpcode::kI16x8ShrU:
1175 : case IrOpcode::kI8x16ShrU: {
1176 636 : LowerShiftOp(node, rep_type);
1177 636 : break;
1178 : }
1179 : case IrOpcode::kF32x4AddHoriz: {
1180 4 : LowerBinaryOp(node, rep_type, machine()->Float32Add(), false);
1181 4 : break;
1182 : }
1183 : #define F32X4_BINOP_CASE(name) \
1184 : case IrOpcode::kF32x4##name: { \
1185 : LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
1186 : break; \
1187 : }
1188 12 : F32X4_BINOP_CASE(Add)
1189 4 : F32X4_BINOP_CASE(Sub)
1190 4 : F32X4_BINOP_CASE(Mul)
1191 4 : F32X4_BINOP_CASE(Min)
1192 4 : F32X4_BINOP_CASE(Max)
1193 : #undef F32X4_BINOP_CASE
1194 : #define F32X4_UNOP_CASE(name) \
1195 : case IrOpcode::kF32x4##name: { \
1196 : LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
1197 : break; \
1198 : }
1199 4 : F32X4_UNOP_CASE(Abs)
1200 4 : F32X4_UNOP_CASE(Neg)
1201 : #undef F32x4_UNOP_CASE
1202 : case IrOpcode::kF32x4RecipApprox:
1203 : case IrOpcode::kF32x4RecipSqrtApprox: {
1204 : DCHECK_EQ(1, node->InputCount());
1205 8 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
1206 8 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1207 8 : Node* float_one = graph()->NewNode(common()->Float32Constant(1.0));
1208 72 : for (int i = 0; i < num_lanes; ++i) {
1209 32 : Node* tmp = rep[i];
1210 32 : if (node->opcode() == IrOpcode::kF32x4RecipSqrtApprox) {
1211 16 : tmp = graph()->NewNode(machine()->Float32Sqrt(), rep[i]);
1212 : }
1213 64 : rep_node[i] = graph()->NewNode(machine()->Float32Div(), float_one, tmp);
1214 : }
1215 8 : ReplaceNode(node, rep_node, num_lanes);
1216 8 : break;
1217 : }
1218 : case IrOpcode::kF32x4SConvertI32x4: {
1219 4 : LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundInt32ToFloat32());
1220 4 : break;
1221 : }
1222 : case IrOpcode::kF32x4UConvertI32x4: {
1223 4 : LowerUnaryOp(node, SimdType::kInt32x4, machine()->RoundUint32ToFloat32());
1224 4 : break;
1225 : }
1226 : case IrOpcode::kI32x4Splat:
1227 : case IrOpcode::kF32x4Splat:
1228 : case IrOpcode::kI16x8Splat:
1229 : case IrOpcode::kI8x16Splat: {
1230 1428 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1231 22740 : for (int i = 0; i < num_lanes; ++i) {
1232 10656 : if (HasReplacement(0, node->InputAt(0))) {
1233 32 : rep_node[i] = GetReplacements(node->InputAt(0))[0];
1234 : } else {
1235 21280 : rep_node[i] = node->InputAt(0);
1236 : }
1237 : }
1238 1428 : ReplaceNode(node, rep_node, num_lanes);
1239 1428 : break;
1240 : }
1241 : case IrOpcode::kI32x4ExtractLane:
1242 : case IrOpcode::kF32x4ExtractLane:
1243 : case IrOpcode::kI16x8ExtractLane:
1244 : case IrOpcode::kI8x16ExtractLane: {
1245 140 : int32_t lane = OpParameter<int32_t>(node->op());
1246 140 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1247 140 : rep_node[0] = GetReplacementsWithType(node->InputAt(0), rep_type)[lane];
1248 1492 : for (int i = 1; i < num_lanes; ++i) {
1249 676 : rep_node[i] = nullptr;
1250 : }
1251 140 : ReplaceNode(node, rep_node, num_lanes);
1252 140 : break;
1253 : }
1254 : case IrOpcode::kI32x4ReplaceLane:
1255 : case IrOpcode::kF32x4ReplaceLane:
1256 : case IrOpcode::kI16x8ReplaceLane:
1257 : case IrOpcode::kI8x16ReplaceLane: {
1258 : DCHECK_EQ(2, node->InputCount());
1259 : Node* repNode = node->InputAt(1);
1260 200 : int32_t lane = OpParameter<int32_t>(node->op());
1261 200 : Node** old_rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
1262 200 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1263 3976 : for (int i = 0; i < num_lanes; ++i) {
1264 1888 : rep_node[i] = old_rep_node[i];
1265 : }
1266 200 : if (HasReplacement(0, repNode)) {
1267 0 : rep_node[lane] = GetReplacements(repNode)[0];
1268 : } else {
1269 200 : rep_node[lane] = repNode;
1270 : }
1271 200 : ReplaceNode(node, rep_node, num_lanes);
1272 200 : break;
1273 : }
1274 : #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \
1275 : case IrOpcode::simd_op: { \
1276 : LowerCompareOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
1277 : break; \
1278 : }
1279 4 : COMPARISON_CASE(Float32x4, kF32x4Eq, Float32Equal, false)
1280 8 : COMPARISON_CASE(Float32x4, kF32x4Lt, Float32LessThan, false)
1281 8 : COMPARISON_CASE(Float32x4, kF32x4Le, Float32LessThanOrEqual, false)
1282 0 : COMPARISON_CASE(Float32x4, kF32x4Gt, Float32LessThan, true)
1283 0 : COMPARISON_CASE(Float32x4, kF32x4Ge, Float32LessThanOrEqual, true)
1284 20 : COMPARISON_CASE(Int32x4, kI32x4Eq, Word32Equal, false)
1285 0 : COMPARISON_CASE(Int32x4, kI32x4LtS, Int32LessThan, false)
1286 0 : COMPARISON_CASE(Int32x4, kI32x4LeS, Int32LessThanOrEqual, false)
1287 8 : COMPARISON_CASE(Int32x4, kI32x4GtS, Int32LessThan, true)
1288 8 : COMPARISON_CASE(Int32x4, kI32x4GeS, Int32LessThanOrEqual, true)
1289 0 : COMPARISON_CASE(Int32x4, kI32x4LtU, Uint32LessThan, false)
1290 0 : COMPARISON_CASE(Int32x4, kI32x4LeU, Uint32LessThanOrEqual, false)
1291 8 : COMPARISON_CASE(Int32x4, kI32x4GtU, Uint32LessThan, true)
1292 8 : COMPARISON_CASE(Int32x4, kI32x4GeU, Uint32LessThanOrEqual, true)
1293 20 : COMPARISON_CASE(Int16x8, kI16x8Eq, Word32Equal, false)
1294 0 : COMPARISON_CASE(Int16x8, kI16x8LtS, Int32LessThan, false)
1295 0 : COMPARISON_CASE(Int16x8, kI16x8LeS, Int32LessThanOrEqual, false)
1296 8 : COMPARISON_CASE(Int16x8, kI16x8GtS, Int32LessThan, true)
1297 8 : COMPARISON_CASE(Int16x8, kI16x8GeS, Int32LessThanOrEqual, true)
1298 0 : COMPARISON_CASE(Int16x8, kI16x8LtU, Uint32LessThan, false)
1299 0 : COMPARISON_CASE(Int16x8, kI16x8LeU, Uint32LessThanOrEqual, false)
1300 8 : COMPARISON_CASE(Int16x8, kI16x8GtU, Uint32LessThan, true)
1301 8 : COMPARISON_CASE(Int16x8, kI16x8GeU, Uint32LessThanOrEqual, true)
1302 20 : COMPARISON_CASE(Int8x16, kI8x16Eq, Word32Equal, false)
1303 0 : COMPARISON_CASE(Int8x16, kI8x16LtS, Int32LessThan, false)
1304 0 : COMPARISON_CASE(Int8x16, kI8x16LeS, Int32LessThanOrEqual, false)
1305 8 : COMPARISON_CASE(Int8x16, kI8x16GtS, Int32LessThan, true)
1306 8 : COMPARISON_CASE(Int8x16, kI8x16GeS, Int32LessThanOrEqual, true)
1307 0 : COMPARISON_CASE(Int8x16, kI8x16LtU, Uint32LessThan, false)
1308 0 : COMPARISON_CASE(Int8x16, kI8x16LeU, Uint32LessThanOrEqual, false)
1309 8 : COMPARISON_CASE(Int8x16, kI8x16GtU, Uint32LessThan, true)
1310 8 : COMPARISON_CASE(Int8x16, kI8x16GeU, Uint32LessThanOrEqual, true)
1311 : #undef COMPARISON_CASE
1312 : case IrOpcode::kF32x4Ne: {
1313 4 : LowerNotEqual(node, SimdType::kFloat32x4, machine()->Float32Equal());
1314 4 : break;
1315 : }
1316 : case IrOpcode::kI32x4Ne: {
1317 24 : LowerNotEqual(node, SimdType::kInt32x4, machine()->Word32Equal());
1318 24 : break;
1319 : }
1320 : case IrOpcode::kI16x8Ne: {
1321 24 : LowerNotEqual(node, SimdType::kInt16x8, machine()->Word32Equal());
1322 24 : break;
1323 : }
1324 : case IrOpcode::kI8x16Ne: {
1325 24 : LowerNotEqual(node, SimdType::kInt8x16, machine()->Word32Equal());
1326 24 : break;
1327 : }
1328 : case IrOpcode::kS128Select: {
1329 : DCHECK_EQ(3, node->InputCount());
1330 : DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kInt32x4 ||
1331 : ReplacementType(node->InputAt(0)) == SimdType::kInt16x8 ||
1332 : ReplacementType(node->InputAt(0)) == SimdType::kInt8x16);
1333 : Node** boolean_input = GetReplacements(node->InputAt(0));
1334 16 : Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
1335 16 : Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
1336 16 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1337 272 : for (int i = 0; i < num_lanes; ++i) {
1338 : Node* tmp1 =
1339 128 : graph()->NewNode(machine()->Word32Xor(), rep_left[i], rep_right[i]);
1340 : Node* tmp2 =
1341 128 : graph()->NewNode(machine()->Word32And(), boolean_input[i], tmp1);
1342 128 : rep_node[i] =
1343 256 : graph()->NewNode(machine()->Word32Xor(), rep_right[i], tmp2);
1344 : }
1345 16 : ReplaceNode(node, rep_node, num_lanes);
1346 16 : break;
1347 : }
1348 : case IrOpcode::kS8x16Shuffle: {
1349 : DCHECK_EQ(2, node->InputCount());
1350 6616 : const uint8_t* shuffle = OpParameter<uint8_t*>(node->op());
1351 6616 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), rep_type);
1352 6616 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), rep_type);
1353 : Node** rep_node = zone()->NewArray<Node*>(16);
1354 218328 : for (int i = 0; i < 16; i++) {
1355 105856 : int lane = shuffle[i];
1356 105856 : rep_node[i] = lane < 16 ? rep_left[lane] : rep_right[lane - 16];
1357 : }
1358 6616 : ReplaceNode(node, rep_node, 16);
1359 6616 : break;
1360 : }
1361 : case IrOpcode::kS1x4AnyTrue:
1362 : case IrOpcode::kS1x4AllTrue:
1363 : case IrOpcode::kS1x8AnyTrue:
1364 : case IrOpcode::kS1x8AllTrue:
1365 : case IrOpcode::kS1x16AnyTrue:
1366 : case IrOpcode::kS1x16AllTrue: {
1367 : DCHECK_EQ(1, node->InputCount());
1368 : SimdType input_rep_type = ReplacementType(node->InputAt(0));
1369 : int input_num_lanes = NumLanes(input_rep_type);
1370 : Node** rep = GetReplacements(node->InputAt(0));
1371 96 : Node** rep_node = zone()->NewArray<Node*>(num_lanes);
1372 96 : Node* true_node = mcgraph_->Int32Constant(-1);
1373 96 : Node* false_node = mcgraph_->Int32Constant(0);
1374 : Node* tmp_result = false_node;
1375 176 : if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1376 160 : node->opcode() == IrOpcode::kS1x8AllTrue ||
1377 : node->opcode() == IrOpcode::kS1x16AllTrue) {
1378 : tmp_result = true_node;
1379 : }
1380 1888 : for (int i = 0; i < input_num_lanes; ++i) {
1381 : Diamond is_false(
1382 : graph(), common(),
1383 1792 : graph()->NewNode(machine()->Word32Equal(), rep[i], false_node));
1384 1728 : if (node->opcode() == IrOpcode::kS1x4AllTrue ||
1385 1600 : node->opcode() == IrOpcode::kS1x8AllTrue ||
1386 : node->opcode() == IrOpcode::kS1x16AllTrue) {
1387 : tmp_result = is_false.Phi(MachineRepresentation::kWord32, false_node,
1388 448 : tmp_result);
1389 : } else {
1390 : tmp_result = is_false.Phi(MachineRepresentation::kWord32, tmp_result,
1391 448 : true_node);
1392 : }
1393 : }
1394 96 : rep_node[0] = tmp_result;
1395 672 : for (int i = 1; i < num_lanes; ++i) {
1396 288 : rep_node[i] = nullptr;
1397 : }
1398 96 : ReplaceNode(node, rep_node, num_lanes);
1399 96 : break;
1400 : }
1401 33808 : default: { DefaultLowering(node); }
1402 : }
1403 83216 : }
1404 :
1405 50304 : bool SimdScalarLowering::DefaultLowering(Node* node) {
1406 : bool something_changed = false;
1407 94128 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
1408 : Node* input = node->InputAt(i);
1409 43824 : if (HasReplacement(0, input)) {
1410 : something_changed = true;
1411 232 : node->ReplaceInput(i, GetReplacements(input)[0]);
1412 : }
1413 43824 : if (HasReplacement(1, input)) {
1414 : something_changed = true;
1415 0 : for (int j = 1; j < ReplacementCount(input); ++j) {
1416 0 : node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
1417 : }
1418 : }
1419 : }
1420 50304 : return something_changed;
1421 : }
1422 :
1423 23424 : void SimdScalarLowering::ReplaceNode(Node* old, Node** new_nodes, int count) {
1424 70272 : replacements_[old->id()].node = zone()->NewArray<Node*>(count);
1425 697760 : for (int i = 0; i < count; ++i) {
1426 674336 : replacements_[old->id()].node[i] = new_nodes[i];
1427 : }
1428 46848 : replacements_[old->id()].num_replacements = count;
1429 23424 : }
1430 :
1431 0 : bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
1432 197488 : return replacements_[node->id()].node != nullptr &&
1433 480 : replacements_[node->id()].node[index] != nullptr;
1434 : }
1435 :
1436 0 : SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
1437 194088 : return replacements_[node->id()].type;
1438 : }
1439 :
1440 0 : Node** SimdScalarLowering::GetReplacements(Node* node) {
1441 38752 : Node** result = replacements_[node->id()].node;
1442 : DCHECK(result);
1443 0 : return result;
1444 : }
1445 :
1446 0 : int SimdScalarLowering::ReplacementCount(Node* node) {
1447 0 : return replacements_[node->id()].num_replacements;
1448 : }
1449 :
1450 28 : void SimdScalarLowering::Int32ToFloat32(Node** replacements, Node** result) {
1451 252 : for (int i = 0; i < kNumLanes32; ++i) {
1452 112 : if (replacements[i] != nullptr) {
1453 112 : result[i] =
1454 224 : graph()->NewNode(machine()->BitcastInt32ToFloat32(), replacements[i]);
1455 : } else {
1456 0 : result[i] = nullptr;
1457 : }
1458 : }
1459 28 : }
1460 :
1461 12 : void SimdScalarLowering::Float32ToInt32(Node** replacements, Node** result) {
1462 108 : for (int i = 0; i < kNumLanes32; ++i) {
1463 48 : if (replacements[i] != nullptr) {
1464 48 : result[i] =
1465 96 : graph()->NewNode(machine()->BitcastFloat32ToInt32(), replacements[i]);
1466 : } else {
1467 0 : result[i] = nullptr;
1468 : }
1469 : }
1470 12 : }
1471 :
1472 : template <typename T>
1473 0 : void SimdScalarLowering::Int32ToSmallerInt(Node** replacements, Node** result) {
1474 : const int num_ints = sizeof(int32_t) / sizeof(T);
1475 : const int bit_size = sizeof(T) * 8;
1476 : const Operator* sign_extend;
1477 : switch (sizeof(T)) {
1478 : case 1:
1479 0 : sign_extend = machine()->SignExtendWord8ToInt32();
1480 : break;
1481 : case 2:
1482 0 : sign_extend = machine()->SignExtendWord16ToInt32();
1483 : break;
1484 : default:
1485 : UNREACHABLE();
1486 : }
1487 :
1488 0 : for (int i = 0; i < kNumLanes32; i++) {
1489 0 : if (replacements[i] != nullptr) {
1490 0 : for (int j = 0; j < num_ints; j++) {
1491 0 : result[num_ints * i + j] = graph()->NewNode(
1492 : sign_extend,
1493 : graph()->NewNode(machine()->Word32Sar(), replacements[i],
1494 0 : mcgraph_->Int32Constant(j * bit_size)));
1495 : }
1496 : } else {
1497 0 : for (int j = 0; j < num_ints; j++) {
1498 0 : result[num_ints * i + j] = nullptr;
1499 : }
1500 : }
1501 : }
1502 0 : }
1503 :
1504 : template <typename T>
1505 0 : void SimdScalarLowering::SmallerIntToInt32(Node** replacements, Node** result) {
1506 : const int num_ints = sizeof(int32_t) / sizeof(T);
1507 : const int bit_size = sizeof(T) * 8;
1508 : const int bit_mask = (1 << bit_size) - 1;
1509 :
1510 0 : for (int i = 0; i < kNumLanes32; ++i) {
1511 0 : result[i] = mcgraph_->Int32Constant(0);
1512 0 : for (int j = 0; j < num_ints; j++) {
1513 0 : if (replacements[num_ints * i + j] != nullptr) {
1514 0 : Node* clean_bits = graph()->NewNode(machine()->Word32And(),
1515 : replacements[num_ints * i + j],
1516 0 : mcgraph_->Int32Constant(bit_mask));
1517 0 : Node* shift = graph()->NewNode(machine()->Word32Shl(), clean_bits,
1518 0 : mcgraph_->Int32Constant(j * bit_size));
1519 0 : result[i] = graph()->NewNode(machine()->Word32Or(), result[i], shift);
1520 : }
1521 : }
1522 : }
1523 0 : }
1524 :
1525 19304 : Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
1526 : Node** replacements = GetReplacements(node);
1527 19304 : if (ReplacementType(node) == type) {
1528 : return GetReplacements(node);
1529 : }
1530 : int num_lanes = NumLanes(type);
1531 40 : Node** result = zone()->NewArray<Node*>(num_lanes);
1532 40 : if (type == SimdType::kInt32x4) {
1533 12 : if (ReplacementType(node) == SimdType::kFloat32x4) {
1534 12 : Float32ToInt32(replacements, result);
1535 0 : } else if (ReplacementType(node) == SimdType::kInt16x8) {
1536 0 : SmallerIntToInt32<int16_t>(replacements, result);
1537 0 : } else if (ReplacementType(node) == SimdType::kInt8x16) {
1538 0 : SmallerIntToInt32<int8_t>(replacements, result);
1539 : } else {
1540 0 : UNREACHABLE();
1541 : }
1542 28 : } else if (type == SimdType::kFloat32x4) {
1543 28 : if (ReplacementType(node) == SimdType::kInt32x4) {
1544 28 : Int32ToFloat32(replacements, result);
1545 0 : } else if (ReplacementType(node) == SimdType::kInt16x8) {
1546 0 : UNIMPLEMENTED();
1547 : } else {
1548 0 : UNREACHABLE();
1549 : }
1550 0 : } else if (type == SimdType::kInt16x8) {
1551 0 : if (ReplacementType(node) == SimdType::kInt32x4) {
1552 0 : Int32ToSmallerInt<int16_t>(replacements, result);
1553 0 : } else if (ReplacementType(node) == SimdType::kFloat32x4) {
1554 0 : UNIMPLEMENTED();
1555 : } else {
1556 0 : UNREACHABLE();
1557 : }
1558 0 : } else if (type == SimdType::kInt8x16) {
1559 0 : if (ReplacementType(node) == SimdType::kInt32x4) {
1560 0 : Int32ToSmallerInt<int8_t>(replacements, result);
1561 : } else {
1562 0 : UNIMPLEMENTED();
1563 : }
1564 : } else {
1565 0 : UNREACHABLE();
1566 : }
1567 : return result;
1568 : }
1569 :
1570 88 : void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
1571 88 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
1572 88 : if (rep == MachineRepresentation::kSimd128) {
1573 : // We have to create the replacements for a phi node before we actually
1574 : // lower the phi to break potential cycles in the graph. The replacements of
1575 : // input nodes do not exist yet, so we use a placeholder node to pass the
1576 : // graph verifier.
1577 : int value_count = phi->op()->ValueInputCount();
1578 : SimdType type = ReplacementType(phi);
1579 : int num_lanes = NumLanes(type);
1580 8 : Node*** inputs_rep = zone()->NewArray<Node**>(num_lanes);
1581 72 : for (int i = 0; i < num_lanes; ++i) {
1582 64 : inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
1583 32 : inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
1584 : }
1585 40 : for (int i = 0; i < value_count; ++i) {
1586 144 : for (int j = 0; j < num_lanes; ++j) {
1587 64 : inputs_rep[j][i] = placeholder_;
1588 : }
1589 : }
1590 : Node** rep_nodes = zone()->NewArray<Node*>(num_lanes);
1591 72 : for (int i = 0; i < num_lanes; ++i) {
1592 96 : rep_nodes[i] = graph()->NewNode(
1593 64 : common()->Phi(MachineTypeFrom(type).representation(), value_count),
1594 64 : value_count + 1, inputs_rep[i], false);
1595 : }
1596 8 : ReplaceNode(phi, rep_nodes, num_lanes);
1597 : }
1598 88 : }
1599 : } // namespace compiler
1600 : } // namespace internal
1601 121996 : } // namespace v8
|
