Line data Source code
1 : // Copyright 2016 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/simd-scalar-lowering.h"
6 : #include "src/compiler/diamond.h"
7 : #include "src/compiler/linkage.h"
8 : #include "src/compiler/node-matchers.h"
9 : #include "src/compiler/node-properties.h"
10 :
11 : #include "src/compiler/node.h"
12 : #include "src/objects-inl.h"
13 : #include "src/wasm/wasm-module.h"
14 :
15 : namespace v8 {
16 : namespace internal {
17 : namespace compiler {
18 :
19 0 : SimdScalarLowering::SimdScalarLowering(
20 0 : JSGraph* jsgraph, Signature<MachineRepresentation>* signature)
21 : : jsgraph_(jsgraph),
22 : state_(jsgraph->graph(), 3),
23 : stack_(jsgraph_->zone()),
24 : replacements_(nullptr),
25 : signature_(signature),
26 : placeholder_(graph()->NewNode(common()->Parameter(-2, "placeholder"),
27 0 : graph()->start())),
28 0 : parameter_count_after_lowering_(-1) {
29 : DCHECK_NOT_NULL(graph());
30 : DCHECK_NOT_NULL(graph()->end());
31 0 : replacements_ = zone()->NewArray<Replacement>(graph()->NodeCount());
32 0 : memset(replacements_, 0, sizeof(Replacement) * graph()->NodeCount());
33 0 : }
34 :
35 0 : void SimdScalarLowering::LowerGraph() {
36 0 : stack_.push_back({graph()->end(), 0});
37 0 : state_.Set(graph()->end(), State::kOnStack);
38 0 : replacements_[graph()->end()->id()].type = SimdType::kInt32;
39 :
40 0 : while (!stack_.empty()) {
41 : NodeState& top = stack_.back();
42 0 : if (top.input_index == top.node->InputCount()) {
43 : // All inputs of top have already been lowered, now lower top.
44 : stack_.pop_back();
45 0 : state_.Set(top.node, State::kVisited);
46 0 : LowerNode(top.node);
47 : } else {
48 : // Push the next input onto the stack.
49 0 : Node* input = top.node->InputAt(top.input_index++);
50 0 : if (state_.Get(input) == State::kUnvisited) {
51 0 : SetLoweredType(input, top.node);
52 0 : if (input->opcode() == IrOpcode::kPhi) {
53 : // To break cycles with phi nodes we push phis on a separate stack so
54 : // that they are processed after all other nodes.
55 0 : PreparePhiReplacement(input);
56 0 : stack_.push_front({input, 0});
57 0 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
58 : input->opcode() == IrOpcode::kLoop) {
59 0 : stack_.push_front({input, 0});
60 : } else {
61 0 : stack_.push_back({input, 0});
62 : }
63 : state_.Set(input, State::kOnStack);
64 : }
65 : }
66 : }
67 0 : }
68 :
69 : #define FOREACH_INT32X4_OPCODE(V) \
70 : V(I32x4Splat) \
71 : V(I32x4ExtractLane) \
72 : V(I32x4ReplaceLane) \
73 : V(I32x4SConvertF32x4) \
74 : V(I32x4UConvertF32x4) \
75 : V(I32x4Neg) \
76 : V(I32x4Add) \
77 : V(I32x4Sub) \
78 : V(I32x4Mul) \
79 : V(I32x4MinS) \
80 : V(I32x4MaxS) \
81 : V(I32x4MinU) \
82 : V(I32x4MaxU) \
83 : V(S128And) \
84 : V(S128Or) \
85 : V(S128Xor) \
86 : V(S128Not)
87 :
88 : #define FOREACH_FLOAT32X4_OPCODE(V) \
89 : V(F32x4Splat) \
90 : V(F32x4ExtractLane) \
91 : V(F32x4ReplaceLane) \
92 : V(F32x4SConvertI32x4) \
93 : V(F32x4UConvertI32x4) \
94 : V(F32x4Abs) \
95 : V(F32x4Neg) \
96 : V(F32x4Add) \
97 : V(F32x4Sub) \
98 : V(F32x4Mul) \
99 : V(F32x4Min) \
100 : V(F32x4Max)
101 :
102 : #define FOREACH_FLOAT32X4_TO_SIMD1X4OPCODE(V) \
103 : V(F32x4Eq) \
104 : V(F32x4Ne) \
105 : V(F32x4Lt) \
106 : V(F32x4Le) \
107 : V(F32x4Gt) \
108 : V(F32x4Ge)
109 :
110 : #define FOREACH_INT32X4_TO_SIMD1X4OPCODE(V) \
111 : V(I32x4Eq) \
112 : V(I32x4Ne) \
113 : V(I32x4LtS) \
114 : V(I32x4LeS) \
115 : V(I32x4GtS) \
116 : V(I32x4GeS) \
117 : V(I32x4LtU) \
118 : V(I32x4LeU) \
119 : V(I32x4GtU) \
120 : V(I32x4GeU)
121 :
122 0 : void SimdScalarLowering::SetLoweredType(Node* node, Node* output) {
123 0 : switch (node->opcode()) {
124 : #define CASE_STMT(name) case IrOpcode::k##name:
125 : FOREACH_INT32X4_OPCODE(CASE_STMT)
126 : case IrOpcode::kReturn:
127 : case IrOpcode::kParameter:
128 : case IrOpcode::kCall: {
129 0 : replacements_[node->id()].type = SimdType::kInt32;
130 0 : break;
131 : }
132 : FOREACH_FLOAT32X4_OPCODE(CASE_STMT) {
133 0 : replacements_[node->id()].type = SimdType::kFloat32;
134 0 : break;
135 : }
136 : FOREACH_FLOAT32X4_TO_SIMD1X4OPCODE(CASE_STMT)
137 : FOREACH_INT32X4_TO_SIMD1X4OPCODE(CASE_STMT) {
138 0 : replacements_[node->id()].type = SimdType::kSimd1x4;
139 0 : break;
140 : }
141 : default: {
142 0 : switch (output->opcode()) {
143 : FOREACH_FLOAT32X4_TO_SIMD1X4OPCODE(CASE_STMT)
144 : case IrOpcode::kF32x4SConvertI32x4:
145 : case IrOpcode::kF32x4UConvertI32x4: {
146 0 : replacements_[node->id()].type = SimdType::kInt32;
147 0 : break;
148 : }
149 : FOREACH_INT32X4_TO_SIMD1X4OPCODE(CASE_STMT)
150 : case IrOpcode::kI32x4SConvertF32x4:
151 : case IrOpcode::kI32x4UConvertF32x4: {
152 0 : replacements_[node->id()].type = SimdType::kFloat32;
153 0 : break;
154 : }
155 : case IrOpcode::kS32x4Select: {
156 0 : replacements_[node->id()].type = SimdType::kSimd1x4;
157 0 : break;
158 : }
159 : default: {
160 0 : replacements_[node->id()].type = replacements_[output->id()].type;
161 : }
162 : }
163 : }
164 : #undef CASE_STMT
165 : }
166 0 : }
167 :
168 : static int GetParameterIndexAfterLowering(
169 0 : Signature<MachineRepresentation>* signature, int old_index) {
170 : // In function calls, the simd128 types are passed as 4 Int32 types. The
171 : // parameters are typecast to the types as needed for various operations.
172 : int result = old_index;
173 0 : for (int i = 0; i < old_index; ++i) {
174 0 : if (signature->GetParam(i) == MachineRepresentation::kSimd128) {
175 0 : result += 3;
176 : }
177 : }
178 : return result;
179 : }
180 :
181 0 : int SimdScalarLowering::GetParameterCountAfterLowering() {
182 0 : if (parameter_count_after_lowering_ == -1) {
183 : // GetParameterIndexAfterLowering(parameter_count) returns the parameter
184 : // count after lowering.
185 : parameter_count_after_lowering_ = GetParameterIndexAfterLowering(
186 0 : signature(), static_cast<int>(signature()->parameter_count()));
187 : }
188 0 : return parameter_count_after_lowering_;
189 : }
190 :
191 : static int GetReturnCountAfterLowering(
192 0 : Signature<MachineRepresentation>* signature) {
193 0 : int result = static_cast<int>(signature->return_count());
194 0 : for (int i = 0; i < static_cast<int>(signature->return_count()); ++i) {
195 0 : if (signature->GetReturn(i) == MachineRepresentation::kSimd128) {
196 0 : result += 3;
197 : }
198 : }
199 : return result;
200 : }
201 :
202 0 : void SimdScalarLowering::GetIndexNodes(Node* index, Node** new_indices) {
203 0 : new_indices[0] = index;
204 0 : for (size_t i = 1; i < kMaxLanes; ++i) {
205 0 : new_indices[i] = graph()->NewNode(machine()->Int32Add(), index,
206 : graph()->NewNode(common()->Int32Constant(
207 0 : static_cast<int>(i) * kLaneWidth)));
208 : }
209 0 : }
210 :
211 0 : void SimdScalarLowering::LowerLoadOp(MachineRepresentation rep, Node* node,
212 0 : const Operator* load_op) {
213 0 : if (rep == MachineRepresentation::kSimd128) {
214 : Node* base = node->InputAt(0);
215 : Node* index = node->InputAt(1);
216 : Node* indices[kMaxLanes];
217 0 : GetIndexNodes(index, indices);
218 : Node* rep_nodes[kMaxLanes];
219 0 : rep_nodes[0] = node;
220 0 : NodeProperties::ChangeOp(rep_nodes[0], load_op);
221 0 : if (node->InputCount() > 2) {
222 : DCHECK(node->InputCount() > 3);
223 : Node* effect_input = node->InputAt(2);
224 : Node* control_input = node->InputAt(3);
225 : rep_nodes[3] = graph()->NewNode(load_op, base, indices[3], effect_input,
226 0 : control_input);
227 : rep_nodes[2] = graph()->NewNode(load_op, base, indices[2], rep_nodes[3],
228 0 : control_input);
229 : rep_nodes[1] = graph()->NewNode(load_op, base, indices[1], rep_nodes[2],
230 0 : control_input);
231 0 : rep_nodes[0]->ReplaceInput(2, rep_nodes[1]);
232 : } else {
233 0 : for (size_t i = 1; i < kMaxLanes; ++i) {
234 0 : rep_nodes[i] = graph()->NewNode(load_op, base, indices[i]);
235 : }
236 : }
237 : ReplaceNode(node, rep_nodes);
238 : } else {
239 0 : DefaultLowering(node);
240 : }
241 0 : }
242 :
243 0 : void SimdScalarLowering::LowerStoreOp(MachineRepresentation rep, Node* node,
244 : const Operator* store_op,
245 0 : SimdType rep_type) {
246 0 : if (rep == MachineRepresentation::kSimd128) {
247 : Node* base = node->InputAt(0);
248 : Node* index = node->InputAt(1);
249 : Node* indices[kMaxLanes];
250 0 : GetIndexNodes(index, indices);
251 : DCHECK(node->InputCount() > 2);
252 : Node* value = node->InputAt(2);
253 : DCHECK(HasReplacement(1, value));
254 : Node* rep_nodes[kMaxLanes];
255 0 : rep_nodes[0] = node;
256 0 : Node** rep_inputs = GetReplacementsWithType(value, rep_type);
257 0 : rep_nodes[0]->ReplaceInput(2, rep_inputs[0]);
258 0 : NodeProperties::ChangeOp(node, store_op);
259 0 : if (node->InputCount() > 3) {
260 : DCHECK(node->InputCount() > 4);
261 : Node* effect_input = node->InputAt(3);
262 : Node* control_input = node->InputAt(4);
263 : rep_nodes[3] = graph()->NewNode(store_op, base, indices[3], rep_inputs[3],
264 0 : effect_input, control_input);
265 : rep_nodes[2] = graph()->NewNode(store_op, base, indices[2], rep_inputs[2],
266 0 : rep_nodes[3], control_input);
267 : rep_nodes[1] = graph()->NewNode(store_op, base, indices[1], rep_inputs[1],
268 0 : rep_nodes[2], control_input);
269 0 : rep_nodes[0]->ReplaceInput(3, rep_nodes[1]);
270 :
271 : } else {
272 0 : for (size_t i = 1; i < kMaxLanes; ++i) {
273 : rep_nodes[i] =
274 0 : graph()->NewNode(store_op, base, indices[i], rep_inputs[i]);
275 : }
276 : }
277 :
278 : ReplaceNode(node, rep_nodes);
279 : } else {
280 0 : DefaultLowering(node);
281 : }
282 0 : }
283 :
284 0 : void SimdScalarLowering::LowerBinaryOp(Node* node, SimdType input_rep_type,
285 0 : const Operator* op, bool invert_inputs) {
286 : DCHECK(node->InputCount() == 2);
287 0 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
288 0 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
289 : Node* rep_node[kMaxLanes];
290 0 : for (int i = 0; i < kMaxLanes; ++i) {
291 0 : if (invert_inputs) {
292 0 : rep_node[i] = graph()->NewNode(op, rep_right[i], rep_left[i]);
293 : } else {
294 0 : rep_node[i] = graph()->NewNode(op, rep_left[i], rep_right[i]);
295 : }
296 : }
297 : ReplaceNode(node, rep_node);
298 0 : }
299 :
300 0 : void SimdScalarLowering::LowerUnaryOp(Node* node, SimdType input_rep_type,
301 0 : const Operator* op) {
302 : DCHECK(node->InputCount() == 1);
303 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), input_rep_type);
304 : Node* rep_node[kMaxLanes];
305 0 : for (int i = 0; i < kMaxLanes; ++i) {
306 0 : rep_node[i] = graph()->NewNode(op, rep[i]);
307 : }
308 : ReplaceNode(node, rep_node);
309 0 : }
310 :
311 0 : void SimdScalarLowering::LowerIntMinMax(Node* node, const Operator* op,
312 0 : bool is_max) {
313 : DCHECK(node->InputCount() == 2);
314 0 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), SimdType::kInt32);
315 : Node** rep_right =
316 0 : GetReplacementsWithType(node->InputAt(1), SimdType::kInt32);
317 : Node* rep_node[kMaxLanes];
318 0 : for (int i = 0; i < kMaxLanes; ++i) {
319 : Diamond d(graph(), common(),
320 0 : graph()->NewNode(op, rep_left[i], rep_right[i]));
321 0 : if (is_max) {
322 : rep_node[i] =
323 0 : d.Phi(MachineRepresentation::kWord32, rep_right[i], rep_left[i]);
324 : } else {
325 : rep_node[i] =
326 0 : d.Phi(MachineRepresentation::kWord32, rep_left[i], rep_right[i]);
327 : }
328 : }
329 : ReplaceNode(node, rep_node);
330 0 : }
331 :
332 0 : Node* SimdScalarLowering::BuildF64Trunc(Node* input) {
333 0 : if (machine()->Float64RoundTruncate().IsSupported()) {
334 0 : return graph()->NewNode(machine()->Float64RoundTruncate().op(), input);
335 : } else {
336 : ExternalReference ref =
337 0 : ExternalReference::wasm_f64_trunc(jsgraph_->isolate());
338 : Node* stack_slot =
339 0 : graph()->NewNode(machine()->StackSlot(MachineRepresentation::kFloat64));
340 : const Operator* store_op = machine()->Store(
341 0 : StoreRepresentation(MachineRepresentation::kFloat64, kNoWriteBarrier));
342 : Node* effect =
343 : graph()->NewNode(store_op, stack_slot, jsgraph_->Int32Constant(0),
344 0 : input, graph()->start(), graph()->start());
345 0 : Node* function = graph()->NewNode(common()->ExternalConstant(ref));
346 : Node** args = zone()->NewArray<Node*>(4);
347 0 : args[0] = function;
348 0 : args[1] = stack_slot;
349 0 : args[2] = effect;
350 0 : args[3] = graph()->start();
351 : Signature<MachineType>::Builder sig_builder(zone(), 0, 1);
352 : sig_builder.AddParam(MachineType::Pointer());
353 : CallDescriptor* desc =
354 0 : Linkage::GetSimplifiedCDescriptor(zone(), sig_builder.Build());
355 0 : Node* call = graph()->NewNode(common()->Call(desc), 4, args);
356 : return graph()->NewNode(machine()->Load(LoadRepresentation::Float64()),
357 : stack_slot, jsgraph_->Int32Constant(0), call,
358 0 : graph()->start());
359 : }
360 : }
361 :
362 0 : void SimdScalarLowering::LowerConvertFromFloat(Node* node, bool is_signed) {
363 : DCHECK(node->InputCount() == 1);
364 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kFloat32);
365 : Node* rep_node[kMaxLanes];
366 0 : Node* double_zero = graph()->NewNode(common()->Float64Constant(0.0));
367 : Node* min = graph()->NewNode(
368 0 : common()->Float64Constant(static_cast<double>(is_signed ? kMinInt : 0)));
369 : Node* max = graph()->NewNode(common()->Float64Constant(
370 0 : static_cast<double>(is_signed ? kMaxInt : 0xffffffffu)));
371 0 : for (int i = 0; i < kMaxLanes; ++i) {
372 : Node* double_rep =
373 0 : graph()->NewNode(machine()->ChangeFloat32ToFloat64(), rep[i]);
374 : Diamond nan_d(graph(), common(), graph()->NewNode(machine()->Float64Equal(),
375 0 : double_rep, double_rep));
376 : Node* temp =
377 0 : nan_d.Phi(MachineRepresentation::kFloat64, double_rep, double_zero);
378 : Diamond min_d(graph(), common(),
379 0 : graph()->NewNode(machine()->Float64LessThan(), temp, min));
380 0 : temp = min_d.Phi(MachineRepresentation::kFloat64, min, temp);
381 : Diamond max_d(graph(), common(),
382 0 : graph()->NewNode(machine()->Float64LessThan(), max, temp));
383 0 : temp = max_d.Phi(MachineRepresentation::kFloat64, max, temp);
384 0 : Node* trunc = BuildF64Trunc(temp);
385 0 : if (is_signed) {
386 0 : rep_node[i] = graph()->NewNode(machine()->ChangeFloat64ToInt32(), trunc);
387 : } else {
388 : rep_node[i] =
389 0 : graph()->NewNode(machine()->TruncateFloat64ToUint32(), trunc);
390 : }
391 : }
392 : ReplaceNode(node, rep_node);
393 0 : }
394 :
395 0 : void SimdScalarLowering::LowerShiftOp(Node* node, const Operator* op) {
396 : static int32_t shift_mask = 0x1f;
397 : DCHECK_EQ(1, node->InputCount());
398 0 : int32_t shift_amount = OpParameter<int32_t>(node);
399 : Node* shift_node =
400 0 : graph()->NewNode(common()->Int32Constant(shift_amount & shift_mask));
401 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), SimdType::kInt32);
402 : Node* rep_node[kMaxLanes];
403 0 : for (int i = 0; i < kMaxLanes; ++i) {
404 0 : rep_node[i] = graph()->NewNode(op, rep[i], shift_node);
405 : }
406 : ReplaceNode(node, rep_node);
407 0 : }
408 :
409 0 : void SimdScalarLowering::LowerNotEqual(Node* node, SimdType input_rep_type,
410 0 : const Operator* op) {
411 : DCHECK(node->InputCount() == 2);
412 0 : Node** rep_left = GetReplacementsWithType(node->InputAt(0), input_rep_type);
413 0 : Node** rep_right = GetReplacementsWithType(node->InputAt(1), input_rep_type);
414 : Node* rep_node[kMaxLanes];
415 0 : for (int i = 0; i < kMaxLanes; ++i) {
416 : Diamond d(graph(), common(),
417 0 : graph()->NewNode(op, rep_left[i], rep_right[i]));
418 : rep_node[i] = d.Phi(MachineRepresentation::kWord32,
419 0 : jsgraph_->Int32Constant(0), jsgraph_->Int32Constant(1));
420 : }
421 : ReplaceNode(node, rep_node);
422 0 : }
423 :
424 0 : void SimdScalarLowering::LowerNode(Node* node) {
425 : SimdType rep_type = ReplacementType(node);
426 0 : switch (node->opcode()) {
427 : case IrOpcode::kStart: {
428 0 : int parameter_count = GetParameterCountAfterLowering();
429 : // Only exchange the node if the parameter count actually changed.
430 0 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
431 : int delta =
432 0 : parameter_count - static_cast<int>(signature()->parameter_count());
433 0 : int new_output_count = node->op()->ValueOutputCount() + delta;
434 0 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
435 : }
436 : break;
437 : }
438 : case IrOpcode::kParameter: {
439 : DCHECK(node->InputCount() == 1);
440 : // Only exchange the node if the parameter count actually changed. We do
441 : // not even have to do the default lowering because the the start node,
442 : // the only input of a parameter node, only changes if the parameter count
443 : // changes.
444 0 : if (GetParameterCountAfterLowering() !=
445 0 : static_cast<int>(signature()->parameter_count())) {
446 0 : int old_index = ParameterIndexOf(node->op());
447 : int new_index = GetParameterIndexAfterLowering(signature(), old_index);
448 0 : if (old_index == new_index) {
449 0 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
450 :
451 : Node* new_node[kMaxLanes];
452 0 : for (int i = 0; i < kMaxLanes; ++i) {
453 0 : new_node[i] = nullptr;
454 : }
455 0 : new_node[0] = node;
456 0 : if (signature()->GetParam(old_index) ==
457 : MachineRepresentation::kSimd128) {
458 0 : for (int i = 1; i < kMaxLanes; ++i) {
459 : new_node[i] = graph()->NewNode(common()->Parameter(new_index + i),
460 0 : graph()->start());
461 : }
462 : }
463 : ReplaceNode(node, new_node);
464 : }
465 : }
466 : break;
467 : }
468 : case IrOpcode::kLoad: {
469 : MachineRepresentation rep =
470 0 : LoadRepresentationOf(node->op()).representation();
471 : const Operator* load_op;
472 0 : if (rep_type == SimdType::kInt32) {
473 0 : load_op = machine()->Load(MachineType::Int32());
474 0 : } else if (rep_type == SimdType::kFloat32) {
475 0 : load_op = machine()->Load(MachineType::Float32());
476 : }
477 0 : LowerLoadOp(rep, node, load_op);
478 0 : break;
479 : }
480 : case IrOpcode::kUnalignedLoad: {
481 : MachineRepresentation rep =
482 0 : UnalignedLoadRepresentationOf(node->op()).representation();
483 : const Operator* load_op;
484 0 : if (rep_type == SimdType::kInt32) {
485 0 : load_op = machine()->UnalignedLoad(MachineType::Int32());
486 0 : } else if (rep_type == SimdType::kFloat32) {
487 0 : load_op = machine()->UnalignedLoad(MachineType::Float32());
488 : }
489 0 : LowerLoadOp(rep, node, load_op);
490 0 : break;
491 : }
492 : case IrOpcode::kStore: {
493 : MachineRepresentation rep =
494 0 : StoreRepresentationOf(node->op()).representation();
495 : WriteBarrierKind write_barrier_kind =
496 0 : StoreRepresentationOf(node->op()).write_barrier_kind();
497 : const Operator* store_op;
498 0 : if (rep_type == SimdType::kInt32) {
499 : store_op = machine()->Store(StoreRepresentation(
500 0 : MachineRepresentation::kWord32, write_barrier_kind));
501 : } else {
502 : store_op = machine()->Store(StoreRepresentation(
503 0 : MachineRepresentation::kFloat32, write_barrier_kind));
504 : }
505 0 : LowerStoreOp(rep, node, store_op, rep_type);
506 0 : break;
507 : }
508 : case IrOpcode::kUnalignedStore: {
509 0 : MachineRepresentation rep = UnalignedStoreRepresentationOf(node->op());
510 : const Operator* store_op;
511 0 : if (rep_type == SimdType::kInt32) {
512 0 : store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
513 : } else {
514 0 : store_op = machine()->UnalignedStore(MachineRepresentation::kFloat32);
515 : }
516 0 : LowerStoreOp(rep, node, store_op, rep_type);
517 0 : break;
518 : }
519 : case IrOpcode::kReturn: {
520 0 : DefaultLowering(node);
521 : int new_return_count = GetReturnCountAfterLowering(signature());
522 0 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
523 0 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
524 : }
525 : break;
526 : }
527 : case IrOpcode::kCall: {
528 : // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor.
529 0 : CallDescriptor* descriptor =
530 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
531 0 : if (DefaultLowering(node) ||
532 0 : (descriptor->ReturnCount() == 1 &&
533 : descriptor->GetReturnType(0) == MachineType::Simd128())) {
534 : // We have to adjust the call descriptor.
535 : const Operator* op =
536 : common()->Call(wasm::ModuleEnv::GetI32WasmCallDescriptorForSimd(
537 0 : zone(), descriptor));
538 0 : NodeProperties::ChangeOp(node, op);
539 : }
540 0 : if (descriptor->ReturnCount() == 1 &&
541 : descriptor->GetReturnType(0) == MachineType::Simd128()) {
542 : // We access the additional return values through projections.
543 : Node* rep_node[kMaxLanes];
544 0 : for (int i = 0; i < kMaxLanes; ++i) {
545 : rep_node[i] =
546 0 : graph()->NewNode(common()->Projection(i), node, graph()->start());
547 : }
548 : ReplaceNode(node, rep_node);
549 : }
550 : break;
551 : }
552 : case IrOpcode::kPhi: {
553 0 : MachineRepresentation rep = PhiRepresentationOf(node->op());
554 0 : if (rep == MachineRepresentation::kSimd128) {
555 : // The replacement nodes have already been created, we only have to
556 : // replace placeholder nodes.
557 0 : Node** rep_node = GetReplacements(node);
558 0 : for (int i = 0; i < node->op()->ValueInputCount(); ++i) {
559 : Node** rep_input =
560 0 : GetReplacementsWithType(node->InputAt(i), rep_type);
561 0 : for (int j = 0; j < kMaxLanes; j++) {
562 0 : rep_node[j]->ReplaceInput(i, rep_input[j]);
563 : }
564 : }
565 : } else {
566 0 : DefaultLowering(node);
567 : }
568 : break;
569 : }
570 : #define I32X4_BINOP_CASE(opcode, instruction) \
571 : case IrOpcode::opcode: { \
572 : LowerBinaryOp(node, rep_type, machine()->instruction()); \
573 : break; \
574 : }
575 0 : I32X4_BINOP_CASE(kI32x4Add, Int32Add)
576 0 : I32X4_BINOP_CASE(kI32x4Sub, Int32Sub)
577 0 : I32X4_BINOP_CASE(kI32x4Mul, Int32Mul)
578 0 : I32X4_BINOP_CASE(kS128And, Word32And)
579 0 : I32X4_BINOP_CASE(kS128Or, Word32Or)
580 0 : I32X4_BINOP_CASE(kS128Xor, Word32Xor)
581 : #undef I32X4_BINOP_CASE
582 : case IrOpcode::kI32x4MaxS: {
583 0 : LowerIntMinMax(node, machine()->Int32LessThan(), true);
584 0 : break;
585 : }
586 : case IrOpcode::kI32x4MinS: {
587 0 : LowerIntMinMax(node, machine()->Int32LessThan(), false);
588 0 : break;
589 : }
590 : case IrOpcode::kI32x4MaxU: {
591 0 : LowerIntMinMax(node, machine()->Uint32LessThan(), true);
592 0 : break;
593 : }
594 : case IrOpcode::kI32x4MinU: {
595 0 : LowerIntMinMax(node, machine()->Uint32LessThan(), false);
596 0 : break;
597 : }
598 : case IrOpcode::kI32x4Neg: {
599 : DCHECK(node->InputCount() == 1);
600 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
601 : Node* rep_node[kMaxLanes];
602 0 : Node* zero = graph()->NewNode(common()->Int32Constant(0));
603 0 : for (int i = 0; i < kMaxLanes; ++i) {
604 0 : rep_node[i] = graph()->NewNode(machine()->Int32Sub(), zero, rep[i]);
605 : }
606 : ReplaceNode(node, rep_node);
607 : break;
608 : }
609 : case IrOpcode::kS128Not: {
610 : DCHECK(node->InputCount() == 1);
611 0 : Node** rep = GetReplacementsWithType(node->InputAt(0), rep_type);
612 : Node* rep_node[kMaxLanes];
613 0 : Node* mask = graph()->NewNode(common()->Int32Constant(0xffffffff));
614 0 : for (int i = 0; i < kMaxLanes; ++i) {
615 0 : rep_node[i] = graph()->NewNode(machine()->Word32Xor(), rep[i], mask);
616 : }
617 : ReplaceNode(node, rep_node);
618 : break;
619 : }
620 : case IrOpcode::kI32x4SConvertF32x4: {
621 0 : LowerConvertFromFloat(node, true);
622 0 : break;
623 : }
624 : case IrOpcode::kI32x4UConvertF32x4: {
625 0 : LowerConvertFromFloat(node, false);
626 0 : break;
627 : }
628 : case IrOpcode::kI32x4Shl: {
629 0 : LowerShiftOp(node, machine()->Word32Shl());
630 0 : break;
631 : }
632 : case IrOpcode::kI32x4ShrS: {
633 0 : LowerShiftOp(node, machine()->Word32Sar());
634 0 : break;
635 : }
636 : case IrOpcode::kI32x4ShrU: {
637 0 : LowerShiftOp(node, machine()->Word32Shr());
638 0 : break;
639 : }
640 : #define F32X4_BINOP_CASE(name) \
641 : case IrOpcode::kF32x4##name: { \
642 : LowerBinaryOp(node, rep_type, machine()->Float32##name()); \
643 : break; \
644 : }
645 0 : F32X4_BINOP_CASE(Add)
646 0 : F32X4_BINOP_CASE(Sub)
647 0 : F32X4_BINOP_CASE(Mul)
648 0 : F32X4_BINOP_CASE(Min)
649 0 : F32X4_BINOP_CASE(Max)
650 : #undef F32X4_BINOP_CASE
651 : #define F32X4_UNOP_CASE(name) \
652 : case IrOpcode::kF32x4##name: { \
653 : LowerUnaryOp(node, rep_type, machine()->Float32##name()); \
654 : break; \
655 : }
656 0 : F32X4_UNOP_CASE(Abs)
657 0 : F32X4_UNOP_CASE(Neg)
658 : #undef F32x4_UNOP_CASE
659 : case IrOpcode::kF32x4SConvertI32x4: {
660 0 : LowerUnaryOp(node, SimdType::kInt32, machine()->RoundInt32ToFloat32());
661 0 : break;
662 : }
663 : case IrOpcode::kF32x4UConvertI32x4: {
664 0 : LowerUnaryOp(node, SimdType::kInt32, machine()->RoundUint32ToFloat32());
665 0 : break;
666 : }
667 : case IrOpcode::kI32x4Splat:
668 : case IrOpcode::kF32x4Splat: {
669 : Node* rep_node[kMaxLanes];
670 0 : for (int i = 0; i < kMaxLanes; ++i) {
671 0 : if (HasReplacement(0, node->InputAt(0))) {
672 0 : rep_node[i] = GetReplacements(node->InputAt(0))[0];
673 : } else {
674 0 : rep_node[i] = node->InputAt(0);
675 : }
676 : }
677 : ReplaceNode(node, rep_node);
678 : break;
679 : }
680 : case IrOpcode::kI32x4ExtractLane:
681 : case IrOpcode::kF32x4ExtractLane: {
682 0 : int32_t lane = OpParameter<int32_t>(node);
683 : Node* rep_node[kMaxLanes] = {
684 0 : GetReplacementsWithType(node->InputAt(0), rep_type)[lane], nullptr,
685 0 : nullptr, nullptr};
686 : ReplaceNode(node, rep_node);
687 : break;
688 : }
689 : case IrOpcode::kI32x4ReplaceLane:
690 : case IrOpcode::kF32x4ReplaceLane: {
691 : DCHECK_EQ(2, node->InputCount());
692 : Node* repNode = node->InputAt(1);
693 0 : int32_t lane = OpParameter<int32_t>(node);
694 : DCHECK(lane >= 0 && lane <= 3);
695 0 : Node** rep_node = GetReplacementsWithType(node->InputAt(0), rep_type);
696 0 : if (HasReplacement(0, repNode)) {
697 0 : rep_node[lane] = GetReplacements(repNode)[0];
698 : } else {
699 0 : rep_node[lane] = repNode;
700 : }
701 : ReplaceNode(node, rep_node);
702 : break;
703 : }
704 : #define COMPARISON_CASE(type, simd_op, lowering_op, invert) \
705 : case IrOpcode::simd_op: { \
706 : LowerBinaryOp(node, SimdType::k##type, machine()->lowering_op(), invert); \
707 : break; \
708 : }
709 0 : COMPARISON_CASE(Float32, kF32x4Eq, Float32Equal, false)
710 0 : COMPARISON_CASE(Float32, kF32x4Lt, Float32LessThan, false)
711 0 : COMPARISON_CASE(Float32, kF32x4Le, Float32LessThanOrEqual, false)
712 0 : COMPARISON_CASE(Float32, kF32x4Gt, Float32LessThan, true)
713 0 : COMPARISON_CASE(Float32, kF32x4Ge, Float32LessThanOrEqual, true)
714 0 : COMPARISON_CASE(Int32, kI32x4Eq, Word32Equal, false)
715 0 : COMPARISON_CASE(Int32, kI32x4LtS, Int32LessThan, false)
716 0 : COMPARISON_CASE(Int32, kI32x4LeS, Int32LessThanOrEqual, false)
717 0 : COMPARISON_CASE(Int32, kI32x4GtS, Int32LessThan, true)
718 0 : COMPARISON_CASE(Int32, kI32x4GeS, Int32LessThanOrEqual, true)
719 0 : COMPARISON_CASE(Int32, kI32x4LtU, Uint32LessThan, false)
720 0 : COMPARISON_CASE(Int32, kI32x4LeU, Uint32LessThanOrEqual, false)
721 0 : COMPARISON_CASE(Int32, kI32x4GtU, Uint32LessThan, true)
722 0 : COMPARISON_CASE(Int32, kI32x4GeU, Uint32LessThanOrEqual, true)
723 : #undef COMPARISON_CASE
724 : case IrOpcode::kF32x4Ne: {
725 0 : LowerNotEqual(node, SimdType::kFloat32, machine()->Float32Equal());
726 0 : break;
727 : }
728 : case IrOpcode::kI32x4Ne: {
729 0 : LowerNotEqual(node, SimdType::kInt32, machine()->Word32Equal());
730 0 : break;
731 : }
732 : case IrOpcode::kS32x4Select: {
733 : DCHECK(node->InputCount() == 3);
734 : DCHECK(ReplacementType(node->InputAt(0)) == SimdType::kSimd1x4);
735 0 : Node** boolean_input = GetReplacements(node->InputAt(0));
736 0 : Node** rep_left = GetReplacementsWithType(node->InputAt(1), rep_type);
737 0 : Node** rep_right = GetReplacementsWithType(node->InputAt(2), rep_type);
738 : Node* rep_node[kMaxLanes];
739 0 : for (int i = 0; i < kMaxLanes; ++i) {
740 : Diamond d(graph(), common(),
741 0 : graph()->NewNode(machine()->Word32Equal(), boolean_input[i],
742 0 : jsgraph_->Int32Constant(0)));
743 0 : if (rep_type == SimdType::kFloat32) {
744 : rep_node[i] =
745 0 : d.Phi(MachineRepresentation::kFloat32, rep_right[1], rep_left[0]);
746 0 : } else if (rep_type == SimdType::kInt32) {
747 : rep_node[i] =
748 0 : d.Phi(MachineRepresentation::kWord32, rep_right[1], rep_left[0]);
749 : } else {
750 0 : UNREACHABLE();
751 : }
752 : }
753 : ReplaceNode(node, rep_node);
754 : break;
755 : }
756 0 : default: { DefaultLowering(node); }
757 : }
758 0 : }
759 :
760 0 : bool SimdScalarLowering::DefaultLowering(Node* node) {
761 : bool something_changed = false;
762 0 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
763 : Node* input = node->InputAt(i);
764 0 : if (HasReplacement(0, input)) {
765 : something_changed = true;
766 0 : node->ReplaceInput(i, GetReplacements(input)[0]);
767 : }
768 0 : if (HasReplacement(1, input)) {
769 : something_changed = true;
770 0 : for (int j = 1; j < kMaxLanes; j++) {
771 0 : node->InsertInput(zone(), i + j, GetReplacements(input)[j]);
772 : }
773 : }
774 : }
775 0 : return something_changed;
776 : }
777 :
778 0 : void SimdScalarLowering::ReplaceNode(Node* old, Node** new_node) {
779 : // if new_low == nullptr, then also new_high == nullptr.
780 : DCHECK(new_node[0] != nullptr ||
781 : (new_node[1] == nullptr && new_node[2] == nullptr &&
782 : new_node[3] == nullptr));
783 0 : for (int i = 0; i < kMaxLanes; ++i) {
784 0 : replacements_[old->id()].node[i] = new_node[i];
785 : }
786 0 : }
787 :
788 0 : bool SimdScalarLowering::HasReplacement(size_t index, Node* node) {
789 0 : return replacements_[node->id()].node[index] != nullptr;
790 : }
791 :
792 0 : SimdScalarLowering::SimdType SimdScalarLowering::ReplacementType(Node* node) {
793 0 : return replacements_[node->id()].type;
794 : }
795 :
796 0 : Node** SimdScalarLowering::GetReplacements(Node* node) {
797 0 : Node** result = replacements_[node->id()].node;
798 : DCHECK(result);
799 0 : return result;
800 : }
801 :
802 0 : Node** SimdScalarLowering::GetReplacementsWithType(Node* node, SimdType type) {
803 0 : Node** replacements = GetReplacements(node);
804 0 : if (ReplacementType(node) == type) {
805 : return GetReplacements(node);
806 : }
807 : Node** result = zone()->NewArray<Node*>(kMaxLanes);
808 0 : if (ReplacementType(node) == SimdType::kInt32 && type == SimdType::kFloat32) {
809 0 : for (int i = 0; i < kMaxLanes; ++i) {
810 0 : if (replacements[i] != nullptr) {
811 0 : result[i] = graph()->NewNode(machine()->BitcastInt32ToFloat32(),
812 0 : replacements[i]);
813 : } else {
814 0 : result[i] = nullptr;
815 : }
816 : }
817 0 : } else if (ReplacementType(node) == SimdType::kFloat32 &&
818 : type == SimdType::kInt32) {
819 0 : for (int i = 0; i < kMaxLanes; ++i) {
820 0 : if (replacements[i] != nullptr) {
821 0 : result[i] = graph()->NewNode(machine()->BitcastFloat32ToInt32(),
822 0 : replacements[i]);
823 : } else {
824 0 : result[i] = nullptr;
825 : }
826 : }
827 : } else {
828 0 : UNREACHABLE();
829 : }
830 0 : return result;
831 : }
832 :
833 0 : void SimdScalarLowering::PreparePhiReplacement(Node* phi) {
834 0 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
835 0 : if (rep == MachineRepresentation::kSimd128) {
836 : // We have to create the replacements for a phi node before we actually
837 : // lower the phi to break potential cycles in the graph. The replacements of
838 : // input nodes do not exist yet, so we use a placeholder node to pass the
839 : // graph verifier.
840 0 : int value_count = phi->op()->ValueInputCount();
841 : SimdType type = ReplacementType(phi);
842 : Node** inputs_rep[kMaxLanes];
843 0 : for (int i = 0; i < kMaxLanes; ++i) {
844 0 : inputs_rep[i] = zone()->NewArray<Node*>(value_count + 1);
845 0 : inputs_rep[i][value_count] = NodeProperties::GetControlInput(phi, 0);
846 : }
847 0 : for (int i = 0; i < value_count; ++i) {
848 0 : for (int j = 0; j < kMaxLanes; j++) {
849 0 : inputs_rep[j][i] = placeholder_;
850 : }
851 : }
852 : Node* rep_nodes[kMaxLanes];
853 0 : for (int i = 0; i < kMaxLanes; ++i) {
854 0 : if (type == SimdType::kInt32) {
855 : rep_nodes[i] = graph()->NewNode(
856 : common()->Phi(MachineRepresentation::kWord32, value_count),
857 0 : value_count + 1, inputs_rep[i], false);
858 0 : } else if (type == SimdType::kFloat32) {
859 : rep_nodes[i] = graph()->NewNode(
860 : common()->Phi(MachineRepresentation::kFloat32, value_count),
861 0 : value_count + 1, inputs_rep[i], false);
862 : } else {
863 0 : UNREACHABLE();
864 : }
865 : }
866 : ReplaceNode(phi, rep_nodes);
867 : }
868 0 : }
869 : } // namespace compiler
870 : } // namespace internal
871 : } // namespace v8
|
