Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/int64-lowering.h"
6 :
7 : #include "src/compiler/common-operator.h"
8 : #include "src/compiler/diamond.h"
9 : #include "src/compiler/graph.h"
10 : #include "src/compiler/linkage.h"
11 : #include "src/compiler/machine-operator.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-properties.h"
14 : #include "src/compiler/node.h"
15 : #include "src/compiler/wasm-compiler.h"
16 : // TODO(wasm): Remove this include.
17 : #include "src/wasm/wasm-linkage.h"
18 : #include "src/zone/zone.h"
19 :
20 : namespace v8 {
21 : namespace internal {
22 : namespace compiler {
23 :
24 47 : Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
25 : CommonOperatorBuilder* common, Zone* zone,
26 : Signature<MachineRepresentation>* signature)
27 : : zone_(zone),
28 : graph_(graph),
29 : machine_(machine),
30 : common_(common),
31 : state_(graph, 3),
32 : stack_(zone),
33 : replacements_(nullptr),
34 : signature_(signature),
35 47 : placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
36 141 : graph->start())) {
37 : DCHECK_NOT_NULL(graph);
38 : DCHECK_NOT_NULL(graph->end());
39 47 : replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
40 47 : memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
41 47 : }
42 :
43 47 : void Int64Lowering::LowerGraph() {
44 47 : if (!machine()->Is32()) {
45 : return;
46 : }
47 94 : stack_.push_back({graph()->end(), 0});
48 : state_.Set(graph()->end(), State::kOnStack);
49 :
50 771 : while (!stack_.empty()) {
51 : NodeState& top = stack_.back();
52 1448 : if (top.input_index == top.node->InputCount()) {
53 : // All inputs of top have already been lowered, now lower top.
54 325 : stack_.pop_back();
55 325 : state_.Set(top.node, State::kVisited);
56 325 : LowerNode(top.node);
57 : } else {
58 : // Push the next input onto the stack.
59 399 : Node* input = top.node->InputAt(top.input_index++);
60 399 : if (state_.Get(input) == State::kUnvisited) {
61 278 : if (input->opcode() == IrOpcode::kPhi) {
62 : // To break cycles with phi nodes we push phis on a separate stack so
63 : // that they are processed after all other nodes.
64 4 : PreparePhiReplacement(input);
65 8 : stack_.push_front({input, 0});
66 274 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
67 : input->opcode() == IrOpcode::kLoop) {
68 4 : stack_.push_front({input, 0});
69 : } else {
70 544 : stack_.push_back({input, 0});
71 : }
72 : state_.Set(input, State::kOnStack);
73 : }
74 : }
75 : }
76 : }
77 :
78 : namespace {
79 :
80 : int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor,
81 : int old_index) {
82 : int result = old_index;
83 0 : for (int i = 0; i < old_index; i++) {
84 0 : if (call_descriptor->GetReturnType(i).representation() ==
85 : MachineRepresentation::kWord64) {
86 0 : result++;
87 : }
88 : }
89 : return result;
90 : }
91 :
92 : int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) {
93 0 : return GetReturnIndexAfterLowering(
94 : call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
95 : }
96 :
97 : int GetParameterIndexAfterLowering(
98 : Signature<MachineRepresentation>* signature, int old_index) {
99 : int result = old_index;
100 106 : for (int i = 0; i < old_index; i++) {
101 52 : if (signature->GetParam(i) == MachineRepresentation::kWord64) {
102 18 : result++;
103 : }
104 : }
105 : return result;
106 : }
107 :
108 : int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
109 30 : int result = static_cast<int>(signature->return_count());
110 90 : for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
111 60 : if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
112 30 : result++;
113 : }
114 : }
115 : return result;
116 : }
117 :
118 : } // namespace
119 :
120 0 : void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
121 : DCHECK_EQ(5, node->InputCount());
122 0 : LowerMemoryBaseAndIndex(node);
123 : Node* value = node->InputAt(2);
124 0 : node->ReplaceInput(2, GetReplacementLow(value));
125 0 : node->InsertInput(zone(), 3, GetReplacementHigh(value));
126 0 : NodeProperties::ChangeOp(node, op);
127 0 : ReplaceNodeWithProjections(node);
128 0 : }
129 :
130 0 : void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
131 0 : DefaultLowering(node, true);
132 0 : NodeProperties::ChangeOp(node, op);
133 0 : ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
134 0 : }
135 :
136 : // static
137 0 : int Int64Lowering::GetParameterCountAfterLowering(
138 : Signature<MachineRepresentation>* signature) {
139 : // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
140 : // after lowering.
141 47 : return GetParameterIndexAfterLowering(
142 0 : signature, static_cast<int>(signature->parameter_count()));
143 : }
144 :
145 7 : void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
146 : Node*& index_high) {
147 : #if defined(V8_TARGET_LITTLE_ENDIAN)
148 7 : index_low = index;
149 14 : index_high = graph()->NewNode(machine()->Int32Add(), index,
150 7 : graph()->NewNode(common()->Int32Constant(4)));
151 : #elif defined(V8_TARGET_BIG_ENDIAN)
152 : index_low = graph()->NewNode(machine()->Int32Add(), index,
153 : graph()->NewNode(common()->Int32Constant(4)));
154 : index_high = index;
155 : #endif
156 7 : }
157 :
158 325 : void Int64Lowering::LowerNode(Node* node) {
159 325 : switch (node->opcode()) {
160 : case IrOpcode::kInt64Constant: {
161 55 : int64_t value = OpParameter<int64_t>(node->op());
162 55 : Node* low_node = graph()->NewNode(
163 : common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
164 55 : Node* high_node = graph()->NewNode(
165 55 : common()->Int32Constant(static_cast<int32_t>(value >> 32)));
166 : ReplaceNode(node, low_node, high_node);
167 : break;
168 : }
169 : case IrOpcode::kLoad:
170 : case IrOpcode::kUnalignedLoad: {
171 : MachineRepresentation rep;
172 4 : if (node->opcode() == IrOpcode::kLoad) {
173 3 : rep = LoadRepresentationOf(node->op()).representation();
174 : } else {
175 : DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
176 1 : rep = LoadRepresentationOf(node->op()).representation();
177 : }
178 :
179 4 : if (rep == MachineRepresentation::kWord64) {
180 4 : LowerMemoryBaseAndIndex(node);
181 : Node* base = node->InputAt(0);
182 : Node* index = node->InputAt(1);
183 : Node* index_low;
184 : Node* index_high;
185 4 : GetIndexNodes(index, index_low, index_high);
186 : const Operator* load_op;
187 :
188 4 : if (node->opcode() == IrOpcode::kLoad) {
189 3 : load_op = machine()->Load(MachineType::Int32());
190 : } else {
191 : DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
192 1 : load_op = machine()->UnalignedLoad(MachineType::Int32());
193 : }
194 :
195 : Node* high_node;
196 4 : if (node->InputCount() > 2) {
197 : Node* effect_high = node->InputAt(2);
198 : Node* control_high = node->InputAt(3);
199 4 : high_node = graph()->NewNode(load_op, base, index_high, effect_high,
200 : control_high);
201 : // change the effect change from old_node --> old_effect to
202 : // old_node --> high_node --> old_effect.
203 4 : node->ReplaceInput(2, high_node);
204 : } else {
205 0 : high_node = graph()->NewNode(load_op, base, index_high);
206 : }
207 4 : node->ReplaceInput(1, index_low);
208 4 : NodeProperties::ChangeOp(node, load_op);
209 : ReplaceNode(node, node, high_node);
210 : } else {
211 0 : DefaultLowering(node);
212 : }
213 : break;
214 : }
215 : case IrOpcode::kStore:
216 : case IrOpcode::kUnalignedStore: {
217 : MachineRepresentation rep;
218 4 : if (node->opcode() == IrOpcode::kStore) {
219 3 : rep = StoreRepresentationOf(node->op()).representation();
220 : } else {
221 : DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
222 1 : rep = UnalignedStoreRepresentationOf(node->op());
223 : }
224 :
225 4 : if (rep == MachineRepresentation::kWord64) {
226 : // We change the original store node to store the low word, and create
227 : // a new store node to store the high word. The effect and control edges
228 : // are copied from the original store to the new store node, the effect
229 : // edge of the original store is redirected to the new store.
230 3 : LowerMemoryBaseAndIndex(node);
231 : Node* base = node->InputAt(0);
232 : Node* index = node->InputAt(1);
233 : Node* index_low;
234 : Node* index_high;
235 3 : GetIndexNodes(index, index_low, index_high);
236 : Node* value = node->InputAt(2);
237 : DCHECK(HasReplacementLow(value));
238 : DCHECK(HasReplacementHigh(value));
239 :
240 : const Operator* store_op;
241 3 : if (node->opcode() == IrOpcode::kStore) {
242 : WriteBarrierKind write_barrier_kind =
243 2 : StoreRepresentationOf(node->op()).write_barrier_kind();
244 2 : store_op = machine()->Store(StoreRepresentation(
245 2 : MachineRepresentation::kWord32, write_barrier_kind));
246 : } else {
247 : DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
248 1 : store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
249 : }
250 :
251 : Node* high_node;
252 3 : if (node->InputCount() > 3) {
253 : Node* effect_high = node->InputAt(3);
254 : Node* control_high = node->InputAt(4);
255 3 : high_node = graph()->NewNode(store_op, base, index_high,
256 : GetReplacementHigh(value), effect_high,
257 : control_high);
258 3 : node->ReplaceInput(3, high_node);
259 :
260 : } else {
261 0 : high_node = graph()->NewNode(store_op, base, index_high,
262 : GetReplacementHigh(value));
263 : }
264 :
265 3 : node->ReplaceInput(1, index_low);
266 3 : node->ReplaceInput(2, GetReplacementLow(value));
267 3 : NodeProperties::ChangeOp(node, store_op);
268 : ReplaceNode(node, node, high_node);
269 : } else {
270 1 : DefaultLowering(node, true);
271 : }
272 : break;
273 : }
274 : case IrOpcode::kStart: {
275 : int parameter_count = GetParameterCountAfterLowering(signature());
276 : // Only exchange the node if the parameter count actually changed.
277 47 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
278 : int delta =
279 5 : parameter_count - static_cast<int>(signature()->parameter_count());
280 5 : int new_output_count = node->op()->ValueOutputCount() + delta;
281 5 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
282 : }
283 : break;
284 : }
285 : case IrOpcode::kParameter: {
286 : DCHECK_EQ(1, node->InputCount());
287 5 : int param_count = static_cast<int>(signature()->parameter_count());
288 : // Only exchange the node if the parameter count actually changed. We do
289 : // not even have to do the default lowering because the the start node,
290 : // the only input of a parameter node, only changes if the parameter count
291 : // changes.
292 5 : if (GetParameterCountAfterLowering(signature()) != param_count) {
293 5 : int old_index = ParameterIndexOf(node->op());
294 : // Prevent special lowering of wasm's instance or JS
295 : // context/closure parameters.
296 5 : if (old_index <= 0 || old_index > param_count) {
297 3 : DefaultLowering(node);
298 3 : break;
299 : }
300 : // Adjust old_index to be compliant with the signature.
301 2 : --old_index;
302 : int new_index = GetParameterIndexAfterLowering(signature(), old_index);
303 : // Adjust new_index to consider the instance parameter.
304 2 : ++new_index;
305 2 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
306 :
307 4 : if (signature()->GetParam(old_index) ==
308 : MachineRepresentation::kWord64) {
309 1 : Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
310 : graph()->start());
311 : ReplaceNode(node, node, high_node);
312 : }
313 : }
314 : break;
315 : }
316 : case IrOpcode::kReturn: {
317 : int input_count = node->InputCount();
318 47 : DefaultLowering(node);
319 47 : if (input_count != node->InputCount()) {
320 : int new_return_count = GetReturnCountAfterLowering(signature());
321 30 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
322 30 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
323 : }
324 : }
325 : break;
326 : }
327 : case IrOpcode::kTailCall: {
328 : auto call_descriptor =
329 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
330 : bool returns_require_lowering =
331 : GetReturnCountAfterLowering(call_descriptor) !=
332 : static_cast<int>(call_descriptor->ReturnCount());
333 0 : if (DefaultLowering(node) || returns_require_lowering) {
334 : // Tail calls do not have return values, so adjusting the call
335 : // descriptor is enough.
336 0 : auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor);
337 0 : NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
338 : }
339 : break;
340 : }
341 : case IrOpcode::kCall: {
342 : auto call_descriptor =
343 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
344 : bool returns_require_lowering =
345 : GetReturnCountAfterLowering(call_descriptor) !=
346 : static_cast<int>(call_descriptor->ReturnCount());
347 0 : if (DefaultLowering(node) || returns_require_lowering) {
348 : // We have to adjust the call descriptor.
349 0 : NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor(
350 0 : zone(), call_descriptor)));
351 : }
352 0 : if (returns_require_lowering) {
353 : size_t return_arity = call_descriptor->ReturnCount();
354 0 : if (return_arity == 1) {
355 : // We access the additional return values through projections.
356 0 : ReplaceNodeWithProjections(node);
357 : } else {
358 : ZoneVector<Node*> projections(return_arity, zone());
359 : NodeProperties::CollectValueProjections(node, projections.data(),
360 0 : return_arity);
361 0 : for (size_t old_index = 0, new_index = 0; old_index < return_arity;
362 : ++old_index, ++new_index) {
363 0 : Node* use_node = projections[old_index];
364 : DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
365 : DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
366 : static_cast<int>(old_index)),
367 : static_cast<int>(new_index));
368 0 : if (new_index != old_index) {
369 0 : NodeProperties::ChangeOp(
370 0 : use_node, common()->Projection(new_index));
371 : }
372 0 : if (call_descriptor->GetReturnType(old_index).representation() ==
373 : MachineRepresentation::kWord64) {
374 0 : Node* high_node = graph()->NewNode(
375 : common()->Projection(new_index + 1), node,
376 : graph()->start());
377 : ReplaceNode(use_node, use_node, high_node);
378 : ++new_index;
379 : }
380 : }
381 : }
382 : }
383 : break;
384 : }
385 : case IrOpcode::kWord64And: {
386 : DCHECK_EQ(2, node->InputCount());
387 : Node* left = node->InputAt(0);
388 : Node* right = node->InputAt(1);
389 :
390 : Node* low_node =
391 3 : graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
392 : GetReplacementLow(right));
393 : Node* high_node =
394 3 : graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
395 : GetReplacementHigh(right));
396 : ReplaceNode(node, low_node, high_node);
397 : break;
398 : }
399 : case IrOpcode::kTruncateInt64ToInt32: {
400 : DCHECK_EQ(1, node->InputCount());
401 : Node* input = node->InputAt(0);
402 : ReplaceNode(node, GetReplacementLow(input), nullptr);
403 4 : node->NullAllInputs();
404 4 : break;
405 : }
406 : case IrOpcode::kInt64Add: {
407 : DCHECK_EQ(2, node->InputCount());
408 :
409 : Node* right = node->InputAt(1);
410 0 : node->ReplaceInput(1, GetReplacementLow(right));
411 0 : node->AppendInput(zone(), GetReplacementHigh(right));
412 :
413 : Node* left = node->InputAt(0);
414 0 : node->ReplaceInput(0, GetReplacementLow(left));
415 0 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
416 :
417 0 : NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
418 : // We access the additional return values through projections.
419 0 : ReplaceNodeWithProjections(node);
420 0 : break;
421 : }
422 : case IrOpcode::kInt64Sub: {
423 : DCHECK_EQ(2, node->InputCount());
424 :
425 : Node* right = node->InputAt(1);
426 1 : node->ReplaceInput(1, GetReplacementLow(right));
427 1 : node->AppendInput(zone(), GetReplacementHigh(right));
428 :
429 : Node* left = node->InputAt(0);
430 1 : node->ReplaceInput(0, GetReplacementLow(left));
431 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
432 :
433 1 : NodeProperties::ChangeOp(node, machine()->Int32PairSub());
434 : // We access the additional return values through projections.
435 1 : ReplaceNodeWithProjections(node);
436 1 : break;
437 : }
438 : case IrOpcode::kInt64Mul: {
439 : DCHECK_EQ(2, node->InputCount());
440 :
441 : Node* right = node->InputAt(1);
442 1 : node->ReplaceInput(1, GetReplacementLow(right));
443 1 : node->AppendInput(zone(), GetReplacementHigh(right));
444 :
445 : Node* left = node->InputAt(0);
446 1 : node->ReplaceInput(0, GetReplacementLow(left));
447 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
448 :
449 1 : NodeProperties::ChangeOp(node, machine()->Int32PairMul());
450 : // We access the additional return values through projections.
451 1 : ReplaceNodeWithProjections(node);
452 1 : break;
453 : }
454 : case IrOpcode::kWord64Or: {
455 : DCHECK_EQ(2, node->InputCount());
456 : Node* left = node->InputAt(0);
457 : Node* right = node->InputAt(1);
458 :
459 : Node* low_node =
460 1 : graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
461 : GetReplacementLow(right));
462 : Node* high_node =
463 1 : graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
464 : GetReplacementHigh(right));
465 : ReplaceNode(node, low_node, high_node);
466 : break;
467 : }
468 : case IrOpcode::kWord64Xor: {
469 : DCHECK_EQ(2, node->InputCount());
470 : Node* left = node->InputAt(0);
471 : Node* right = node->InputAt(1);
472 :
473 : Node* low_node =
474 1 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
475 : GetReplacementLow(right));
476 : Node* high_node =
477 1 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
478 : GetReplacementHigh(right));
479 : ReplaceNode(node, low_node, high_node);
480 : break;
481 : }
482 : case IrOpcode::kWord64Shl: {
483 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
484 : // of the output to 0 and just calculate the high word.
485 : DCHECK_EQ(2, node->InputCount());
486 : Node* shift = node->InputAt(1);
487 1 : if (HasReplacementLow(shift)) {
488 : // We do not have to care about the high word replacement, because
489 : // the shift can only be between 0 and 63 anyways.
490 1 : node->ReplaceInput(1, GetReplacementLow(shift));
491 : }
492 :
493 : Node* value = node->InputAt(0);
494 1 : node->ReplaceInput(0, GetReplacementLow(value));
495 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
496 :
497 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShl());
498 : // We access the additional return values through projections.
499 1 : ReplaceNodeWithProjections(node);
500 1 : break;
501 : }
502 : case IrOpcode::kWord64Shr: {
503 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
504 : // of the output to 0 and just calculate the high word.
505 : DCHECK_EQ(2, node->InputCount());
506 : Node* shift = node->InputAt(1);
507 1 : if (HasReplacementLow(shift)) {
508 : // We do not have to care about the high word replacement, because
509 : // the shift can only be between 0 and 63 anyways.
510 1 : node->ReplaceInput(1, GetReplacementLow(shift));
511 : }
512 :
513 : Node* value = node->InputAt(0);
514 1 : node->ReplaceInput(0, GetReplacementLow(value));
515 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
516 :
517 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShr());
518 : // We access the additional return values through projections.
519 1 : ReplaceNodeWithProjections(node);
520 1 : break;
521 : }
522 : case IrOpcode::kWord64Sar: {
523 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
524 : // of the output to 0 and just calculate the high word.
525 : DCHECK_EQ(2, node->InputCount());
526 : Node* shift = node->InputAt(1);
527 1 : if (HasReplacementLow(shift)) {
528 : // We do not have to care about the high word replacement, because
529 : // the shift can only be between 0 and 63 anyways.
530 1 : node->ReplaceInput(1, GetReplacementLow(shift));
531 : }
532 :
533 : Node* value = node->InputAt(0);
534 1 : node->ReplaceInput(0, GetReplacementLow(value));
535 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
536 :
537 1 : NodeProperties::ChangeOp(node, machine()->Word32PairSar());
538 : // We access the additional return values through projections.
539 1 : ReplaceNodeWithProjections(node);
540 1 : break;
541 : }
542 : case IrOpcode::kWord64Equal: {
543 : DCHECK_EQ(2, node->InputCount());
544 : Node* left = node->InputAt(0);
545 : Node* right = node->InputAt(1);
546 :
547 : // TODO(wasm): Use explicit comparisons and && here?
548 10 : Node* replacement = graph()->NewNode(
549 : machine()->Word32Equal(),
550 : graph()->NewNode(
551 : machine()->Word32Or(),
552 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
553 : GetReplacementLow(right)),
554 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
555 : GetReplacementHigh(right))),
556 : graph()->NewNode(common()->Int32Constant(0)));
557 :
558 : ReplaceNode(node, replacement, nullptr);
559 : break;
560 : }
561 : case IrOpcode::kInt64LessThan: {
562 1 : LowerComparison(node, machine()->Int32LessThan(),
563 1 : machine()->Uint32LessThan());
564 1 : break;
565 : }
566 : case IrOpcode::kInt64LessThanOrEqual: {
567 1 : LowerComparison(node, machine()->Int32LessThan(),
568 1 : machine()->Uint32LessThanOrEqual());
569 1 : break;
570 : }
571 : case IrOpcode::kUint64LessThan: {
572 1 : LowerComparison(node, machine()->Uint32LessThan(),
573 1 : machine()->Uint32LessThan());
574 1 : break;
575 : }
576 : case IrOpcode::kUint64LessThanOrEqual: {
577 1 : LowerComparison(node, machine()->Uint32LessThan(),
578 1 : machine()->Uint32LessThanOrEqual());
579 1 : break;
580 : }
581 : case IrOpcode::kSignExtendWord32ToInt64:
582 : case IrOpcode::kChangeInt32ToInt64: {
583 : DCHECK_EQ(1, node->InputCount());
584 : Node* input = node->InputAt(0);
585 2 : if (HasReplacementLow(input)) {
586 : input = GetReplacementLow(input);
587 : }
588 : // We use SAR to preserve the sign in the high word.
589 4 : ReplaceNode(
590 : node, input,
591 : graph()->NewNode(machine()->Word32Sar(), input,
592 : graph()->NewNode(common()->Int32Constant(31))));
593 2 : node->NullAllInputs();
594 2 : break;
595 : }
596 : case IrOpcode::kChangeUint32ToUint64: {
597 : DCHECK_EQ(1, node->InputCount());
598 : Node* input = node->InputAt(0);
599 2 : if (HasReplacementLow(input)) {
600 : input = GetReplacementLow(input);
601 : }
602 2 : ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
603 2 : node->NullAllInputs();
604 2 : break;
605 : }
606 : case IrOpcode::kBitcastInt64ToFloat64: {
607 : DCHECK_EQ(1, node->InputCount());
608 : Node* input = node->InputAt(0);
609 1 : Node* stack_slot = graph()->NewNode(
610 : machine()->StackSlot(MachineRepresentation::kWord64));
611 :
612 3 : Node* store_high_word = graph()->NewNode(
613 : machine()->Store(
614 : StoreRepresentation(MachineRepresentation::kWord32,
615 : WriteBarrierKind::kNoWriteBarrier)),
616 : stack_slot,
617 : graph()->NewNode(
618 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
619 : GetReplacementHigh(input), graph()->start(), graph()->start());
620 :
621 3 : Node* store_low_word = graph()->NewNode(
622 : machine()->Store(
623 : StoreRepresentation(MachineRepresentation::kWord32,
624 : WriteBarrierKind::kNoWriteBarrier)),
625 : stack_slot,
626 : graph()->NewNode(
627 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
628 : GetReplacementLow(input), store_high_word, graph()->start());
629 :
630 : Node* load =
631 2 : graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
632 : graph()->NewNode(common()->Int32Constant(0)),
633 : store_low_word, graph()->start());
634 :
635 : ReplaceNode(node, load, nullptr);
636 : break;
637 : }
638 : case IrOpcode::kBitcastFloat64ToInt64: {
639 : DCHECK_EQ(1, node->InputCount());
640 : Node* input = node->InputAt(0);
641 1 : if (HasReplacementLow(input)) {
642 : input = GetReplacementLow(input);
643 : }
644 1 : Node* stack_slot = graph()->NewNode(
645 : machine()->StackSlot(MachineRepresentation::kWord64));
646 3 : Node* store = graph()->NewNode(
647 : machine()->Store(
648 : StoreRepresentation(MachineRepresentation::kFloat64,
649 : WriteBarrierKind::kNoWriteBarrier)),
650 : stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
651 : graph()->start(), graph()->start());
652 :
653 2 : Node* high_node = graph()->NewNode(
654 : machine()->Load(MachineType::Int32()), stack_slot,
655 : graph()->NewNode(
656 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
657 : store, graph()->start());
658 :
659 2 : Node* low_node = graph()->NewNode(
660 : machine()->Load(MachineType::Int32()), stack_slot,
661 : graph()->NewNode(
662 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
663 : store, graph()->start());
664 : ReplaceNode(node, low_node, high_node);
665 : break;
666 : }
667 : case IrOpcode::kWord64Ror: {
668 : DCHECK_EQ(2, node->InputCount());
669 : Node* input = node->InputAt(0);
670 : Node* shift = HasReplacementLow(node->InputAt(1))
671 : ? GetReplacementLow(node->InputAt(1))
672 5 : : node->InputAt(1);
673 : Int32Matcher m(shift);
674 5 : if (m.HasValue()) {
675 : // Precondition: 0 <= shift < 64.
676 4 : int32_t shift_value = m.Value() & 0x3F;
677 4 : if (shift_value == 0) {
678 : ReplaceNode(node, GetReplacementLow(input),
679 : GetReplacementHigh(input));
680 3 : } else if (shift_value == 32) {
681 : ReplaceNode(node, GetReplacementHigh(input),
682 : GetReplacementLow(input));
683 : } else {
684 : Node* low_input;
685 : Node* high_input;
686 2 : if (shift_value < 32) {
687 : low_input = GetReplacementLow(input);
688 : high_input = GetReplacementHigh(input);
689 : } else {
690 : low_input = GetReplacementHigh(input);
691 : high_input = GetReplacementLow(input);
692 : }
693 2 : int32_t masked_shift_value = shift_value & 0x1F;
694 : Node* masked_shift =
695 2 : graph()->NewNode(common()->Int32Constant(masked_shift_value));
696 2 : Node* inv_shift = graph()->NewNode(
697 : common()->Int32Constant(32 - masked_shift_value));
698 :
699 6 : Node* low_node = graph()->NewNode(
700 : machine()->Word32Or(),
701 : graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
702 : graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
703 6 : Node* high_node = graph()->NewNode(
704 : machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
705 : high_input, masked_shift),
706 : graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
707 : ReplaceNode(node, low_node, high_node);
708 : }
709 : } else {
710 : Node* safe_shift = shift;
711 1 : if (!machine()->Word32ShiftIsSafe()) {
712 : safe_shift =
713 2 : graph()->NewNode(machine()->Word32And(), shift,
714 : graph()->NewNode(common()->Int32Constant(0x1F)));
715 : }
716 :
717 : // By creating this bit-mask with SAR and SHL we do not have to deal
718 : // with shift == 0 as a special case.
719 4 : Node* inv_mask = graph()->NewNode(
720 : machine()->Word32Shl(),
721 : graph()->NewNode(machine()->Word32Sar(),
722 : graph()->NewNode(common()->Int32Constant(
723 : std::numeric_limits<int32_t>::min())),
724 : safe_shift),
725 : graph()->NewNode(common()->Int32Constant(1)));
726 :
727 : Node* bit_mask =
728 2 : graph()->NewNode(machine()->Word32Xor(), inv_mask,
729 : graph()->NewNode(common()->Int32Constant(-1)));
730 :
731 : // We have to mask the shift value for this comparison. If
732 : // !machine()->Word32ShiftIsSafe() then the masking should already be
733 : // part of the graph.
734 : Node* masked_shift6 = shift;
735 1 : if (machine()->Word32ShiftIsSafe()) {
736 : masked_shift6 =
737 0 : graph()->NewNode(machine()->Word32And(), shift,
738 : graph()->NewNode(common()->Int32Constant(0x3F)));
739 : }
740 :
741 : Diamond lt32(
742 : graph(), common(),
743 : graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
744 3 : graph()->NewNode(common()->Int32Constant(32))));
745 :
746 : // The low word and the high word can be swapped either at the input or
747 : // at the output. We swap the inputs so that shift does not have to be
748 : // kept for so long in a register.
749 : Node* input_low =
750 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
751 1 : GetReplacementHigh(input));
752 : Node* input_high =
753 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
754 1 : GetReplacementLow(input));
755 :
756 : Node* rotate_low =
757 1 : graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
758 : Node* rotate_high =
759 1 : graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
760 :
761 3 : Node* low_node = graph()->NewNode(
762 : machine()->Word32Or(),
763 : graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
764 : graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
765 :
766 3 : Node* high_node = graph()->NewNode(
767 : machine()->Word32Or(),
768 : graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
769 : graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
770 :
771 : ReplaceNode(node, low_node, high_node);
772 : }
773 : break;
774 : }
775 : case IrOpcode::kWord64Clz: {
776 : DCHECK_EQ(1, node->InputCount());
777 : Node* input = node->InputAt(0);
778 : Diamond d(
779 : graph(), common(),
780 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
781 3 : graph()->NewNode(common()->Int32Constant(0))));
782 :
783 4 : Node* low_node = d.Phi(
784 : MachineRepresentation::kWord32,
785 : graph()->NewNode(machine()->Int32Add(),
786 : graph()->NewNode(machine()->Word32Clz(),
787 : GetReplacementLow(input)),
788 : graph()->NewNode(common()->Int32Constant(32))),
789 1 : graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
790 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
791 : break;
792 : }
793 : case IrOpcode::kWord64Ctz: {
794 : DCHECK_EQ(1, node->InputCount());
795 : DCHECK(machine()->Word32Ctz().IsSupported());
796 : Node* input = node->InputAt(0);
797 : Diamond d(
798 : graph(), common(),
799 : graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
800 3 : graph()->NewNode(common()->Int32Constant(0))));
801 : Node* low_node =
802 2 : d.Phi(MachineRepresentation::kWord32,
803 : graph()->NewNode(machine()->Int32Add(),
804 2 : graph()->NewNode(machine()->Word32Ctz().op(),
805 : GetReplacementHigh(input)),
806 : graph()->NewNode(common()->Int32Constant(32))),
807 2 : graph()->NewNode(machine()->Word32Ctz().op(),
808 1 : GetReplacementLow(input)));
809 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
810 : break;
811 : }
812 : case IrOpcode::kWord64Popcnt: {
813 : DCHECK_EQ(1, node->InputCount());
814 : Node* input = node->InputAt(0);
815 : // We assume that a Word64Popcnt node only has been created if
816 : // Word32Popcnt is actually supported.
817 : DCHECK(machine()->Word32Popcnt().IsSupported());
818 2 : ReplaceNode(node, graph()->NewNode(
819 : machine()->Int32Add(),
820 2 : graph()->NewNode(machine()->Word32Popcnt().op(),
821 : GetReplacementLow(input)),
822 2 : graph()->NewNode(machine()->Word32Popcnt().op(),
823 : GetReplacementHigh(input))),
824 : graph()->NewNode(common()->Int32Constant(0)));
825 1 : break;
826 : }
827 : case IrOpcode::kPhi: {
828 4 : MachineRepresentation rep = PhiRepresentationOf(node->op());
829 4 : if (rep == MachineRepresentation::kWord64) {
830 : // The replacement nodes have already been created, we only have to
831 : // replace placeholder nodes.
832 : Node* low_node = GetReplacementLow(node);
833 : Node* high_node = GetReplacementHigh(node);
834 5 : for (int i = 0; i < node->op()->ValueInputCount(); i++) {
835 2 : low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
836 2 : high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
837 : }
838 : } else {
839 3 : DefaultLowering(node);
840 : }
841 : break;
842 : }
843 : case IrOpcode::kWord64ReverseBytes: {
844 : Node* input = node->InputAt(0);
845 2 : ReplaceNode(node,
846 : graph()->NewNode(machine()->Word32ReverseBytes(),
847 : GetReplacementHigh(input)),
848 : graph()->NewNode(machine()->Word32ReverseBytes(),
849 : GetReplacementLow(input)));
850 : break;
851 : }
852 : case IrOpcode::kSignExtendWord8ToInt64: {
853 : DCHECK_EQ(1, node->InputCount());
854 : Node* input = node->InputAt(0);
855 0 : if (HasReplacementLow(input)) {
856 : input = GetReplacementLow(input);
857 : }
858 : // Sign extend low node to Int32
859 0 : input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
860 :
861 : // We use SAR to preserve the sign in the high word.
862 0 : ReplaceNode(
863 : node, input,
864 : graph()->NewNode(machine()->Word32Sar(), input,
865 : graph()->NewNode(common()->Int32Constant(31))));
866 0 : node->NullAllInputs();
867 0 : break;
868 : }
869 : case IrOpcode::kSignExtendWord16ToInt64: {
870 : DCHECK_EQ(1, node->InputCount());
871 : Node* input = node->InputAt(0);
872 0 : if (HasReplacementLow(input)) {
873 : input = GetReplacementLow(input);
874 : }
875 : // Sign extend low node to Int32
876 0 : input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
877 :
878 : // We use SAR to preserve the sign in the high word.
879 0 : ReplaceNode(
880 : node, input,
881 : graph()->NewNode(machine()->Word32Sar(), input,
882 : graph()->NewNode(common()->Int32Constant(31))));
883 0 : node->NullAllInputs();
884 0 : break;
885 : }
886 : case IrOpcode::kWord64AtomicLoad: {
887 : DCHECK_EQ(4, node->InputCount());
888 0 : MachineType type = AtomicOpType(node->op());
889 0 : DefaultLowering(node, true);
890 0 : if (type == MachineType::Uint64()) {
891 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
892 0 : ReplaceNodeWithProjections(node);
893 : } else {
894 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
895 0 : ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
896 : }
897 : break;
898 : }
899 : case IrOpcode::kWord64AtomicStore: {
900 : DCHECK_EQ(5, node->InputCount());
901 0 : MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
902 0 : if (rep == MachineRepresentation::kWord64) {
903 0 : LowerMemoryBaseAndIndex(node);
904 : Node* value = node->InputAt(2);
905 0 : node->ReplaceInput(2, GetReplacementLow(value));
906 0 : node->InsertInput(zone(), 3, GetReplacementHigh(value));
907 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
908 : } else {
909 0 : DefaultLowering(node, true);
910 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
911 : }
912 : break;
913 : }
914 : #define ATOMIC_CASE(name) \
915 : case IrOpcode::kWord64Atomic##name: { \
916 : MachineType type = AtomicOpType(node->op()); \
917 : if (type == MachineType::Uint64()) { \
918 : LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \
919 : } else { \
920 : LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
921 : } \
922 : break; \
923 : }
924 0 : ATOMIC_CASE(Add)
925 0 : ATOMIC_CASE(Sub)
926 0 : ATOMIC_CASE(And)
927 0 : ATOMIC_CASE(Or)
928 0 : ATOMIC_CASE(Xor)
929 0 : ATOMIC_CASE(Exchange)
930 : #undef ATOMIC_CASE
931 : case IrOpcode::kWord64AtomicCompareExchange: {
932 0 : MachineType type = AtomicOpType(node->op());
933 0 : if (type == MachineType::Uint64()) {
934 0 : LowerMemoryBaseAndIndex(node);
935 : Node* old_value = node->InputAt(2);
936 : Node* new_value = node->InputAt(3);
937 0 : node->ReplaceInput(2, GetReplacementLow(old_value));
938 0 : node->ReplaceInput(3, GetReplacementHigh(old_value));
939 0 : node->InsertInput(zone(), 4, GetReplacementLow(new_value));
940 0 : node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
941 0 : NodeProperties::ChangeOp(node,
942 0 : machine()->Word32AtomicPairCompareExchange());
943 0 : ReplaceNodeWithProjections(node);
944 : } else {
945 : DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
946 : type == MachineType::Uint8());
947 0 : DefaultLowering(node, true);
948 0 : NodeProperties::ChangeOp(node,
949 0 : machine()->Word32AtomicCompareExchange(type));
950 0 : ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
951 : }
952 : break;
953 : }
954 :
955 124 : default: { DefaultLowering(node); }
956 : }
957 325 : } // NOLINT(readability/fn_size)
958 :
959 4 : void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
960 : const Operator* low_word_op) {
961 : DCHECK_EQ(2, node->InputCount());
962 : Node* left = node->InputAt(0);
963 : Node* right = node->InputAt(1);
964 12 : Node* replacement = graph()->NewNode(
965 : machine()->Word32Or(),
966 : graph()->NewNode(high_word_op, GetReplacementHigh(left),
967 : GetReplacementHigh(right)),
968 : graph()->NewNode(
969 : machine()->Word32And(),
970 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
971 : GetReplacementHigh(right)),
972 : graph()->NewNode(low_word_op, GetReplacementLow(left),
973 : GetReplacementLow(right))));
974 :
975 : ReplaceNode(node, replacement, nullptr);
976 4 : }
977 :
978 178 : bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
979 : bool something_changed = false;
980 285 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
981 : Node* input = node->InputAt(i);
982 107 : if (HasReplacementLow(input)) {
983 : something_changed = true;
984 40 : node->ReplaceInput(i, GetReplacementLow(input));
985 : }
986 211 : if (!low_word_only && HasReplacementHigh(input)) {
987 : something_changed = true;
988 30 : node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
989 : }
990 : }
991 178 : return something_changed;
992 : }
993 :
994 0 : void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
995 : // if new_low == nullptr, then also new_high == nullptr.
996 : DCHECK(new_low != nullptr || new_high == nullptr);
997 192 : replacements_[old->id()].low = new_low;
998 198 : replacements_[old->id()].high = new_high;
999 0 : }
1000 :
1001 0 : bool Int64Lowering::HasReplacementLow(Node* node) {
1002 268 : return replacements_[node->id()].low != nullptr;
1003 : }
1004 :
1005 0 : Node* Int64Lowering::GetReplacementLow(Node* node) {
1006 86 : Node* result = replacements_[node->id()].low;
1007 : DCHECK(result);
1008 0 : return result;
1009 : }
1010 :
1011 0 : bool Int64Lowering::HasReplacementHigh(Node* node) {
1012 208 : return replacements_[node->id()].high != nullptr;
1013 : }
1014 :
1015 0 : Node* Int64Lowering::GetReplacementHigh(Node* node) {
1016 149 : Node* result = replacements_[node->id()].high;
1017 : DCHECK(result);
1018 0 : return result;
1019 : }
1020 :
1021 4 : void Int64Lowering::PreparePhiReplacement(Node* phi) {
1022 4 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
1023 4 : if (rep == MachineRepresentation::kWord64) {
1024 : // We have to create the replacements for a phi node before we actually
1025 : // lower the phi to break potential cycles in the graph. The replacements of
1026 : // input nodes do not exist yet, so we use a placeholder node to pass the
1027 : // graph verifier.
1028 : int value_count = phi->op()->ValueInputCount();
1029 1 : Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1030 : Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1031 5 : for (int i = 0; i < value_count; i++) {
1032 2 : inputs_low[i] = placeholder_;
1033 2 : inputs_high[i] = placeholder_;
1034 : }
1035 1 : inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1036 1 : inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1037 1 : ReplaceNode(phi,
1038 : graph()->NewNode(
1039 : common()->Phi(MachineRepresentation::kWord32, value_count),
1040 : value_count + 1, inputs_low, false),
1041 : graph()->NewNode(
1042 : common()->Phi(MachineRepresentation::kWord32, value_count),
1043 : value_count + 1, inputs_high, false));
1044 : }
1045 4 : }
1046 :
1047 5 : void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1048 : DCHECK(node != nullptr);
1049 : Node* low_node =
1050 5 : graph()->NewNode(common()->Projection(0), node, graph()->start());
1051 : Node* high_node =
1052 5 : graph()->NewNode(common()->Projection(1), node, graph()->start());
1053 : ReplaceNode(node, low_node, high_node);
1054 5 : }
1055 :
1056 7 : void Int64Lowering::LowerMemoryBaseAndIndex(Node* node) {
1057 : DCHECK(node != nullptr);
1058 : // Low word only replacements for memory operands for 32-bit address space.
1059 : Node* base = node->InputAt(0);
1060 : Node* index = node->InputAt(1);
1061 7 : if (HasReplacementLow(base)) {
1062 3 : node->ReplaceInput(0, GetReplacementLow(base));
1063 : }
1064 7 : if (HasReplacementLow(index)) {
1065 3 : node->ReplaceInput(1, GetReplacementLow(index));
1066 : }
1067 7 : }
1068 :
1069 : } // namespace compiler
1070 : } // namespace internal
1071 122004 : } // namespace v8
|
