Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/int64-lowering.h"
6 :
7 : #include "src/compiler/common-operator.h"
8 : #include "src/compiler/diamond.h"
9 : #include "src/compiler/graph.h"
10 : #include "src/compiler/linkage.h"
11 : #include "src/compiler/machine-operator.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-properties.h"
14 : #include "src/compiler/node.h"
15 : #include "src/compiler/wasm-compiler.h"
16 : // TODO(wasm): Remove this include.
17 : #include "src/wasm/wasm-linkage.h"
18 : #include "src/zone/zone.h"
19 :
20 : namespace v8 {
21 : namespace internal {
22 : namespace compiler {
23 :
24 180 : Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
25 : CommonOperatorBuilder* common, Zone* zone,
26 : Signature<MachineRepresentation>* signature)
27 : : zone_(zone),
28 : graph_(graph),
29 : machine_(machine),
30 : common_(common),
31 : state_(graph, 3),
32 : stack_(zone),
33 : replacements_(nullptr),
34 : signature_(signature),
35 : placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
36 180 : graph->start())) {
37 : DCHECK_NOT_NULL(graph);
38 : DCHECK_NOT_NULL(graph->end());
39 45 : replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
40 45 : memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
41 45 : }
42 :
43 135 : void Int64Lowering::LowerGraph() {
44 45 : if (!machine()->Is32()) {
45 45 : return;
46 : }
47 90 : stack_.push_back({graph()->end(), 0});
48 45 : state_.Set(graph()->end(), State::kOnStack);
49 :
50 745 : while (!stack_.empty()) {
51 : NodeState& top = stack_.back();
52 1400 : if (top.input_index == top.node->InputCount()) {
53 : // All inputs of top have already been lowered, now lower top.
54 : stack_.pop_back();
55 315 : state_.Set(top.node, State::kVisited);
56 315 : LowerNode(top.node);
57 : } else {
58 : // Push the next input onto the stack.
59 655 : Node* input = top.node->InputAt(top.input_index++);
60 385 : if (state_.Get(input) == State::kUnvisited) {
61 270 : if (input->opcode() == IrOpcode::kPhi) {
62 : // To break cycles with phi nodes we push phis on a separate stack so
63 : // that they are processed after all other nodes.
64 4 : PreparePhiReplacement(input);
65 8 : stack_.push_front({input, 0});
66 266 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
67 : input->opcode() == IrOpcode::kLoop) {
68 4 : stack_.push_front({input, 0});
69 : } else {
70 528 : stack_.push_back({input, 0});
71 : }
72 : state_.Set(input, State::kOnStack);
73 : }
74 : }
75 : }
76 : }
77 :
78 : namespace {
79 :
80 : int GetReturnIndexAfterLowering(CallDescriptor* call_descriptor,
81 : int old_index) {
82 : int result = old_index;
83 0 : for (int i = 0; i < old_index; i++) {
84 0 : if (call_descriptor->GetReturnType(i).representation() ==
85 : MachineRepresentation::kWord64) {
86 0 : result++;
87 : }
88 : }
89 : return result;
90 : }
91 :
92 0 : int GetReturnCountAfterLowering(CallDescriptor* call_descriptor) {
93 : return GetReturnIndexAfterLowering(
94 0 : call_descriptor, static_cast<int>(call_descriptor->ReturnCount()));
95 : }
96 :
97 : int GetParameterIndexAfterLowering(
98 18 : Signature<MachineRepresentation>* signature, int old_index) {
99 : int result = old_index;
100 18 : for (int i = 0; i < old_index; i++) {
101 36 : if (signature->GetParam(i) == MachineRepresentation::kWord64) {
102 10 : result++;
103 : }
104 : }
105 : return result;
106 : }
107 :
108 60 : int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
109 30 : int result = static_cast<int>(signature->return_count());
110 30 : for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
111 60 : if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
112 30 : result++;
113 : }
114 : }
115 : return result;
116 : }
117 :
118 : } // namespace
119 :
120 0 : void Int64Lowering::LowerWord64AtomicBinop(Node* node, const Operator* op) {
121 : DCHECK_EQ(5, node->InputCount());
122 : Node* value = node->InputAt(2);
123 0 : node->ReplaceInput(2, GetReplacementLow(value));
124 0 : node->InsertInput(zone(), 3, GetReplacementHigh(value));
125 0 : NodeProperties::ChangeOp(node, op);
126 0 : ReplaceNodeWithProjections(node);
127 0 : }
128 :
129 0 : void Int64Lowering::LowerWord64AtomicNarrowOp(Node* node, const Operator* op) {
130 : Node* value = node->InputAt(2);
131 0 : node->ReplaceInput(2, GetReplacementLow(value));
132 0 : NodeProperties::ChangeOp(node, op);
133 0 : ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
134 0 : }
135 :
136 : // static
137 0 : int Int64Lowering::GetParameterCountAfterLowering(
138 48 : Signature<MachineRepresentation>* signature) {
139 : // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
140 : // after lowering.
141 : return GetParameterIndexAfterLowering(
142 48 : signature, static_cast<int>(signature->parameter_count()));
143 : }
144 :
145 7 : void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
146 28 : Node*& index_high) {
147 7 : if (HasReplacementLow(index)) {
148 : index = GetReplacementLow(index);
149 : }
150 : #if defined(V8_TARGET_LITTLE_ENDIAN)
151 7 : index_low = index;
152 : index_high = graph()->NewNode(machine()->Int32Add(), index,
153 21 : graph()->NewNode(common()->Int32Constant(4)));
154 : #elif defined(V8_TARGET_BIG_ENDIAN)
155 : index_low = graph()->NewNode(machine()->Int32Add(), index,
156 : graph()->NewNode(common()->Int32Constant(4)));
157 : index_high = index;
158 : #endif
159 7 : }
160 :
161 898 : void Int64Lowering::LowerNode(Node* node) {
162 315 : switch (node->opcode()) {
163 : case IrOpcode::kInt64Constant: {
164 55 : int64_t value = OpParameter<int64_t>(node->op());
165 : Node* low_node = graph()->NewNode(
166 110 : common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
167 : Node* high_node = graph()->NewNode(
168 110 : common()->Int32Constant(static_cast<int32_t>(value >> 32)));
169 : ReplaceNode(node, low_node, high_node);
170 : break;
171 : }
172 : case IrOpcode::kLoad:
173 : case IrOpcode::kUnalignedLoad: {
174 : MachineRepresentation rep;
175 4 : if (node->opcode() == IrOpcode::kLoad) {
176 3 : rep = LoadRepresentationOf(node->op()).representation();
177 : } else {
178 : DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
179 1 : rep = LoadRepresentationOf(node->op()).representation();
180 : }
181 :
182 4 : if (rep == MachineRepresentation::kWord64) {
183 : Node* base = node->InputAt(0);
184 : Node* index = node->InputAt(1);
185 : Node* index_low;
186 : Node* index_high;
187 4 : GetIndexNodes(index, index_low, index_high);
188 : const Operator* load_op;
189 :
190 4 : if (node->opcode() == IrOpcode::kLoad) {
191 3 : load_op = machine()->Load(MachineType::Int32());
192 : } else {
193 : DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
194 1 : load_op = machine()->UnalignedLoad(MachineType::Int32());
195 : }
196 :
197 : Node* high_node;
198 4 : if (node->InputCount() > 2) {
199 : Node* effect_high = node->InputAt(2);
200 : Node* control_high = node->InputAt(3);
201 : high_node = graph()->NewNode(load_op, base, index_high, effect_high,
202 4 : control_high);
203 : // change the effect change from old_node --> old_effect to
204 : // old_node --> high_node --> old_effect.
205 4 : node->ReplaceInput(2, high_node);
206 : } else {
207 0 : high_node = graph()->NewNode(load_op, base, index_high);
208 : }
209 4 : node->ReplaceInput(1, index_low);
210 4 : NodeProperties::ChangeOp(node, load_op);
211 : ReplaceNode(node, node, high_node);
212 : } else {
213 0 : DefaultLowering(node);
214 : }
215 : break;
216 : }
217 : case IrOpcode::kStore:
218 : case IrOpcode::kUnalignedStore: {
219 : MachineRepresentation rep;
220 4 : if (node->opcode() == IrOpcode::kStore) {
221 3 : rep = StoreRepresentationOf(node->op()).representation();
222 : } else {
223 : DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
224 1 : rep = UnalignedStoreRepresentationOf(node->op());
225 : }
226 :
227 4 : if (rep == MachineRepresentation::kWord64) {
228 : // We change the original store node to store the low word, and create
229 : // a new store node to store the high word. The effect and control edges
230 : // are copied from the original store to the new store node, the effect
231 : // edge of the original store is redirected to the new store.
232 : Node* base = node->InputAt(0);
233 : Node* index = node->InputAt(1);
234 : Node* index_low;
235 : Node* index_high;
236 3 : GetIndexNodes(index, index_low, index_high);
237 : Node* value = node->InputAt(2);
238 : DCHECK(HasReplacementLow(value));
239 : DCHECK(HasReplacementHigh(value));
240 :
241 : const Operator* store_op;
242 3 : if (node->opcode() == IrOpcode::kStore) {
243 : WriteBarrierKind write_barrier_kind =
244 2 : StoreRepresentationOf(node->op()).write_barrier_kind();
245 : store_op = machine()->Store(StoreRepresentation(
246 2 : MachineRepresentation::kWord32, write_barrier_kind));
247 : } else {
248 : DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
249 1 : store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
250 : }
251 :
252 : Node* high_node;
253 3 : if (node->InputCount() > 3) {
254 : Node* effect_high = node->InputAt(3);
255 : Node* control_high = node->InputAt(4);
256 : high_node = graph()->NewNode(store_op, base, index_high,
257 : GetReplacementHigh(value), effect_high,
258 3 : control_high);
259 3 : node->ReplaceInput(3, high_node);
260 :
261 : } else {
262 : high_node = graph()->NewNode(store_op, base, index_high,
263 0 : GetReplacementHigh(value));
264 : }
265 :
266 3 : node->ReplaceInput(1, index_low);
267 3 : node->ReplaceInput(2, GetReplacementLow(value));
268 3 : NodeProperties::ChangeOp(node, store_op);
269 : ReplaceNode(node, node, high_node);
270 : } else {
271 1 : DefaultLowering(node, true);
272 : }
273 : break;
274 : }
275 : case IrOpcode::kStart: {
276 : int parameter_count = GetParameterCountAfterLowering(signature());
277 : // Only exchange the node if the parameter count actually changed.
278 45 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
279 : int delta =
280 3 : parameter_count - static_cast<int>(signature()->parameter_count());
281 6 : int new_output_count = node->op()->ValueOutputCount() + delta;
282 3 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
283 : }
284 : break;
285 : }
286 : case IrOpcode::kParameter: {
287 : DCHECK_EQ(1, node->InputCount());
288 : // Only exchange the node if the parameter count actually changed. We do
289 : // not even have to do the default lowering because the the start node,
290 : // the only input of a parameter node, only changes if the parameter count
291 : // changes.
292 3 : if (GetParameterCountAfterLowering(signature()) !=
293 : static_cast<int>(signature()->parameter_count())) {
294 3 : int old_index = ParameterIndexOf(node->op());
295 : // TODO(wasm): Make this part not wasm specific.
296 : // Prevent special lowering of the instance parameter.
297 3 : if (old_index == wasm::kWasmInstanceParameterIndex) {
298 1 : DefaultLowering(node);
299 1 : break;
300 : }
301 : // Adjust old_index to be compliant with the signature.
302 2 : --old_index;
303 : int new_index = GetParameterIndexAfterLowering(signature(), old_index);
304 : // Adjust new_index to consider the instance parameter.
305 2 : ++new_index;
306 2 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
307 :
308 6 : if (signature()->GetParam(old_index) ==
309 : MachineRepresentation::kWord64) {
310 : Node* high_node = graph()->NewNode(common()->Parameter(new_index + 1),
311 2 : graph()->start());
312 : ReplaceNode(node, node, high_node);
313 : }
314 : }
315 : break;
316 : }
317 : case IrOpcode::kReturn: {
318 : int input_count = node->InputCount();
319 45 : DefaultLowering(node);
320 45 : if (input_count != node->InputCount()) {
321 : int new_return_count = GetReturnCountAfterLowering(signature());
322 30 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
323 30 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
324 : }
325 : }
326 : break;
327 : }
328 : case IrOpcode::kTailCall: {
329 : auto call_descriptor =
330 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
331 : bool returns_require_lowering =
332 : GetReturnCountAfterLowering(call_descriptor) !=
333 : static_cast<int>(call_descriptor->ReturnCount());
334 0 : if (DefaultLowering(node) || returns_require_lowering) {
335 : // Tail calls do not have return values, so adjusting the call
336 : // descriptor is enough.
337 0 : auto new_descriptor = GetI32WasmCallDescriptor(zone(), call_descriptor);
338 0 : NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
339 : }
340 : break;
341 : }
342 : case IrOpcode::kCall: {
343 0 : auto call_descriptor =
344 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
345 : bool returns_require_lowering =
346 : GetReturnCountAfterLowering(call_descriptor) !=
347 0 : static_cast<int>(call_descriptor->ReturnCount());
348 0 : if (DefaultLowering(node) || returns_require_lowering) {
349 : // We have to adjust the call descriptor.
350 : NodeProperties::ChangeOp(node, common()->Call(GetI32WasmCallDescriptor(
351 0 : zone(), call_descriptor)));
352 : }
353 0 : if (returns_require_lowering) {
354 : size_t return_arity = call_descriptor->ReturnCount();
355 0 : if (return_arity == 1) {
356 : // We access the additional return values through projections.
357 0 : ReplaceNodeWithProjections(node);
358 : } else {
359 : ZoneVector<Node*> projections(return_arity, zone());
360 : NodeProperties::CollectValueProjections(node, projections.data(),
361 0 : return_arity);
362 0 : for (size_t old_index = 0, new_index = 0; old_index < return_arity;
363 : ++old_index, ++new_index) {
364 0 : Node* use_node = projections[old_index];
365 : DCHECK_EQ(ProjectionIndexOf(use_node->op()), old_index);
366 : DCHECK_EQ(GetReturnIndexAfterLowering(call_descriptor,
367 : static_cast<int>(old_index)),
368 : static_cast<int>(new_index));
369 0 : if (new_index != old_index) {
370 : NodeProperties::ChangeOp(
371 0 : use_node, common()->Projection(new_index));
372 : }
373 0 : if (call_descriptor->GetReturnType(old_index).representation() ==
374 : MachineRepresentation::kWord64) {
375 : Node* high_node = graph()->NewNode(
376 : common()->Projection(new_index + 1), node,
377 0 : graph()->start());
378 : ReplaceNode(use_node, use_node, high_node);
379 : ++new_index;
380 : }
381 : }
382 : }
383 : }
384 : break;
385 : }
386 : case IrOpcode::kWord64And: {
387 : DCHECK_EQ(2, node->InputCount());
388 : Node* left = node->InputAt(0);
389 : Node* right = node->InputAt(1);
390 :
391 : Node* low_node =
392 : graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
393 3 : GetReplacementLow(right));
394 : Node* high_node =
395 : graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
396 3 : GetReplacementHigh(right));
397 : ReplaceNode(node, low_node, high_node);
398 : break;
399 : }
400 : case IrOpcode::kTruncateInt64ToInt32: {
401 : DCHECK_EQ(1, node->InputCount());
402 : Node* input = node->InputAt(0);
403 : ReplaceNode(node, GetReplacementLow(input), nullptr);
404 4 : node->NullAllInputs();
405 4 : break;
406 : }
407 : case IrOpcode::kInt64Add: {
408 : DCHECK_EQ(2, node->InputCount());
409 :
410 : Node* right = node->InputAt(1);
411 0 : node->ReplaceInput(1, GetReplacementLow(right));
412 0 : node->AppendInput(zone(), GetReplacementHigh(right));
413 :
414 : Node* left = node->InputAt(0);
415 0 : node->ReplaceInput(0, GetReplacementLow(left));
416 0 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
417 :
418 0 : NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
419 : // We access the additional return values through projections.
420 0 : ReplaceNodeWithProjections(node);
421 0 : break;
422 : }
423 : case IrOpcode::kInt64Sub: {
424 : DCHECK_EQ(2, node->InputCount());
425 :
426 : Node* right = node->InputAt(1);
427 1 : node->ReplaceInput(1, GetReplacementLow(right));
428 1 : node->AppendInput(zone(), GetReplacementHigh(right));
429 :
430 : Node* left = node->InputAt(0);
431 1 : node->ReplaceInput(0, GetReplacementLow(left));
432 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
433 :
434 1 : NodeProperties::ChangeOp(node, machine()->Int32PairSub());
435 : // We access the additional return values through projections.
436 1 : ReplaceNodeWithProjections(node);
437 1 : break;
438 : }
439 : case IrOpcode::kInt64Mul: {
440 : DCHECK_EQ(2, node->InputCount());
441 :
442 : Node* right = node->InputAt(1);
443 1 : node->ReplaceInput(1, GetReplacementLow(right));
444 1 : node->AppendInput(zone(), GetReplacementHigh(right));
445 :
446 : Node* left = node->InputAt(0);
447 1 : node->ReplaceInput(0, GetReplacementLow(left));
448 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
449 :
450 1 : NodeProperties::ChangeOp(node, machine()->Int32PairMul());
451 : // We access the additional return values through projections.
452 1 : ReplaceNodeWithProjections(node);
453 1 : break;
454 : }
455 : case IrOpcode::kWord64Or: {
456 : DCHECK_EQ(2, node->InputCount());
457 : Node* left = node->InputAt(0);
458 : Node* right = node->InputAt(1);
459 :
460 : Node* low_node =
461 : graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
462 1 : GetReplacementLow(right));
463 : Node* high_node =
464 : graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
465 1 : GetReplacementHigh(right));
466 : ReplaceNode(node, low_node, high_node);
467 : break;
468 : }
469 : case IrOpcode::kWord64Xor: {
470 : DCHECK_EQ(2, node->InputCount());
471 : Node* left = node->InputAt(0);
472 : Node* right = node->InputAt(1);
473 :
474 : Node* low_node =
475 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
476 1 : GetReplacementLow(right));
477 : Node* high_node =
478 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
479 1 : GetReplacementHigh(right));
480 : ReplaceNode(node, low_node, high_node);
481 : break;
482 : }
483 : case IrOpcode::kWord64Shl: {
484 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
485 : // of the output to 0 and just calculate the high word.
486 : DCHECK_EQ(2, node->InputCount());
487 : Node* shift = node->InputAt(1);
488 1 : if (HasReplacementLow(shift)) {
489 : // We do not have to care about the high word replacement, because
490 : // the shift can only be between 0 and 63 anyways.
491 1 : node->ReplaceInput(1, GetReplacementLow(shift));
492 : }
493 :
494 : Node* value = node->InputAt(0);
495 1 : node->ReplaceInput(0, GetReplacementLow(value));
496 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
497 :
498 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShl());
499 : // We access the additional return values through projections.
500 1 : ReplaceNodeWithProjections(node);
501 1 : break;
502 : }
503 : case IrOpcode::kWord64Shr: {
504 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
505 : // of the output to 0 and just calculate the high word.
506 : DCHECK_EQ(2, node->InputCount());
507 : Node* shift = node->InputAt(1);
508 1 : if (HasReplacementLow(shift)) {
509 : // We do not have to care about the high word replacement, because
510 : // the shift can only be between 0 and 63 anyways.
511 1 : node->ReplaceInput(1, GetReplacementLow(shift));
512 : }
513 :
514 : Node* value = node->InputAt(0);
515 1 : node->ReplaceInput(0, GetReplacementLow(value));
516 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
517 :
518 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShr());
519 : // We access the additional return values through projections.
520 1 : ReplaceNodeWithProjections(node);
521 1 : break;
522 : }
523 : case IrOpcode::kWord64Sar: {
524 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
525 : // of the output to 0 and just calculate the high word.
526 : DCHECK_EQ(2, node->InputCount());
527 : Node* shift = node->InputAt(1);
528 1 : if (HasReplacementLow(shift)) {
529 : // We do not have to care about the high word replacement, because
530 : // the shift can only be between 0 and 63 anyways.
531 1 : node->ReplaceInput(1, GetReplacementLow(shift));
532 : }
533 :
534 : Node* value = node->InputAt(0);
535 1 : node->ReplaceInput(0, GetReplacementLow(value));
536 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
537 :
538 1 : NodeProperties::ChangeOp(node, machine()->Word32PairSar());
539 : // We access the additional return values through projections.
540 1 : ReplaceNodeWithProjections(node);
541 1 : break;
542 : }
543 : case IrOpcode::kWord64Equal: {
544 : DCHECK_EQ(2, node->InputCount());
545 : Node* left = node->InputAt(0);
546 : Node* right = node->InputAt(1);
547 :
548 : // TODO(wasm): Use explicit comparisons and && here?
549 : Node* replacement = graph()->NewNode(
550 : machine()->Word32Equal(),
551 : graph()->NewNode(
552 : machine()->Word32Or(),
553 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
554 : GetReplacementLow(right)),
555 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
556 : GetReplacementHigh(right))),
557 10 : graph()->NewNode(common()->Int32Constant(0)));
558 :
559 : ReplaceNode(node, replacement, nullptr);
560 : break;
561 : }
562 : case IrOpcode::kInt64LessThan: {
563 : LowerComparison(node, machine()->Int32LessThan(),
564 2 : machine()->Uint32LessThan());
565 1 : break;
566 : }
567 : case IrOpcode::kInt64LessThanOrEqual: {
568 : LowerComparison(node, machine()->Int32LessThan(),
569 2 : machine()->Uint32LessThanOrEqual());
570 1 : break;
571 : }
572 : case IrOpcode::kUint64LessThan: {
573 : LowerComparison(node, machine()->Uint32LessThan(),
574 2 : machine()->Uint32LessThan());
575 1 : break;
576 : }
577 : case IrOpcode::kUint64LessThanOrEqual: {
578 : LowerComparison(node, machine()->Uint32LessThan(),
579 2 : machine()->Uint32LessThanOrEqual());
580 1 : break;
581 : }
582 : case IrOpcode::kSignExtendWord32ToInt64:
583 : case IrOpcode::kChangeInt32ToInt64: {
584 : DCHECK_EQ(1, node->InputCount());
585 : Node* input = node->InputAt(0);
586 2 : if (HasReplacementLow(input)) {
587 : input = GetReplacementLow(input);
588 : }
589 : // We use SAR to preserve the sign in the high word.
590 : ReplaceNode(
591 : node, input,
592 : graph()->NewNode(machine()->Word32Sar(), input,
593 4 : graph()->NewNode(common()->Int32Constant(31))));
594 2 : node->NullAllInputs();
595 2 : break;
596 : }
597 : case IrOpcode::kChangeUint32ToUint64: {
598 : DCHECK_EQ(1, node->InputCount());
599 : Node* input = node->InputAt(0);
600 2 : if (HasReplacementLow(input)) {
601 : input = GetReplacementLow(input);
602 : }
603 2 : ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
604 2 : node->NullAllInputs();
605 2 : break;
606 : }
607 : case IrOpcode::kBitcastInt64ToFloat64: {
608 : DCHECK_EQ(1, node->InputCount());
609 : Node* input = node->InputAt(0);
610 : Node* stack_slot = graph()->NewNode(
611 1 : machine()->StackSlot(MachineRepresentation::kWord64));
612 :
613 : Node* store_high_word = graph()->NewNode(
614 : machine()->Store(
615 : StoreRepresentation(MachineRepresentation::kWord32,
616 : WriteBarrierKind::kNoWriteBarrier)),
617 : stack_slot,
618 : graph()->NewNode(
619 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
620 4 : GetReplacementHigh(input), graph()->start(), graph()->start());
621 :
622 : Node* store_low_word = graph()->NewNode(
623 : machine()->Store(
624 : StoreRepresentation(MachineRepresentation::kWord32,
625 : WriteBarrierKind::kNoWriteBarrier)),
626 : stack_slot,
627 : graph()->NewNode(
628 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
629 4 : GetReplacementLow(input), store_high_word, graph()->start());
630 :
631 : Node* load =
632 : graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
633 : graph()->NewNode(common()->Int32Constant(0)),
634 3 : store_low_word, graph()->start());
635 :
636 : ReplaceNode(node, load, nullptr);
637 : break;
638 : }
639 : case IrOpcode::kBitcastFloat64ToInt64: {
640 : DCHECK_EQ(1, node->InputCount());
641 : Node* input = node->InputAt(0);
642 1 : if (HasReplacementLow(input)) {
643 : input = GetReplacementLow(input);
644 : }
645 : Node* stack_slot = graph()->NewNode(
646 1 : machine()->StackSlot(MachineRepresentation::kWord64));
647 : Node* store = graph()->NewNode(
648 : machine()->Store(
649 : StoreRepresentation(MachineRepresentation::kFloat64,
650 : WriteBarrierKind::kNoWriteBarrier)),
651 : stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
652 4 : graph()->start(), graph()->start());
653 :
654 : Node* high_node = graph()->NewNode(
655 : machine()->Load(MachineType::Int32()), stack_slot,
656 : graph()->NewNode(
657 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
658 3 : store, graph()->start());
659 :
660 : Node* low_node = graph()->NewNode(
661 : machine()->Load(MachineType::Int32()), stack_slot,
662 : graph()->NewNode(
663 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
664 3 : store, graph()->start());
665 : ReplaceNode(node, low_node, high_node);
666 : break;
667 : }
668 : case IrOpcode::kWord64Ror: {
669 : DCHECK_EQ(2, node->InputCount());
670 : Node* input = node->InputAt(0);
671 : Node* shift = HasReplacementLow(node->InputAt(1))
672 : ? GetReplacementLow(node->InputAt(1))
673 5 : : node->InputAt(1);
674 : Int32Matcher m(shift);
675 5 : if (m.HasValue()) {
676 : // Precondition: 0 <= shift < 64.
677 4 : int32_t shift_value = m.Value() & 0x3F;
678 4 : if (shift_value == 0) {
679 : ReplaceNode(node, GetReplacementLow(input),
680 : GetReplacementHigh(input));
681 3 : } else if (shift_value == 32) {
682 : ReplaceNode(node, GetReplacementHigh(input),
683 : GetReplacementLow(input));
684 : } else {
685 : Node* low_input;
686 : Node* high_input;
687 2 : if (shift_value < 32) {
688 : low_input = GetReplacementLow(input);
689 : high_input = GetReplacementHigh(input);
690 : } else {
691 : low_input = GetReplacementHigh(input);
692 : high_input = GetReplacementLow(input);
693 : }
694 2 : int32_t masked_shift_value = shift_value & 0x1F;
695 : Node* masked_shift =
696 2 : graph()->NewNode(common()->Int32Constant(masked_shift_value));
697 : Node* inv_shift = graph()->NewNode(
698 4 : common()->Int32Constant(32 - masked_shift_value));
699 :
700 : Node* low_node = graph()->NewNode(
701 : machine()->Word32Or(),
702 : graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
703 6 : graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
704 : Node* high_node = graph()->NewNode(
705 : machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
706 : high_input, masked_shift),
707 6 : graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
708 : ReplaceNode(node, low_node, high_node);
709 : }
710 : } else {
711 : Node* safe_shift = shift;
712 1 : if (!machine()->Word32ShiftIsSafe()) {
713 : safe_shift =
714 : graph()->NewNode(machine()->Word32And(), shift,
715 2 : graph()->NewNode(common()->Int32Constant(0x1F)));
716 : }
717 :
718 : // By creating this bit-mask with SAR and SHL we do not have to deal
719 : // with shift == 0 as a special case.
720 : Node* inv_mask = graph()->NewNode(
721 : machine()->Word32Shl(),
722 : graph()->NewNode(machine()->Word32Sar(),
723 : graph()->NewNode(common()->Int32Constant(
724 : std::numeric_limits<int32_t>::min())),
725 : safe_shift),
726 4 : graph()->NewNode(common()->Int32Constant(1)));
727 :
728 : Node* bit_mask =
729 : graph()->NewNode(machine()->Word32Xor(), inv_mask,
730 2 : graph()->NewNode(common()->Int32Constant(-1)));
731 :
732 : // We have to mask the shift value for this comparison. If
733 : // !machine()->Word32ShiftIsSafe() then the masking should already be
734 : // part of the graph.
735 : Node* masked_shift6 = shift;
736 1 : if (machine()->Word32ShiftIsSafe()) {
737 : masked_shift6 =
738 : graph()->NewNode(machine()->Word32And(), shift,
739 0 : graph()->NewNode(common()->Int32Constant(0x3F)));
740 : }
741 :
742 : Diamond lt32(
743 : graph(), common(),
744 : graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
745 3 : graph()->NewNode(common()->Int32Constant(32))));
746 :
747 : // The low word and the high word can be swapped either at the input or
748 : // at the output. We swap the inputs so that shift does not have to be
749 : // kept for so long in a register.
750 : Node* input_low =
751 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
752 1 : GetReplacementHigh(input));
753 : Node* input_high =
754 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
755 1 : GetReplacementLow(input));
756 :
757 : Node* rotate_low =
758 1 : graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
759 : Node* rotate_high =
760 1 : graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
761 :
762 : Node* low_node = graph()->NewNode(
763 : machine()->Word32Or(),
764 : graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
765 3 : graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
766 :
767 : Node* high_node = graph()->NewNode(
768 : machine()->Word32Or(),
769 : graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
770 3 : graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
771 :
772 : ReplaceNode(node, low_node, high_node);
773 : }
774 : break;
775 : }
776 : case IrOpcode::kWord64Clz: {
777 : DCHECK_EQ(1, node->InputCount());
778 : Node* input = node->InputAt(0);
779 : Diamond d(
780 : graph(), common(),
781 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
782 3 : graph()->NewNode(common()->Int32Constant(0))));
783 :
784 : Node* low_node = d.Phi(
785 : MachineRepresentation::kWord32,
786 : graph()->NewNode(machine()->Int32Add(),
787 : graph()->NewNode(machine()->Word32Clz(),
788 : GetReplacementLow(input)),
789 : graph()->NewNode(common()->Int32Constant(32))),
790 5 : graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
791 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
792 : break;
793 : }
794 : case IrOpcode::kWord64Ctz: {
795 : DCHECK_EQ(1, node->InputCount());
796 : DCHECK(machine()->Word32Ctz().IsSupported());
797 : Node* input = node->InputAt(0);
798 : Diamond d(
799 : graph(), common(),
800 : graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
801 3 : graph()->NewNode(common()->Int32Constant(0))));
802 : Node* low_node =
803 : d.Phi(MachineRepresentation::kWord32,
804 : graph()->NewNode(machine()->Int32Add(),
805 : graph()->NewNode(machine()->Word32Ctz().op(),
806 : GetReplacementHigh(input)),
807 : graph()->NewNode(common()->Int32Constant(32))),
808 : graph()->NewNode(machine()->Word32Ctz().op(),
809 5 : GetReplacementLow(input)));
810 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
811 : break;
812 : }
813 : case IrOpcode::kWord64Popcnt: {
814 : DCHECK_EQ(1, node->InputCount());
815 : Node* input = node->InputAt(0);
816 : // We assume that a Word64Popcnt node only has been created if
817 : // Word32Popcnt is actually supported.
818 : DCHECK(machine()->Word32Popcnt().IsSupported());
819 : ReplaceNode(node, graph()->NewNode(
820 : machine()->Int32Add(),
821 : graph()->NewNode(machine()->Word32Popcnt().op(),
822 : GetReplacementLow(input)),
823 : graph()->NewNode(machine()->Word32Popcnt().op(),
824 : GetReplacementHigh(input))),
825 5 : graph()->NewNode(common()->Int32Constant(0)));
826 1 : break;
827 : }
828 : case IrOpcode::kPhi: {
829 4 : MachineRepresentation rep = PhiRepresentationOf(node->op());
830 4 : if (rep == MachineRepresentation::kWord64) {
831 : // The replacement nodes have already been created, we only have to
832 : // replace placeholder nodes.
833 : Node* low_node = GetReplacementLow(node);
834 : Node* high_node = GetReplacementHigh(node);
835 9 : for (int i = 0; i < node->op()->ValueInputCount(); i++) {
836 2 : low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
837 2 : high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
838 : }
839 : } else {
840 3 : DefaultLowering(node);
841 : }
842 : break;
843 : }
844 : case IrOpcode::kWord64ReverseBytes: {
845 : Node* input = node->InputAt(0);
846 : ReplaceNode(node,
847 : graph()->NewNode(machine()->Word32ReverseBytes(),
848 : GetReplacementHigh(input)),
849 : graph()->NewNode(machine()->Word32ReverseBytes(),
850 2 : GetReplacementLow(input)));
851 : break;
852 : }
853 : case IrOpcode::kSignExtendWord8ToInt64: {
854 : DCHECK_EQ(1, node->InputCount());
855 : Node* input = node->InputAt(0);
856 0 : if (HasReplacementLow(input)) {
857 : input = GetReplacementLow(input);
858 : }
859 : // Sign extend low node to Int32
860 0 : input = graph()->NewNode(machine()->SignExtendWord8ToInt32(), input);
861 :
862 : // We use SAR to preserve the sign in the high word.
863 : ReplaceNode(
864 : node, input,
865 : graph()->NewNode(machine()->Word32Sar(), input,
866 0 : graph()->NewNode(common()->Int32Constant(31))));
867 0 : node->NullAllInputs();
868 0 : break;
869 : }
870 : case IrOpcode::kSignExtendWord16ToInt64: {
871 : DCHECK_EQ(1, node->InputCount());
872 : Node* input = node->InputAt(0);
873 0 : if (HasReplacementLow(input)) {
874 : input = GetReplacementLow(input);
875 : }
876 : // Sign extend low node to Int32
877 0 : input = graph()->NewNode(machine()->SignExtendWord16ToInt32(), input);
878 :
879 : // We use SAR to preserve the sign in the high word.
880 : ReplaceNode(
881 : node, input,
882 : graph()->NewNode(machine()->Word32Sar(), input,
883 0 : graph()->NewNode(common()->Int32Constant(31))));
884 0 : node->NullAllInputs();
885 0 : break;
886 : }
887 : case IrOpcode::kWord64AtomicLoad: {
888 : DCHECK_EQ(4, node->InputCount());
889 0 : MachineType type = AtomicOpType(node->op());
890 0 : if (type == MachineType::Uint64()) {
891 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
892 0 : ReplaceNodeWithProjections(node);
893 : } else {
894 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
895 0 : ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
896 : }
897 : break;
898 : }
899 : case IrOpcode::kWord64AtomicStore: {
900 : DCHECK_EQ(5, node->InputCount());
901 0 : MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
902 0 : if (rep == MachineRepresentation::kWord64) {
903 : Node* value = node->InputAt(2);
904 0 : node->ReplaceInput(2, GetReplacementLow(value));
905 0 : node->InsertInput(zone(), 3, GetReplacementHigh(value));
906 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
907 : } else {
908 0 : DefaultLowering(node, true);
909 0 : NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
910 : }
911 : break;
912 : }
913 : #define ATOMIC_CASE(name) \
914 : case IrOpcode::kWord64Atomic##name: { \
915 : MachineType type = AtomicOpType(node->op()); \
916 : if (type == MachineType::Uint64()) { \
917 : LowerWord64AtomicBinop(node, machine()->Word32AtomicPair##name()); \
918 : } else { \
919 : LowerWord64AtomicNarrowOp(node, machine()->Word32Atomic##name(type)); \
920 : } \
921 : break; \
922 : }
923 0 : ATOMIC_CASE(Add)
924 0 : ATOMIC_CASE(Sub)
925 0 : ATOMIC_CASE(And)
926 0 : ATOMIC_CASE(Or)
927 0 : ATOMIC_CASE(Xor)
928 0 : ATOMIC_CASE(Exchange)
929 : #undef ATOMIC_CASE
930 : case IrOpcode::kWord64AtomicCompareExchange: {
931 0 : MachineType type = AtomicOpType(node->op());
932 0 : if (type == MachineType::Uint64()) {
933 : Node* old_value = node->InputAt(2);
934 : Node* new_value = node->InputAt(3);
935 0 : node->ReplaceInput(2, GetReplacementLow(old_value));
936 0 : node->ReplaceInput(3, GetReplacementHigh(old_value));
937 0 : node->InsertInput(zone(), 4, GetReplacementLow(new_value));
938 0 : node->InsertInput(zone(), 5, GetReplacementHigh(new_value));
939 : NodeProperties::ChangeOp(node,
940 0 : machine()->Word32AtomicPairCompareExchange());
941 0 : ReplaceNodeWithProjections(node);
942 : } else {
943 : DCHECK(type == MachineType::Uint32() || type == MachineType::Uint16() ||
944 : type == MachineType::Uint8());
945 : Node* old_value = node->InputAt(2);
946 0 : node->ReplaceInput(2, GetReplacementLow(old_value));
947 : Node* new_value = node->InputAt(3);
948 0 : node->ReplaceInput(3, GetReplacementLow(new_value));
949 : NodeProperties::ChangeOp(node,
950 0 : machine()->Word32AtomicCompareExchange(type));
951 0 : ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
952 : }
953 : break;
954 : }
955 :
956 120 : default: { DefaultLowering(node); }
957 : }
958 315 : } // NOLINT(readability/fn_size)
959 :
960 4 : void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
961 32 : const Operator* low_word_op) {
962 : DCHECK_EQ(2, node->InputCount());
963 : Node* left = node->InputAt(0);
964 : Node* right = node->InputAt(1);
965 : Node* replacement = graph()->NewNode(
966 : machine()->Word32Or(),
967 : graph()->NewNode(high_word_op, GetReplacementHigh(left),
968 : GetReplacementHigh(right)),
969 : graph()->NewNode(
970 : machine()->Word32And(),
971 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
972 : GetReplacementHigh(right)),
973 : graph()->NewNode(low_word_op, GetReplacementLow(left),
974 12 : GetReplacementLow(right))));
975 :
976 : ReplaceNode(node, replacement, nullptr);
977 4 : }
978 :
979 200 : bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
980 : bool something_changed = false;
981 271 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
982 : Node* input = node->InputAt(i);
983 101 : if (HasReplacementLow(input)) {
984 : something_changed = true;
985 40 : node->ReplaceInput(i, GetReplacementLow(input));
986 : }
987 199 : if (!low_word_only && HasReplacementHigh(input)) {
988 : something_changed = true;
989 60 : node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
990 : }
991 : }
992 170 : return something_changed;
993 : }
994 :
995 192 : void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
996 : // if new_low == nullptr, then also new_high == nullptr.
997 : DCHECK(new_low != nullptr || new_high == nullptr);
998 192 : replacements_[old->id()].low = new_low;
999 198 : replacements_[old->id()].high = new_high;
1000 0 : }
1001 :
1002 121 : bool Int64Lowering::HasReplacementLow(Node* node) {
1003 242 : return replacements_[node->id()].low != nullptr;
1004 : }
1005 :
1006 48 : Node* Int64Lowering::GetReplacementLow(Node* node) {
1007 86 : Node* result = replacements_[node->id()].low;
1008 : DCHECK(result);
1009 0 : return result;
1010 : }
1011 :
1012 98 : bool Int64Lowering::HasReplacementHigh(Node* node) {
1013 196 : return replacements_[node->id()].high != nullptr;
1014 : }
1015 :
1016 81 : Node* Int64Lowering::GetReplacementHigh(Node* node) {
1017 149 : Node* result = replacements_[node->id()].high;
1018 : DCHECK(result);
1019 0 : return result;
1020 : }
1021 :
1022 11 : void Int64Lowering::PreparePhiReplacement(Node* phi) {
1023 4 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
1024 4 : if (rep == MachineRepresentation::kWord64) {
1025 : // We have to create the replacements for a phi node before we actually
1026 : // lower the phi to break potential cycles in the graph. The replacements of
1027 : // input nodes do not exist yet, so we use a placeholder node to pass the
1028 : // graph verifier.
1029 1 : int value_count = phi->op()->ValueInputCount();
1030 1 : Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
1031 : Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
1032 3 : for (int i = 0; i < value_count; i++) {
1033 2 : inputs_low[i] = placeholder_;
1034 2 : inputs_high[i] = placeholder_;
1035 : }
1036 1 : inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
1037 1 : inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
1038 : ReplaceNode(phi,
1039 : graph()->NewNode(
1040 : common()->Phi(MachineRepresentation::kWord32, value_count),
1041 : value_count + 1, inputs_low, false),
1042 : graph()->NewNode(
1043 : common()->Phi(MachineRepresentation::kWord32, value_count),
1044 4 : value_count + 1, inputs_high, false));
1045 : }
1046 4 : }
1047 :
1048 30 : void Int64Lowering::ReplaceNodeWithProjections(Node* node) {
1049 : DCHECK(node != nullptr);
1050 : Node* low_node =
1051 10 : graph()->NewNode(common()->Projection(0), node, graph()->start());
1052 : Node* high_node =
1053 10 : graph()->NewNode(common()->Projection(1), node, graph()->start());
1054 : ReplaceNode(node, low_node, high_node);
1055 5 : }
1056 :
1057 : } // namespace compiler
1058 : } // namespace internal
1059 183867 : } // namespace v8
|
