Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/int64-lowering.h"
6 :
7 : #include "src/compiler/common-operator.h"
8 : #include "src/compiler/diamond.h"
9 : #include "src/compiler/graph.h"
10 : #include "src/compiler/linkage.h"
11 : #include "src/compiler/machine-operator.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-properties.h"
14 : #include "src/compiler/node.h"
15 : #include "src/compiler/wasm-compiler.h"
16 : #include "src/zone/zone.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 : namespace compiler {
21 :
22 180 : Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
23 : CommonOperatorBuilder* common, Zone* zone,
24 : Signature<MachineRepresentation>* signature)
25 : : zone_(zone),
26 : graph_(graph),
27 : machine_(machine),
28 : common_(common),
29 : state_(graph, 3),
30 : stack_(zone),
31 : replacements_(nullptr),
32 : signature_(signature),
33 : placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
34 180 : graph->start())) {
35 : DCHECK_NOT_NULL(graph);
36 : DCHECK_NOT_NULL(graph->end());
37 45 : replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
38 45 : memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
39 45 : }
40 :
41 135 : void Int64Lowering::LowerGraph() {
42 45 : if (!machine()->Is32()) {
43 45 : return;
44 : }
45 90 : stack_.push_back({graph()->end(), 0});
46 45 : state_.Set(graph()->end(), State::kOnStack);
47 :
48 745 : while (!stack_.empty()) {
49 : NodeState& top = stack_.back();
50 1400 : if (top.input_index == top.node->InputCount()) {
51 : // All inputs of top have already been lowered, now lower top.
52 : stack_.pop_back();
53 315 : state_.Set(top.node, State::kVisited);
54 315 : LowerNode(top.node);
55 : } else {
56 : // Push the next input onto the stack.
57 655 : Node* input = top.node->InputAt(top.input_index++);
58 385 : if (state_.Get(input) == State::kUnvisited) {
59 270 : if (input->opcode() == IrOpcode::kPhi) {
60 : // To break cycles with phi nodes we push phis on a separate stack so
61 : // that they are processed after all other nodes.
62 4 : PreparePhiReplacement(input);
63 8 : stack_.push_front({input, 0});
64 266 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
65 : input->opcode() == IrOpcode::kLoop) {
66 4 : stack_.push_front({input, 0});
67 : } else {
68 528 : stack_.push_back({input, 0});
69 : }
70 : state_.Set(input, State::kOnStack);
71 : }
72 : }
73 : }
74 : }
75 :
76 : namespace {
77 :
78 : static int GetParameterIndexAfterLowering(
79 18 : Signature<MachineRepresentation>* signature, int old_index) {
80 : int result = old_index;
81 18 : for (int i = 0; i < old_index; i++) {
82 36 : if (signature->GetParam(i) == MachineRepresentation::kWord64) {
83 10 : result++;
84 : }
85 : }
86 : return result;
87 : }
88 :
89 60 : int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
90 30 : int result = static_cast<int>(signature->return_count());
91 30 : for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
92 60 : if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
93 30 : result++;
94 : }
95 : }
96 : return result;
97 : }
98 :
99 : } // namespace
100 :
101 : // static
102 0 : int Int64Lowering::GetParameterCountAfterLowering(
103 48 : Signature<MachineRepresentation>* signature) {
104 : // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
105 : // after lowering.
106 : return GetParameterIndexAfterLowering(
107 48 : signature, static_cast<int>(signature->parameter_count()));
108 : }
109 :
110 : // static
111 0 : bool Int64Lowering::IsI64AsTwoParameters(MachineOperatorBuilder* machine,
112 : MachineRepresentation type) {
113 0 : return machine->Is32() && type == MachineRepresentation::kWord64;
114 : }
115 :
116 7 : void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
117 28 : Node*& index_high) {
118 7 : if (HasReplacementLow(index)) {
119 : index = GetReplacementLow(index);
120 : }
121 : #if defined(V8_TARGET_LITTLE_ENDIAN)
122 7 : index_low = index;
123 : index_high = graph()->NewNode(machine()->Int32Add(), index,
124 21 : graph()->NewNode(common()->Int32Constant(4)));
125 : #elif defined(V8_TARGET_BIG_ENDIAN)
126 : index_low = graph()->NewNode(machine()->Int32Add(), index,
127 : graph()->NewNode(common()->Int32Constant(4)));
128 : index_high = index;
129 : #endif
130 7 : }
131 :
132 928 : void Int64Lowering::LowerNode(Node* node) {
133 315 : switch (node->opcode()) {
134 : case IrOpcode::kInt64Constant: {
135 55 : int64_t value = OpParameter<int64_t>(node);
136 : Node* low_node = graph()->NewNode(
137 110 : common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
138 : Node* high_node = graph()->NewNode(
139 110 : common()->Int32Constant(static_cast<int32_t>(value >> 32)));
140 : ReplaceNode(node, low_node, high_node);
141 : break;
142 : }
143 : case IrOpcode::kLoad:
144 : case IrOpcode::kUnalignedLoad: {
145 : MachineRepresentation rep;
146 4 : if (node->opcode() == IrOpcode::kLoad) {
147 3 : rep = LoadRepresentationOf(node->op()).representation();
148 : } else {
149 : DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
150 1 : rep = UnalignedLoadRepresentationOf(node->op()).representation();
151 : }
152 :
153 4 : if (rep == MachineRepresentation::kWord64) {
154 : Node* base = node->InputAt(0);
155 : Node* index = node->InputAt(1);
156 : Node* index_low;
157 : Node* index_high;
158 4 : GetIndexNodes(index, index_low, index_high);
159 : const Operator* load_op;
160 :
161 4 : if (node->opcode() == IrOpcode::kLoad) {
162 3 : load_op = machine()->Load(MachineType::Int32());
163 : } else {
164 : DCHECK_EQ(IrOpcode::kUnalignedLoad, node->opcode());
165 1 : load_op = machine()->UnalignedLoad(MachineType::Int32());
166 : }
167 :
168 : Node* high_node;
169 4 : if (node->InputCount() > 2) {
170 : Node* effect_high = node->InputAt(2);
171 : Node* control_high = node->InputAt(3);
172 : high_node = graph()->NewNode(load_op, base, index_high, effect_high,
173 4 : control_high);
174 : // change the effect change from old_node --> old_effect to
175 : // old_node --> high_node --> old_effect.
176 4 : node->ReplaceInput(2, high_node);
177 : } else {
178 0 : high_node = graph()->NewNode(load_op, base, index_high);
179 : }
180 4 : node->ReplaceInput(1, index_low);
181 4 : NodeProperties::ChangeOp(node, load_op);
182 : ReplaceNode(node, node, high_node);
183 : } else {
184 0 : DefaultLowering(node);
185 : }
186 : break;
187 : }
188 : case IrOpcode::kStore:
189 : case IrOpcode::kUnalignedStore: {
190 : MachineRepresentation rep;
191 4 : if (node->opcode() == IrOpcode::kStore) {
192 3 : rep = StoreRepresentationOf(node->op()).representation();
193 : } else {
194 : DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
195 1 : rep = UnalignedStoreRepresentationOf(node->op());
196 : }
197 :
198 4 : if (rep == MachineRepresentation::kWord64) {
199 : // We change the original store node to store the low word, and create
200 : // a new store node to store the high word. The effect and control edges
201 : // are copied from the original store to the new store node, the effect
202 : // edge of the original store is redirected to the new store.
203 : Node* base = node->InputAt(0);
204 : Node* index = node->InputAt(1);
205 : Node* index_low;
206 : Node* index_high;
207 3 : GetIndexNodes(index, index_low, index_high);
208 : Node* value = node->InputAt(2);
209 : DCHECK(HasReplacementLow(value));
210 : DCHECK(HasReplacementHigh(value));
211 :
212 : const Operator* store_op;
213 3 : if (node->opcode() == IrOpcode::kStore) {
214 : WriteBarrierKind write_barrier_kind =
215 2 : StoreRepresentationOf(node->op()).write_barrier_kind();
216 : store_op = machine()->Store(StoreRepresentation(
217 2 : MachineRepresentation::kWord32, write_barrier_kind));
218 : } else {
219 : DCHECK_EQ(IrOpcode::kUnalignedStore, node->opcode());
220 1 : store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
221 : }
222 :
223 : Node* high_node;
224 3 : if (node->InputCount() > 3) {
225 : Node* effect_high = node->InputAt(3);
226 : Node* control_high = node->InputAt(4);
227 : high_node = graph()->NewNode(store_op, base, index_high,
228 : GetReplacementHigh(value), effect_high,
229 3 : control_high);
230 3 : node->ReplaceInput(3, high_node);
231 :
232 : } else {
233 : high_node = graph()->NewNode(store_op, base, index_high,
234 0 : GetReplacementHigh(value));
235 : }
236 :
237 3 : node->ReplaceInput(1, index_low);
238 3 : node->ReplaceInput(2, GetReplacementLow(value));
239 3 : NodeProperties::ChangeOp(node, store_op);
240 : ReplaceNode(node, node, high_node);
241 : } else {
242 1 : DefaultLowering(node, true);
243 : }
244 : break;
245 : }
246 : case IrOpcode::kStart: {
247 : int parameter_count = GetParameterCountAfterLowering(signature());
248 : // Only exchange the node if the parameter count actually changed.
249 45 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
250 : int delta =
251 3 : parameter_count - static_cast<int>(signature()->parameter_count());
252 6 : int new_output_count = node->op()->ValueOutputCount() + delta;
253 3 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
254 : }
255 : break;
256 : }
257 : case IrOpcode::kParameter: {
258 : DCHECK_EQ(1, node->InputCount());
259 : // Only exchange the node if the parameter count actually changed. We do
260 : // not even have to do the default lowering because the the start node,
261 : // the only input of a parameter node, only changes if the parameter count
262 : // changes.
263 3 : if (GetParameterCountAfterLowering(signature()) !=
264 : static_cast<int>(signature()->parameter_count())) {
265 3 : int old_index = ParameterIndexOf(node->op());
266 : // TODO(wasm): Make this part not wasm specific.
267 : // Prevent special lowering of the WasmContext parameter.
268 3 : if (old_index == kWasmContextParameterIndex) {
269 1 : DefaultLowering(node);
270 1 : break;
271 : }
272 : // Adjust old_index to be compliant with the signature.
273 2 : --old_index;
274 : int new_index = GetParameterIndexAfterLowering(signature(), old_index);
275 : // Adjust new_index to consider the WasmContext parameter.
276 2 : ++new_index;
277 2 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
278 :
279 : Node* high_node = nullptr;
280 6 : if (signature()->GetParam(old_index) ==
281 : MachineRepresentation::kWord64) {
282 : high_node = graph()->NewNode(common()->Parameter(new_index + 1),
283 2 : graph()->start());
284 : }
285 : ReplaceNode(node, node, high_node);
286 : }
287 : break;
288 : }
289 : case IrOpcode::kReturn: {
290 : int input_count = node->InputCount();
291 45 : DefaultLowering(node);
292 45 : if (input_count != node->InputCount()) {
293 : int new_return_count = GetReturnCountAfterLowering(signature());
294 30 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
295 30 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
296 : }
297 : }
298 : break;
299 : }
300 : case IrOpcode::kTailCall: {
301 0 : CallDescriptor* descriptor =
302 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
303 0 : if (DefaultLowering(node) ||
304 0 : (descriptor->ReturnCount() == 1 &&
305 : descriptor->GetReturnType(0) == MachineType::Int64())) {
306 : // Tail calls do not have return values, so adjusting the call
307 : // descriptor is enough.
308 0 : auto new_descriptor = GetI32WasmCallDescriptor(zone(), descriptor);
309 0 : NodeProperties::ChangeOp(node, common()->TailCall(new_descriptor));
310 : }
311 : break;
312 : }
313 : case IrOpcode::kCall: {
314 0 : CallDescriptor* descriptor =
315 0 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
316 0 : if (DefaultLowering(node) ||
317 0 : (descriptor->ReturnCount() == 1 &&
318 : descriptor->GetReturnType(0) == MachineType::Int64())) {
319 : // We have to adjust the call descriptor.
320 : NodeProperties::ChangeOp(
321 0 : node, common()->Call(GetI32WasmCallDescriptor(zone(), descriptor)));
322 : }
323 0 : if (descriptor->ReturnCount() == 1 &&
324 : descriptor->GetReturnType(0) == MachineType::Int64()) {
325 : // We access the additional return values through projections.
326 : Node* low_node =
327 0 : graph()->NewNode(common()->Projection(0), node, graph()->start());
328 : Node* high_node =
329 0 : graph()->NewNode(common()->Projection(1), node, graph()->start());
330 : ReplaceNode(node, low_node, high_node);
331 : }
332 : break;
333 : }
334 : case IrOpcode::kWord64And: {
335 : DCHECK_EQ(2, node->InputCount());
336 : Node* left = node->InputAt(0);
337 : Node* right = node->InputAt(1);
338 :
339 : Node* low_node =
340 : graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
341 3 : GetReplacementLow(right));
342 : Node* high_node =
343 : graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
344 3 : GetReplacementHigh(right));
345 : ReplaceNode(node, low_node, high_node);
346 : break;
347 : }
348 : case IrOpcode::kTruncateInt64ToInt32: {
349 : DCHECK_EQ(1, node->InputCount());
350 : Node* input = node->InputAt(0);
351 : ReplaceNode(node, GetReplacementLow(input), nullptr);
352 4 : node->NullAllInputs();
353 4 : break;
354 : }
355 : case IrOpcode::kInt64Add: {
356 : DCHECK_EQ(2, node->InputCount());
357 :
358 : Node* right = node->InputAt(1);
359 0 : node->ReplaceInput(1, GetReplacementLow(right));
360 0 : node->AppendInput(zone(), GetReplacementHigh(right));
361 :
362 : Node* left = node->InputAt(0);
363 0 : node->ReplaceInput(0, GetReplacementLow(left));
364 0 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
365 :
366 0 : NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
367 : // We access the additional return values through projections.
368 : Node* low_node =
369 0 : graph()->NewNode(common()->Projection(0), node, graph()->start());
370 : Node* high_node =
371 0 : graph()->NewNode(common()->Projection(1), node, graph()->start());
372 : ReplaceNode(node, low_node, high_node);
373 : break;
374 : }
375 : case IrOpcode::kInt64Sub: {
376 : DCHECK_EQ(2, node->InputCount());
377 :
378 : Node* right = node->InputAt(1);
379 1 : node->ReplaceInput(1, GetReplacementLow(right));
380 1 : node->AppendInput(zone(), GetReplacementHigh(right));
381 :
382 : Node* left = node->InputAt(0);
383 1 : node->ReplaceInput(0, GetReplacementLow(left));
384 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
385 :
386 1 : NodeProperties::ChangeOp(node, machine()->Int32PairSub());
387 : // We access the additional return values through projections.
388 : Node* low_node =
389 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
390 : Node* high_node =
391 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
392 : ReplaceNode(node, low_node, high_node);
393 : break;
394 : }
395 : case IrOpcode::kInt64Mul: {
396 : DCHECK_EQ(2, node->InputCount());
397 :
398 : Node* right = node->InputAt(1);
399 1 : node->ReplaceInput(1, GetReplacementLow(right));
400 1 : node->AppendInput(zone(), GetReplacementHigh(right));
401 :
402 : Node* left = node->InputAt(0);
403 1 : node->ReplaceInput(0, GetReplacementLow(left));
404 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
405 :
406 1 : NodeProperties::ChangeOp(node, machine()->Int32PairMul());
407 : // We access the additional return values through projections.
408 : Node* low_node =
409 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
410 : Node* high_node =
411 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
412 : ReplaceNode(node, low_node, high_node);
413 : break;
414 : }
415 : case IrOpcode::kWord64Or: {
416 : DCHECK_EQ(2, node->InputCount());
417 : Node* left = node->InputAt(0);
418 : Node* right = node->InputAt(1);
419 :
420 : Node* low_node =
421 : graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
422 1 : GetReplacementLow(right));
423 : Node* high_node =
424 : graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
425 1 : GetReplacementHigh(right));
426 : ReplaceNode(node, low_node, high_node);
427 : break;
428 : }
429 : case IrOpcode::kWord64Xor: {
430 : DCHECK_EQ(2, node->InputCount());
431 : Node* left = node->InputAt(0);
432 : Node* right = node->InputAt(1);
433 :
434 : Node* low_node =
435 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
436 1 : GetReplacementLow(right));
437 : Node* high_node =
438 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
439 1 : GetReplacementHigh(right));
440 : ReplaceNode(node, low_node, high_node);
441 : break;
442 : }
443 : case IrOpcode::kWord64Shl: {
444 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
445 : // of the output to 0 and just calculate the high word.
446 : DCHECK_EQ(2, node->InputCount());
447 : Node* shift = node->InputAt(1);
448 1 : if (HasReplacementLow(shift)) {
449 : // We do not have to care about the high word replacement, because
450 : // the shift can only be between 0 and 63 anyways.
451 1 : node->ReplaceInput(1, GetReplacementLow(shift));
452 : }
453 :
454 : Node* value = node->InputAt(0);
455 1 : node->ReplaceInput(0, GetReplacementLow(value));
456 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
457 :
458 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShl());
459 : // We access the additional return values through projections.
460 : Node* low_node =
461 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
462 : Node* high_node =
463 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
464 : ReplaceNode(node, low_node, high_node);
465 : break;
466 : }
467 : case IrOpcode::kWord64Shr: {
468 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
469 : // of the output to 0 and just calculate the high word.
470 : DCHECK_EQ(2, node->InputCount());
471 : Node* shift = node->InputAt(1);
472 1 : if (HasReplacementLow(shift)) {
473 : // We do not have to care about the high word replacement, because
474 : // the shift can only be between 0 and 63 anyways.
475 1 : node->ReplaceInput(1, GetReplacementLow(shift));
476 : }
477 :
478 : Node* value = node->InputAt(0);
479 1 : node->ReplaceInput(0, GetReplacementLow(value));
480 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
481 :
482 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShr());
483 : // We access the additional return values through projections.
484 : Node* low_node =
485 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
486 : Node* high_node =
487 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
488 : ReplaceNode(node, low_node, high_node);
489 : break;
490 : }
491 : case IrOpcode::kWord64Sar: {
492 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
493 : // of the output to 0 and just calculate the high word.
494 : DCHECK_EQ(2, node->InputCount());
495 : Node* shift = node->InputAt(1);
496 1 : if (HasReplacementLow(shift)) {
497 : // We do not have to care about the high word replacement, because
498 : // the shift can only be between 0 and 63 anyways.
499 1 : node->ReplaceInput(1, GetReplacementLow(shift));
500 : }
501 :
502 : Node* value = node->InputAt(0);
503 1 : node->ReplaceInput(0, GetReplacementLow(value));
504 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
505 :
506 1 : NodeProperties::ChangeOp(node, machine()->Word32PairSar());
507 : // We access the additional return values through projections.
508 : Node* low_node =
509 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
510 : Node* high_node =
511 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
512 : ReplaceNode(node, low_node, high_node);
513 : break;
514 : }
515 : case IrOpcode::kWord64Equal: {
516 : DCHECK_EQ(2, node->InputCount());
517 : Node* left = node->InputAt(0);
518 : Node* right = node->InputAt(1);
519 :
520 : // TODO(wasm): Use explicit comparisons and && here?
521 : Node* replacement = graph()->NewNode(
522 : machine()->Word32Equal(),
523 : graph()->NewNode(
524 : machine()->Word32Or(),
525 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
526 : GetReplacementLow(right)),
527 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
528 : GetReplacementHigh(right))),
529 10 : graph()->NewNode(common()->Int32Constant(0)));
530 :
531 : ReplaceNode(node, replacement, nullptr);
532 : break;
533 : }
534 : case IrOpcode::kInt64LessThan: {
535 : LowerComparison(node, machine()->Int32LessThan(),
536 2 : machine()->Uint32LessThan());
537 1 : break;
538 : }
539 : case IrOpcode::kInt64LessThanOrEqual: {
540 : LowerComparison(node, machine()->Int32LessThan(),
541 2 : machine()->Uint32LessThanOrEqual());
542 1 : break;
543 : }
544 : case IrOpcode::kUint64LessThan: {
545 : LowerComparison(node, machine()->Uint32LessThan(),
546 2 : machine()->Uint32LessThan());
547 1 : break;
548 : }
549 : case IrOpcode::kUint64LessThanOrEqual: {
550 : LowerComparison(node, machine()->Uint32LessThan(),
551 2 : machine()->Uint32LessThanOrEqual());
552 1 : break;
553 : }
554 : case IrOpcode::kChangeInt32ToInt64: {
555 : DCHECK_EQ(1, node->InputCount());
556 : Node* input = node->InputAt(0);
557 2 : if (HasReplacementLow(input)) {
558 : input = GetReplacementLow(input);
559 : }
560 : // We use SAR to preserve the sign in the high word.
561 : ReplaceNode(
562 : node, input,
563 : graph()->NewNode(machine()->Word32Sar(), input,
564 4 : graph()->NewNode(common()->Int32Constant(31))));
565 2 : node->NullAllInputs();
566 2 : break;
567 : }
568 : case IrOpcode::kChangeUint32ToUint64: {
569 : DCHECK_EQ(1, node->InputCount());
570 : Node* input = node->InputAt(0);
571 2 : if (HasReplacementLow(input)) {
572 : input = GetReplacementLow(input);
573 : }
574 2 : ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
575 2 : node->NullAllInputs();
576 2 : break;
577 : }
578 : case IrOpcode::kBitcastInt64ToFloat64: {
579 : DCHECK_EQ(1, node->InputCount());
580 : Node* input = node->InputAt(0);
581 : Node* stack_slot = graph()->NewNode(
582 1 : machine()->StackSlot(MachineRepresentation::kWord64));
583 :
584 : Node* store_high_word = graph()->NewNode(
585 : machine()->Store(
586 : StoreRepresentation(MachineRepresentation::kWord32,
587 : WriteBarrierKind::kNoWriteBarrier)),
588 : stack_slot,
589 : graph()->NewNode(
590 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
591 4 : GetReplacementHigh(input), graph()->start(), graph()->start());
592 :
593 : Node* store_low_word = graph()->NewNode(
594 : machine()->Store(
595 : StoreRepresentation(MachineRepresentation::kWord32,
596 : WriteBarrierKind::kNoWriteBarrier)),
597 : stack_slot,
598 : graph()->NewNode(
599 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
600 4 : GetReplacementLow(input), store_high_word, graph()->start());
601 :
602 : Node* load =
603 : graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
604 : graph()->NewNode(common()->Int32Constant(0)),
605 3 : store_low_word, graph()->start());
606 :
607 : ReplaceNode(node, load, nullptr);
608 : break;
609 : }
610 : case IrOpcode::kBitcastFloat64ToInt64: {
611 : DCHECK_EQ(1, node->InputCount());
612 : Node* input = node->InputAt(0);
613 1 : if (HasReplacementLow(input)) {
614 : input = GetReplacementLow(input);
615 : }
616 : Node* stack_slot = graph()->NewNode(
617 1 : machine()->StackSlot(MachineRepresentation::kWord64));
618 : Node* store = graph()->NewNode(
619 : machine()->Store(
620 : StoreRepresentation(MachineRepresentation::kFloat64,
621 : WriteBarrierKind::kNoWriteBarrier)),
622 : stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
623 4 : graph()->start(), graph()->start());
624 :
625 : Node* high_node = graph()->NewNode(
626 : machine()->Load(MachineType::Int32()), stack_slot,
627 : graph()->NewNode(
628 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
629 3 : store, graph()->start());
630 :
631 : Node* low_node = graph()->NewNode(
632 : machine()->Load(MachineType::Int32()), stack_slot,
633 : graph()->NewNode(
634 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
635 3 : store, graph()->start());
636 : ReplaceNode(node, low_node, high_node);
637 : break;
638 : }
639 : case IrOpcode::kWord64Ror: {
640 : DCHECK_EQ(2, node->InputCount());
641 : Node* input = node->InputAt(0);
642 : Node* shift = HasReplacementLow(node->InputAt(1))
643 : ? GetReplacementLow(node->InputAt(1))
644 5 : : node->InputAt(1);
645 : Int32Matcher m(shift);
646 5 : if (m.HasValue()) {
647 : // Precondition: 0 <= shift < 64.
648 4 : int32_t shift_value = m.Value() & 0x3f;
649 4 : if (shift_value == 0) {
650 : ReplaceNode(node, GetReplacementLow(input),
651 : GetReplacementHigh(input));
652 3 : } else if (shift_value == 32) {
653 : ReplaceNode(node, GetReplacementHigh(input),
654 : GetReplacementLow(input));
655 : } else {
656 : Node* low_input;
657 : Node* high_input;
658 2 : if (shift_value < 32) {
659 : low_input = GetReplacementLow(input);
660 : high_input = GetReplacementHigh(input);
661 : } else {
662 : low_input = GetReplacementHigh(input);
663 : high_input = GetReplacementLow(input);
664 : }
665 2 : int32_t masked_shift_value = shift_value & 0x1f;
666 : Node* masked_shift =
667 2 : graph()->NewNode(common()->Int32Constant(masked_shift_value));
668 : Node* inv_shift = graph()->NewNode(
669 4 : common()->Int32Constant(32 - masked_shift_value));
670 :
671 : Node* low_node = graph()->NewNode(
672 : machine()->Word32Or(),
673 : graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
674 6 : graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
675 : Node* high_node = graph()->NewNode(
676 : machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
677 : high_input, masked_shift),
678 6 : graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
679 : ReplaceNode(node, low_node, high_node);
680 : }
681 : } else {
682 : Node* safe_shift = shift;
683 1 : if (!machine()->Word32ShiftIsSafe()) {
684 : safe_shift =
685 : graph()->NewNode(machine()->Word32And(), shift,
686 2 : graph()->NewNode(common()->Int32Constant(0x1f)));
687 : }
688 :
689 : // By creating this bit-mask with SAR and SHL we do not have to deal
690 : // with shift == 0 as a special case.
691 : Node* inv_mask = graph()->NewNode(
692 : machine()->Word32Shl(),
693 : graph()->NewNode(machine()->Word32Sar(),
694 : graph()->NewNode(common()->Int32Constant(
695 : std::numeric_limits<int32_t>::min())),
696 : safe_shift),
697 4 : graph()->NewNode(common()->Int32Constant(1)));
698 :
699 : Node* bit_mask =
700 : graph()->NewNode(machine()->Word32Xor(), inv_mask,
701 2 : graph()->NewNode(common()->Int32Constant(-1)));
702 :
703 : // We have to mask the shift value for this comparison. If
704 : // !machine()->Word32ShiftIsSafe() then the masking should already be
705 : // part of the graph.
706 : Node* masked_shift6 = shift;
707 1 : if (machine()->Word32ShiftIsSafe()) {
708 : masked_shift6 =
709 : graph()->NewNode(machine()->Word32And(), shift,
710 0 : graph()->NewNode(common()->Int32Constant(0x3f)));
711 : }
712 :
713 : Diamond lt32(
714 : graph(), common(),
715 : graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
716 3 : graph()->NewNode(common()->Int32Constant(32))));
717 :
718 : // The low word and the high word can be swapped either at the input or
719 : // at the output. We swap the inputs so that shift does not have to be
720 : // kept for so long in a register.
721 : Node* input_low =
722 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
723 1 : GetReplacementHigh(input));
724 : Node* input_high =
725 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
726 1 : GetReplacementLow(input));
727 :
728 : Node* rotate_low =
729 1 : graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
730 : Node* rotate_high =
731 1 : graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
732 :
733 : Node* low_node = graph()->NewNode(
734 : machine()->Word32Or(),
735 : graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
736 3 : graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
737 :
738 : Node* high_node = graph()->NewNode(
739 : machine()->Word32Or(),
740 : graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
741 3 : graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
742 :
743 : ReplaceNode(node, low_node, high_node);
744 : }
745 : break;
746 : }
747 : case IrOpcode::kWord64Clz: {
748 : DCHECK_EQ(1, node->InputCount());
749 : Node* input = node->InputAt(0);
750 : Diamond d(
751 : graph(), common(),
752 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
753 3 : graph()->NewNode(common()->Int32Constant(0))));
754 :
755 : Node* low_node = d.Phi(
756 : MachineRepresentation::kWord32,
757 : graph()->NewNode(machine()->Int32Add(),
758 : graph()->NewNode(machine()->Word32Clz(),
759 : GetReplacementLow(input)),
760 : graph()->NewNode(common()->Int32Constant(32))),
761 5 : graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
762 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
763 : break;
764 : }
765 : case IrOpcode::kWord64Ctz: {
766 : DCHECK_EQ(1, node->InputCount());
767 : DCHECK(machine()->Word32Ctz().IsSupported());
768 : Node* input = node->InputAt(0);
769 : Diamond d(
770 : graph(), common(),
771 : graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
772 3 : graph()->NewNode(common()->Int32Constant(0))));
773 : Node* low_node =
774 : d.Phi(MachineRepresentation::kWord32,
775 : graph()->NewNode(machine()->Int32Add(),
776 : graph()->NewNode(machine()->Word32Ctz().op(),
777 : GetReplacementHigh(input)),
778 : graph()->NewNode(common()->Int32Constant(32))),
779 : graph()->NewNode(machine()->Word32Ctz().op(),
780 5 : GetReplacementLow(input)));
781 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
782 : break;
783 : }
784 : case IrOpcode::kWord64Popcnt: {
785 : DCHECK_EQ(1, node->InputCount());
786 : Node* input = node->InputAt(0);
787 : // We assume that a Word64Popcnt node only has been created if
788 : // Word32Popcnt is actually supported.
789 : DCHECK(machine()->Word32Popcnt().IsSupported());
790 : ReplaceNode(node, graph()->NewNode(
791 : machine()->Int32Add(),
792 : graph()->NewNode(machine()->Word32Popcnt().op(),
793 : GetReplacementLow(input)),
794 : graph()->NewNode(machine()->Word32Popcnt().op(),
795 : GetReplacementHigh(input))),
796 5 : graph()->NewNode(common()->Int32Constant(0)));
797 1 : break;
798 : }
799 : case IrOpcode::kPhi: {
800 4 : MachineRepresentation rep = PhiRepresentationOf(node->op());
801 4 : if (rep == MachineRepresentation::kWord64) {
802 : // The replacement nodes have already been created, we only have to
803 : // replace placeholder nodes.
804 : Node* low_node = GetReplacementLow(node);
805 : Node* high_node = GetReplacementHigh(node);
806 9 : for (int i = 0; i < node->op()->ValueInputCount(); i++) {
807 2 : low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
808 2 : high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
809 : }
810 : } else {
811 3 : DefaultLowering(node);
812 : }
813 : break;
814 : }
815 : case IrOpcode::kProjection: {
816 0 : Node* call = node->InputAt(0);
817 : DCHECK_EQ(IrOpcode::kCall, call->opcode());
818 0 : CallDescriptor* descriptor =
819 0 : const_cast<CallDescriptor*>(CallDescriptorOf(call->op()));
820 0 : for (size_t i = 0; i < descriptor->ReturnCount(); i++) {
821 0 : if (descriptor->GetReturnType(i) == MachineType::Int64()) {
822 0 : UNREACHABLE(); // TODO(titzer): implement multiple i64 returns.
823 : }
824 : }
825 : break;
826 : }
827 : case IrOpcode::kWord64ReverseBytes: {
828 : Node* input = node->InputAt(0);
829 : ReplaceNode(node, graph()->NewNode(machine()->Word32ReverseBytes().op(),
830 : GetReplacementHigh(input)),
831 : graph()->NewNode(machine()->Word32ReverseBytes().op(),
832 3 : GetReplacementLow(input)));
833 1 : break;
834 : }
835 :
836 120 : default: { DefaultLowering(node); }
837 : }
838 315 : } // NOLINT(readability/fn_size)
839 :
840 4 : void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
841 32 : const Operator* low_word_op) {
842 : DCHECK_EQ(2, node->InputCount());
843 : Node* left = node->InputAt(0);
844 : Node* right = node->InputAt(1);
845 : Node* replacement = graph()->NewNode(
846 : machine()->Word32Or(),
847 : graph()->NewNode(high_word_op, GetReplacementHigh(left),
848 : GetReplacementHigh(right)),
849 : graph()->NewNode(
850 : machine()->Word32And(),
851 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
852 : GetReplacementHigh(right)),
853 : graph()->NewNode(low_word_op, GetReplacementLow(left),
854 12 : GetReplacementLow(right))));
855 :
856 : ReplaceNode(node, replacement, nullptr);
857 4 : }
858 :
859 200 : bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
860 : bool something_changed = false;
861 271 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
862 : Node* input = node->InputAt(i);
863 101 : if (HasReplacementLow(input)) {
864 : something_changed = true;
865 41 : node->ReplaceInput(i, GetReplacementLow(input));
866 : }
867 199 : if (!low_word_only && HasReplacementHigh(input)) {
868 : something_changed = true;
869 60 : node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
870 : }
871 : }
872 170 : return something_changed;
873 : }
874 :
875 194 : void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
876 : // if new_low == nullptr, then also new_high == nullptr.
877 : DCHECK(new_low != nullptr || new_high == nullptr);
878 194 : replacements_[old->id()].low = new_low;
879 200 : replacements_[old->id()].high = new_high;
880 0 : }
881 :
882 121 : bool Int64Lowering::HasReplacementLow(Node* node) {
883 242 : return replacements_[node->id()].low != nullptr;
884 : }
885 :
886 48 : Node* Int64Lowering::GetReplacementLow(Node* node) {
887 86 : Node* result = replacements_[node->id()].low;
888 : DCHECK(result);
889 0 : return result;
890 : }
891 :
892 98 : bool Int64Lowering::HasReplacementHigh(Node* node) {
893 196 : return replacements_[node->id()].high != nullptr;
894 : }
895 :
896 81 : Node* Int64Lowering::GetReplacementHigh(Node* node) {
897 149 : Node* result = replacements_[node->id()].high;
898 : DCHECK(result);
899 0 : return result;
900 : }
901 :
902 11 : void Int64Lowering::PreparePhiReplacement(Node* phi) {
903 4 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
904 4 : if (rep == MachineRepresentation::kWord64) {
905 : // We have to create the replacements for a phi node before we actually
906 : // lower the phi to break potential cycles in the graph. The replacements of
907 : // input nodes do not exist yet, so we use a placeholder node to pass the
908 : // graph verifier.
909 1 : int value_count = phi->op()->ValueInputCount();
910 1 : Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
911 : Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
912 3 : for (int i = 0; i < value_count; i++) {
913 2 : inputs_low[i] = placeholder_;
914 2 : inputs_high[i] = placeholder_;
915 : }
916 1 : inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
917 1 : inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
918 : ReplaceNode(phi,
919 : graph()->NewNode(
920 : common()->Phi(MachineRepresentation::kWord32, value_count),
921 : value_count + 1, inputs_low, false),
922 : graph()->NewNode(
923 : common()->Phi(MachineRepresentation::kWord32, value_count),
924 4 : value_count + 1, inputs_high, false));
925 : }
926 4 : }
927 : } // namespace compiler
928 : } // namespace internal
929 : } // namespace v8
|
