Line data Source code
1 : // Copyright 2014 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/int64-lowering.h"
6 : #include "src/compiler/common-operator.h"
7 : #include "src/compiler/diamond.h"
8 : #include "src/compiler/graph.h"
9 : #include "src/compiler/linkage.h"
10 : #include "src/compiler/machine-operator.h"
11 : #include "src/compiler/node-matchers.h"
12 : #include "src/compiler/node-properties.h"
13 :
14 : #include "src/compiler/node.h"
15 : #include "src/objects-inl.h"
16 : #include "src/wasm/wasm-module.h"
17 : #include "src/zone/zone.h"
18 :
19 : namespace v8 {
20 : namespace internal {
21 : namespace compiler {
22 :
23 192 : Int64Lowering::Int64Lowering(Graph* graph, MachineOperatorBuilder* machine,
24 : CommonOperatorBuilder* common, Zone* zone,
25 : Signature<MachineRepresentation>* signature)
26 : : zone_(zone),
27 : graph_(graph),
28 : machine_(machine),
29 : common_(common),
30 : state_(graph, 3),
31 : stack_(zone),
32 : replacements_(nullptr),
33 : signature_(signature),
34 : placeholder_(graph->NewNode(common->Parameter(-2, "placeholder"),
35 192 : graph->start())) {
36 : DCHECK_NOT_NULL(graph);
37 : DCHECK_NOT_NULL(graph->end());
38 48 : replacements_ = zone->NewArray<Replacement>(graph->NodeCount());
39 48 : memset(replacements_, 0, sizeof(Replacement) * graph->NodeCount());
40 48 : }
41 :
42 144 : void Int64Lowering::LowerGraph() {
43 48 : if (!machine()->Is32()) {
44 48 : return;
45 : }
46 96 : stack_.push_back({graph()->end(), 0});
47 48 : state_.Set(graph()->end(), State::kOnStack);
48 :
49 799 : while (!stack_.empty()) {
50 : NodeState& top = stack_.back();
51 1502 : if (top.input_index == top.node->InputCount()) {
52 : // All inputs of top have already been lowered, now lower top.
53 : stack_.pop_back();
54 337 : state_.Set(top.node, State::kVisited);
55 337 : LowerNode(top.node);
56 : } else {
57 : // Push the next input onto the stack.
58 703 : Node* input = top.node->InputAt(top.input_index++);
59 414 : if (state_.Get(input) == State::kUnvisited) {
60 289 : if (input->opcode() == IrOpcode::kPhi) {
61 : // To break cycles with phi nodes we push phis on a separate stack so
62 : // that they are processed after all other nodes.
63 4 : PreparePhiReplacement(input);
64 8 : stack_.push_front({input, 0});
65 285 : } else if (input->opcode() == IrOpcode::kEffectPhi ||
66 : input->opcode() == IrOpcode::kLoop) {
67 4 : stack_.push_front({input, 0});
68 : } else {
69 566 : stack_.push_back({input, 0});
70 : }
71 : state_.Set(input, State::kOnStack);
72 : }
73 : }
74 : }
75 : }
76 :
77 : namespace {
78 :
79 : static int GetParameterIndexAfterLowering(
80 18 : Signature<MachineRepresentation>* signature, int old_index) {
81 : int result = old_index;
82 18 : for (int i = 0; i < old_index; i++) {
83 36 : if (signature->GetParam(i) == MachineRepresentation::kWord64) {
84 10 : result++;
85 : }
86 : }
87 : return result;
88 : }
89 :
90 64 : int GetReturnCountAfterLowering(Signature<MachineRepresentation>* signature) {
91 32 : int result = static_cast<int>(signature->return_count());
92 32 : for (int i = 0; i < static_cast<int>(signature->return_count()); i++) {
93 64 : if (signature->GetReturn(i) == MachineRepresentation::kWord64) {
94 32 : result++;
95 : }
96 : }
97 : return result;
98 : }
99 :
100 : } // namespace
101 :
102 : // static
103 0 : int Int64Lowering::GetParameterCountAfterLowering(
104 51 : Signature<MachineRepresentation>* signature) {
105 : // GetParameterIndexAfterLowering(parameter_count) returns the parameter count
106 : // after lowering.
107 : return GetParameterIndexAfterLowering(
108 51 : signature, static_cast<int>(signature->parameter_count()));
109 : }
110 :
111 : // static
112 2380 : bool Int64Lowering::IsI64AsTwoParameters(MachineOperatorBuilder* machine,
113 : MachineRepresentation type) {
114 2380 : return machine->Is32() && type == MachineRepresentation::kWord64;
115 : }
116 :
117 7 : void Int64Lowering::GetIndexNodes(Node* index, Node*& index_low,
118 28 : Node*& index_high) {
119 7 : if (HasReplacementLow(index)) {
120 : index = GetReplacementLow(index);
121 : }
122 : #if defined(V8_TARGET_LITTLE_ENDIAN)
123 7 : index_low = index;
124 : index_high = graph()->NewNode(machine()->Int32Add(), index,
125 21 : graph()->NewNode(common()->Int32Constant(4)));
126 : #elif defined(V8_TARGET_BIG_ENDIAN)
127 : index_low = graph()->NewNode(machine()->Int32Add(), index,
128 : graph()->NewNode(common()->Int32Constant(4)));
129 : index_high = index;
130 : #endif
131 7 : }
132 :
133 998 : void Int64Lowering::LowerNode(Node* node) {
134 337 : switch (node->opcode()) {
135 : case IrOpcode::kInt64Constant: {
136 59 : int64_t value = OpParameter<int64_t>(node);
137 : Node* low_node = graph()->NewNode(
138 118 : common()->Int32Constant(static_cast<int32_t>(value & 0xFFFFFFFF)));
139 : Node* high_node = graph()->NewNode(
140 118 : common()->Int32Constant(static_cast<int32_t>(value >> 32)));
141 : ReplaceNode(node, low_node, high_node);
142 : break;
143 : }
144 : case IrOpcode::kLoad:
145 : case IrOpcode::kUnalignedLoad: {
146 : MachineRepresentation rep;
147 4 : if (node->opcode() == IrOpcode::kLoad) {
148 3 : rep = LoadRepresentationOf(node->op()).representation();
149 : } else {
150 : DCHECK(node->opcode() == IrOpcode::kUnalignedLoad);
151 1 : rep = UnalignedLoadRepresentationOf(node->op()).representation();
152 : }
153 :
154 4 : if (rep == MachineRepresentation::kWord64) {
155 : Node* base = node->InputAt(0);
156 : Node* index = node->InputAt(1);
157 : Node* index_low;
158 : Node* index_high;
159 4 : GetIndexNodes(index, index_low, index_high);
160 : const Operator* load_op;
161 :
162 4 : if (node->opcode() == IrOpcode::kLoad) {
163 3 : load_op = machine()->Load(MachineType::Int32());
164 : } else {
165 : DCHECK(node->opcode() == IrOpcode::kUnalignedLoad);
166 1 : load_op = machine()->UnalignedLoad(MachineType::Int32());
167 : }
168 :
169 : Node* high_node;
170 4 : if (node->InputCount() > 2) {
171 : Node* effect_high = node->InputAt(2);
172 : Node* control_high = node->InputAt(3);
173 : high_node = graph()->NewNode(load_op, base, index_high, effect_high,
174 4 : control_high);
175 : // change the effect change from old_node --> old_effect to
176 : // old_node --> high_node --> old_effect.
177 4 : node->ReplaceInput(2, high_node);
178 : } else {
179 0 : high_node = graph()->NewNode(load_op, base, index_high);
180 : }
181 4 : node->ReplaceInput(1, index_low);
182 4 : NodeProperties::ChangeOp(node, load_op);
183 : ReplaceNode(node, node, high_node);
184 : } else {
185 0 : DefaultLowering(node);
186 : }
187 : break;
188 : }
189 : case IrOpcode::kStore:
190 : case IrOpcode::kUnalignedStore: {
191 : MachineRepresentation rep;
192 4 : if (node->opcode() == IrOpcode::kStore) {
193 3 : rep = StoreRepresentationOf(node->op()).representation();
194 : } else {
195 : DCHECK(node->opcode() == IrOpcode::kUnalignedStore);
196 1 : rep = UnalignedStoreRepresentationOf(node->op());
197 : }
198 :
199 4 : if (rep == MachineRepresentation::kWord64) {
200 : // We change the original store node to store the low word, and create
201 : // a new store node to store the high word. The effect and control edges
202 : // are copied from the original store to the new store node, the effect
203 : // edge of the original store is redirected to the new store.
204 : Node* base = node->InputAt(0);
205 : Node* index = node->InputAt(1);
206 : Node* index_low;
207 : Node* index_high;
208 3 : GetIndexNodes(index, index_low, index_high);
209 : Node* value = node->InputAt(2);
210 : DCHECK(HasReplacementLow(value));
211 : DCHECK(HasReplacementHigh(value));
212 :
213 : const Operator* store_op;
214 3 : if (node->opcode() == IrOpcode::kStore) {
215 : WriteBarrierKind write_barrier_kind =
216 2 : StoreRepresentationOf(node->op()).write_barrier_kind();
217 : store_op = machine()->Store(StoreRepresentation(
218 2 : MachineRepresentation::kWord32, write_barrier_kind));
219 : } else {
220 : DCHECK(node->opcode() == IrOpcode::kUnalignedStore);
221 1 : store_op = machine()->UnalignedStore(MachineRepresentation::kWord32);
222 : }
223 :
224 : Node* high_node;
225 3 : if (node->InputCount() > 3) {
226 : Node* effect_high = node->InputAt(3);
227 : Node* control_high = node->InputAt(4);
228 : high_node = graph()->NewNode(store_op, base, index_high,
229 : GetReplacementHigh(value), effect_high,
230 3 : control_high);
231 3 : node->ReplaceInput(3, high_node);
232 :
233 : } else {
234 : high_node = graph()->NewNode(store_op, base, index_high,
235 0 : GetReplacementHigh(value));
236 : }
237 :
238 3 : node->ReplaceInput(1, index_low);
239 3 : node->ReplaceInput(2, GetReplacementLow(value));
240 3 : NodeProperties::ChangeOp(node, store_op);
241 : ReplaceNode(node, node, high_node);
242 : } else {
243 1 : DefaultLowering(node, true);
244 : }
245 : break;
246 : }
247 : case IrOpcode::kStart: {
248 : int parameter_count = GetParameterCountAfterLowering(signature());
249 : // Only exchange the node if the parameter count actually changed.
250 48 : if (parameter_count != static_cast<int>(signature()->parameter_count())) {
251 : int delta =
252 3 : parameter_count - static_cast<int>(signature()->parameter_count());
253 6 : int new_output_count = node->op()->ValueOutputCount() + delta;
254 3 : NodeProperties::ChangeOp(node, common()->Start(new_output_count));
255 : }
256 : break;
257 : }
258 : case IrOpcode::kParameter: {
259 : DCHECK(node->InputCount() == 1);
260 : // Only exchange the node if the parameter count actually changed. We do
261 : // not even have to do the default lowering because the the start node,
262 : // the only input of a parameter node, only changes if the parameter count
263 : // changes.
264 3 : if (GetParameterCountAfterLowering(signature()) !=
265 : static_cast<int>(signature()->parameter_count())) {
266 3 : int old_index = ParameterIndexOf(node->op());
267 : int new_index = GetParameterIndexAfterLowering(signature(), old_index);
268 3 : NodeProperties::ChangeOp(node, common()->Parameter(new_index));
269 :
270 : Node* high_node = nullptr;
271 9 : if (signature()->GetParam(old_index) ==
272 : MachineRepresentation::kWord64) {
273 : high_node = graph()->NewNode(common()->Parameter(new_index + 1),
274 4 : graph()->start());
275 : }
276 : ReplaceNode(node, node, high_node);
277 : }
278 : break;
279 : }
280 : case IrOpcode::kReturn: {
281 : int input_count = node->InputCount();
282 48 : DefaultLowering(node);
283 48 : if (input_count != node->InputCount()) {
284 : int new_return_count = GetReturnCountAfterLowering(signature());
285 32 : if (static_cast<int>(signature()->return_count()) != new_return_count) {
286 32 : NodeProperties::ChangeOp(node, common()->Return(new_return_count));
287 : }
288 : }
289 : break;
290 : }
291 : case IrOpcode::kCall: {
292 : // TODO(turbofan): Make WASM code const-correct wrt. CallDescriptor.
293 3 : CallDescriptor* descriptor =
294 2 : const_cast<CallDescriptor*>(CallDescriptorOf(node->op()));
295 5 : if (DefaultLowering(node) ||
296 1 : (descriptor->ReturnCount() == 1 &&
297 : descriptor->GetReturnType(0) == MachineType::Int64())) {
298 : // We have to adjust the call descriptor.
299 : const Operator* op = common()->Call(
300 4 : wasm::ModuleEnv::GetI32WasmCallDescriptor(zone(), descriptor));
301 2 : NodeProperties::ChangeOp(node, op);
302 : }
303 4 : if (descriptor->ReturnCount() == 1 &&
304 : descriptor->GetReturnType(0) == MachineType::Int64()) {
305 : // We access the additional return values through projections.
306 : Node* low_node =
307 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
308 : Node* high_node =
309 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
310 : ReplaceNode(node, low_node, high_node);
311 : }
312 : break;
313 : }
314 : case IrOpcode::kWord64And: {
315 : DCHECK(node->InputCount() == 2);
316 : Node* left = node->InputAt(0);
317 : Node* right = node->InputAt(1);
318 :
319 : Node* low_node =
320 : graph()->NewNode(machine()->Word32And(), GetReplacementLow(left),
321 3 : GetReplacementLow(right));
322 : Node* high_node =
323 : graph()->NewNode(machine()->Word32And(), GetReplacementHigh(left),
324 3 : GetReplacementHigh(right));
325 : ReplaceNode(node, low_node, high_node);
326 : break;
327 : }
328 : case IrOpcode::kTruncateInt64ToInt32: {
329 : DCHECK(node->InputCount() == 1);
330 : Node* input = node->InputAt(0);
331 : ReplaceNode(node, GetReplacementLow(input), nullptr);
332 4 : node->NullAllInputs();
333 4 : break;
334 : }
335 : case IrOpcode::kInt64Add: {
336 : DCHECK(node->InputCount() == 2);
337 :
338 : Node* right = node->InputAt(1);
339 1 : node->ReplaceInput(1, GetReplacementLow(right));
340 1 : node->AppendInput(zone(), GetReplacementHigh(right));
341 :
342 : Node* left = node->InputAt(0);
343 1 : node->ReplaceInput(0, GetReplacementLow(left));
344 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
345 :
346 1 : NodeProperties::ChangeOp(node, machine()->Int32PairAdd());
347 : // We access the additional return values through projections.
348 : Node* low_node =
349 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
350 : Node* high_node =
351 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
352 : ReplaceNode(node, low_node, high_node);
353 : break;
354 : }
355 : case IrOpcode::kInt64Sub: {
356 : DCHECK(node->InputCount() == 2);
357 :
358 : Node* right = node->InputAt(1);
359 1 : node->ReplaceInput(1, GetReplacementLow(right));
360 1 : node->AppendInput(zone(), GetReplacementHigh(right));
361 :
362 : Node* left = node->InputAt(0);
363 1 : node->ReplaceInput(0, GetReplacementLow(left));
364 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
365 :
366 1 : NodeProperties::ChangeOp(node, machine()->Int32PairSub());
367 : // We access the additional return values through projections.
368 : Node* low_node =
369 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
370 : Node* high_node =
371 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
372 : ReplaceNode(node, low_node, high_node);
373 : break;
374 : }
375 : case IrOpcode::kInt64Mul: {
376 : DCHECK(node->InputCount() == 2);
377 :
378 : Node* right = node->InputAt(1);
379 1 : node->ReplaceInput(1, GetReplacementLow(right));
380 1 : node->AppendInput(zone(), GetReplacementHigh(right));
381 :
382 : Node* left = node->InputAt(0);
383 1 : node->ReplaceInput(0, GetReplacementLow(left));
384 1 : node->InsertInput(zone(), 1, GetReplacementHigh(left));
385 :
386 1 : NodeProperties::ChangeOp(node, machine()->Int32PairMul());
387 : // We access the additional return values through projections.
388 : Node* low_node =
389 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
390 : Node* high_node =
391 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
392 : ReplaceNode(node, low_node, high_node);
393 : break;
394 : }
395 : case IrOpcode::kWord64Or: {
396 : DCHECK(node->InputCount() == 2);
397 : Node* left = node->InputAt(0);
398 : Node* right = node->InputAt(1);
399 :
400 : Node* low_node =
401 : graph()->NewNode(machine()->Word32Or(), GetReplacementLow(left),
402 1 : GetReplacementLow(right));
403 : Node* high_node =
404 : graph()->NewNode(machine()->Word32Or(), GetReplacementHigh(left),
405 1 : GetReplacementHigh(right));
406 : ReplaceNode(node, low_node, high_node);
407 : break;
408 : }
409 : case IrOpcode::kWord64Xor: {
410 : DCHECK(node->InputCount() == 2);
411 : Node* left = node->InputAt(0);
412 : Node* right = node->InputAt(1);
413 :
414 : Node* low_node =
415 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
416 1 : GetReplacementLow(right));
417 : Node* high_node =
418 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
419 1 : GetReplacementHigh(right));
420 : ReplaceNode(node, low_node, high_node);
421 : break;
422 : }
423 : case IrOpcode::kWord64Shl: {
424 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
425 : // of the output to 0 and just calculate the high word.
426 : DCHECK(node->InputCount() == 2);
427 : Node* shift = node->InputAt(1);
428 1 : if (HasReplacementLow(shift)) {
429 : // We do not have to care about the high word replacement, because
430 : // the shift can only be between 0 and 63 anyways.
431 1 : node->ReplaceInput(1, GetReplacementLow(shift));
432 : }
433 :
434 : Node* value = node->InputAt(0);
435 1 : node->ReplaceInput(0, GetReplacementLow(value));
436 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
437 :
438 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShl());
439 : // We access the additional return values through projections.
440 : Node* low_node =
441 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
442 : Node* high_node =
443 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
444 : ReplaceNode(node, low_node, high_node);
445 : break;
446 : }
447 : case IrOpcode::kWord64Shr: {
448 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
449 : // of the output to 0 and just calculate the high word.
450 : DCHECK(node->InputCount() == 2);
451 : Node* shift = node->InputAt(1);
452 1 : if (HasReplacementLow(shift)) {
453 : // We do not have to care about the high word replacement, because
454 : // the shift can only be between 0 and 63 anyways.
455 1 : node->ReplaceInput(1, GetReplacementLow(shift));
456 : }
457 :
458 : Node* value = node->InputAt(0);
459 1 : node->ReplaceInput(0, GetReplacementLow(value));
460 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
461 :
462 1 : NodeProperties::ChangeOp(node, machine()->Word32PairShr());
463 : // We access the additional return values through projections.
464 : Node* low_node =
465 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
466 : Node* high_node =
467 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
468 : ReplaceNode(node, low_node, high_node);
469 : break;
470 : }
471 : case IrOpcode::kWord64Sar: {
472 : // TODO(turbofan): if the shift count >= 32, then we can set the low word
473 : // of the output to 0 and just calculate the high word.
474 : DCHECK(node->InputCount() == 2);
475 : Node* shift = node->InputAt(1);
476 1 : if (HasReplacementLow(shift)) {
477 : // We do not have to care about the high word replacement, because
478 : // the shift can only be between 0 and 63 anyways.
479 1 : node->ReplaceInput(1, GetReplacementLow(shift));
480 : }
481 :
482 : Node* value = node->InputAt(0);
483 1 : node->ReplaceInput(0, GetReplacementLow(value));
484 1 : node->InsertInput(zone(), 1, GetReplacementHigh(value));
485 :
486 1 : NodeProperties::ChangeOp(node, machine()->Word32PairSar());
487 : // We access the additional return values through projections.
488 : Node* low_node =
489 2 : graph()->NewNode(common()->Projection(0), node, graph()->start());
490 : Node* high_node =
491 2 : graph()->NewNode(common()->Projection(1), node, graph()->start());
492 : ReplaceNode(node, low_node, high_node);
493 : break;
494 : }
495 : case IrOpcode::kWord64Equal: {
496 : DCHECK(node->InputCount() == 2);
497 : Node* left = node->InputAt(0);
498 : Node* right = node->InputAt(1);
499 :
500 : // TODO(wasm): Use explicit comparisons and && here?
501 : Node* replacement = graph()->NewNode(
502 : machine()->Word32Equal(),
503 : graph()->NewNode(
504 : machine()->Word32Or(),
505 : graph()->NewNode(machine()->Word32Xor(), GetReplacementLow(left),
506 : GetReplacementLow(right)),
507 : graph()->NewNode(machine()->Word32Xor(), GetReplacementHigh(left),
508 : GetReplacementHigh(right))),
509 10 : graph()->NewNode(common()->Int32Constant(0)));
510 :
511 : ReplaceNode(node, replacement, nullptr);
512 : break;
513 : }
514 : case IrOpcode::kInt64LessThan: {
515 : LowerComparison(node, machine()->Int32LessThan(),
516 2 : machine()->Uint32LessThan());
517 1 : break;
518 : }
519 : case IrOpcode::kInt64LessThanOrEqual: {
520 : LowerComparison(node, machine()->Int32LessThan(),
521 2 : machine()->Uint32LessThanOrEqual());
522 1 : break;
523 : }
524 : case IrOpcode::kUint64LessThan: {
525 : LowerComparison(node, machine()->Uint32LessThan(),
526 2 : machine()->Uint32LessThan());
527 1 : break;
528 : }
529 : case IrOpcode::kUint64LessThanOrEqual: {
530 : LowerComparison(node, machine()->Uint32LessThan(),
531 2 : machine()->Uint32LessThanOrEqual());
532 1 : break;
533 : }
534 : case IrOpcode::kChangeInt32ToInt64: {
535 : DCHECK(node->InputCount() == 1);
536 : Node* input = node->InputAt(0);
537 2 : if (HasReplacementLow(input)) {
538 : input = GetReplacementLow(input);
539 : }
540 : // We use SAR to preserve the sign in the high word.
541 : ReplaceNode(
542 : node, input,
543 : graph()->NewNode(machine()->Word32Sar(), input,
544 4 : graph()->NewNode(common()->Int32Constant(31))));
545 2 : node->NullAllInputs();
546 2 : break;
547 : }
548 : case IrOpcode::kChangeUint32ToUint64: {
549 : DCHECK(node->InputCount() == 1);
550 : Node* input = node->InputAt(0);
551 2 : if (HasReplacementLow(input)) {
552 : input = GetReplacementLow(input);
553 : }
554 2 : ReplaceNode(node, input, graph()->NewNode(common()->Int32Constant(0)));
555 2 : node->NullAllInputs();
556 2 : break;
557 : }
558 : case IrOpcode::kBitcastInt64ToFloat64: {
559 : DCHECK(node->InputCount() == 1);
560 : Node* input = node->InputAt(0);
561 : Node* stack_slot = graph()->NewNode(
562 1 : machine()->StackSlot(MachineRepresentation::kWord64));
563 :
564 : Node* store_high_word = graph()->NewNode(
565 : machine()->Store(
566 : StoreRepresentation(MachineRepresentation::kWord32,
567 : WriteBarrierKind::kNoWriteBarrier)),
568 : stack_slot,
569 : graph()->NewNode(
570 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
571 4 : GetReplacementHigh(input), graph()->start(), graph()->start());
572 :
573 : Node* store_low_word = graph()->NewNode(
574 : machine()->Store(
575 : StoreRepresentation(MachineRepresentation::kWord32,
576 : WriteBarrierKind::kNoWriteBarrier)),
577 : stack_slot,
578 : graph()->NewNode(
579 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
580 4 : GetReplacementLow(input), store_high_word, graph()->start());
581 :
582 : Node* load =
583 : graph()->NewNode(machine()->Load(MachineType::Float64()), stack_slot,
584 : graph()->NewNode(common()->Int32Constant(0)),
585 3 : store_low_word, graph()->start());
586 :
587 : ReplaceNode(node, load, nullptr);
588 : break;
589 : }
590 : case IrOpcode::kBitcastFloat64ToInt64: {
591 : DCHECK(node->InputCount() == 1);
592 : Node* input = node->InputAt(0);
593 1 : if (HasReplacementLow(input)) {
594 : input = GetReplacementLow(input);
595 : }
596 : Node* stack_slot = graph()->NewNode(
597 1 : machine()->StackSlot(MachineRepresentation::kWord64));
598 : Node* store = graph()->NewNode(
599 : machine()->Store(
600 : StoreRepresentation(MachineRepresentation::kFloat64,
601 : WriteBarrierKind::kNoWriteBarrier)),
602 : stack_slot, graph()->NewNode(common()->Int32Constant(0)), input,
603 4 : graph()->start(), graph()->start());
604 :
605 : Node* high_node = graph()->NewNode(
606 : machine()->Load(MachineType::Int32()), stack_slot,
607 : graph()->NewNode(
608 : common()->Int32Constant(kInt64UpperHalfMemoryOffset)),
609 3 : store, graph()->start());
610 :
611 : Node* low_node = graph()->NewNode(
612 : machine()->Load(MachineType::Int32()), stack_slot,
613 : graph()->NewNode(
614 : common()->Int32Constant(kInt64LowerHalfMemoryOffset)),
615 3 : store, graph()->start());
616 : ReplaceNode(node, low_node, high_node);
617 : break;
618 : }
619 : case IrOpcode::kWord64Ror: {
620 : DCHECK(node->InputCount() == 2);
621 : Node* input = node->InputAt(0);
622 : Node* shift = HasReplacementLow(node->InputAt(1))
623 : ? GetReplacementLow(node->InputAt(1))
624 5 : : node->InputAt(1);
625 : Int32Matcher m(shift);
626 5 : if (m.HasValue()) {
627 : // Precondition: 0 <= shift < 64.
628 4 : int32_t shift_value = m.Value() & 0x3f;
629 4 : if (shift_value == 0) {
630 : ReplaceNode(node, GetReplacementLow(input),
631 : GetReplacementHigh(input));
632 3 : } else if (shift_value == 32) {
633 : ReplaceNode(node, GetReplacementHigh(input),
634 : GetReplacementLow(input));
635 : } else {
636 : Node* low_input;
637 : Node* high_input;
638 2 : if (shift_value < 32) {
639 : low_input = GetReplacementLow(input);
640 : high_input = GetReplacementHigh(input);
641 : } else {
642 : low_input = GetReplacementHigh(input);
643 : high_input = GetReplacementLow(input);
644 : }
645 2 : int32_t masked_shift_value = shift_value & 0x1f;
646 : Node* masked_shift =
647 2 : graph()->NewNode(common()->Int32Constant(masked_shift_value));
648 : Node* inv_shift = graph()->NewNode(
649 4 : common()->Int32Constant(32 - masked_shift_value));
650 :
651 : Node* low_node = graph()->NewNode(
652 : machine()->Word32Or(),
653 : graph()->NewNode(machine()->Word32Shr(), low_input, masked_shift),
654 6 : graph()->NewNode(machine()->Word32Shl(), high_input, inv_shift));
655 : Node* high_node = graph()->NewNode(
656 : machine()->Word32Or(), graph()->NewNode(machine()->Word32Shr(),
657 : high_input, masked_shift),
658 6 : graph()->NewNode(machine()->Word32Shl(), low_input, inv_shift));
659 : ReplaceNode(node, low_node, high_node);
660 : }
661 : } else {
662 : Node* safe_shift = shift;
663 1 : if (!machine()->Word32ShiftIsSafe()) {
664 : safe_shift =
665 : graph()->NewNode(machine()->Word32And(), shift,
666 2 : graph()->NewNode(common()->Int32Constant(0x1f)));
667 : }
668 :
669 : // By creating this bit-mask with SAR and SHL we do not have to deal
670 : // with shift == 0 as a special case.
671 : Node* inv_mask = graph()->NewNode(
672 : machine()->Word32Shl(),
673 : graph()->NewNode(machine()->Word32Sar(),
674 : graph()->NewNode(common()->Int32Constant(
675 : std::numeric_limits<int32_t>::min())),
676 : safe_shift),
677 4 : graph()->NewNode(common()->Int32Constant(1)));
678 :
679 : Node* bit_mask =
680 : graph()->NewNode(machine()->Word32Xor(), inv_mask,
681 2 : graph()->NewNode(common()->Int32Constant(-1)));
682 :
683 : // We have to mask the shift value for this comparison. If
684 : // !machine()->Word32ShiftIsSafe() then the masking should already be
685 : // part of the graph.
686 : Node* masked_shift6 = shift;
687 1 : if (machine()->Word32ShiftIsSafe()) {
688 : masked_shift6 =
689 : graph()->NewNode(machine()->Word32And(), shift,
690 0 : graph()->NewNode(common()->Int32Constant(0x3f)));
691 : }
692 :
693 : Diamond lt32(
694 : graph(), common(),
695 : graph()->NewNode(machine()->Int32LessThan(), masked_shift6,
696 3 : graph()->NewNode(common()->Int32Constant(32))));
697 :
698 : // The low word and the high word can be swapped either at the input or
699 : // at the output. We swap the inputs so that shift does not have to be
700 : // kept for so long in a register.
701 : Node* input_low =
702 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementLow(input),
703 1 : GetReplacementHigh(input));
704 : Node* input_high =
705 : lt32.Phi(MachineRepresentation::kWord32, GetReplacementHigh(input),
706 1 : GetReplacementLow(input));
707 :
708 : Node* rotate_low =
709 1 : graph()->NewNode(machine()->Word32Ror(), input_low, safe_shift);
710 : Node* rotate_high =
711 1 : graph()->NewNode(machine()->Word32Ror(), input_high, safe_shift);
712 :
713 : Node* low_node = graph()->NewNode(
714 : machine()->Word32Or(),
715 : graph()->NewNode(machine()->Word32And(), rotate_low, bit_mask),
716 3 : graph()->NewNode(machine()->Word32And(), rotate_high, inv_mask));
717 :
718 : Node* high_node = graph()->NewNode(
719 : machine()->Word32Or(),
720 : graph()->NewNode(machine()->Word32And(), rotate_high, bit_mask),
721 3 : graph()->NewNode(machine()->Word32And(), rotate_low, inv_mask));
722 :
723 : ReplaceNode(node, low_node, high_node);
724 : }
725 : break;
726 : }
727 : case IrOpcode::kWord64Clz: {
728 : DCHECK(node->InputCount() == 1);
729 : Node* input = node->InputAt(0);
730 : Diamond d(
731 : graph(), common(),
732 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(input),
733 3 : graph()->NewNode(common()->Int32Constant(0))));
734 :
735 : Node* low_node = d.Phi(
736 : MachineRepresentation::kWord32,
737 : graph()->NewNode(machine()->Int32Add(),
738 : graph()->NewNode(machine()->Word32Clz(),
739 : GetReplacementLow(input)),
740 : graph()->NewNode(common()->Int32Constant(32))),
741 5 : graph()->NewNode(machine()->Word32Clz(), GetReplacementHigh(input)));
742 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
743 : break;
744 : }
745 : case IrOpcode::kWord64Ctz: {
746 : DCHECK(node->InputCount() == 1);
747 : DCHECK(machine()->Word32Ctz().IsSupported());
748 : Node* input = node->InputAt(0);
749 : Diamond d(
750 : graph(), common(),
751 : graph()->NewNode(machine()->Word32Equal(), GetReplacementLow(input),
752 3 : graph()->NewNode(common()->Int32Constant(0))));
753 : Node* low_node =
754 : d.Phi(MachineRepresentation::kWord32,
755 : graph()->NewNode(machine()->Int32Add(),
756 : graph()->NewNode(machine()->Word32Ctz().op(),
757 : GetReplacementHigh(input)),
758 : graph()->NewNode(common()->Int32Constant(32))),
759 : graph()->NewNode(machine()->Word32Ctz().op(),
760 5 : GetReplacementLow(input)));
761 1 : ReplaceNode(node, low_node, graph()->NewNode(common()->Int32Constant(0)));
762 : break;
763 : }
764 : case IrOpcode::kWord64Popcnt: {
765 : DCHECK(node->InputCount() == 1);
766 : Node* input = node->InputAt(0);
767 : // We assume that a Word64Popcnt node only has been created if
768 : // Word32Popcnt is actually supported.
769 : DCHECK(machine()->Word32Popcnt().IsSupported());
770 : ReplaceNode(node, graph()->NewNode(
771 : machine()->Int32Add(),
772 : graph()->NewNode(machine()->Word32Popcnt().op(),
773 : GetReplacementLow(input)),
774 : graph()->NewNode(machine()->Word32Popcnt().op(),
775 : GetReplacementHigh(input))),
776 5 : graph()->NewNode(common()->Int32Constant(0)));
777 1 : break;
778 : }
779 : case IrOpcode::kPhi: {
780 4 : MachineRepresentation rep = PhiRepresentationOf(node->op());
781 4 : if (rep == MachineRepresentation::kWord64) {
782 : // The replacement nodes have already been created, we only have to
783 : // replace placeholder nodes.
784 : Node* low_node = GetReplacementLow(node);
785 : Node* high_node = GetReplacementHigh(node);
786 9 : for (int i = 0; i < node->op()->ValueInputCount(); i++) {
787 2 : low_node->ReplaceInput(i, GetReplacementLow(node->InputAt(i)));
788 2 : high_node->ReplaceInput(i, GetReplacementHigh(node->InputAt(i)));
789 : }
790 : } else {
791 3 : DefaultLowering(node);
792 : }
793 : break;
794 : }
795 : case IrOpcode::kProjection: {
796 0 : Node* call = node->InputAt(0);
797 : DCHECK_EQ(IrOpcode::kCall, call->opcode());
798 0 : CallDescriptor* descriptor =
799 0 : const_cast<CallDescriptor*>(CallDescriptorOf(call->op()));
800 0 : for (size_t i = 0; i < descriptor->ReturnCount(); i++) {
801 0 : if (descriptor->GetReturnType(i) == MachineType::Int64()) {
802 0 : UNREACHABLE(); // TODO(titzer): implement multiple i64 returns.
803 : }
804 : }
805 : break;
806 : }
807 : case IrOpcode::kWord64ReverseBytes: {
808 : Node* input = node->InputAt(0);
809 : ReplaceNode(node, graph()->NewNode(machine()->Word32ReverseBytes().op(),
810 : GetReplacementHigh(input)),
811 : graph()->NewNode(machine()->Word32ReverseBytes().op(),
812 3 : GetReplacementLow(input)));
813 1 : break;
814 : }
815 :
816 129 : default: { DefaultLowering(node); }
817 : }
818 337 : } // NOLINT(readability/fn_size)
819 :
820 4 : void Int64Lowering::LowerComparison(Node* node, const Operator* high_word_op,
821 32 : const Operator* low_word_op) {
822 : DCHECK(node->InputCount() == 2);
823 : Node* left = node->InputAt(0);
824 : Node* right = node->InputAt(1);
825 : Node* replacement = graph()->NewNode(
826 : machine()->Word32Or(),
827 : graph()->NewNode(high_word_op, GetReplacementHigh(left),
828 : GetReplacementHigh(right)),
829 : graph()->NewNode(
830 : machine()->Word32And(),
831 : graph()->NewNode(machine()->Word32Equal(), GetReplacementHigh(left),
832 : GetReplacementHigh(right)),
833 : graph()->NewNode(low_word_op, GetReplacementLow(left),
834 12 : GetReplacementLow(right))));
835 :
836 : ReplaceNode(node, replacement, nullptr);
837 4 : }
838 :
839 217 : bool Int64Lowering::DefaultLowering(Node* node, bool low_word_only) {
840 : bool something_changed = false;
841 294 : for (int i = NodeProperties::PastValueIndex(node) - 1; i >= 0; i--) {
842 : Node* input = node->InputAt(i);
843 111 : if (HasReplacementLow(input)) {
844 : something_changed = true;
845 45 : node->ReplaceInput(i, GetReplacementLow(input));
846 : }
847 219 : if (!low_word_only && HasReplacementHigh(input)) {
848 : something_changed = true;
849 68 : node->InsertInput(zone(), i + 1, GetReplacementHigh(input));
850 : }
851 : }
852 183 : return something_changed;
853 : }
854 :
855 101 : void Int64Lowering::ReplaceNode(Node* old, Node* new_low, Node* new_high) {
856 : // if new_low == nullptr, then also new_high == nullptr.
857 : DCHECK(new_low != nullptr || new_high == nullptr);
858 208 : replacements_[old->id()].low = new_low;
859 107 : replacements_[old->id()].high = new_high;
860 0 : }
861 :
862 131 : bool Int64Lowering::HasReplacementLow(Node* node) {
863 262 : return replacements_[node->id()].low != nullptr;
864 : }
865 :
866 51 : Node* Int64Lowering::GetReplacementLow(Node* node) {
867 91 : Node* result = replacements_[node->id()].low;
868 : DCHECK(result);
869 0 : return result;
870 : }
871 :
872 108 : bool Int64Lowering::HasReplacementHigh(Node* node) {
873 216 : return replacements_[node->id()].high != nullptr;
874 : }
875 :
876 87 : Node* Int64Lowering::GetReplacementHigh(Node* node) {
877 161 : Node* result = replacements_[node->id()].high;
878 : DCHECK(result);
879 0 : return result;
880 : }
881 :
882 11 : void Int64Lowering::PreparePhiReplacement(Node* phi) {
883 4 : MachineRepresentation rep = PhiRepresentationOf(phi->op());
884 4 : if (rep == MachineRepresentation::kWord64) {
885 : // We have to create the replacements for a phi node before we actually
886 : // lower the phi to break potential cycles in the graph. The replacements of
887 : // input nodes do not exist yet, so we use a placeholder node to pass the
888 : // graph verifier.
889 1 : int value_count = phi->op()->ValueInputCount();
890 1 : Node** inputs_low = zone()->NewArray<Node*>(value_count + 1);
891 : Node** inputs_high = zone()->NewArray<Node*>(value_count + 1);
892 3 : for (int i = 0; i < value_count; i++) {
893 2 : inputs_low[i] = placeholder_;
894 2 : inputs_high[i] = placeholder_;
895 : }
896 1 : inputs_low[value_count] = NodeProperties::GetControlInput(phi, 0);
897 1 : inputs_high[value_count] = NodeProperties::GetControlInput(phi, 0);
898 : ReplaceNode(phi,
899 : graph()->NewNode(
900 : common()->Phi(MachineRepresentation::kWord32, value_count),
901 : value_count + 1, inputs_low, false),
902 : graph()->NewNode(
903 : common()->Phi(MachineRepresentation::kWord32, value_count),
904 4 : value_count + 1, inputs_high, false));
905 : }
906 4 : }
907 : } // namespace compiler
908 : } // namespace internal
909 : } // namespace v8
|
