Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/effect-control-linearizer.h"
6 :
7 : #include "src/code-factory.h"
8 : #include "src/compiler/access-builder.h"
9 : #include "src/compiler/compiler-source-position-table.h"
10 : #include "src/compiler/js-graph.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-origin-table.h"
14 : #include "src/compiler/node-properties.h"
15 : #include "src/compiler/node.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/heap/factory-inl.h"
18 : #include "src/machine-type.h"
19 : #include "src/objects/heap-number.h"
20 : #include "src/objects/oddball.h"
21 : #include "src/objects/ordered-hash-table.h"
22 :
23 : #include "src/ptr-compr-inl.h"
24 :
25 : namespace v8 {
26 : namespace internal {
27 : namespace compiler {
28 :
29 : namespace {
30 : bool UsingCompressedPointers() { return false; }
31 :
32 : } // namespace
33 :
34 463662 : EffectControlLinearizer::EffectControlLinearizer(
35 : JSGraph* js_graph, Schedule* schedule, Zone* temp_zone,
36 : SourcePositionTable* source_positions, NodeOriginTable* node_origins,
37 : MaskArrayIndexEnable mask_array_index,
38 : std::vector<Handle<Map>>* embedded_maps)
39 : : js_graph_(js_graph),
40 : schedule_(schedule),
41 : temp_zone_(temp_zone),
42 : mask_array_index_(mask_array_index),
43 : source_positions_(source_positions),
44 : node_origins_(node_origins),
45 : graph_assembler_(js_graph, nullptr, nullptr, temp_zone),
46 : frame_state_zapper_(nullptr),
47 463662 : embedded_maps_(embedded_maps) {}
48 :
49 651836 : Graph* EffectControlLinearizer::graph() const { return js_graph_->graph(); }
50 0 : CommonOperatorBuilder* EffectControlLinearizer::common() const {
51 1098571 : return js_graph_->common();
52 : }
53 0 : SimplifiedOperatorBuilder* EffectControlLinearizer::simplified() const {
54 0 : return js_graph_->simplified();
55 : }
56 0 : MachineOperatorBuilder* EffectControlLinearizer::machine() const {
57 328254 : return js_graph_->machine();
58 : }
59 :
60 : namespace {
61 :
62 : struct BlockEffectControlData {
63 : Node* current_effect = nullptr; // New effect.
64 : Node* current_control = nullptr; // New control.
65 : Node* current_frame_state = nullptr; // New frame state.
66 : };
67 :
68 463660 : class BlockEffectControlMap {
69 : public:
70 : explicit BlockEffectControlMap(Zone* temp_zone) : map_(temp_zone) {}
71 :
72 : BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) {
73 19425691 : return map_[std::make_pair(from->rpo_number(), to->rpo_number())];
74 : }
75 :
76 : const BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) const {
77 : return map_.at(std::make_pair(from->rpo_number(), to->rpo_number()));
78 : }
79 :
80 : private:
81 : using Key = std::pair<int32_t, int32_t>;
82 : using Map = ZoneMap<Key, BlockEffectControlData>;
83 :
84 : Map map_;
85 : };
86 :
87 : // Effect phis that need to be updated after the first pass.
88 : struct PendingEffectPhi {
89 : Node* effect_phi;
90 : BasicBlock* block;
91 :
92 : PendingEffectPhi(Node* effect_phi, BasicBlock* block)
93 39830 : : effect_phi(effect_phi), block(block) {}
94 : };
95 :
96 1102 : void ConnectUnreachableToEnd(Node* effect, Node* control, JSGraph* jsgraph) {
97 : Graph* graph = jsgraph->graph();
98 : CommonOperatorBuilder* common = jsgraph->common();
99 1102 : if (effect->opcode() == IrOpcode::kDead) return;
100 1102 : if (effect->opcode() != IrOpcode::kUnreachable) {
101 0 : effect = graph->NewNode(common->Unreachable(), effect, control);
102 : }
103 1102 : Node* throw_node = graph->NewNode(common->Throw(), effect, control);
104 1102 : NodeProperties::MergeControlToEnd(graph, common, throw_node);
105 : }
106 :
107 796457 : void UpdateEffectPhi(Node* node, BasicBlock* block,
108 : BlockEffectControlMap* block_effects, JSGraph* jsgraph) {
109 : // Update all inputs to an effect phi with the effects from the given
110 : // block->effect map.
111 : DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
112 : DCHECK_EQ(static_cast<size_t>(node->op()->EffectInputCount()),
113 : block->PredecessorCount());
114 4395005 : for (int i = 0; i < node->op()->EffectInputCount(); i++) {
115 : Node* input = node->InputAt(i);
116 1799273 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
117 : const BlockEffectControlData& block_effect =
118 : block_effects->For(predecessor, block);
119 1799273 : Node* effect = block_effect.current_effect;
120 1799273 : if (input != effect) {
121 133230 : node->ReplaceInput(i, effect);
122 : }
123 : }
124 796458 : }
125 :
126 4387998 : void UpdateBlockControl(BasicBlock* block,
127 : BlockEffectControlMap* block_effects) {
128 : Node* control = block->NodeAt(0);
129 : DCHECK(NodeProperties::IsControl(control));
130 :
131 : // Do not rewire the end node.
132 4387998 : if (control->opcode() == IrOpcode::kEnd) return;
133 :
134 : // Update all inputs to the given control node with the correct control.
135 : DCHECK(control->opcode() == IrOpcode::kMerge ||
136 : static_cast<size_t>(control->op()->ControlInputCount()) ==
137 : block->PredecessorCount());
138 7850674 : if (static_cast<size_t>(control->op()->ControlInputCount()) !=
139 : block->PredecessorCount()) {
140 : return; // We already re-wired the control inputs of this node.
141 : }
142 12399087 : for (int i = 0; i < control->op()->ControlInputCount(); i++) {
143 4335165 : Node* input = NodeProperties::GetControlInput(control, i);
144 4335110 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
145 : const BlockEffectControlData& block_effect =
146 : block_effects->For(predecessor, block);
147 4335112 : if (input != block_effect.current_control) {
148 : NodeProperties::ReplaceControlInput(control, block_effect.current_control,
149 36943 : i);
150 : }
151 : }
152 : }
153 :
154 : bool HasIncomingBackEdges(BasicBlock* block) {
155 12097219 : for (BasicBlock* pred : block->predecessors()) {
156 7003518 : if (pred->rpo_number() >= block->rpo_number()) {
157 : return true;
158 : }
159 : }
160 : return false;
161 : }
162 :
163 291143 : void RemoveRenameNode(Node* node) {
164 : DCHECK(IrOpcode::kFinishRegion == node->opcode() ||
165 : IrOpcode::kBeginRegion == node->opcode() ||
166 : IrOpcode::kTypeGuard == node->opcode());
167 : // Update the value/context uses to the value input of the finish node and
168 : // the effect uses to the effect input.
169 3355833 : for (Edge edge : node->use_edges()) {
170 : DCHECK(!edge.from()->IsDead());
171 1532347 : if (NodeProperties::IsEffectEdge(edge)) {
172 292589 : edge.UpdateTo(NodeProperties::GetEffectInput(node));
173 : } else {
174 : DCHECK(!NodeProperties::IsControlEdge(edge));
175 : DCHECK(!NodeProperties::IsFrameStateEdge(edge));
176 1239757 : edge.UpdateTo(node->InputAt(0));
177 : }
178 : }
179 291139 : node->Kill();
180 291138 : }
181 :
182 1087355 : void TryCloneBranch(Node* node, BasicBlock* block, Zone* temp_zone,
183 : Graph* graph, CommonOperatorBuilder* common,
184 : BlockEffectControlMap* block_effects,
185 : SourcePositionTable* source_positions,
186 : NodeOriginTable* node_origins) {
187 : DCHECK_EQ(IrOpcode::kBranch, node->opcode());
188 :
189 : // This optimization is a special case of (super)block cloning. It takes an
190 : // input graph as shown below and clones the Branch node for every predecessor
191 : // to the Merge, essentially removing the Merge completely. This avoids
192 : // materializing the bit for the Phi and may offer potential for further
193 : // branch folding optimizations (i.e. because one or more inputs to the Phi is
194 : // a constant). Note that there may be more Phi nodes hanging off the Merge,
195 : // but we can only a certain subset of them currently (actually only Phi and
196 : // EffectPhi nodes whose uses have either the IfTrue or IfFalse as control
197 : // input).
198 :
199 : // Control1 ... ControlN
200 : // ^ ^
201 : // | | Cond1 ... CondN
202 : // +----+ +----+ ^ ^
203 : // | | | |
204 : // | | +----+ |
205 : // Merge<--+ | +------------+
206 : // ^ \|/
207 : // | Phi
208 : // | |
209 : // Branch----+
210 : // ^
211 : // |
212 : // +-----+-----+
213 : // | |
214 : // IfTrue IfFalse
215 : // ^ ^
216 : // | |
217 :
218 : // The resulting graph (modulo the Phi and EffectPhi nodes) looks like this:
219 :
220 : // Control1 Cond1 ... ControlN CondN
221 : // ^ ^ ^ ^
222 : // \ / \ /
223 : // Branch ... Branch
224 : // ^ ^
225 : // | |
226 : // +---+---+ +---+----+
227 : // | | | |
228 : // IfTrue IfFalse ... IfTrue IfFalse
229 : // ^ ^ ^ ^
230 : // | | | |
231 : // +--+ +-------------+ |
232 : // | | +--------------+ +--+
233 : // | | | |
234 : // Merge Merge
235 : // ^ ^
236 : // | |
237 :
238 : SourcePositionTable::Scope scope(source_positions,
239 1087355 : source_positions->GetSourcePosition(node));
240 : NodeOriginTable::Scope origin_scope(node_origins, "clone branch", node);
241 : Node* branch = node;
242 1087353 : Node* cond = NodeProperties::GetValueInput(branch, 0);
243 2150235 : if (!cond->OwnedBy(branch) || cond->opcode() != IrOpcode::kPhi) return;
244 110591 : Node* merge = NodeProperties::GetControlInput(branch);
245 218778 : if (merge->opcode() != IrOpcode::kMerge ||
246 108187 : NodeProperties::GetControlInput(cond) != merge) {
247 : return;
248 : }
249 : // Grab the IfTrue/IfFalse projections of the Branch.
250 108114 : BranchMatcher matcher(branch);
251 : // Check/collect other Phi/EffectPhi nodes hanging off the Merge.
252 : NodeVector phis(temp_zone);
253 712824 : for (Node* const use : merge->uses()) {
254 307293 : if (use == branch || use == cond) continue;
255 : // We cannot currently deal with non-Phi/EffectPhi nodes hanging off the
256 : // Merge. Ideally, we would just clone the nodes (and everything that
257 : // depends on it to some distant join point), but that requires knowledge
258 : // about dominance/post-dominance.
259 115705 : if (!NodeProperties::IsPhi(use)) return;
260 107488 : for (Edge edge : use->use_edges()) {
261 : // Right now we can only handle Phi/EffectPhi nodes whose uses are
262 : // directly control-dependend on either the IfTrue or the IfFalse
263 : // successor, because we know exactly how to update those uses.
264 11535 : if (edge.from()->op()->ControlInputCount() != 1) return;
265 10447 : Node* control = NodeProperties::GetControlInput(edge.from());
266 10447 : if (NodeProperties::IsPhi(edge.from())) {
267 734 : control = NodeProperties::GetControlInput(control, edge.index());
268 : }
269 10447 : if (control != matcher.IfTrue() && control != matcher.IfFalse()) return;
270 : }
271 95953 : phis.push_back(use);
272 : }
273 98238 : BranchHint const hint = BranchHintOf(branch->op());
274 : int const input_count = merge->op()->ControlInputCount();
275 : DCHECK_LE(1, input_count);
276 98238 : Node** const inputs = graph->zone()->NewArray<Node*>(2 * input_count);
277 : Node** const merge_true_inputs = &inputs[0];
278 98238 : Node** const merge_false_inputs = &inputs[input_count];
279 940192 : for (int index = 0; index < input_count; ++index) {
280 420977 : Node* cond1 = NodeProperties::GetValueInput(cond, index);
281 420977 : Node* control1 = NodeProperties::GetControlInput(merge, index);
282 420977 : Node* branch1 = graph->NewNode(common->Branch(hint), cond1, control1);
283 841954 : merge_true_inputs[index] = graph->NewNode(common->IfTrue(), branch1);
284 841954 : merge_false_inputs[index] = graph->NewNode(common->IfFalse(), branch1);
285 : }
286 : Node* const merge_true = matcher.IfTrue();
287 : Node* const merge_false = matcher.IfFalse();
288 98238 : merge_true->TrimInputCount(0);
289 98238 : merge_false->TrimInputCount(0);
290 940192 : for (int i = 0; i < input_count; ++i) {
291 420977 : merge_true->AppendInput(graph->zone(), merge_true_inputs[i]);
292 420977 : merge_false->AppendInput(graph->zone(), merge_false_inputs[i]);
293 : }
294 : DCHECK_EQ(2u, block->SuccessorCount());
295 98238 : NodeProperties::ChangeOp(matcher.IfTrue(), common->Merge(input_count));
296 98238 : NodeProperties::ChangeOp(matcher.IfFalse(), common->Merge(input_count));
297 : int const true_index =
298 98238 : block->SuccessorAt(0)->NodeAt(0) == matcher.IfTrue() ? 0 : 1;
299 : BlockEffectControlData* true_block_data =
300 98238 : &block_effects->For(block, block->SuccessorAt(true_index));
301 : BlockEffectControlData* false_block_data =
302 98238 : &block_effects->For(block, block->SuccessorAt(true_index ^ 1));
303 194191 : for (Node* const phi : phis) {
304 928129 : for (int index = 0; index < input_count; ++index) {
305 832176 : inputs[index] = phi->InputAt(index);
306 : }
307 95953 : inputs[input_count] = merge_true;
308 95953 : Node* phi_true = graph->NewNode(phi->op(), input_count + 1, inputs);
309 95953 : inputs[input_count] = merge_false;
310 95953 : Node* phi_false = graph->NewNode(phi->op(), input_count + 1, inputs);
311 95953 : if (phi->UseCount() == 0) {
312 : DCHECK_EQ(phi->opcode(), IrOpcode::kEffectPhi);
313 : } else {
314 22550 : for (Edge edge : phi->use_edges()) {
315 7806 : Node* control = NodeProperties::GetControlInput(edge.from());
316 7806 : if (NodeProperties::IsPhi(edge.from())) {
317 729 : control = NodeProperties::GetControlInput(control, edge.index());
318 : }
319 : DCHECK(control == matcher.IfTrue() || control == matcher.IfFalse());
320 7806 : edge.UpdateTo((control == matcher.IfTrue()) ? phi_true : phi_false);
321 : }
322 : }
323 95953 : if (phi->opcode() == IrOpcode::kEffectPhi) {
324 95953 : true_block_data->current_effect = phi_true;
325 95953 : false_block_data->current_effect = phi_false;
326 : }
327 95953 : phi->Kill();
328 : }
329 : // Fix up IfTrue and IfFalse and kill all dead nodes.
330 98238 : if (branch == block->control_input()) {
331 98238 : true_block_data->current_control = merge_true;
332 98238 : false_block_data->current_control = merge_false;
333 : }
334 98238 : branch->Kill();
335 98238 : cond->Kill();
336 98238 : merge->Kill();
337 : }
338 :
339 : } // namespace
340 :
341 463665 : void EffectControlLinearizer::Run() {
342 : BlockEffectControlMap block_effects(temp_zone());
343 : ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone());
344 : ZoneVector<BasicBlock*> pending_block_controls(temp_zone());
345 : NodeVector inputs_buffer(temp_zone());
346 :
347 4851663 : for (BasicBlock* block : *(schedule()->rpo_order())) {
348 : size_t instr = 0;
349 :
350 : // The control node should be the first.
351 4388002 : Node* control = block->NodeAt(instr);
352 : DCHECK(NodeProperties::IsControl(control));
353 : // Update the control inputs.
354 4388002 : if (HasIncomingBackEdges(block)) {
355 : // If there are back edges, we need to update later because we have not
356 : // computed the control yet. This should only happen for loops.
357 : DCHECK_EQ(IrOpcode::kLoop, control->opcode());
358 39830 : pending_block_controls.push_back(block);
359 : } else {
360 : // If there are no back edges, we can update now.
361 4348172 : UpdateBlockControl(block, &block_effects);
362 : }
363 : instr++;
364 :
365 : // Iterate over the phis and update the effect phis.
366 : Node* effect_phi = nullptr;
367 : Node* terminate = nullptr;
368 12192907 : for (; instr < block->NodeCount(); instr++) {
369 : Node* node = block->NodeAt(instr);
370 : // Only go through the phis and effect phis.
371 3853815 : if (node->opcode() == IrOpcode::kEffectPhi) {
372 : // There should be at most one effect phi in a block.
373 : DCHECK_NULL(effect_phi);
374 : // IfException blocks should not have effect phis.
375 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
376 : effect_phi = node;
377 3068469 : } else if (node->opcode() == IrOpcode::kPhi) {
378 : // Just skip phis.
379 2754637 : } else if (node->opcode() == IrOpcode::kTerminate) {
380 : DCHECK_NULL(terminate);
381 : terminate = node;
382 : } else {
383 : break;
384 : }
385 : }
386 :
387 4387934 : if (effect_phi) {
388 : // Make sure we update the inputs to the incoming blocks' effects.
389 785353 : if (HasIncomingBackEdges(block)) {
390 : // In case of loops, we do not update the effect phi immediately
391 : // because the back predecessor has not been handled yet. We just
392 : // record the effect phi for later processing.
393 39823 : pending_effect_phis.push_back(PendingEffectPhi(effect_phi, block));
394 : } else {
395 745530 : UpdateEffectPhi(effect_phi, block, &block_effects, jsgraph());
396 : }
397 : }
398 :
399 4388052 : Node* effect = effect_phi;
400 4388052 : if (effect == nullptr) {
401 : // There was no effect phi.
402 3602623 : if (block == schedule()->start()) {
403 : // Start block => effect is start.
404 : DCHECK_EQ(graph()->start(), control);
405 463662 : effect = graph()->start();
406 6277922 : } else if (control->opcode() == IrOpcode::kEnd) {
407 : // End block is just a dummy, no effect needed.
408 : DCHECK_EQ(BasicBlock::kNone, block->control());
409 : DCHECK_EQ(1u, block->size());
410 462662 : effect = nullptr;
411 : } else {
412 : // If all the predecessors have the same effect, we can use it as our
413 : // current effect.
414 13583779 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
415 : const BlockEffectControlData& data =
416 : block_effects.For(block->PredecessorAt(i), block);
417 2754833 : if (!effect) effect = data.current_effect;
418 2754833 : if (data.current_effect != effect) {
419 11106 : effect = nullptr;
420 11106 : break;
421 : }
422 : }
423 2676316 : if (effect == nullptr) {
424 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
425 : // The input blocks do not have the same effect. We have
426 : // to create an effect phi node.
427 : inputs_buffer.clear();
428 22212 : inputs_buffer.resize(block->PredecessorCount(), jsgraph()->Dead());
429 11106 : inputs_buffer.push_back(control);
430 44424 : effect = graph()->NewNode(
431 11106 : common()->EffectPhi(static_cast<int>(block->PredecessorCount())),
432 11106 : static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front()));
433 : // For loops, we update the effect phi node later to break cycles.
434 22212 : if (control->opcode() == IrOpcode::kLoop) {
435 14 : pending_effect_phis.push_back(PendingEffectPhi(effect, block));
436 : } else {
437 11099 : UpdateEffectPhi(effect, block, &block_effects, jsgraph());
438 : }
439 5330420 : } else if (control->opcode() == IrOpcode::kIfException) {
440 : // The IfException is connected into the effect chain, so we need
441 : // to update the effect here.
442 200325 : NodeProperties::ReplaceEffectInput(control, effect);
443 200325 : effect = control;
444 : }
445 : }
446 : }
447 :
448 : // Fixup the Terminate node.
449 4388069 : if (terminate != nullptr) {
450 39829 : NodeProperties::ReplaceEffectInput(terminate, effect);
451 : }
452 :
453 : // The frame state at block entry is determined by the frame states leaving
454 : // all predecessors. In case there is no frame state dominating this block,
455 : // we can rely on a checkpoint being present before the next deoptimization.
456 : // TODO(mstarzinger): Eventually we will need to go hunt for a frame state
457 : // once deoptimizing nodes roam freely through the schedule.
458 4388069 : Node* frame_state = nullptr;
459 4388069 : if (block != schedule()->start()) {
460 : // If all the predecessors have the same effect, we can use it
461 : // as our current effect.
462 : frame_state =
463 3924311 : block_effects.For(block->PredecessorAt(0), block).current_frame_state;
464 11130922 : for (size_t i = 1; i < block->PredecessorCount(); i++) {
465 1185594 : if (block_effects.For(block->PredecessorAt(i), block)
466 1185594 : .current_frame_state != frame_state) {
467 91494 : frame_state = nullptr;
468 91494 : frame_state_zapper_ = graph()->end();
469 91494 : break;
470 : }
471 : }
472 : }
473 :
474 : // Process the ordinary instructions.
475 109933909 : for (; instr < block->NodeCount(); instr++) {
476 : Node* node = block->NodeAt(instr);
477 33719349 : ProcessNode(node, &frame_state, &effect, &control);
478 : }
479 :
480 4387973 : switch (block->control()) {
481 : case BasicBlock::kGoto:
482 : case BasicBlock::kNone:
483 : break;
484 :
485 : case BasicBlock::kCall:
486 : case BasicBlock::kTailCall:
487 : case BasicBlock::kSwitch:
488 : case BasicBlock::kReturn:
489 : case BasicBlock::kDeoptimize:
490 : case BasicBlock::kThrow:
491 905801 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
492 905801 : break;
493 :
494 : case BasicBlock::kBranch:
495 1087368 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
496 2174712 : TryCloneBranch(block->control_input(), block, temp_zone(), graph(),
497 : common(), &block_effects, source_positions_,
498 1087356 : node_origins_);
499 1087358 : break;
500 : }
501 :
502 : // Store the effect, control and frame state for later use.
503 14006060 : for (BasicBlock* successor : block->successors()) {
504 5230099 : BlockEffectControlData* data = &block_effects.For(block, successor);
505 5230134 : if (data->current_effect == nullptr) {
506 5038217 : data->current_effect = effect;
507 : }
508 5230134 : if (data->current_control == nullptr) {
509 5033650 : data->current_control = control;
510 : }
511 5230134 : data->current_frame_state = frame_state;
512 : }
513 : }
514 :
515 503491 : for (BasicBlock* pending_block_control : pending_block_controls) {
516 39830 : UpdateBlockControl(pending_block_control, &block_effects);
517 : }
518 : // Update the incoming edges of the effect phis that could not be processed
519 : // during the first pass (because they could have incoming back edges).
520 503491 : for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) {
521 39830 : UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block,
522 39830 : &block_effects, jsgraph());
523 : }
524 463660 : }
525 :
526 35712390 : void EffectControlLinearizer::ProcessNode(Node* node, Node** frame_state,
527 : Node** effect, Node** control) {
528 : SourcePositionTable::Scope scope(source_positions_,
529 35712390 : source_positions_->GetSourcePosition(node));
530 35712351 : NodeOriginTable::Scope origin_scope(node_origins_, "process node", node);
531 :
532 : // If the node needs to be wired into the effect/control chain, do this
533 : // here. Pass current frame state for lowering to eager deoptimization.
534 35712351 : if (TryWireInStateEffect(node, *frame_state, effect, control)) {
535 : return;
536 : }
537 :
538 : // If the node has a visible effect, then there must be a checkpoint in the
539 : // effect chain before we are allowed to place another eager deoptimization
540 : // point. We zap the frame state to ensure this invariant is maintained.
541 67535498 : if (region_observability_ == RegionObservability::kObservable &&
542 : !node->op()->HasProperty(Operator::kNoWrite)) {
543 3659986 : *frame_state = nullptr;
544 3659986 : frame_state_zapper_ = node;
545 : }
546 :
547 : // Remove the end markers of 'atomic' allocation region because the
548 : // region should be wired-in now.
549 34598938 : if (node->opcode() == IrOpcode::kFinishRegion) {
550 : // Reset the current region observability.
551 134316 : region_observability_ = RegionObservability::kObservable;
552 : // Update the value uses to the value input of the finish node and
553 : // the effect uses to the effect input.
554 134316 : return RemoveRenameNode(node);
555 : }
556 34464622 : if (node->opcode() == IrOpcode::kBeginRegion) {
557 : // Determine the observability for this region and use that for all
558 : // nodes inside the region (i.e. ignore the absence of kNoWrite on
559 : // StoreField and other operators).
560 : DCHECK_NE(RegionObservability::kNotObservable, region_observability_);
561 134317 : region_observability_ = RegionObservabilityOf(node->op());
562 : // Update the value uses to the value input of the finish node and
563 : // the effect uses to the effect input.
564 134317 : return RemoveRenameNode(node);
565 : }
566 34330305 : if (node->opcode() == IrOpcode::kTypeGuard) {
567 22505 : return RemoveRenameNode(node);
568 : }
569 :
570 : // Special treatment for checkpoint nodes.
571 34307800 : if (node->opcode() == IrOpcode::kCheckpoint) {
572 : // Unlink the check point; effect uses will be updated to the incoming
573 : // effect that is passed. The frame state is preserved for lowering.
574 : DCHECK_EQ(RegionObservability::kObservable, region_observability_);
575 2750917 : *frame_state = NodeProperties::GetFrameStateInput(node);
576 2750916 : return;
577 : }
578 :
579 : // The IfSuccess nodes should always start a basic block (and basic block
580 : // start nodes are not handled in the ProcessNode method).
581 : DCHECK_NE(IrOpcode::kIfSuccess, node->opcode());
582 :
583 : // If the node takes an effect, replace with the current one.
584 31556883 : if (node->op()->EffectInputCount() > 0) {
585 : DCHECK_EQ(1, node->op()->EffectInputCount());
586 8161610 : Node* input_effect = NodeProperties::GetEffectInput(node);
587 :
588 8161610 : if (input_effect != *effect) {
589 2757644 : NodeProperties::ReplaceEffectInput(node, *effect);
590 : }
591 :
592 : // If the node produces an effect, update our current effect. (However,
593 : // ignore new effect chains started with ValueEffect.)
594 8161593 : if (node->op()->EffectOutputCount() > 0) {
595 : DCHECK_EQ(1, node->op()->EffectOutputCount());
596 7463132 : *effect = node;
597 : }
598 : } else {
599 : // New effect chain is only started with a Start or ValueEffect node.
600 : DCHECK(node->op()->EffectOutputCount() == 0 ||
601 : node->opcode() == IrOpcode::kStart);
602 : }
603 :
604 : // Rewire control inputs.
605 49077056 : for (int i = 0; i < node->op()->ControlInputCount(); i++) {
606 8760113 : NodeProperties::ReplaceControlInput(node, *control, i);
607 : }
608 : // Update the current control.
609 31556848 : if (node->op()->ControlOutputCount() > 0) {
610 4766072 : *control = node;
611 : }
612 :
613 : // Break the effect chain on {Unreachable} and reconnect to the graph end.
614 : // Mark the following code for deletion by connecting to the {Dead} node.
615 31556848 : if (node->opcode() == IrOpcode::kUnreachable) {
616 1102 : ConnectUnreachableToEnd(*effect, *control, jsgraph());
617 1102 : *effect = *control = jsgraph()->Dead();
618 : }
619 : }
620 :
621 35712390 : bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
622 : Node* frame_state,
623 : Node** effect,
624 : Node** control) {
625 71424780 : gasm()->Reset(*effect, *control);
626 : Node* result = nullptr;
627 35712377 : switch (node->opcode()) {
628 : case IrOpcode::kChangeBitToTagged:
629 9644 : result = LowerChangeBitToTagged(node);
630 9644 : break;
631 : case IrOpcode::kChangeInt31ToTaggedSigned:
632 39439 : result = LowerChangeInt31ToTaggedSigned(node);
633 39439 : break;
634 : case IrOpcode::kChangeInt32ToTagged:
635 36026 : result = LowerChangeInt32ToTagged(node);
636 36026 : break;
637 : case IrOpcode::kChangeInt64ToTagged:
638 45 : result = LowerChangeInt64ToTagged(node);
639 45 : break;
640 : case IrOpcode::kChangeUint32ToTagged:
641 651 : result = LowerChangeUint32ToTagged(node);
642 651 : break;
643 : case IrOpcode::kChangeUint64ToTagged:
644 53 : result = LowerChangeUint64ToTagged(node);
645 53 : break;
646 : case IrOpcode::kChangeFloat64ToTagged:
647 53408 : result = LowerChangeFloat64ToTagged(node);
648 53408 : break;
649 : case IrOpcode::kChangeFloat64ToTaggedPointer:
650 91 : result = LowerChangeFloat64ToTaggedPointer(node);
651 91 : break;
652 : case IrOpcode::kChangeTaggedSignedToInt32:
653 51731 : result = LowerChangeTaggedSignedToInt32(node);
654 51731 : break;
655 : case IrOpcode::kChangeTaggedSignedToInt64:
656 297 : result = LowerChangeTaggedSignedToInt64(node);
657 297 : break;
658 : case IrOpcode::kChangeTaggedToBit:
659 139247 : result = LowerChangeTaggedToBit(node);
660 139247 : break;
661 : case IrOpcode::kChangeTaggedToInt32:
662 1758 : result = LowerChangeTaggedToInt32(node);
663 1758 : break;
664 : case IrOpcode::kChangeTaggedToUint32:
665 395 : result = LowerChangeTaggedToUint32(node);
666 395 : break;
667 : case IrOpcode::kChangeTaggedToInt64:
668 2 : result = LowerChangeTaggedToInt64(node);
669 2 : break;
670 : case IrOpcode::kChangeTaggedToFloat64:
671 : result = LowerChangeTaggedToFloat64(node);
672 11255 : break;
673 : case IrOpcode::kChangeTaggedToTaggedSigned:
674 14 : result = LowerChangeTaggedToTaggedSigned(node);
675 14 : break;
676 : case IrOpcode::kChangeCompressedToTaggedSigned:
677 0 : result = LowerChangeCompressedToTaggedSigned(node);
678 0 : break;
679 : case IrOpcode::kChangeTaggedToCompressedSigned:
680 0 : result = LowerChangeTaggedToCompressedSigned(node);
681 0 : break;
682 : case IrOpcode::kTruncateTaggedToBit:
683 47659 : result = LowerTruncateTaggedToBit(node);
684 47659 : break;
685 : case IrOpcode::kTruncateTaggedPointerToBit:
686 399 : result = LowerTruncateTaggedPointerToBit(node);
687 399 : break;
688 : case IrOpcode::kTruncateTaggedToFloat64:
689 728 : result = LowerTruncateTaggedToFloat64(node);
690 728 : break;
691 : case IrOpcode::kPoisonIndex:
692 1034 : result = LowerPoisonIndex(node);
693 1034 : break;
694 : case IrOpcode::kCheckMaps:
695 51530 : LowerCheckMaps(node, frame_state);
696 51530 : break;
697 : case IrOpcode::kCompareMaps:
698 7576 : result = LowerCompareMaps(node);
699 7576 : break;
700 : case IrOpcode::kCheckNumber:
701 484 : result = LowerCheckNumber(node, frame_state);
702 484 : break;
703 : case IrOpcode::kCheckReceiver:
704 813 : result = LowerCheckReceiver(node, frame_state);
705 813 : break;
706 : case IrOpcode::kCheckReceiverOrNullOrUndefined:
707 73 : result = LowerCheckReceiverOrNullOrUndefined(node, frame_state);
708 73 : break;
709 : case IrOpcode::kCheckSymbol:
710 32 : result = LowerCheckSymbol(node, frame_state);
711 32 : break;
712 : case IrOpcode::kCheckString:
713 5367 : result = LowerCheckString(node, frame_state);
714 5367 : break;
715 : case IrOpcode::kCheckInternalizedString:
716 1410 : result = LowerCheckInternalizedString(node, frame_state);
717 1410 : break;
718 : case IrOpcode::kCheckNonEmptyOneByteString:
719 1474 : result = LowerCheckNonEmptyOneByteString(node, frame_state);
720 1474 : break;
721 : case IrOpcode::kCheckNonEmptyTwoByteString:
722 201 : result = LowerCheckNonEmptyTwoByteString(node, frame_state);
723 201 : break;
724 : case IrOpcode::kCheckNonEmptyString:
725 18 : result = LowerCheckNonEmptyString(node, frame_state);
726 18 : break;
727 : case IrOpcode::kCheckIf:
728 16431 : LowerCheckIf(node, frame_state);
729 16431 : break;
730 : case IrOpcode::kCheckedInt32Add:
731 55954 : result = LowerCheckedInt32Add(node, frame_state);
732 55952 : break;
733 : case IrOpcode::kCheckedInt32Sub:
734 7688 : result = LowerCheckedInt32Sub(node, frame_state);
735 7688 : break;
736 : case IrOpcode::kCheckedInt32Div:
737 235 : result = LowerCheckedInt32Div(node, frame_state);
738 235 : break;
739 : case IrOpcode::kCheckedInt32Mod:
740 967 : result = LowerCheckedInt32Mod(node, frame_state);
741 967 : break;
742 : case IrOpcode::kCheckedUint32Div:
743 52 : result = LowerCheckedUint32Div(node, frame_state);
744 52 : break;
745 : case IrOpcode::kCheckedUint32Mod:
746 37 : result = LowerCheckedUint32Mod(node, frame_state);
747 37 : break;
748 : case IrOpcode::kCheckedInt32Mul:
749 5912 : result = LowerCheckedInt32Mul(node, frame_state);
750 5912 : break;
751 : case IrOpcode::kCheckedInt32ToTaggedSigned:
752 0 : result = LowerCheckedInt32ToTaggedSigned(node, frame_state);
753 0 : break;
754 : case IrOpcode::kCheckedInt64ToInt32:
755 9 : result = LowerCheckedInt64ToInt32(node, frame_state);
756 9 : break;
757 : case IrOpcode::kCheckedInt64ToTaggedSigned:
758 151 : result = LowerCheckedInt64ToTaggedSigned(node, frame_state);
759 151 : break;
760 : case IrOpcode::kCheckedUint32Bounds:
761 52596 : result = LowerCheckedUint32Bounds(node, frame_state);
762 52596 : break;
763 : case IrOpcode::kCheckedUint32ToInt32:
764 396 : result = LowerCheckedUint32ToInt32(node, frame_state);
765 396 : break;
766 : case IrOpcode::kCheckedUint32ToTaggedSigned:
767 7 : result = LowerCheckedUint32ToTaggedSigned(node, frame_state);
768 7 : break;
769 : case IrOpcode::kCheckedUint64Bounds:
770 195 : result = LowerCheckedUint64Bounds(node, frame_state);
771 195 : break;
772 : case IrOpcode::kCheckedUint64ToInt32:
773 0 : result = LowerCheckedUint64ToInt32(node, frame_state);
774 0 : break;
775 : case IrOpcode::kCheckedUint64ToTaggedSigned:
776 0 : result = LowerCheckedUint64ToTaggedSigned(node, frame_state);
777 0 : break;
778 : case IrOpcode::kCheckedFloat64ToInt32:
779 2315 : result = LowerCheckedFloat64ToInt32(node, frame_state);
780 2315 : break;
781 : case IrOpcode::kCheckedFloat64ToInt64:
782 16 : result = LowerCheckedFloat64ToInt64(node, frame_state);
783 16 : break;
784 : case IrOpcode::kCheckedTaggedSignedToInt32:
785 24837 : if (frame_state == nullptr) {
786 0 : FATAL("No frame state (zapped by #%d: %s)", frame_state_zapper_->id(),
787 0 : frame_state_zapper_->op()->mnemonic());
788 : }
789 24837 : result = LowerCheckedTaggedSignedToInt32(node, frame_state);
790 24838 : break;
791 : case IrOpcode::kCheckedTaggedToInt32:
792 2299 : result = LowerCheckedTaggedToInt32(node, frame_state);
793 2299 : break;
794 : case IrOpcode::kCheckedTaggedToInt64:
795 87 : result = LowerCheckedTaggedToInt64(node, frame_state);
796 87 : break;
797 : case IrOpcode::kCheckedTaggedToFloat64:
798 51109 : result = LowerCheckedTaggedToFloat64(node, frame_state);
799 51109 : break;
800 : case IrOpcode::kCheckedTaggedToTaggedSigned:
801 18738 : result = LowerCheckedTaggedToTaggedSigned(node, frame_state);
802 18738 : break;
803 : case IrOpcode::kCheckedTaggedToTaggedPointer:
804 34577 : result = LowerCheckedTaggedToTaggedPointer(node, frame_state);
805 34577 : break;
806 : case IrOpcode::kCheckedCompressedToTaggedSigned:
807 0 : result = LowerCheckedCompressedToTaggedSigned(node, frame_state);
808 0 : break;
809 : case IrOpcode::kCheckedCompressedToTaggedPointer:
810 0 : result = LowerCheckedCompressedToTaggedPointer(node, frame_state);
811 0 : break;
812 : case IrOpcode::kCheckedTaggedToCompressedSigned:
813 0 : result = LowerCheckedTaggedToCompressedSigned(node, frame_state);
814 0 : break;
815 : case IrOpcode::kCheckedTaggedToCompressedPointer:
816 0 : result = LowerCheckedTaggedToCompressedPointer(node, frame_state);
817 0 : break;
818 : case IrOpcode::kTruncateTaggedToWord32:
819 502 : result = LowerTruncateTaggedToWord32(node);
820 502 : break;
821 : case IrOpcode::kCheckedTruncateTaggedToWord32:
822 1176 : result = LowerCheckedTruncateTaggedToWord32(node, frame_state);
823 1176 : break;
824 : case IrOpcode::kNumberToString:
825 989 : result = LowerNumberToString(node);
826 989 : break;
827 : case IrOpcode::kObjectIsArrayBufferView:
828 16 : result = LowerObjectIsArrayBufferView(node);
829 16 : break;
830 : case IrOpcode::kObjectIsBigInt:
831 25 : result = LowerObjectIsBigInt(node);
832 25 : break;
833 : case IrOpcode::kObjectIsCallable:
834 103 : result = LowerObjectIsCallable(node);
835 103 : break;
836 : case IrOpcode::kObjectIsConstructor:
837 377 : result = LowerObjectIsConstructor(node);
838 377 : break;
839 : case IrOpcode::kObjectIsDetectableCallable:
840 14084 : result = LowerObjectIsDetectableCallable(node);
841 14084 : break;
842 : case IrOpcode::kObjectIsMinusZero:
843 23 : result = LowerObjectIsMinusZero(node);
844 23 : break;
845 : case IrOpcode::kNumberIsMinusZero:
846 98 : result = LowerNumberIsMinusZero(node);
847 98 : break;
848 : case IrOpcode::kObjectIsNaN:
849 716 : result = LowerObjectIsNaN(node);
850 716 : break;
851 : case IrOpcode::kNumberIsNaN:
852 2511 : result = LowerNumberIsNaN(node);
853 2511 : break;
854 : case IrOpcode::kObjectIsNonCallable:
855 6056 : result = LowerObjectIsNonCallable(node);
856 6056 : break;
857 : case IrOpcode::kObjectIsNumber:
858 7592 : result = LowerObjectIsNumber(node);
859 7592 : break;
860 : case IrOpcode::kObjectIsReceiver:
861 19514 : result = LowerObjectIsReceiver(node);
862 19514 : break;
863 : case IrOpcode::kObjectIsSmi:
864 4132 : result = LowerObjectIsSmi(node);
865 4132 : break;
866 : case IrOpcode::kObjectIsString:
867 2237 : result = LowerObjectIsString(node);
868 2237 : break;
869 : case IrOpcode::kObjectIsSymbol:
870 23 : result = LowerObjectIsSymbol(node);
871 23 : break;
872 : case IrOpcode::kObjectIsUndetectable:
873 1426 : result = LowerObjectIsUndetectable(node);
874 1426 : break;
875 : case IrOpcode::kArgumentsFrame:
876 16900 : result = LowerArgumentsFrame(node);
877 16900 : break;
878 : case IrOpcode::kArgumentsLength:
879 16927 : result = LowerArgumentsLength(node);
880 16927 : break;
881 : case IrOpcode::kToBoolean:
882 198 : result = LowerToBoolean(node);
883 198 : break;
884 : case IrOpcode::kTypeOf:
885 21305 : result = LowerTypeOf(node);
886 21305 : break;
887 : case IrOpcode::kNewDoubleElements:
888 28 : result = LowerNewDoubleElements(node);
889 28 : break;
890 : case IrOpcode::kNewSmiOrObjectElements:
891 471 : result = LowerNewSmiOrObjectElements(node);
892 471 : break;
893 : case IrOpcode::kNewArgumentsElements:
894 16423 : result = LowerNewArgumentsElements(node);
895 16423 : break;
896 : case IrOpcode::kNewConsOneByteString:
897 3832 : result = LowerNewConsOneByteString(node);
898 3832 : break;
899 : case IrOpcode::kNewConsTwoByteString:
900 33 : result = LowerNewConsTwoByteString(node);
901 33 : break;
902 : case IrOpcode::kNewConsString:
903 1090 : result = LowerNewConsString(node);
904 1090 : break;
905 : case IrOpcode::kSameValue:
906 0 : result = LowerSameValue(node);
907 0 : break;
908 : case IrOpcode::kSameValueNumbersOnly:
909 166 : result = LowerSameValueNumbersOnly(node);
910 166 : break;
911 : case IrOpcode::kNumberSameValue:
912 77 : result = LowerNumberSameValue(node);
913 77 : break;
914 : case IrOpcode::kDeadValue:
915 1336 : result = LowerDeadValue(node);
916 1336 : break;
917 : case IrOpcode::kStringConcat:
918 21134 : result = LowerStringConcat(node);
919 21134 : break;
920 : case IrOpcode::kStringFromSingleCharCode:
921 958 : result = LowerStringFromSingleCharCode(node);
922 958 : break;
923 : case IrOpcode::kStringFromSingleCodePoint:
924 207 : result = LowerStringFromSingleCodePoint(node);
925 207 : break;
926 : case IrOpcode::kStringIndexOf:
927 266 : result = LowerStringIndexOf(node);
928 266 : break;
929 : case IrOpcode::kStringLength:
930 23488 : result = LowerStringLength(node);
931 23488 : break;
932 : case IrOpcode::kStringToNumber:
933 330 : result = LowerStringToNumber(node);
934 330 : break;
935 : case IrOpcode::kStringCharCodeAt:
936 2236 : result = LowerStringCharCodeAt(node);
937 2236 : break;
938 : case IrOpcode::kStringCodePointAt:
939 223 : result = LowerStringCodePointAt(node, UnicodeEncodingOf(node->op()));
940 223 : break;
941 : case IrOpcode::kStringToLowerCaseIntl:
942 105 : result = LowerStringToLowerCaseIntl(node);
943 105 : break;
944 : case IrOpcode::kStringToUpperCaseIntl:
945 43 : result = LowerStringToUpperCaseIntl(node);
946 43 : break;
947 : case IrOpcode::kStringSubstring:
948 1100 : result = LowerStringSubstring(node);
949 1100 : break;
950 : case IrOpcode::kStringEqual:
951 10405 : result = LowerStringEqual(node);
952 10405 : break;
953 : case IrOpcode::kStringLessThan:
954 121 : result = LowerStringLessThan(node);
955 121 : break;
956 : case IrOpcode::kStringLessThanOrEqual:
957 701 : result = LowerStringLessThanOrEqual(node);
958 701 : break;
959 : case IrOpcode::kNumberIsFloat64Hole:
960 61 : result = LowerNumberIsFloat64Hole(node);
961 61 : break;
962 : case IrOpcode::kNumberIsFinite:
963 126 : result = LowerNumberIsFinite(node);
964 126 : break;
965 : case IrOpcode::kObjectIsFiniteNumber:
966 7 : result = LowerObjectIsFiniteNumber(node);
967 7 : break;
968 : case IrOpcode::kNumberIsInteger:
969 119 : result = LowerNumberIsInteger(node);
970 119 : break;
971 : case IrOpcode::kObjectIsInteger:
972 7 : result = LowerObjectIsInteger(node);
973 7 : break;
974 : case IrOpcode::kNumberIsSafeInteger:
975 7 : result = LowerNumberIsSafeInteger(node);
976 7 : break;
977 : case IrOpcode::kObjectIsSafeInteger:
978 7 : result = LowerObjectIsSafeInteger(node);
979 7 : break;
980 : case IrOpcode::kCheckFloat64Hole:
981 474 : result = LowerCheckFloat64Hole(node, frame_state);
982 474 : break;
983 : case IrOpcode::kCheckNotTaggedHole:
984 102 : result = LowerCheckNotTaggedHole(node, frame_state);
985 102 : break;
986 : case IrOpcode::kConvertTaggedHoleToUndefined:
987 1681 : result = LowerConvertTaggedHoleToUndefined(node);
988 1681 : break;
989 : case IrOpcode::kCheckEqualsInternalizedString:
990 109 : LowerCheckEqualsInternalizedString(node, frame_state);
991 109 : break;
992 : case IrOpcode::kAllocate:
993 114421 : result = LowerAllocate(node);
994 114421 : break;
995 : case IrOpcode::kCheckEqualsSymbol:
996 18 : LowerCheckEqualsSymbol(node, frame_state);
997 18 : break;
998 : case IrOpcode::kPlainPrimitiveToNumber:
999 20 : result = LowerPlainPrimitiveToNumber(node);
1000 20 : break;
1001 : case IrOpcode::kPlainPrimitiveToWord32:
1002 9 : result = LowerPlainPrimitiveToWord32(node);
1003 9 : break;
1004 : case IrOpcode::kPlainPrimitiveToFloat64:
1005 0 : result = LowerPlainPrimitiveToFloat64(node);
1006 0 : break;
1007 : case IrOpcode::kEnsureWritableFastElements:
1008 1686 : result = LowerEnsureWritableFastElements(node);
1009 1686 : break;
1010 : case IrOpcode::kMaybeGrowFastElements:
1011 3527 : result = LowerMaybeGrowFastElements(node, frame_state);
1012 3527 : break;
1013 : case IrOpcode::kTransitionElementsKind:
1014 644 : LowerTransitionElementsKind(node);
1015 644 : break;
1016 : case IrOpcode::kLoadMessage:
1017 7226 : result = LowerLoadMessage(node);
1018 7226 : break;
1019 : case IrOpcode::kStoreMessage:
1020 29747 : LowerStoreMessage(node);
1021 29747 : break;
1022 : case IrOpcode::kLoadFieldByIndex:
1023 582 : result = LowerLoadFieldByIndex(node);
1024 582 : break;
1025 : case IrOpcode::kLoadTypedElement:
1026 4546 : result = LowerLoadTypedElement(node);
1027 4546 : break;
1028 : case IrOpcode::kLoadDataViewElement:
1029 268 : result = LowerLoadDataViewElement(node);
1030 268 : break;
1031 : case IrOpcode::kLoadStackArgument:
1032 572 : result = LowerLoadStackArgument(node);
1033 572 : break;
1034 : case IrOpcode::kStoreTypedElement:
1035 4386 : LowerStoreTypedElement(node);
1036 4386 : break;
1037 : case IrOpcode::kStoreDataViewElement:
1038 196 : LowerStoreDataViewElement(node);
1039 196 : break;
1040 : case IrOpcode::kStoreSignedSmallElement:
1041 71 : LowerStoreSignedSmallElement(node);
1042 71 : break;
1043 : case IrOpcode::kFindOrderedHashMapEntry:
1044 174 : result = LowerFindOrderedHashMapEntry(node);
1045 174 : break;
1046 : case IrOpcode::kFindOrderedHashMapEntryForInt32Key:
1047 17 : result = LowerFindOrderedHashMapEntryForInt32Key(node);
1048 17 : break;
1049 : case IrOpcode::kTransitionAndStoreNumberElement:
1050 28 : LowerTransitionAndStoreNumberElement(node);
1051 28 : break;
1052 : case IrOpcode::kTransitionAndStoreNonNumberElement:
1053 20 : LowerTransitionAndStoreNonNumberElement(node);
1054 20 : break;
1055 : case IrOpcode::kTransitionAndStoreElement:
1056 86 : LowerTransitionAndStoreElement(node);
1057 86 : break;
1058 : case IrOpcode::kRuntimeAbort:
1059 2539 : LowerRuntimeAbort(node);
1060 2539 : break;
1061 : case IrOpcode::kConvertReceiver:
1062 875 : result = LowerConvertReceiver(node);
1063 875 : break;
1064 : case IrOpcode::kFloat64RoundUp:
1065 16444 : if (!LowerFloat64RoundUp(node).To(&result)) {
1066 : return false;
1067 : }
1068 : break;
1069 : case IrOpcode::kFloat64RoundDown:
1070 53588 : if (!LowerFloat64RoundDown(node).To(&result)) {
1071 : return false;
1072 : }
1073 : break;
1074 : case IrOpcode::kFloat64RoundTruncate:
1075 13828 : if (!LowerFloat64RoundTruncate(node).To(&result)) {
1076 : return false;
1077 : }
1078 : break;
1079 : case IrOpcode::kFloat64RoundTiesEven:
1080 532 : if (!LowerFloat64RoundTiesEven(node).To(&result)) {
1081 : return false;
1082 : }
1083 : break;
1084 : case IrOpcode::kDateNow:
1085 13 : result = LowerDateNow(node);
1086 13 : break;
1087 : default:
1088 : return false;
1089 : }
1090 :
1091 2227064 : if ((result ? 1 : 0) != node->op()->ValueOutputCount()) {
1092 : FATAL(
1093 : "Effect control linearizer lowering of '%s':"
1094 : " value output count does not agree.",
1095 0 : node->op()->mnemonic());
1096 : }
1097 :
1098 1113532 : *effect = gasm()->ExtractCurrentEffect();
1099 1113533 : *control = gasm()->ExtractCurrentControl();
1100 1113533 : NodeProperties::ReplaceUses(node, result, *effect, *control);
1101 1113531 : return true;
1102 : }
1103 :
1104 : #define __ gasm()->
1105 :
1106 53409 : Node* EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node) {
1107 53409 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1108 : Node* value = node->InputAt(0);
1109 :
1110 106816 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1111 53408 : auto if_heapnumber = __ MakeDeferredLabel();
1112 53408 : auto if_int32 = __ MakeLabel();
1113 :
1114 53408 : Node* value32 = __ RoundFloat64ToInt32(value);
1115 53409 : __ GotoIf(__ Float64Equal(value, __ ChangeInt32ToFloat64(value32)),
1116 53409 : &if_int32);
1117 : __ Goto(&if_heapnumber);
1118 :
1119 : __ Bind(&if_int32);
1120 : {
1121 53408 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1122 40499 : Node* zero = __ Int32Constant(0);
1123 40499 : auto if_zero = __ MakeDeferredLabel();
1124 40499 : auto if_smi = __ MakeLabel();
1125 :
1126 40499 : __ GotoIf(__ Word32Equal(value32, zero), &if_zero);
1127 : __ Goto(&if_smi);
1128 :
1129 : __ Bind(&if_zero);
1130 : {
1131 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
1132 40499 : __ GotoIf(__ Int32LessThan(__ Float64ExtractHighWord32(value), zero),
1133 40499 : &if_heapnumber);
1134 : __ Goto(&if_smi);
1135 : }
1136 :
1137 : __ Bind(&if_smi);
1138 : }
1139 :
1140 : if (SmiValuesAre32Bits()) {
1141 53408 : Node* value_smi = ChangeInt32ToSmi(value32);
1142 : __ Goto(&done, value_smi);
1143 : } else {
1144 : DCHECK(SmiValuesAre31Bits());
1145 : Node* add = __ Int32AddWithOverflow(value32, value32);
1146 : Node* ovf = __ Projection(1, add);
1147 : __ GotoIf(ovf, &if_heapnumber);
1148 : Node* value_smi = __ Projection(0, add);
1149 : value_smi = ChangeInt32ToIntPtr(value_smi);
1150 : __ Goto(&done, value_smi);
1151 : }
1152 : }
1153 :
1154 : __ Bind(&if_heapnumber);
1155 : {
1156 53408 : Node* value_number = AllocateHeapNumberWithValue(value);
1157 : __ Goto(&done, value_number);
1158 : }
1159 :
1160 : __ Bind(&done);
1161 53409 : return done.PhiAt(0);
1162 : }
1163 :
1164 91 : Node* EffectControlLinearizer::LowerChangeFloat64ToTaggedPointer(Node* node) {
1165 : Node* value = node->InputAt(0);
1166 91 : return AllocateHeapNumberWithValue(value);
1167 : }
1168 :
1169 9644 : Node* EffectControlLinearizer::LowerChangeBitToTagged(Node* node) {
1170 : Node* value = node->InputAt(0);
1171 :
1172 9644 : auto if_true = __ MakeLabel();
1173 19288 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1174 :
1175 9644 : __ GotoIf(value, &if_true);
1176 9644 : __ Goto(&done, __ FalseConstant());
1177 :
1178 : __ Bind(&if_true);
1179 9644 : __ Goto(&done, __ TrueConstant());
1180 :
1181 : __ Bind(&done);
1182 9644 : return done.PhiAt(0);
1183 : }
1184 :
1185 39439 : Node* EffectControlLinearizer::LowerChangeInt31ToTaggedSigned(Node* node) {
1186 : Node* value = node->InputAt(0);
1187 39439 : return ChangeInt32ToSmi(value);
1188 : }
1189 :
1190 36026 : Node* EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node) {
1191 : Node* value = node->InputAt(0);
1192 :
1193 : if (SmiValuesAre32Bits()) {
1194 36026 : return ChangeInt32ToSmi(value);
1195 : }
1196 : DCHECK(SmiValuesAre31Bits());
1197 :
1198 : auto if_overflow = __ MakeDeferredLabel();
1199 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1200 :
1201 : Node* add = __ Int32AddWithOverflow(value, value);
1202 : Node* ovf = __ Projection(1, add);
1203 : __ GotoIf(ovf, &if_overflow);
1204 : Node* value_smi = __ Projection(0, add);
1205 : value_smi = ChangeInt32ToIntPtr(value_smi);
1206 : __ Goto(&done, value_smi);
1207 :
1208 : __ Bind(&if_overflow);
1209 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt32ToFloat64(value));
1210 : __ Goto(&done, number);
1211 :
1212 : __ Bind(&done);
1213 : return done.PhiAt(0);
1214 : }
1215 :
1216 45 : Node* EffectControlLinearizer::LowerChangeInt64ToTagged(Node* node) {
1217 : Node* value = node->InputAt(0);
1218 :
1219 45 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1220 90 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1221 :
1222 45 : Node* value32 = __ TruncateInt64ToInt32(value);
1223 45 : __ GotoIfNot(__ Word64Equal(__ ChangeInt32ToInt64(value32), value),
1224 45 : &if_not_in_smi_range);
1225 :
1226 : if (SmiValuesAre32Bits()) {
1227 : Node* value_smi = ChangeInt64ToSmi(value);
1228 : __ Goto(&done, value_smi);
1229 : } else {
1230 : Node* add = __ Int32AddWithOverflow(value32, value32);
1231 : Node* ovf = __ Projection(1, add);
1232 : __ GotoIf(ovf, &if_not_in_smi_range);
1233 : Node* value_smi = ChangeInt32ToIntPtr(__ Projection(0, add));
1234 : __ Goto(&done, value_smi);
1235 : }
1236 :
1237 : __ Bind(&if_not_in_smi_range);
1238 45 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1239 : __ Goto(&done, number);
1240 :
1241 : __ Bind(&done);
1242 45 : return done.PhiAt(0);
1243 : }
1244 :
1245 651 : Node* EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node) {
1246 : Node* value = node->InputAt(0);
1247 :
1248 651 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1249 1302 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1250 :
1251 651 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
1252 651 : __ GotoIfNot(check, &if_not_in_smi_range);
1253 651 : __ Goto(&done, ChangeUint32ToSmi(value));
1254 :
1255 : __ Bind(&if_not_in_smi_range);
1256 651 : Node* number = AllocateHeapNumberWithValue(__ ChangeUint32ToFloat64(value));
1257 :
1258 : __ Goto(&done, number);
1259 : __ Bind(&done);
1260 :
1261 651 : return done.PhiAt(0);
1262 : }
1263 :
1264 53 : Node* EffectControlLinearizer::LowerChangeUint64ToTagged(Node* node) {
1265 : Node* value = node->InputAt(0);
1266 :
1267 53 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1268 106 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1269 :
1270 : Node* check =
1271 53 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
1272 53 : __ GotoIfNot(check, &if_not_in_smi_range);
1273 : __ Goto(&done, ChangeInt64ToSmi(value));
1274 :
1275 : __ Bind(&if_not_in_smi_range);
1276 53 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1277 :
1278 : __ Goto(&done, number);
1279 : __ Bind(&done);
1280 :
1281 53 : return done.PhiAt(0);
1282 : }
1283 :
1284 51731 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt32(Node* node) {
1285 : Node* value = node->InputAt(0);
1286 51731 : return ChangeSmiToInt32(value);
1287 : }
1288 :
1289 297 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt64(Node* node) {
1290 : Node* value = node->InputAt(0);
1291 297 : return ChangeSmiToInt64(value);
1292 : }
1293 :
1294 139247 : Node* EffectControlLinearizer::LowerChangeTaggedToBit(Node* node) {
1295 : Node* value = node->InputAt(0);
1296 139247 : return __ WordEqual(value, __ TrueConstant());
1297 : }
1298 :
1299 48058 : void EffectControlLinearizer::TruncateTaggedPointerToBit(
1300 : Node* node, GraphAssemblerLabel<1>* done) {
1301 : Node* value = node->InputAt(0);
1302 :
1303 48058 : auto if_heapnumber = __ MakeDeferredLabel();
1304 48058 : auto if_bigint = __ MakeDeferredLabel();
1305 :
1306 48058 : Node* zero = __ Int32Constant(0);
1307 48058 : Node* fzero = __ Float64Constant(0.0);
1308 :
1309 : // Check if {value} is false.
1310 48058 : __ GotoIf(__ WordEqual(value, __ FalseConstant()), done, zero);
1311 :
1312 : // Check if {value} is the empty string.
1313 48058 : __ GotoIf(__ WordEqual(value, __ EmptyStringConstant()), done, zero);
1314 :
1315 : // Load the map of {value}.
1316 48058 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1317 :
1318 : // Check if the {value} is undetectable and immediately return false.
1319 : // This includes undefined and null.
1320 : Node* value_map_bitfield =
1321 48058 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1322 48058 : __ GotoIfNot(
1323 : __ Word32Equal(
1324 : __ Word32And(value_map_bitfield,
1325 : __ Int32Constant(Map::IsUndetectableBit::kMask)),
1326 : zero),
1327 48058 : done, zero);
1328 :
1329 : // Check if {value} is a HeapNumber.
1330 48058 : __ GotoIf(__ WordEqual(value_map, __ HeapNumberMapConstant()),
1331 48058 : &if_heapnumber);
1332 :
1333 : // Check if {value} is a BigInt.
1334 48058 : __ GotoIf(__ WordEqual(value_map, __ BigIntMapConstant()), &if_bigint);
1335 :
1336 : // All other values that reach here are true.
1337 48058 : __ Goto(done, __ Int32Constant(1));
1338 :
1339 : __ Bind(&if_heapnumber);
1340 : {
1341 : // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or
1342 : // NaN.
1343 : Node* value_value =
1344 48058 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1345 48058 : __ Goto(done, __ Float64LessThan(fzero, __ Float64Abs(value_value)));
1346 : }
1347 :
1348 : __ Bind(&if_bigint);
1349 : {
1350 48058 : Node* bitfield = __ LoadField(AccessBuilder::ForBigIntBitfield(), value);
1351 48058 : Node* length_is_zero = __ Word32Equal(
1352 : __ Word32And(bitfield, __ Int32Constant(BigInt::LengthBits::kMask)),
1353 48058 : __ Int32Constant(0));
1354 48058 : __ Goto(done, __ Word32Equal(length_is_zero, zero));
1355 : }
1356 48058 : }
1357 :
1358 47659 : Node* EffectControlLinearizer::LowerTruncateTaggedToBit(Node* node) {
1359 95318 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1360 47659 : auto if_smi = __ MakeDeferredLabel();
1361 :
1362 : Node* value = node->InputAt(0);
1363 95318 : __ GotoIf(ObjectIsSmi(value), &if_smi);
1364 :
1365 47659 : TruncateTaggedPointerToBit(node, &done);
1366 :
1367 : __ Bind(&if_smi);
1368 : {
1369 : // If {value} is a Smi, then we only need to check that it's not zero.
1370 47659 : __ Goto(&done, __ Word32Equal(__ WordEqual(value, __ IntPtrConstant(0)),
1371 : __ Int32Constant(0)));
1372 : }
1373 :
1374 : __ Bind(&done);
1375 47659 : return done.PhiAt(0);
1376 : }
1377 :
1378 399 : Node* EffectControlLinearizer::LowerTruncateTaggedPointerToBit(Node* node) {
1379 798 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1380 :
1381 399 : TruncateTaggedPointerToBit(node, &done);
1382 :
1383 : __ Bind(&done);
1384 399 : return done.PhiAt(0);
1385 : }
1386 :
1387 1758 : Node* EffectControlLinearizer::LowerChangeTaggedToInt32(Node* node) {
1388 : Node* value = node->InputAt(0);
1389 :
1390 1758 : auto if_not_smi = __ MakeDeferredLabel();
1391 3516 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1392 :
1393 1758 : Node* check = ObjectIsSmi(value);
1394 1758 : __ GotoIfNot(check, &if_not_smi);
1395 1758 : __ Goto(&done, ChangeSmiToInt32(value));
1396 :
1397 : __ Bind(&if_not_smi);
1398 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1399 : Oddball::kToNumberRawOffset);
1400 1758 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1401 1758 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1402 : __ Goto(&done, vfalse);
1403 :
1404 : __ Bind(&done);
1405 1758 : return done.PhiAt(0);
1406 : }
1407 :
1408 395 : Node* EffectControlLinearizer::LowerChangeTaggedToUint32(Node* node) {
1409 : Node* value = node->InputAt(0);
1410 :
1411 395 : auto if_not_smi = __ MakeDeferredLabel();
1412 790 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1413 :
1414 395 : Node* check = ObjectIsSmi(value);
1415 395 : __ GotoIfNot(check, &if_not_smi);
1416 395 : __ Goto(&done, ChangeSmiToInt32(value));
1417 :
1418 : __ Bind(&if_not_smi);
1419 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1420 : Oddball::kToNumberRawOffset);
1421 395 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1422 395 : vfalse = __ ChangeFloat64ToUint32(vfalse);
1423 : __ Goto(&done, vfalse);
1424 :
1425 : __ Bind(&done);
1426 395 : return done.PhiAt(0);
1427 : }
1428 :
1429 2 : Node* EffectControlLinearizer::LowerChangeTaggedToInt64(Node* node) {
1430 : Node* value = node->InputAt(0);
1431 :
1432 2 : auto if_not_smi = __ MakeDeferredLabel();
1433 4 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
1434 :
1435 2 : Node* check = ObjectIsSmi(value);
1436 2 : __ GotoIfNot(check, &if_not_smi);
1437 2 : __ Goto(&done, ChangeSmiToInt64(value));
1438 :
1439 : __ Bind(&if_not_smi);
1440 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1441 : Oddball::kToNumberRawOffset);
1442 2 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1443 2 : vfalse = __ ChangeFloat64ToInt64(vfalse);
1444 : __ Goto(&done, vfalse);
1445 :
1446 : __ Bind(&done);
1447 2 : return done.PhiAt(0);
1448 : }
1449 :
1450 0 : Node* EffectControlLinearizer::LowerChangeTaggedToFloat64(Node* node) {
1451 11255 : return LowerTruncateTaggedToFloat64(node);
1452 : }
1453 :
1454 14 : Node* EffectControlLinearizer::LowerChangeTaggedToTaggedSigned(Node* node) {
1455 : Node* value = node->InputAt(0);
1456 :
1457 14 : auto if_not_smi = __ MakeDeferredLabel();
1458 28 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1459 :
1460 14 : Node* check = ObjectIsSmi(value);
1461 14 : __ GotoIfNot(check, &if_not_smi);
1462 : __ Goto(&done, value);
1463 :
1464 : __ Bind(&if_not_smi);
1465 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1466 : Oddball::kToNumberRawOffset);
1467 14 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1468 14 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1469 14 : vfalse = ChangeInt32ToSmi(vfalse);
1470 : __ Goto(&done, vfalse);
1471 :
1472 : __ Bind(&done);
1473 14 : return done.PhiAt(0);
1474 : }
1475 :
1476 0 : Node* EffectControlLinearizer::LowerChangeCompressedToTaggedSigned(Node* node) {
1477 : Node* value = node->InputAt(0);
1478 :
1479 0 : auto if_not_smi = __ MakeDeferredLabel();
1480 0 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1481 :
1482 0 : Node* check = ObjectIsSmi(value);
1483 0 : __ GotoIfNot(check, &if_not_smi);
1484 0 : __ Goto(&done, __ ChangeCompressedSignedToTaggedSigned(value));
1485 :
1486 : __ Bind(&if_not_smi);
1487 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1488 0 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(),
1489 0 : __ ChangeCompressedToTagged(value));
1490 0 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1491 0 : vfalse = ChangeInt32ToSmi(vfalse);
1492 : __ Goto(&done, vfalse);
1493 :
1494 : __ Bind(&done);
1495 0 : return done.PhiAt(0);
1496 : }
1497 :
1498 0 : Node* EffectControlLinearizer::LowerChangeTaggedToCompressedSigned(Node* node) {
1499 : Node* value = node->InputAt(0);
1500 :
1501 0 : auto if_not_smi = __ MakeDeferredLabel();
1502 0 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1503 :
1504 0 : Node* check = ObjectIsSmi(value);
1505 0 : __ GotoIfNot(check, &if_not_smi);
1506 0 : __ Goto(&done, __ ChangeTaggedSignedToCompressedSigned(value));
1507 :
1508 : __ Bind(&if_not_smi);
1509 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1510 0 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1511 0 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1512 0 : vfalse = ChangeInt32ToSmi(vfalse);
1513 0 : vfalse = __ ChangeTaggedSignedToCompressedSigned(vfalse);
1514 : __ Goto(&done, vfalse);
1515 :
1516 : __ Bind(&done);
1517 0 : return done.PhiAt(0);
1518 : }
1519 :
1520 11983 : Node* EffectControlLinearizer::LowerTruncateTaggedToFloat64(Node* node) {
1521 : Node* value = node->InputAt(0);
1522 :
1523 11983 : auto if_not_smi = __ MakeDeferredLabel();
1524 23966 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
1525 :
1526 11983 : Node* check = ObjectIsSmi(value);
1527 11983 : __ GotoIfNot(check, &if_not_smi);
1528 11983 : Node* vtrue = ChangeSmiToInt32(value);
1529 11983 : vtrue = __ ChangeInt32ToFloat64(vtrue);
1530 : __ Goto(&done, vtrue);
1531 :
1532 : __ Bind(&if_not_smi);
1533 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1534 : Oddball::kToNumberRawOffset);
1535 11983 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1536 : __ Goto(&done, vfalse);
1537 :
1538 : __ Bind(&done);
1539 11983 : return done.PhiAt(0);
1540 : }
1541 :
1542 1034 : Node* EffectControlLinearizer::LowerPoisonIndex(Node* node) {
1543 : Node* index = node->InputAt(0);
1544 1034 : if (mask_array_index_ == kMaskArrayIndex) {
1545 0 : index = __ Word32PoisonOnSpeculation(index);
1546 : }
1547 1034 : return index;
1548 : }
1549 :
1550 51530 : void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
1551 51530 : CheckMapsParameters const& p = CheckMapsParametersOf(node->op());
1552 : Node* value = node->InputAt(0);
1553 :
1554 : ZoneHandleSet<Map> const& maps = p.maps();
1555 : size_t const map_count = maps.size();
1556 :
1557 51530 : if (p.flags() & CheckMapsFlag::kTryMigrateInstance) {
1558 66 : auto done = __ MakeLabel();
1559 66 : auto migrate = __ MakeDeferredLabel();
1560 :
1561 : // Load the current map of the {value}.
1562 132 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1563 :
1564 : // Perform the map checks.
1565 204 : for (size_t i = 0; i < map_count; ++i) {
1566 69 : Node* map = __ HeapConstant(maps[i]);
1567 69 : Node* check = __ WordEqual(value_map, map);
1568 69 : if (i == map_count - 1) {
1569 66 : __ Branch(check, &done, &migrate, IsSafetyCheck::kCriticalSafetyCheck);
1570 : } else {
1571 3 : auto next_map = __ MakeLabel();
1572 3 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1573 : __ Bind(&next_map);
1574 : }
1575 : }
1576 :
1577 : // Perform the (deferred) instance migration.
1578 : __ Bind(&migrate);
1579 : {
1580 : // If map is not deprecated the migration attempt does not make sense.
1581 : Node* bitfield3 =
1582 66 : __ LoadField(AccessBuilder::ForMapBitField3(), value_map);
1583 66 : Node* if_not_deprecated = __ WordEqual(
1584 : __ Word32And(bitfield3,
1585 : __ Int32Constant(Map::IsDeprecatedBit::kMask)),
1586 66 : __ Int32Constant(0));
1587 : __ DeoptimizeIf(DeoptimizeReason::kWrongMap, p.feedback(),
1588 : if_not_deprecated, frame_state,
1589 66 : IsSafetyCheck::kCriticalSafetyCheck);
1590 :
1591 66 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
1592 : Runtime::FunctionId id = Runtime::kTryMigrateInstance;
1593 66 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1594 66 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
1595 66 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1), value,
1596 : __ ExternalConstant(ExternalReference::Create(id)),
1597 66 : __ Int32Constant(1), __ NoContextConstant());
1598 66 : Node* check = ObjectIsSmi(result);
1599 : __ DeoptimizeIf(DeoptimizeReason::kInstanceMigrationFailed, p.feedback(),
1600 66 : check, frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1601 : }
1602 :
1603 : // Reload the current map of the {value}.
1604 66 : value_map = __ LoadField(AccessBuilder::ForMap(), value);
1605 :
1606 : // Perform the map checks again.
1607 204 : for (size_t i = 0; i < map_count; ++i) {
1608 69 : Node* map = __ HeapConstant(maps[i]);
1609 69 : Node* check = __ WordEqual(value_map, map);
1610 69 : if (i == map_count - 1) {
1611 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1612 66 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1613 : } else {
1614 3 : auto next_map = __ MakeLabel();
1615 3 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1616 : __ Bind(&next_map);
1617 : }
1618 : }
1619 :
1620 : __ Goto(&done);
1621 : __ Bind(&done);
1622 : } else {
1623 51464 : auto done = __ MakeLabel();
1624 :
1625 : // Load the current map of the {value}.
1626 : Node* value_map =
1627 : UsingCompressedPointers()
1628 : ? __ LoadField(AccessBuilder::ForCompressedMap(), value)
1629 102928 : : __ LoadField(AccessBuilder::ForMap(), value);
1630 :
1631 166114 : for (size_t i = 0; i < map_count; ++i) {
1632 : Node* check;
1633 :
1634 : if (UsingCompressedPointers()) {
1635 : // We need the dereference scope to embed the map pointer value as an
1636 : // int32. We don't visit the pointer.
1637 : AllowHandleDereference allow_map_dereference;
1638 : int32_t int32Map = static_cast<int32_t>(CompressTagged(maps[i]->ptr()));
1639 : Node* map = __ Int32Constant(int32Map);
1640 : check = __ Word32Equal(value_map, map);
1641 : this->embedded_maps()->push_back(maps[i]);
1642 : } else {
1643 57325 : Node* map = __ HeapConstant(maps[i]);
1644 57325 : check = __ WordEqual(value_map, map);
1645 : }
1646 :
1647 57325 : if (i == map_count - 1) {
1648 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1649 51464 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1650 : } else {
1651 5861 : auto next_map = __ MakeLabel();
1652 5861 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1653 : __ Bind(&next_map);
1654 : }
1655 : }
1656 : __ Goto(&done);
1657 : __ Bind(&done);
1658 : }
1659 51530 : }
1660 :
1661 7576 : Node* EffectControlLinearizer::LowerCompareMaps(Node* node) {
1662 7576 : ZoneHandleSet<Map> const& maps = CompareMapsParametersOf(node->op());
1663 : size_t const map_count = maps.size();
1664 : Node* value = node->InputAt(0);
1665 :
1666 15152 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1667 :
1668 : // Load the current map of the {value}.
1669 : Node* value_map = UsingCompressedPointers()
1670 : ? __ LoadField(AccessBuilder::ForCompressedMap(), value)
1671 15152 : : __ LoadField(AccessBuilder::ForMap(), value);
1672 :
1673 22744 : for (size_t i = 0; i < map_count; ++i) {
1674 : Node* check;
1675 :
1676 : if (UsingCompressedPointers()) {
1677 : // We need the dereference scope to embed the map pointer value as an
1678 : // int32. We don't visit the pointer.
1679 : AllowHandleDereference allow_map_dereference;
1680 : int32_t int32Map = static_cast<int32_t>(CompressTagged(maps[i]->ptr()));
1681 : Node* map = __ Int32Constant(int32Map);
1682 : check = __ Word32Equal(value_map, map);
1683 : this->embedded_maps()->push_back(maps[i]);
1684 : } else {
1685 7584 : Node* map = __ HeapConstant(maps[i]);
1686 7584 : check = __ WordEqual(value_map, map);
1687 : }
1688 :
1689 7584 : auto next_map = __ MakeLabel();
1690 7584 : auto passed = __ MakeLabel();
1691 7584 : __ Branch(check, &passed, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1692 :
1693 : __ Bind(&passed);
1694 7584 : __ Goto(&done, __ Int32Constant(1));
1695 :
1696 : __ Bind(&next_map);
1697 : }
1698 7576 : __ Goto(&done, __ Int32Constant(0));
1699 :
1700 : __ Bind(&done);
1701 7576 : return done.PhiAt(0);
1702 : }
1703 :
1704 484 : Node* EffectControlLinearizer::LowerCheckNumber(Node* node, Node* frame_state) {
1705 : Node* value = node->InputAt(0);
1706 484 : const CheckParameters& params = CheckParametersOf(node->op());
1707 :
1708 484 : auto if_not_smi = __ MakeDeferredLabel();
1709 484 : auto done = __ MakeLabel();
1710 :
1711 484 : Node* check0 = ObjectIsSmi(value);
1712 484 : __ GotoIfNot(check0, &if_not_smi);
1713 : __ Goto(&done);
1714 :
1715 : __ Bind(&if_not_smi);
1716 484 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1717 484 : Node* check1 = __ WordEqual(value_map, __ HeapNumberMapConstant());
1718 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
1719 484 : check1, frame_state);
1720 : __ Goto(&done);
1721 :
1722 : __ Bind(&done);
1723 484 : return value;
1724 : }
1725 :
1726 813 : Node* EffectControlLinearizer::LowerCheckReceiver(Node* node,
1727 : Node* frame_state) {
1728 : Node* value = node->InputAt(0);
1729 :
1730 1626 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1731 : Node* value_instance_type =
1732 813 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1733 :
1734 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1735 813 : Node* check = __ Uint32LessThanOrEqual(
1736 813 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1737 1626 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObject, VectorSlotPair(),
1738 813 : check, frame_state);
1739 813 : return value;
1740 : }
1741 :
1742 73 : Node* EffectControlLinearizer::LowerCheckReceiverOrNullOrUndefined(
1743 : Node* node, Node* frame_state) {
1744 : Node* value = node->InputAt(0);
1745 :
1746 146 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1747 : Node* value_instance_type =
1748 73 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1749 :
1750 : // Rule out all primitives except oddballs (true, false, undefined, null).
1751 : STATIC_ASSERT(LAST_PRIMITIVE_TYPE == ODDBALL_TYPE);
1752 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1753 73 : Node* check0 = __ Uint32LessThanOrEqual(__ Uint32Constant(ODDBALL_TYPE),
1754 73 : value_instance_type);
1755 146 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1756 73 : VectorSlotPair(), check0, frame_state);
1757 :
1758 : // Rule out booleans.
1759 73 : Node* check1 = __ WordEqual(value_map, __ BooleanMapConstant());
1760 146 : __ DeoptimizeIf(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1761 73 : VectorSlotPair(), check1, frame_state);
1762 73 : return value;
1763 : }
1764 :
1765 32 : Node* EffectControlLinearizer::LowerCheckSymbol(Node* node, Node* frame_state) {
1766 : Node* value = node->InputAt(0);
1767 :
1768 64 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1769 :
1770 : Node* check =
1771 32 : __ WordEqual(value_map, __ HeapConstant(factory()->symbol_map()));
1772 64 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASymbol, VectorSlotPair(), check,
1773 32 : frame_state);
1774 32 : return value;
1775 : }
1776 :
1777 5367 : Node* EffectControlLinearizer::LowerCheckString(Node* node, Node* frame_state) {
1778 : Node* value = node->InputAt(0);
1779 5367 : const CheckParameters& params = CheckParametersOf(node->op());
1780 :
1781 10734 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1782 : Node* value_instance_type =
1783 5367 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1784 :
1785 5367 : Node* check = __ Uint32LessThan(value_instance_type,
1786 5367 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
1787 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAString, params.feedback(), check,
1788 5367 : frame_state);
1789 5367 : return value;
1790 : }
1791 :
1792 1410 : Node* EffectControlLinearizer::LowerCheckInternalizedString(Node* node,
1793 : Node* frame_state) {
1794 : Node* value = node->InputAt(0);
1795 :
1796 2820 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1797 : Node* value_instance_type =
1798 1410 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1799 :
1800 1410 : Node* check = __ Word32Equal(
1801 : __ Word32And(value_instance_type,
1802 : __ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
1803 1410 : __ Int32Constant(kInternalizedTag));
1804 2820 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1805 1410 : check, frame_state);
1806 :
1807 1410 : return value;
1808 : }
1809 :
1810 1474 : Node* EffectControlLinearizer::LowerCheckNonEmptyOneByteString(
1811 : Node* node, Node* frame_state) {
1812 : Node* value = node->InputAt(0);
1813 :
1814 2948 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1815 : Node* value_instance_type =
1816 1474 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1817 :
1818 1474 : Node* check = __ Word32Equal(
1819 : __ Word32And(value_instance_type,
1820 : __ Int32Constant(kIsNotStringMask | kStringEncodingMask |
1821 : kIsEmptyStringMask)),
1822 1474 : __ Int32Constant(kStringTag | kOneByteStringTag | kIsNotEmptyStringTag));
1823 2948 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1824 1474 : check, frame_state);
1825 :
1826 1474 : return value;
1827 : }
1828 :
1829 201 : Node* EffectControlLinearizer::LowerCheckNonEmptyTwoByteString(
1830 : Node* node, Node* frame_state) {
1831 : Node* value = node->InputAt(0);
1832 :
1833 402 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1834 : Node* value_instance_type =
1835 201 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1836 :
1837 201 : Node* check = __ Word32Equal(
1838 : __ Word32And(value_instance_type,
1839 : __ Int32Constant(kIsNotStringMask | kStringEncodingMask |
1840 : kIsEmptyStringMask)),
1841 201 : __ Int32Constant(kStringTag | kTwoByteStringTag | kIsNotEmptyStringTag));
1842 402 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1843 201 : check, frame_state);
1844 :
1845 201 : return value;
1846 : }
1847 :
1848 18 : Node* EffectControlLinearizer::LowerCheckNonEmptyString(Node* node,
1849 : Node* frame_state) {
1850 : Node* value = node->InputAt(0);
1851 :
1852 36 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1853 : Node* value_instance_type =
1854 18 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1855 :
1856 18 : Node* check = __ Word32Equal(
1857 : __ Word32And(value_instance_type,
1858 : __ Int32Constant(kIsNotStringMask | kIsEmptyStringMask)),
1859 18 : __ Int32Constant(kStringTag | kIsNotEmptyStringTag));
1860 36 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1861 18 : check, frame_state);
1862 :
1863 18 : return value;
1864 : }
1865 :
1866 16431 : void EffectControlLinearizer::LowerCheckIf(Node* node, Node* frame_state) {
1867 : Node* value = node->InputAt(0);
1868 16431 : const CheckIfParameters& p = CheckIfParametersOf(node->op());
1869 16431 : __ DeoptimizeIfNot(p.reason(), p.feedback(), value, frame_state);
1870 16431 : }
1871 :
1872 21134 : Node* EffectControlLinearizer::LowerStringConcat(Node* node) {
1873 : Node* lhs = node->InputAt(1);
1874 : Node* rhs = node->InputAt(2);
1875 :
1876 : Callable const callable =
1877 21134 : CodeFactory::StringAdd(isolate(), STRING_ADD_CHECK_NONE);
1878 21134 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1879 21134 : graph()->zone(), callable.descriptor(),
1880 : callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags,
1881 21134 : Operator::kNoDeopt | Operator::kNoWrite | Operator::kNoThrow);
1882 :
1883 : Node* value =
1884 63402 : __ Call(call_descriptor, jsgraph()->HeapConstant(callable.code()), lhs,
1885 21134 : rhs, __ NoContextConstant());
1886 :
1887 21134 : return value;
1888 : }
1889 :
1890 55954 : Node* EffectControlLinearizer::LowerCheckedInt32Add(Node* node,
1891 : Node* frame_state) {
1892 : Node* lhs = node->InputAt(0);
1893 : Node* rhs = node->InputAt(1);
1894 :
1895 55954 : Node* value = __ Int32AddWithOverflow(lhs, rhs);
1896 55952 : Node* check = __ Projection(1, value);
1897 111905 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1898 55953 : frame_state);
1899 55952 : return __ Projection(0, value);
1900 : }
1901 :
1902 7688 : Node* EffectControlLinearizer::LowerCheckedInt32Sub(Node* node,
1903 : Node* frame_state) {
1904 : Node* lhs = node->InputAt(0);
1905 : Node* rhs = node->InputAt(1);
1906 :
1907 7688 : Node* value = __ Int32SubWithOverflow(lhs, rhs);
1908 7688 : Node* check = __ Projection(1, value);
1909 15376 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1910 7688 : frame_state);
1911 7688 : return __ Projection(0, value);
1912 : }
1913 :
1914 235 : Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node,
1915 : Node* frame_state) {
1916 : Node* lhs = node->InputAt(0);
1917 : Node* rhs = node->InputAt(1);
1918 235 : Node* zero = __ Int32Constant(0);
1919 :
1920 : // Check if the {rhs} is a known power of two.
1921 : Int32Matcher m(rhs);
1922 235 : if (m.IsPowerOf2()) {
1923 : // Since we know that {rhs} is a power of two, we can perform a fast
1924 : // check to see if the relevant least significant bits of the {lhs}
1925 : // are all zero, and if so we know that we can perform a division
1926 : // safely (and fast by doing an arithmetic - aka sign preserving -
1927 : // right shift on {lhs}).
1928 : int32_t divisor = m.Value();
1929 134 : Node* mask = __ Int32Constant(divisor - 1);
1930 134 : Node* shift = __ Int32Constant(WhichPowerOf2(divisor));
1931 134 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
1932 268 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1933 134 : check, frame_state);
1934 134 : return __ Word32Sar(lhs, shift);
1935 : } else {
1936 101 : auto if_rhs_positive = __ MakeLabel();
1937 101 : auto if_rhs_negative = __ MakeDeferredLabel();
1938 202 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1939 :
1940 : // Check if {rhs} is positive (and not zero).
1941 101 : Node* check_rhs_positive = __ Int32LessThan(zero, rhs);
1942 101 : __ Branch(check_rhs_positive, &if_rhs_positive, &if_rhs_negative);
1943 :
1944 : __ Bind(&if_rhs_positive);
1945 : {
1946 : // Fast case, no additional checking required.
1947 101 : __ Goto(&done, __ Int32Div(lhs, rhs));
1948 : }
1949 :
1950 : __ Bind(&if_rhs_negative);
1951 : {
1952 101 : auto if_lhs_minint = __ MakeDeferredLabel();
1953 101 : auto if_lhs_notminint = __ MakeLabel();
1954 :
1955 : // Check if {rhs} is zero.
1956 101 : Node* check_rhs_zero = __ Word32Equal(rhs, zero);
1957 202 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
1958 101 : check_rhs_zero, frame_state);
1959 :
1960 : // Check if {lhs} is zero, as that would produce minus zero.
1961 101 : Node* check_lhs_zero = __ Word32Equal(lhs, zero);
1962 202 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
1963 101 : check_lhs_zero, frame_state);
1964 :
1965 : // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
1966 : // to return -kMinInt, which is not representable as Word32.
1967 202 : Node* check_lhs_minint = graph()->NewNode(machine()->Word32Equal(), lhs,
1968 : __ Int32Constant(kMinInt));
1969 101 : __ Branch(check_lhs_minint, &if_lhs_minint, &if_lhs_notminint);
1970 :
1971 : __ Bind(&if_lhs_minint);
1972 : {
1973 : // Check that {rhs} is not -1, otherwise result would be -kMinInt.
1974 101 : Node* check_rhs_minusone = __ Word32Equal(rhs, __ Int32Constant(-1));
1975 202 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(),
1976 101 : check_rhs_minusone, frame_state);
1977 :
1978 : // Perform the actual integer division.
1979 101 : __ Goto(&done, __ Int32Div(lhs, rhs));
1980 : }
1981 :
1982 : __ Bind(&if_lhs_notminint);
1983 : {
1984 : // Perform the actual integer division.
1985 101 : __ Goto(&done, __ Int32Div(lhs, rhs));
1986 : }
1987 : }
1988 :
1989 : __ Bind(&done);
1990 : Node* value = done.PhiAt(0);
1991 :
1992 : // Check if the remainder is non-zero.
1993 101 : Node* check = __ Word32Equal(lhs, __ Int32Mul(value, rhs));
1994 202 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1995 101 : check, frame_state);
1996 :
1997 : return value;
1998 : }
1999 : }
2000 :
2001 1004 : Node* EffectControlLinearizer::BuildUint32Mod(Node* lhs, Node* rhs) {
2002 1004 : auto if_rhs_power_of_two = __ MakeLabel();
2003 2008 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2004 :
2005 : // Compute the mask for the {rhs}.
2006 1004 : Node* one = __ Int32Constant(1);
2007 1004 : Node* msk = __ Int32Sub(rhs, one);
2008 :
2009 : // Check if the {rhs} is a power of two.
2010 1004 : __ GotoIf(__ Word32Equal(__ Word32And(rhs, msk), __ Int32Constant(0)),
2011 1004 : &if_rhs_power_of_two);
2012 : {
2013 : // The {rhs} is not a power of two, do a generic Uint32Mod.
2014 1004 : __ Goto(&done, __ Uint32Mod(lhs, rhs));
2015 : }
2016 :
2017 : __ Bind(&if_rhs_power_of_two);
2018 : {
2019 : // The {rhs} is a power of two, just do a fast bit masking.
2020 1004 : __ Goto(&done, __ Word32And(lhs, msk));
2021 : }
2022 :
2023 : __ Bind(&done);
2024 1004 : return done.PhiAt(0);
2025 : }
2026 :
2027 967 : Node* EffectControlLinearizer::LowerCheckedInt32Mod(Node* node,
2028 : Node* frame_state) {
2029 : // General case for signed integer modulus, with optimization for (unknown)
2030 : // power of 2 right hand side.
2031 : //
2032 : // if rhs <= 0 then
2033 : // rhs = -rhs
2034 : // deopt if rhs == 0
2035 : // let msk = rhs - 1 in
2036 : // if lhs < 0 then
2037 : // let lhs_abs = -lsh in
2038 : // let res = if rhs & msk == 0 then
2039 : // lhs_abs & msk
2040 : // else
2041 : // lhs_abs % rhs in
2042 : // if lhs < 0 then
2043 : // deopt if res == 0
2044 : // -res
2045 : // else
2046 : // res
2047 : // else
2048 : // if rhs & msk == 0 then
2049 : // lhs & msk
2050 : // else
2051 : // lhs % rhs
2052 : //
2053 : Node* lhs = node->InputAt(0);
2054 : Node* rhs = node->InputAt(1);
2055 :
2056 967 : auto if_rhs_not_positive = __ MakeDeferredLabel();
2057 967 : auto if_lhs_negative = __ MakeDeferredLabel();
2058 : auto if_rhs_power_of_two = __ MakeLabel();
2059 1934 : auto rhs_checked = __ MakeLabel(MachineRepresentation::kWord32);
2060 1934 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2061 :
2062 967 : Node* zero = __ Int32Constant(0);
2063 :
2064 : // Check if {rhs} is not strictly positive.
2065 967 : Node* check0 = __ Int32LessThanOrEqual(rhs, zero);
2066 967 : __ GotoIf(check0, &if_rhs_not_positive);
2067 : __ Goto(&rhs_checked, rhs);
2068 :
2069 : __ Bind(&if_rhs_not_positive);
2070 : {
2071 : // Negate {rhs}, might still produce a negative result in case of
2072 : // -2^31, but that is handled safely below.
2073 967 : Node* vtrue0 = __ Int32Sub(zero, rhs);
2074 :
2075 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
2076 1934 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
2077 967 : __ Word32Equal(vtrue0, zero), frame_state);
2078 : __ Goto(&rhs_checked, vtrue0);
2079 : }
2080 :
2081 : __ Bind(&rhs_checked);
2082 : rhs = rhs_checked.PhiAt(0);
2083 :
2084 967 : __ GotoIf(__ Int32LessThan(lhs, zero), &if_lhs_negative);
2085 : {
2086 : // The {lhs} is a non-negative integer.
2087 967 : __ Goto(&done, BuildUint32Mod(lhs, rhs));
2088 : }
2089 :
2090 : __ Bind(&if_lhs_negative);
2091 : {
2092 : // The {lhs} is a negative integer. This is very unlikely and
2093 : // we intentionally don't use the BuildUint32Mod() here, which
2094 : // would try to figure out whether {rhs} is a power of two,
2095 : // since this is intended to be a slow-path.
2096 967 : Node* res = __ Uint32Mod(__ Int32Sub(zero, lhs), rhs);
2097 :
2098 : // Check if we would have to return -0.
2099 1934 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
2100 967 : __ Word32Equal(res, zero), frame_state);
2101 967 : __ Goto(&done, __ Int32Sub(zero, res));
2102 : }
2103 :
2104 : __ Bind(&done);
2105 967 : return done.PhiAt(0);
2106 : }
2107 :
2108 52 : Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node,
2109 : Node* frame_state) {
2110 : Node* lhs = node->InputAt(0);
2111 : Node* rhs = node->InputAt(1);
2112 52 : Node* zero = __ Int32Constant(0);
2113 :
2114 : // Check if the {rhs} is a known power of two.
2115 : Uint32Matcher m(rhs);
2116 52 : if (m.IsPowerOf2()) {
2117 : // Since we know that {rhs} is a power of two, we can perform a fast
2118 : // check to see if the relevant least significant bits of the {lhs}
2119 : // are all zero, and if so we know that we can perform a division
2120 : // safely (and fast by doing a logical - aka zero extending - right
2121 : // shift on {lhs}).
2122 : uint32_t divisor = m.Value();
2123 41 : Node* mask = __ Uint32Constant(divisor - 1);
2124 41 : Node* shift = __ Uint32Constant(WhichPowerOf2(divisor));
2125 41 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
2126 82 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
2127 41 : check, frame_state);
2128 41 : return __ Word32Shr(lhs, shift);
2129 : } else {
2130 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
2131 11 : Node* check = __ Word32Equal(rhs, zero);
2132 22 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
2133 11 : frame_state);
2134 :
2135 : // Perform the actual unsigned integer division.
2136 11 : Node* value = __ Uint32Div(lhs, rhs);
2137 :
2138 : // Check if the remainder is non-zero.
2139 11 : check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
2140 22 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
2141 11 : check, frame_state);
2142 11 : return value;
2143 : }
2144 : }
2145 :
2146 37 : Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node,
2147 : Node* frame_state) {
2148 : Node* lhs = node->InputAt(0);
2149 : Node* rhs = node->InputAt(1);
2150 :
2151 37 : Node* zero = __ Int32Constant(0);
2152 :
2153 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
2154 37 : Node* check = __ Word32Equal(rhs, zero);
2155 74 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
2156 37 : frame_state);
2157 :
2158 : // Perform the actual unsigned integer modulus.
2159 37 : return BuildUint32Mod(lhs, rhs);
2160 : }
2161 :
2162 5912 : Node* EffectControlLinearizer::LowerCheckedInt32Mul(Node* node,
2163 : Node* frame_state) {
2164 5912 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
2165 : Node* lhs = node->InputAt(0);
2166 : Node* rhs = node->InputAt(1);
2167 :
2168 5912 : Node* projection = __ Int32MulWithOverflow(lhs, rhs);
2169 5912 : Node* check = __ Projection(1, projection);
2170 11824 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
2171 5912 : frame_state);
2172 :
2173 5912 : Node* value = __ Projection(0, projection);
2174 :
2175 5912 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2176 3931 : auto if_zero = __ MakeDeferredLabel();
2177 3931 : auto check_done = __ MakeLabel();
2178 3931 : Node* zero = __ Int32Constant(0);
2179 3931 : Node* check_zero = __ Word32Equal(value, zero);
2180 3931 : __ GotoIf(check_zero, &if_zero);
2181 : __ Goto(&check_done);
2182 :
2183 : __ Bind(&if_zero);
2184 : // We may need to return negative zero.
2185 3931 : Node* check_or = __ Int32LessThan(__ Word32Or(lhs, rhs), zero);
2186 7862 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(), check_or,
2187 3931 : frame_state);
2188 : __ Goto(&check_done);
2189 :
2190 : __ Bind(&check_done);
2191 : }
2192 :
2193 5912 : return value;
2194 : }
2195 :
2196 0 : Node* EffectControlLinearizer::LowerCheckedInt32ToTaggedSigned(
2197 : Node* node, Node* frame_state) {
2198 : DCHECK(SmiValuesAre31Bits());
2199 : Node* value = node->InputAt(0);
2200 0 : const CheckParameters& params = CheckParametersOf(node->op());
2201 :
2202 0 : Node* add = __ Int32AddWithOverflow(value, value);
2203 0 : Node* check = __ Projection(1, add);
2204 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2205 0 : frame_state);
2206 0 : Node* result = __ Projection(0, add);
2207 : result = ChangeInt32ToIntPtr(result);
2208 0 : return result;
2209 : }
2210 :
2211 9 : Node* EffectControlLinearizer::LowerCheckedInt64ToInt32(Node* node,
2212 : Node* frame_state) {
2213 : Node* value = node->InputAt(0);
2214 9 : const CheckParameters& params = CheckParametersOf(node->op());
2215 :
2216 9 : Node* value32 = __ TruncateInt64ToInt32(value);
2217 9 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2218 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2219 9 : frame_state);
2220 9 : return value32;
2221 : }
2222 :
2223 151 : Node* EffectControlLinearizer::LowerCheckedInt64ToTaggedSigned(
2224 : Node* node, Node* frame_state) {
2225 : Node* value = node->InputAt(0);
2226 151 : const CheckParameters& params = CheckParametersOf(node->op());
2227 :
2228 151 : Node* value32 = __ TruncateInt64ToInt32(value);
2229 151 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2230 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2231 151 : frame_state);
2232 :
2233 : if (SmiValuesAre32Bits()) {
2234 : return ChangeInt64ToSmi(value);
2235 : } else {
2236 : Node* add = __ Int32AddWithOverflow(value32, value32);
2237 : Node* check = __ Projection(1, add);
2238 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2239 : frame_state);
2240 : Node* result = __ Projection(0, add);
2241 : result = ChangeInt32ToIntPtr(result);
2242 : return result;
2243 : }
2244 : }
2245 :
2246 52596 : Node* EffectControlLinearizer::LowerCheckedUint32Bounds(Node* node,
2247 : Node* frame_state) {
2248 : Node* index = node->InputAt(0);
2249 : Node* limit = node->InputAt(1);
2250 52596 : const CheckBoundsParameters& params = CheckBoundsParametersOf(node->op());
2251 :
2252 52596 : Node* check = __ Uint32LessThan(index, limit);
2253 52596 : switch (params.mode()) {
2254 : case CheckBoundsParameters::kDeoptOnOutOfBounds:
2255 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds,
2256 : params.check_parameters().feedback(), check,
2257 47974 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2258 47974 : break;
2259 : case CheckBoundsParameters::kAbortOnOutOfBounds: {
2260 4622 : auto if_abort = __ MakeDeferredLabel();
2261 4622 : auto done = __ MakeLabel();
2262 :
2263 4622 : __ Branch(check, &done, &if_abort);
2264 :
2265 : __ Bind(&if_abort);
2266 4622 : __ Unreachable();
2267 : __ Goto(&done);
2268 :
2269 : __ Bind(&done);
2270 : break;
2271 : }
2272 : }
2273 :
2274 52596 : return index;
2275 : }
2276 :
2277 396 : Node* EffectControlLinearizer::LowerCheckedUint32ToInt32(Node* node,
2278 : Node* frame_state) {
2279 : Node* value = node->InputAt(0);
2280 396 : const CheckParameters& params = CheckParametersOf(node->op());
2281 396 : Node* unsafe = __ Int32LessThan(value, __ Int32Constant(0));
2282 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), unsafe,
2283 396 : frame_state);
2284 396 : return value;
2285 : }
2286 :
2287 7 : Node* EffectControlLinearizer::LowerCheckedUint32ToTaggedSigned(
2288 : Node* node, Node* frame_state) {
2289 : Node* value = node->InputAt(0);
2290 7 : const CheckParameters& params = CheckParametersOf(node->op());
2291 7 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
2292 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2293 7 : frame_state);
2294 7 : return ChangeUint32ToSmi(value);
2295 : }
2296 :
2297 195 : Node* EffectControlLinearizer::LowerCheckedUint64Bounds(Node* node,
2298 : Node* frame_state) {
2299 195 : CheckParameters const& params = CheckParametersOf(node->op());
2300 : Node* const index = node->InputAt(0);
2301 : Node* const limit = node->InputAt(1);
2302 :
2303 195 : Node* check = __ Uint64LessThan(index, limit);
2304 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, params.feedback(), check,
2305 195 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2306 195 : return index;
2307 : }
2308 :
2309 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToInt32(Node* node,
2310 : Node* frame_state) {
2311 : Node* value = node->InputAt(0);
2312 0 : const CheckParameters& params = CheckParametersOf(node->op());
2313 :
2314 0 : Node* check = __ Uint64LessThanOrEqual(value, __ Int64Constant(kMaxInt));
2315 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2316 0 : frame_state);
2317 0 : return __ TruncateInt64ToInt32(value);
2318 : }
2319 :
2320 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToTaggedSigned(
2321 : Node* node, Node* frame_state) {
2322 : Node* value = node->InputAt(0);
2323 0 : const CheckParameters& params = CheckParametersOf(node->op());
2324 :
2325 : Node* check =
2326 0 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
2327 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2328 0 : frame_state);
2329 0 : return ChangeInt64ToSmi(value);
2330 : }
2331 :
2332 4614 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt32(
2333 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2334 : Node* frame_state) {
2335 4614 : Node* value32 = __ RoundFloat64ToInt32(value);
2336 4614 : Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32));
2337 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2338 4614 : check_same, frame_state);
2339 :
2340 4614 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2341 : // Check if {value} is -0.
2342 2156 : auto if_zero = __ MakeDeferredLabel();
2343 2156 : auto check_done = __ MakeLabel();
2344 :
2345 2156 : Node* check_zero = __ Word32Equal(value32, __ Int32Constant(0));
2346 2156 : __ GotoIf(check_zero, &if_zero);
2347 : __ Goto(&check_done);
2348 :
2349 : __ Bind(&if_zero);
2350 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2351 2156 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2352 2156 : __ Int32Constant(0));
2353 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2354 2156 : frame_state);
2355 : __ Goto(&check_done);
2356 :
2357 : __ Bind(&check_done);
2358 : }
2359 4614 : return value32;
2360 : }
2361 :
2362 2315 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt32(Node* node,
2363 : Node* frame_state) {
2364 : const CheckMinusZeroParameters& params =
2365 2315 : CheckMinusZeroParametersOf(node->op());
2366 : Node* value = node->InputAt(0);
2367 : return BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), value,
2368 2315 : frame_state);
2369 : }
2370 :
2371 103 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt64(
2372 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2373 : Node* frame_state) {
2374 103 : Node* value64 = __ TruncateFloat64ToInt64(value);
2375 103 : Node* check_same = __ Float64Equal(value, __ ChangeInt64ToFloat64(value64));
2376 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2377 103 : check_same, frame_state);
2378 :
2379 103 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2380 : // Check if {value} is -0.
2381 0 : auto if_zero = __ MakeDeferredLabel();
2382 0 : auto check_done = __ MakeLabel();
2383 :
2384 0 : Node* check_zero = __ Word64Equal(value64, __ Int64Constant(0));
2385 0 : __ GotoIf(check_zero, &if_zero);
2386 : __ Goto(&check_done);
2387 :
2388 : __ Bind(&if_zero);
2389 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2390 0 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2391 0 : __ Int32Constant(0));
2392 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2393 0 : frame_state);
2394 : __ Goto(&check_done);
2395 :
2396 : __ Bind(&check_done);
2397 : }
2398 103 : return value64;
2399 : }
2400 :
2401 16 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt64(Node* node,
2402 : Node* frame_state) {
2403 : const CheckMinusZeroParameters& params =
2404 16 : CheckMinusZeroParametersOf(node->op());
2405 : Node* value = node->InputAt(0);
2406 : return BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), value,
2407 16 : frame_state);
2408 : }
2409 :
2410 24837 : Node* EffectControlLinearizer::LowerCheckedTaggedSignedToInt32(
2411 : Node* node, Node* frame_state) {
2412 : Node* value = node->InputAt(0);
2413 24837 : const CheckParameters& params = CheckParametersOf(node->op());
2414 24838 : Node* check = ObjectIsSmi(value);
2415 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2416 24838 : frame_state);
2417 24838 : return ChangeSmiToInt32(value);
2418 : }
2419 :
2420 2299 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt32(Node* node,
2421 : Node* frame_state) {
2422 : const CheckMinusZeroParameters& params =
2423 2299 : CheckMinusZeroParametersOf(node->op());
2424 : Node* value = node->InputAt(0);
2425 :
2426 2299 : auto if_not_smi = __ MakeDeferredLabel();
2427 4598 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2428 :
2429 2299 : Node* check = ObjectIsSmi(value);
2430 2299 : __ GotoIfNot(check, &if_not_smi);
2431 : // In the Smi case, just convert to int32.
2432 2299 : __ Goto(&done, ChangeSmiToInt32(value));
2433 :
2434 : // In the non-Smi case, check the heap numberness, load the number and convert
2435 : // to int32.
2436 : __ Bind(&if_not_smi);
2437 2299 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2438 2299 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2439 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2440 2299 : check_map, frame_state);
2441 2299 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2442 : vfalse = BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), vfalse,
2443 2299 : frame_state);
2444 : __ Goto(&done, vfalse);
2445 :
2446 : __ Bind(&done);
2447 2299 : return done.PhiAt(0);
2448 : }
2449 :
2450 87 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt64(Node* node,
2451 : Node* frame_state) {
2452 : const CheckMinusZeroParameters& params =
2453 87 : CheckMinusZeroParametersOf(node->op());
2454 : Node* value = node->InputAt(0);
2455 :
2456 87 : auto if_not_smi = __ MakeDeferredLabel();
2457 174 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
2458 :
2459 87 : Node* check = ObjectIsSmi(value);
2460 87 : __ GotoIfNot(check, &if_not_smi);
2461 : // In the Smi case, just convert to int64.
2462 87 : __ Goto(&done, ChangeSmiToInt64(value));
2463 :
2464 : // In the non-Smi case, check the heap numberness, load the number and convert
2465 : // to int64.
2466 : __ Bind(&if_not_smi);
2467 87 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2468 87 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2469 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2470 87 : check_map, frame_state);
2471 87 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2472 : vfalse = BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), vfalse,
2473 87 : frame_state);
2474 : __ Goto(&done, vfalse);
2475 :
2476 : __ Bind(&done);
2477 87 : return done.PhiAt(0);
2478 : }
2479 :
2480 52285 : Node* EffectControlLinearizer::BuildCheckedHeapNumberOrOddballToFloat64(
2481 : CheckTaggedInputMode mode, const VectorSlotPair& feedback, Node* value,
2482 : Node* frame_state) {
2483 104570 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2484 52285 : Node* check_number = __ WordEqual(value_map, __ HeapNumberMapConstant());
2485 52285 : switch (mode) {
2486 : case CheckTaggedInputMode::kNumber: {
2487 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, feedback,
2488 8292 : check_number, frame_state);
2489 8292 : break;
2490 : }
2491 : case CheckTaggedInputMode::kNumberOrOddball: {
2492 43993 : auto check_done = __ MakeLabel();
2493 :
2494 43993 : __ GotoIf(check_number, &check_done);
2495 : // For oddballs also contain the numeric value, let us just check that
2496 : // we have an oddball here.
2497 : Node* instance_type =
2498 43993 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2499 : Node* check_oddball =
2500 43993 : __ Word32Equal(instance_type, __ Int32Constant(ODDBALL_TYPE));
2501 : __ DeoptimizeIfNot(DeoptimizeReason::kNotANumberOrOddball, feedback,
2502 43993 : check_oddball, frame_state);
2503 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
2504 : Oddball::kToNumberRawOffset);
2505 : __ Goto(&check_done);
2506 :
2507 : __ Bind(&check_done);
2508 : break;
2509 : }
2510 : }
2511 52285 : return __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2512 : }
2513 :
2514 51109 : Node* EffectControlLinearizer::LowerCheckedTaggedToFloat64(Node* node,
2515 : Node* frame_state) {
2516 : CheckTaggedInputParameters const& p =
2517 51109 : CheckTaggedInputParametersOf(node->op());
2518 : Node* value = node->InputAt(0);
2519 :
2520 51109 : auto if_smi = __ MakeLabel();
2521 102218 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
2522 :
2523 51109 : Node* check = ObjectIsSmi(value);
2524 51109 : __ GotoIf(check, &if_smi);
2525 :
2526 : // In the Smi case, just convert to int32 and then float64.
2527 : // Otherwise, check heap numberness and load the number.
2528 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2529 51109 : p.mode(), p.feedback(), value, frame_state);
2530 : __ Goto(&done, number);
2531 :
2532 : __ Bind(&if_smi);
2533 51109 : Node* from_smi = ChangeSmiToInt32(value);
2534 51109 : from_smi = __ ChangeInt32ToFloat64(from_smi);
2535 : __ Goto(&done, from_smi);
2536 :
2537 : __ Bind(&done);
2538 51109 : return done.PhiAt(0);
2539 : }
2540 :
2541 18738 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedSigned(
2542 : Node* node, Node* frame_state) {
2543 : Node* value = node->InputAt(0);
2544 18738 : const CheckParameters& params = CheckParametersOf(node->op());
2545 :
2546 18738 : Node* check = ObjectIsSmi(value);
2547 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2548 18738 : frame_state);
2549 :
2550 18738 : return value;
2551 : }
2552 :
2553 34577 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedPointer(
2554 : Node* node, Node* frame_state) {
2555 : Node* value = node->InputAt(0);
2556 34577 : const CheckParameters& params = CheckParametersOf(node->op());
2557 :
2558 34577 : Node* check = ObjectIsSmi(value);
2559 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2560 34577 : frame_state);
2561 34577 : return value;
2562 : }
2563 :
2564 0 : Node* EffectControlLinearizer::LowerCheckedCompressedToTaggedSigned(
2565 : Node* node, Node* frame_state) {
2566 : Node* value = node->InputAt(0);
2567 0 : const CheckParameters& params = CheckParametersOf(node->op());
2568 :
2569 0 : Node* check = ObjectIsSmi(value);
2570 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2571 0 : frame_state);
2572 :
2573 0 : return __ ChangeCompressedSignedToTaggedSigned(value);
2574 : }
2575 :
2576 0 : Node* EffectControlLinearizer::LowerCheckedCompressedToTaggedPointer(
2577 : Node* node, Node* frame_state) {
2578 : Node* value = node->InputAt(0);
2579 0 : const CheckParameters& params = CheckParametersOf(node->op());
2580 :
2581 0 : Node* check = ObjectIsSmi(value);
2582 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2583 0 : frame_state);
2584 0 : return __ ChangeCompressedPointerToTaggedPointer(value);
2585 : }
2586 :
2587 0 : Node* EffectControlLinearizer::LowerCheckedTaggedToCompressedSigned(
2588 : Node* node, Node* frame_state) {
2589 : Node* value = node->InputAt(0);
2590 0 : const CheckParameters& params = CheckParametersOf(node->op());
2591 :
2592 0 : Node* check = ObjectIsSmi(value);
2593 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2594 0 : frame_state);
2595 :
2596 0 : return __ ChangeTaggedSignedToCompressedSigned(value);
2597 : }
2598 :
2599 0 : Node* EffectControlLinearizer::LowerCheckedTaggedToCompressedPointer(
2600 : Node* node, Node* frame_state) {
2601 : Node* value = node->InputAt(0);
2602 0 : const CheckParameters& params = CheckParametersOf(node->op());
2603 :
2604 0 : Node* check = ObjectIsSmi(value);
2605 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2606 0 : frame_state);
2607 0 : return __ ChangeTaggedPointerToCompressedPointer(value);
2608 : }
2609 :
2610 502 : Node* EffectControlLinearizer::LowerTruncateTaggedToWord32(Node* node) {
2611 : Node* value = node->InputAt(0);
2612 :
2613 502 : auto if_not_smi = __ MakeDeferredLabel();
2614 1004 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2615 :
2616 502 : Node* check = ObjectIsSmi(value);
2617 502 : __ GotoIfNot(check, &if_not_smi);
2618 502 : __ Goto(&done, ChangeSmiToInt32(value));
2619 :
2620 : __ Bind(&if_not_smi);
2621 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
2622 : Oddball::kToNumberRawOffset);
2623 502 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2624 502 : vfalse = __ TruncateFloat64ToWord32(vfalse);
2625 : __ Goto(&done, vfalse);
2626 :
2627 : __ Bind(&done);
2628 502 : return done.PhiAt(0);
2629 : }
2630 :
2631 1176 : Node* EffectControlLinearizer::LowerCheckedTruncateTaggedToWord32(
2632 : Node* node, Node* frame_state) {
2633 : const CheckTaggedInputParameters& params =
2634 1176 : CheckTaggedInputParametersOf(node->op());
2635 : Node* value = node->InputAt(0);
2636 :
2637 1176 : auto if_not_smi = __ MakeLabel();
2638 2352 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2639 :
2640 1176 : Node* check = ObjectIsSmi(value);
2641 1176 : __ GotoIfNot(check, &if_not_smi);
2642 : // In the Smi case, just convert to int32.
2643 1176 : __ Goto(&done, ChangeSmiToInt32(value));
2644 :
2645 : // Otherwise, check that it's a heap number or oddball and truncate the value
2646 : // to int32.
2647 : __ Bind(&if_not_smi);
2648 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2649 1176 : params.mode(), params.feedback(), value, frame_state);
2650 1176 : number = __ TruncateFloat64ToWord32(number);
2651 : __ Goto(&done, number);
2652 :
2653 : __ Bind(&done);
2654 1176 : return done.PhiAt(0);
2655 : }
2656 :
2657 114421 : Node* EffectControlLinearizer::LowerAllocate(Node* node) {
2658 : Node* size = node->InputAt(0);
2659 114421 : AllocationType allocation = AllocationTypeOf(node->op());
2660 114421 : Node* new_node = __ Allocate(allocation, size);
2661 114421 : return new_node;
2662 : }
2663 :
2664 989 : Node* EffectControlLinearizer::LowerNumberToString(Node* node) {
2665 : Node* argument = node->InputAt(0);
2666 :
2667 : Callable const callable =
2668 989 : Builtins::CallableFor(isolate(), Builtins::kNumberToString);
2669 : Operator::Properties properties = Operator::kEliminatable;
2670 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2671 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2672 989 : graph()->zone(), callable.descriptor(),
2673 989 : callable.descriptor().GetStackParameterCount(), flags, properties);
2674 2967 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), argument,
2675 1978 : __ NoContextConstant());
2676 : }
2677 :
2678 16 : Node* EffectControlLinearizer::LowerObjectIsArrayBufferView(Node* node) {
2679 : Node* value = node->InputAt(0);
2680 :
2681 16 : auto if_smi = __ MakeDeferredLabel();
2682 32 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2683 :
2684 16 : Node* check = ObjectIsSmi(value);
2685 16 : __ GotoIf(check, &if_smi);
2686 :
2687 16 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2688 : Node* value_instance_type =
2689 16 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2690 : STATIC_ASSERT(JS_TYPED_ARRAY_TYPE + 1 == JS_DATA_VIEW_TYPE);
2691 16 : Node* vfalse = __ Uint32LessThan(
2692 : __ Int32Sub(value_instance_type, __ Int32Constant(JS_TYPED_ARRAY_TYPE)),
2693 16 : __ Int32Constant(2));
2694 : __ Goto(&done, vfalse);
2695 :
2696 : __ Bind(&if_smi);
2697 16 : __ Goto(&done, __ Int32Constant(0));
2698 :
2699 : __ Bind(&done);
2700 16 : return done.PhiAt(0);
2701 : }
2702 :
2703 25 : Node* EffectControlLinearizer::LowerObjectIsBigInt(Node* node) {
2704 : Node* value = node->InputAt(0);
2705 :
2706 25 : auto if_smi = __ MakeDeferredLabel();
2707 50 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2708 :
2709 25 : Node* check = ObjectIsSmi(value);
2710 25 : __ GotoIf(check, &if_smi);
2711 25 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2712 25 : Node* vfalse = __ WordEqual(value_map, __ BigIntMapConstant());
2713 : __ Goto(&done, vfalse);
2714 :
2715 : __ Bind(&if_smi);
2716 25 : __ Goto(&done, __ Int32Constant(0));
2717 :
2718 : __ Bind(&done);
2719 25 : return done.PhiAt(0);
2720 : }
2721 :
2722 103 : Node* EffectControlLinearizer::LowerObjectIsCallable(Node* node) {
2723 : Node* value = node->InputAt(0);
2724 :
2725 103 : auto if_smi = __ MakeDeferredLabel();
2726 206 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2727 :
2728 103 : Node* check = ObjectIsSmi(value);
2729 103 : __ GotoIf(check, &if_smi);
2730 :
2731 103 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2732 : Node* value_bit_field =
2733 103 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2734 : Node* vfalse =
2735 103 : __ Word32Equal(__ Int32Constant(Map::IsCallableBit::kMask),
2736 : __ Word32And(value_bit_field,
2737 103 : __ Int32Constant(Map::IsCallableBit::kMask)));
2738 : __ Goto(&done, vfalse);
2739 :
2740 : __ Bind(&if_smi);
2741 103 : __ Goto(&done, __ Int32Constant(0));
2742 :
2743 : __ Bind(&done);
2744 103 : return done.PhiAt(0);
2745 : }
2746 :
2747 377 : Node* EffectControlLinearizer::LowerObjectIsConstructor(Node* node) {
2748 : Node* value = node->InputAt(0);
2749 :
2750 377 : auto if_smi = __ MakeDeferredLabel();
2751 754 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2752 :
2753 377 : Node* check = ObjectIsSmi(value);
2754 377 : __ GotoIf(check, &if_smi);
2755 :
2756 377 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2757 : Node* value_bit_field =
2758 377 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2759 377 : Node* vfalse = __ Word32Equal(
2760 : __ Int32Constant(Map::IsConstructorBit::kMask),
2761 : __ Word32And(value_bit_field,
2762 377 : __ Int32Constant(Map::IsConstructorBit::kMask)));
2763 : __ Goto(&done, vfalse);
2764 :
2765 : __ Bind(&if_smi);
2766 377 : __ Goto(&done, __ Int32Constant(0));
2767 :
2768 : __ Bind(&done);
2769 377 : return done.PhiAt(0);
2770 : }
2771 :
2772 14084 : Node* EffectControlLinearizer::LowerObjectIsDetectableCallable(Node* node) {
2773 : Node* value = node->InputAt(0);
2774 :
2775 14084 : auto if_smi = __ MakeDeferredLabel();
2776 28168 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2777 :
2778 14084 : Node* check = ObjectIsSmi(value);
2779 14084 : __ GotoIf(check, &if_smi);
2780 :
2781 14084 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2782 : Node* value_bit_field =
2783 14084 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2784 14084 : Node* vfalse = __ Word32Equal(
2785 : __ Int32Constant(Map::IsCallableBit::kMask),
2786 : __ Word32And(value_bit_field,
2787 : __ Int32Constant((Map::IsCallableBit::kMask) |
2788 14084 : (Map::IsUndetectableBit::kMask))));
2789 : __ Goto(&done, vfalse);
2790 :
2791 : __ Bind(&if_smi);
2792 14084 : __ Goto(&done, __ Int32Constant(0));
2793 :
2794 : __ Bind(&done);
2795 14084 : return done.PhiAt(0);
2796 : }
2797 :
2798 61 : Node* EffectControlLinearizer::LowerNumberIsFloat64Hole(Node* node) {
2799 : Node* value = node->InputAt(0);
2800 61 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
2801 61 : __ Int32Constant(kHoleNanUpper32));
2802 61 : return check;
2803 : }
2804 :
2805 126 : Node* EffectControlLinearizer::LowerNumberIsFinite(Node* node) {
2806 : Node* number = node->InputAt(0);
2807 126 : Node* diff = __ Float64Sub(number, number);
2808 126 : Node* check = __ Float64Equal(diff, diff);
2809 126 : return check;
2810 : }
2811 :
2812 7 : Node* EffectControlLinearizer::LowerObjectIsFiniteNumber(Node* node) {
2813 : Node* object = node->InputAt(0);
2814 7 : Node* zero = __ Int32Constant(0);
2815 7 : Node* one = __ Int32Constant(1);
2816 :
2817 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2818 :
2819 : // Check if {object} is a Smi.
2820 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2821 :
2822 : // Check if {object} is a HeapNumber.
2823 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2824 7 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2825 7 : zero);
2826 :
2827 : // {object} is a HeapNumber.
2828 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2829 7 : Node* diff = __ Float64Sub(value, value);
2830 7 : Node* check = __ Float64Equal(diff, diff);
2831 : __ Goto(&done, check);
2832 :
2833 : __ Bind(&done);
2834 7 : return done.PhiAt(0);
2835 : }
2836 :
2837 119 : Node* EffectControlLinearizer::LowerNumberIsInteger(Node* node) {
2838 : Node* number = node->InputAt(0);
2839 119 : Node* trunc = BuildFloat64RoundTruncate(number);
2840 119 : Node* diff = __ Float64Sub(number, trunc);
2841 119 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2842 119 : return check;
2843 : }
2844 :
2845 7 : Node* EffectControlLinearizer::LowerObjectIsInteger(Node* node) {
2846 : Node* object = node->InputAt(0);
2847 7 : Node* zero = __ Int32Constant(0);
2848 7 : Node* one = __ Int32Constant(1);
2849 :
2850 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2851 :
2852 : // Check if {object} is a Smi.
2853 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2854 :
2855 : // Check if {object} is a HeapNumber.
2856 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2857 7 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2858 7 : zero);
2859 :
2860 : // {object} is a HeapNumber.
2861 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2862 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2863 7 : Node* diff = __ Float64Sub(value, trunc);
2864 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2865 : __ Goto(&done, check);
2866 :
2867 : __ Bind(&done);
2868 7 : return done.PhiAt(0);
2869 : }
2870 :
2871 7 : Node* EffectControlLinearizer::LowerNumberIsSafeInteger(Node* node) {
2872 : Node* number = node->InputAt(0);
2873 7 : Node* zero = __ Int32Constant(0);
2874 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2875 :
2876 7 : Node* trunc = BuildFloat64RoundTruncate(number);
2877 7 : Node* diff = __ Float64Sub(number, trunc);
2878 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2879 7 : __ GotoIfNot(check, &done, zero);
2880 7 : Node* in_range = __ Float64LessThanOrEqual(
2881 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2882 : __ Goto(&done, in_range);
2883 :
2884 : __ Bind(&done);
2885 7 : return done.PhiAt(0);
2886 : }
2887 :
2888 7 : Node* EffectControlLinearizer::LowerObjectIsSafeInteger(Node* node) {
2889 : Node* object = node->InputAt(0);
2890 7 : Node* zero = __ Int32Constant(0);
2891 7 : Node* one = __ Int32Constant(1);
2892 :
2893 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2894 :
2895 : // Check if {object} is a Smi.
2896 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2897 :
2898 : // Check if {object} is a HeapNumber.
2899 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2900 7 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2901 7 : zero);
2902 :
2903 : // {object} is a HeapNumber.
2904 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2905 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2906 7 : Node* diff = __ Float64Sub(value, trunc);
2907 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2908 7 : __ GotoIfNot(check, &done, zero);
2909 7 : Node* in_range = __ Float64LessThanOrEqual(
2910 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2911 : __ Goto(&done, in_range);
2912 :
2913 : __ Bind(&done);
2914 7 : return done.PhiAt(0);
2915 : }
2916 :
2917 : namespace {
2918 :
2919 61014 : const int64_t kMinusZeroBits = bit_cast<int64_t>(-0.0);
2920 61014 : const int32_t kMinusZeroLoBits = static_cast<int32_t>(kMinusZeroBits);
2921 61014 : const int32_t kMinusZeroHiBits = static_cast<int32_t>(kMinusZeroBits >> 32);
2922 :
2923 : } // namespace
2924 :
2925 23 : Node* EffectControlLinearizer::LowerObjectIsMinusZero(Node* node) {
2926 : Node* value = node->InputAt(0);
2927 23 : Node* zero = __ Int32Constant(0);
2928 :
2929 46 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2930 :
2931 : // Check if {value} is a Smi.
2932 23 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2933 :
2934 : // Check if {value} is a HeapNumber.
2935 23 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2936 23 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2937 23 : zero);
2938 :
2939 : // Check if {value} contains -0.
2940 23 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2941 23 : if (machine()->Is64()) {
2942 23 : Node* value64 = __ BitcastFloat64ToInt64(value_value);
2943 23 : __ Goto(&done, __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits)));
2944 : } else {
2945 0 : Node* value_lo = __ Float64ExtractLowWord32(value_value);
2946 0 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2947 0 : &done, zero);
2948 0 : Node* value_hi = __ Float64ExtractHighWord32(value_value);
2949 0 : __ Goto(&done,
2950 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2951 : }
2952 :
2953 : __ Bind(&done);
2954 23 : return done.PhiAt(0);
2955 : }
2956 :
2957 98 : Node* EffectControlLinearizer::LowerNumberIsMinusZero(Node* node) {
2958 : Node* value = node->InputAt(0);
2959 :
2960 98 : if (machine()->Is64()) {
2961 98 : Node* value64 = __ BitcastFloat64ToInt64(value);
2962 98 : return __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits));
2963 : } else {
2964 0 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2965 :
2966 0 : Node* value_lo = __ Float64ExtractLowWord32(value);
2967 0 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2968 0 : &done, __ Int32Constant(0));
2969 0 : Node* value_hi = __ Float64ExtractHighWord32(value);
2970 0 : __ Goto(&done,
2971 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2972 :
2973 : __ Bind(&done);
2974 : return done.PhiAt(0);
2975 : }
2976 : }
2977 :
2978 716 : Node* EffectControlLinearizer::LowerObjectIsNaN(Node* node) {
2979 : Node* value = node->InputAt(0);
2980 716 : Node* zero = __ Int32Constant(0);
2981 :
2982 1432 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2983 :
2984 : // Check if {value} is a Smi.
2985 716 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2986 :
2987 : // Check if {value} is a HeapNumber.
2988 716 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2989 716 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2990 716 : zero);
2991 :
2992 : // Check if {value} contains a NaN.
2993 716 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2994 716 : __ Goto(&done,
2995 : __ Word32Equal(__ Float64Equal(value_value, value_value), zero));
2996 :
2997 : __ Bind(&done);
2998 716 : return done.PhiAt(0);
2999 : }
3000 :
3001 2511 : Node* EffectControlLinearizer::LowerNumberIsNaN(Node* node) {
3002 : Node* number = node->InputAt(0);
3003 2511 : Node* diff = __ Float64Equal(number, number);
3004 2511 : Node* check = __ Word32Equal(diff, __ Int32Constant(0));
3005 2511 : return check;
3006 : }
3007 :
3008 6056 : Node* EffectControlLinearizer::LowerObjectIsNonCallable(Node* node) {
3009 : Node* value = node->InputAt(0);
3010 :
3011 6056 : auto if_primitive = __ MakeDeferredLabel();
3012 12112 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3013 :
3014 6056 : Node* check0 = ObjectIsSmi(value);
3015 6056 : __ GotoIf(check0, &if_primitive);
3016 :
3017 6056 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3018 : Node* value_instance_type =
3019 6056 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3020 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
3021 6056 : Node* check1 = __ Uint32LessThanOrEqual(
3022 6056 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
3023 6056 : __ GotoIfNot(check1, &if_primitive);
3024 :
3025 : Node* value_bit_field =
3026 6056 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
3027 : Node* check2 =
3028 6056 : __ Word32Equal(__ Int32Constant(0),
3029 : __ Word32And(value_bit_field,
3030 6056 : __ Int32Constant(Map::IsCallableBit::kMask)));
3031 : __ Goto(&done, check2);
3032 :
3033 : __ Bind(&if_primitive);
3034 6056 : __ Goto(&done, __ Int32Constant(0));
3035 :
3036 : __ Bind(&done);
3037 6056 : return done.PhiAt(0);
3038 : }
3039 :
3040 7592 : Node* EffectControlLinearizer::LowerObjectIsNumber(Node* node) {
3041 : Node* value = node->InputAt(0);
3042 :
3043 7592 : auto if_smi = __ MakeLabel();
3044 15184 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3045 :
3046 15184 : __ GotoIf(ObjectIsSmi(value), &if_smi);
3047 7592 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3048 7592 : __ Goto(&done, __ WordEqual(value_map, __ HeapNumberMapConstant()));
3049 :
3050 : __ Bind(&if_smi);
3051 7592 : __ Goto(&done, __ Int32Constant(1));
3052 :
3053 : __ Bind(&done);
3054 7592 : return done.PhiAt(0);
3055 : }
3056 :
3057 19514 : Node* EffectControlLinearizer::LowerObjectIsReceiver(Node* node) {
3058 : Node* value = node->InputAt(0);
3059 :
3060 19514 : auto if_smi = __ MakeDeferredLabel();
3061 39028 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3062 :
3063 39028 : __ GotoIf(ObjectIsSmi(value), &if_smi);
3064 :
3065 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
3066 19514 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3067 : Node* value_instance_type =
3068 19514 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3069 19514 : Node* result = __ Uint32LessThanOrEqual(
3070 19514 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
3071 : __ Goto(&done, result);
3072 :
3073 : __ Bind(&if_smi);
3074 19514 : __ Goto(&done, __ Int32Constant(0));
3075 :
3076 : __ Bind(&done);
3077 19514 : return done.PhiAt(0);
3078 : }
3079 :
3080 4132 : Node* EffectControlLinearizer::LowerObjectIsSmi(Node* node) {
3081 : Node* value = node->InputAt(0);
3082 4132 : return ObjectIsSmi(value);
3083 : }
3084 :
3085 2237 : Node* EffectControlLinearizer::LowerObjectIsString(Node* node) {
3086 : Node* value = node->InputAt(0);
3087 :
3088 2237 : auto if_smi = __ MakeDeferredLabel();
3089 4474 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3090 :
3091 2237 : Node* check = ObjectIsSmi(value);
3092 2237 : __ GotoIf(check, &if_smi);
3093 2237 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3094 : Node* value_instance_type =
3095 2237 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3096 2237 : Node* vfalse = __ Uint32LessThan(value_instance_type,
3097 2237 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
3098 : __ Goto(&done, vfalse);
3099 :
3100 : __ Bind(&if_smi);
3101 2237 : __ Goto(&done, __ Int32Constant(0));
3102 :
3103 : __ Bind(&done);
3104 2237 : return done.PhiAt(0);
3105 : }
3106 :
3107 23 : Node* EffectControlLinearizer::LowerObjectIsSymbol(Node* node) {
3108 : Node* value = node->InputAt(0);
3109 :
3110 23 : auto if_smi = __ MakeDeferredLabel();
3111 46 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3112 :
3113 23 : Node* check = ObjectIsSmi(value);
3114 23 : __ GotoIf(check, &if_smi);
3115 23 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3116 : Node* value_instance_type =
3117 23 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3118 : Node* vfalse =
3119 23 : __ Word32Equal(value_instance_type, __ Uint32Constant(SYMBOL_TYPE));
3120 : __ Goto(&done, vfalse);
3121 :
3122 : __ Bind(&if_smi);
3123 23 : __ Goto(&done, __ Int32Constant(0));
3124 :
3125 : __ Bind(&done);
3126 23 : return done.PhiAt(0);
3127 : }
3128 :
3129 1426 : Node* EffectControlLinearizer::LowerObjectIsUndetectable(Node* node) {
3130 : Node* value = node->InputAt(0);
3131 :
3132 1426 : auto if_smi = __ MakeDeferredLabel();
3133 2852 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3134 :
3135 1426 : Node* check = ObjectIsSmi(value);
3136 1426 : __ GotoIf(check, &if_smi);
3137 :
3138 1426 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3139 : Node* value_bit_field =
3140 1426 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
3141 1426 : Node* vfalse = __ Word32Equal(
3142 : __ Word32Equal(
3143 : __ Int32Constant(0),
3144 : __ Word32And(value_bit_field,
3145 : __ Int32Constant(Map::IsUndetectableBit::kMask))),
3146 1426 : __ Int32Constant(0));
3147 : __ Goto(&done, vfalse);
3148 :
3149 : __ Bind(&if_smi);
3150 1426 : __ Goto(&done, __ Int32Constant(0));
3151 :
3152 : __ Bind(&done);
3153 1426 : return done.PhiAt(0);
3154 : }
3155 :
3156 21305 : Node* EffectControlLinearizer::LowerTypeOf(Node* node) {
3157 : Node* obj = node->InputAt(0);
3158 21305 : Callable const callable = Builtins::CallableFor(isolate(), Builtins::kTypeof);
3159 : Operator::Properties const properties = Operator::kEliminatable;
3160 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
3161 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3162 21305 : graph()->zone(), callable.descriptor(),
3163 21305 : callable.descriptor().GetStackParameterCount(), flags, properties);
3164 63915 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
3165 42610 : __ NoContextConstant());
3166 : }
3167 :
3168 198 : Node* EffectControlLinearizer::LowerToBoolean(Node* node) {
3169 : Node* obj = node->InputAt(0);
3170 : Callable const callable =
3171 198 : Builtins::CallableFor(isolate(), Builtins::kToBoolean);
3172 : Operator::Properties const properties = Operator::kEliminatable;
3173 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
3174 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3175 198 : graph()->zone(), callable.descriptor(),
3176 198 : callable.descriptor().GetStackParameterCount(), flags, properties);
3177 594 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
3178 396 : __ NoContextConstant());
3179 : }
3180 :
3181 16927 : Node* EffectControlLinearizer::LowerArgumentsLength(Node* node) {
3182 16927 : Node* arguments_frame = NodeProperties::GetValueInput(node, 0);
3183 16927 : int formal_parameter_count = FormalParameterCountOf(node->op());
3184 16927 : bool is_rest_length = IsRestLengthOf(node->op());
3185 : DCHECK_LE(0, formal_parameter_count);
3186 :
3187 16927 : if (is_rest_length) {
3188 : // The ArgumentsLength node is computing the number of rest parameters,
3189 : // which is max(0, actual_parameter_count - formal_parameter_count).
3190 : // We have to distinguish the case, when there is an arguments adaptor frame
3191 : // (i.e., arguments_frame != LoadFramePointer()).
3192 229 : auto if_adaptor_frame = __ MakeLabel();
3193 458 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
3194 :
3195 229 : Node* frame = __ LoadFramePointer();
3196 229 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done, __ SmiConstant(0));
3197 : __ Goto(&if_adaptor_frame);
3198 :
3199 : __ Bind(&if_adaptor_frame);
3200 229 : Node* arguments_length = __ Load(
3201 : MachineType::TaggedSigned(), arguments_frame,
3202 229 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
3203 :
3204 : Node* rest_length =
3205 229 : __ IntSub(arguments_length, __ SmiConstant(formal_parameter_count));
3206 229 : __ GotoIf(__ IntLessThan(rest_length, __ SmiConstant(0)), &done,
3207 229 : __ SmiConstant(0));
3208 : __ Goto(&done, rest_length);
3209 :
3210 : __ Bind(&done);
3211 : return done.PhiAt(0);
3212 : } else {
3213 : // The ArgumentsLength node is computing the actual number of arguments.
3214 : // We have to distinguish the case when there is an arguments adaptor frame
3215 : // (i.e., arguments_frame != LoadFramePointer()).
3216 16698 : auto if_adaptor_frame = __ MakeLabel();
3217 33396 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
3218 :
3219 16698 : Node* frame = __ LoadFramePointer();
3220 16698 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done,
3221 16698 : __ SmiConstant(formal_parameter_count));
3222 : __ Goto(&if_adaptor_frame);
3223 :
3224 : __ Bind(&if_adaptor_frame);
3225 16698 : Node* arguments_length = __ Load(
3226 : MachineType::TaggedSigned(), arguments_frame,
3227 16698 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
3228 : __ Goto(&done, arguments_length);
3229 :
3230 : __ Bind(&done);
3231 : return done.PhiAt(0);
3232 : }
3233 : }
3234 :
3235 16900 : Node* EffectControlLinearizer::LowerArgumentsFrame(Node* node) {
3236 33800 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
3237 :
3238 16900 : Node* frame = __ LoadFramePointer();
3239 : Node* parent_frame =
3240 16900 : __ Load(MachineType::Pointer(), frame,
3241 16900 : __ IntPtrConstant(StandardFrameConstants::kCallerFPOffset));
3242 16900 : Node* parent_frame_type = __ Load(
3243 : MachineType::AnyTagged(), parent_frame,
3244 16900 : __ IntPtrConstant(CommonFrameConstants::kContextOrFrameTypeOffset));
3245 16900 : __ GotoIf(__ WordEqual(parent_frame_type,
3246 : __ IntPtrConstant(StackFrame::TypeToMarker(
3247 : StackFrame::ARGUMENTS_ADAPTOR))),
3248 16900 : &done, parent_frame);
3249 : __ Goto(&done, frame);
3250 :
3251 : __ Bind(&done);
3252 16900 : return done.PhiAt(0);
3253 : }
3254 :
3255 28 : Node* EffectControlLinearizer::LowerNewDoubleElements(Node* node) {
3256 28 : AllocationType const allocation = AllocationTypeOf(node->op());
3257 : Node* length = node->InputAt(0);
3258 :
3259 56 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3260 28 : Node* zero_length = __ WordEqual(length, __ IntPtrConstant(0));
3261 56 : __ GotoIf(zero_length, &done,
3262 28 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3263 :
3264 : // Compute the effective size of the backing store.
3265 28 : Node* size = __ IntAdd(__ WordShl(length, __ IntPtrConstant(kDoubleSizeLog2)),
3266 28 : __ IntPtrConstant(FixedDoubleArray::kHeaderSize));
3267 :
3268 : // Allocate the result and initialize the header.
3269 28 : Node* result = __ Allocate(allocation, size);
3270 56 : __ StoreField(AccessBuilder::ForMap(), result,
3271 28 : __ FixedDoubleArrayMapConstant());
3272 56 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3273 28 : ChangeIntPtrToSmi(length));
3274 :
3275 : // Initialize the backing store with holes.
3276 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
3277 : Oddball::kToNumberRawOffset);
3278 : Node* the_hole =
3279 28 : __ LoadField(AccessBuilder::ForHeapNumberValue(), __ TheHoleConstant());
3280 56 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3281 28 : __ Goto(&loop, __ IntPtrConstant(0));
3282 : __ Bind(&loop);
3283 : {
3284 : // Check if we've initialized everything.
3285 : Node* index = loop.PhiAt(0);
3286 28 : Node* check = __ UintLessThan(index, length);
3287 28 : __ GotoIfNot(check, &done, result);
3288 :
3289 : ElementAccess const access = {kTaggedBase, FixedDoubleArray::kHeaderSize,
3290 : Type::NumberOrHole(), MachineType::Float64(),
3291 : kNoWriteBarrier};
3292 28 : __ StoreElement(access, result, index, the_hole);
3293 :
3294 : // Advance the {index}.
3295 28 : index = __ IntAdd(index, __ IntPtrConstant(1));
3296 : __ Goto(&loop, index);
3297 : }
3298 :
3299 : __ Bind(&done);
3300 28 : return done.PhiAt(0);
3301 : }
3302 :
3303 471 : Node* EffectControlLinearizer::LowerNewSmiOrObjectElements(Node* node) {
3304 471 : AllocationType const allocation = AllocationTypeOf(node->op());
3305 : Node* length = node->InputAt(0);
3306 :
3307 942 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3308 471 : Node* zero_length = __ WordEqual(length, __ IntPtrConstant(0));
3309 942 : __ GotoIf(zero_length, &done,
3310 471 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3311 :
3312 : // Compute the effective size of the backing store.
3313 471 : Node* size = __ IntAdd(__ WordShl(length, __ IntPtrConstant(kTaggedSizeLog2)),
3314 471 : __ IntPtrConstant(FixedArray::kHeaderSize));
3315 :
3316 : // Allocate the result and initialize the header.
3317 471 : Node* result = __ Allocate(allocation, size);
3318 471 : __ StoreField(AccessBuilder::ForMap(), result, __ FixedArrayMapConstant());
3319 942 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3320 471 : ChangeIntPtrToSmi(length));
3321 :
3322 : // Initialize the backing store with holes.
3323 471 : Node* the_hole = __ TheHoleConstant();
3324 942 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3325 471 : __ Goto(&loop, __ IntPtrConstant(0));
3326 : __ Bind(&loop);
3327 : {
3328 : // Check if we've initialized everything.
3329 : Node* index = loop.PhiAt(0);
3330 471 : Node* check = __ UintLessThan(index, length);
3331 471 : __ GotoIfNot(check, &done, result);
3332 :
3333 : // Storing "the_hole" doesn't need a write barrier.
3334 : ElementAccess const access = {kTaggedBase, FixedArray::kHeaderSize,
3335 : Type::Any(), MachineType::AnyTagged(),
3336 : kNoWriteBarrier};
3337 471 : __ StoreElement(access, result, index, the_hole);
3338 :
3339 : // Advance the {index}.
3340 471 : index = __ IntAdd(index, __ IntPtrConstant(1));
3341 : __ Goto(&loop, index);
3342 : }
3343 :
3344 : __ Bind(&done);
3345 471 : return done.PhiAt(0);
3346 : }
3347 :
3348 16423 : Node* EffectControlLinearizer::LowerNewArgumentsElements(Node* node) {
3349 16423 : Node* frame = NodeProperties::GetValueInput(node, 0);
3350 16423 : Node* length = NodeProperties::GetValueInput(node, 1);
3351 16423 : int mapped_count = NewArgumentsElementsMappedCountOf(node->op());
3352 :
3353 : Callable const callable =
3354 16423 : Builtins::CallableFor(isolate(), Builtins::kNewArgumentsElements);
3355 16423 : Operator::Properties const properties = node->op()->properties();
3356 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
3357 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3358 16423 : graph()->zone(), callable.descriptor(),
3359 16423 : callable.descriptor().GetStackParameterCount(), flags, properties);
3360 49269 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), frame,
3361 32846 : length, __ SmiConstant(mapped_count), __ NoContextConstant());
3362 : }
3363 :
3364 3832 : Node* EffectControlLinearizer::LowerNewConsOneByteString(Node* node) {
3365 3832 : Node* map = jsgraph()->HeapConstant(factory()->cons_one_byte_string_map());
3366 : Node* length = node->InputAt(0);
3367 : Node* first = node->InputAt(1);
3368 : Node* second = node->InputAt(2);
3369 :
3370 3832 : return AllocateConsString(map, length, first, second);
3371 : }
3372 :
3373 33 : Node* EffectControlLinearizer::LowerNewConsTwoByteString(Node* node) {
3374 33 : Node* map = jsgraph()->HeapConstant(factory()->cons_string_map());
3375 : Node* length = node->InputAt(0);
3376 : Node* first = node->InputAt(1);
3377 : Node* second = node->InputAt(2);
3378 :
3379 33 : return AllocateConsString(map, length, first, second);
3380 : }
3381 :
3382 1090 : Node* EffectControlLinearizer::LowerNewConsString(Node* node) {
3383 : Node* length = node->InputAt(0);
3384 : Node* first = node->InputAt(1);
3385 : Node* second = node->InputAt(2);
3386 :
3387 : // Determine the instance types of {first} and {second}.
3388 2180 : Node* first_map = __ LoadField(AccessBuilder::ForMap(), first);
3389 : Node* first_instance_type =
3390 1090 : __ LoadField(AccessBuilder::ForMapInstanceType(), first_map);
3391 1090 : Node* second_map = __ LoadField(AccessBuilder::ForMap(), second);
3392 : Node* second_instance_type =
3393 1090 : __ LoadField(AccessBuilder::ForMapInstanceType(), second_map);
3394 :
3395 : // Determine the proper map for the resulting ConsString.
3396 : // If both {first} and {second} are one-byte strings, we
3397 : // create a new ConsOneByteString, otherwise we create a
3398 : // new ConsString instead.
3399 1090 : auto if_onebyte = __ MakeLabel();
3400 1090 : auto if_twobyte = __ MakeLabel();
3401 2180 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3402 : STATIC_ASSERT(kOneByteStringTag != 0);
3403 : STATIC_ASSERT(kTwoByteStringTag == 0);
3404 1090 : Node* instance_type = __ Word32And(first_instance_type, second_instance_type);
3405 : Node* encoding =
3406 1090 : __ Word32And(instance_type, __ Int32Constant(kStringEncodingMask));
3407 1090 : __ Branch(__ Word32Equal(encoding, __ Int32Constant(kTwoByteStringTag)),
3408 1090 : &if_twobyte, &if_onebyte);
3409 : __ Bind(&if_onebyte);
3410 2180 : __ Goto(&done,
3411 : jsgraph()->HeapConstant(factory()->cons_one_byte_string_map()));
3412 : __ Bind(&if_twobyte);
3413 2180 : __ Goto(&done, jsgraph()->HeapConstant(factory()->cons_string_map()));
3414 : __ Bind(&done);
3415 : Node* result_map = done.PhiAt(0);
3416 :
3417 : // Allocate the resulting ConsString.
3418 2180 : return AllocateConsString(result_map, length, first, second);
3419 : }
3420 :
3421 4955 : Node* EffectControlLinearizer::AllocateConsString(Node* map, Node* length,
3422 : Node* first, Node* second) {
3423 : Node* result =
3424 4955 : __ Allocate(AllocationType::kYoung, __ IntPtrConstant(ConsString::kSize));
3425 4955 : __ StoreField(AccessBuilder::ForMap(), result, map);
3426 9910 : __ StoreField(AccessBuilder::ForNameHashField(), result,
3427 4955 : __ Int32Constant(Name::kEmptyHashField));
3428 4955 : __ StoreField(AccessBuilder::ForStringLength(), result, length);
3429 4955 : __ StoreField(AccessBuilder::ForConsStringFirst(), result, first);
3430 4955 : __ StoreField(AccessBuilder::ForConsStringSecond(), result, second);
3431 4955 : return result;
3432 : }
3433 :
3434 0 : Node* EffectControlLinearizer::LowerSameValue(Node* node) {
3435 : Node* lhs = node->InputAt(0);
3436 : Node* rhs = node->InputAt(1);
3437 :
3438 : Callable const callable =
3439 0 : Builtins::CallableFor(isolate(), Builtins::kSameValue);
3440 : Operator::Properties properties = Operator::kEliminatable;
3441 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3442 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3443 0 : graph()->zone(), callable.descriptor(),
3444 0 : callable.descriptor().GetStackParameterCount(), flags, properties);
3445 0 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3446 0 : __ NoContextConstant());
3447 : }
3448 :
3449 166 : Node* EffectControlLinearizer::LowerSameValueNumbersOnly(Node* node) {
3450 : Node* lhs = node->InputAt(0);
3451 : Node* rhs = node->InputAt(1);
3452 :
3453 : Callable const callable =
3454 166 : Builtins::CallableFor(isolate(), Builtins::kSameValueNumbersOnly);
3455 : Operator::Properties properties = Operator::kEliminatable;
3456 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3457 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3458 166 : graph()->zone(), callable.descriptor(),
3459 166 : callable.descriptor().GetStackParameterCount(), flags, properties);
3460 498 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3461 332 : __ NoContextConstant());
3462 : }
3463 :
3464 77 : Node* EffectControlLinearizer::LowerNumberSameValue(Node* node) {
3465 : Node* lhs = node->InputAt(0);
3466 : Node* rhs = node->InputAt(1);
3467 :
3468 77 : auto is_float64_equal = __ MakeLabel();
3469 154 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3470 :
3471 77 : __ GotoIf(__ Float64Equal(lhs, rhs), &is_float64_equal);
3472 :
3473 : // Return true iff both {lhs} and {rhs} are NaN.
3474 77 : __ GotoIf(__ Float64Equal(lhs, lhs), &done, __ Int32Constant(0));
3475 77 : __ GotoIf(__ Float64Equal(rhs, rhs), &done, __ Int32Constant(0));
3476 77 : __ Goto(&done, __ Int32Constant(1));
3477 :
3478 : __ Bind(&is_float64_equal);
3479 : // Even if the values are float64-equal, we still need to distinguish
3480 : // zero and minus zero.
3481 77 : Node* lhs_hi = __ Float64ExtractHighWord32(lhs);
3482 77 : Node* rhs_hi = __ Float64ExtractHighWord32(rhs);
3483 77 : __ Goto(&done, __ Word32Equal(lhs_hi, rhs_hi));
3484 :
3485 : __ Bind(&done);
3486 77 : return done.PhiAt(0);
3487 : }
3488 :
3489 1336 : Node* EffectControlLinearizer::LowerDeadValue(Node* node) {
3490 1336 : Node* input = NodeProperties::GetValueInput(node, 0);
3491 1336 : if (input->opcode() != IrOpcode::kUnreachable) {
3492 1006 : Node* unreachable = __ Unreachable();
3493 1006 : NodeProperties::ReplaceValueInput(node, unreachable, 0);
3494 : }
3495 1336 : return node;
3496 : }
3497 :
3498 330 : Node* EffectControlLinearizer::LowerStringToNumber(Node* node) {
3499 : Node* string = node->InputAt(0);
3500 :
3501 : Callable const callable =
3502 330 : Builtins::CallableFor(isolate(), Builtins::kStringToNumber);
3503 : Operator::Properties properties = Operator::kEliminatable;
3504 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3505 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3506 330 : graph()->zone(), callable.descriptor(),
3507 330 : callable.descriptor().GetStackParameterCount(), flags, properties);
3508 990 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), string,
3509 660 : __ NoContextConstant());
3510 : }
3511 :
3512 2236 : Node* EffectControlLinearizer::LowerStringCharCodeAt(Node* node) {
3513 : Node* receiver = node->InputAt(0);
3514 : Node* position = node->InputAt(1);
3515 :
3516 : // We need a loop here to properly deal with indirect strings
3517 : // (SlicedString, ConsString and ThinString).
3518 : auto loop = __ MakeLoopLabel(MachineRepresentation::kTagged,
3519 4472 : MachineType::PointerRepresentation());
3520 : auto loop_next = __ MakeLabel(MachineRepresentation::kTagged,
3521 4472 : MachineType::PointerRepresentation());
3522 4472 : auto loop_done = __ MakeLabel(MachineRepresentation::kWord32);
3523 : __ Goto(&loop, receiver, position);
3524 : __ Bind(&loop);
3525 : {
3526 : Node* receiver = loop.PhiAt(0);
3527 : Node* position = loop.PhiAt(1);
3528 2236 : Node* receiver_map = __ LoadField(AccessBuilder::ForMap(), receiver);
3529 : Node* receiver_instance_type =
3530 2236 : __ LoadField(AccessBuilder::ForMapInstanceType(), receiver_map);
3531 2236 : Node* receiver_representation = __ Word32And(
3532 2236 : receiver_instance_type, __ Int32Constant(kStringRepresentationMask));
3533 :
3534 : // Dispatch on the current {receiver}s string representation.
3535 2236 : auto if_seqstring = __ MakeLabel();
3536 2236 : auto if_consstring = __ MakeLabel();
3537 2236 : auto if_thinstring = __ MakeLabel();
3538 2236 : auto if_externalstring = __ MakeLabel();
3539 2236 : auto if_slicedstring = __ MakeLabel();
3540 2236 : auto if_runtime = __ MakeDeferredLabel();
3541 2236 : __ GotoIf(__ Word32Equal(receiver_representation,
3542 : __ Int32Constant(kSeqStringTag)),
3543 2236 : &if_seqstring);
3544 2236 : __ GotoIf(__ Word32Equal(receiver_representation,
3545 : __ Int32Constant(kConsStringTag)),
3546 2236 : &if_consstring);
3547 2236 : __ GotoIf(__ Word32Equal(receiver_representation,
3548 : __ Int32Constant(kThinStringTag)),
3549 2236 : &if_thinstring);
3550 2236 : __ GotoIf(__ Word32Equal(receiver_representation,
3551 : __ Int32Constant(kExternalStringTag)),
3552 2236 : &if_externalstring);
3553 2236 : __ Branch(__ Word32Equal(receiver_representation,
3554 : __ Int32Constant(kSlicedStringTag)),
3555 2236 : &if_slicedstring, &if_runtime);
3556 :
3557 : __ Bind(&if_seqstring);
3558 : {
3559 2236 : Node* receiver_is_onebyte = __ Word32Equal(
3560 : __ Word32Equal(__ Word32And(receiver_instance_type,
3561 : __ Int32Constant(kStringEncodingMask)),
3562 : __ Int32Constant(kTwoByteStringTag)),
3563 2236 : __ Int32Constant(0));
3564 2236 : Node* result = LoadFromSeqString(receiver, position, receiver_is_onebyte);
3565 : __ Goto(&loop_done, result);
3566 : }
3567 :
3568 : __ Bind(&if_thinstring);
3569 : {
3570 : Node* receiver_actual =
3571 2236 : __ LoadField(AccessBuilder::ForThinStringActual(), receiver);
3572 : __ Goto(&loop_next, receiver_actual, position);
3573 : }
3574 :
3575 : __ Bind(&if_consstring);
3576 : {
3577 : Node* receiver_second =
3578 2236 : __ LoadField(AccessBuilder::ForConsStringSecond(), receiver);
3579 2236 : __ GotoIfNot(__ WordEqual(receiver_second, __ EmptyStringConstant()),
3580 2236 : &if_runtime);
3581 : Node* receiver_first =
3582 2236 : __ LoadField(AccessBuilder::ForConsStringFirst(), receiver);
3583 : __ Goto(&loop_next, receiver_first, position);
3584 : }
3585 :
3586 : __ Bind(&if_externalstring);
3587 : {
3588 : // We need to bailout to the runtime for uncached external strings.
3589 2236 : __ GotoIf(__ Word32Equal(
3590 : __ Word32And(receiver_instance_type,
3591 : __ Int32Constant(kUncachedExternalStringMask)),
3592 : __ Int32Constant(kUncachedExternalStringTag)),
3593 2236 : &if_runtime);
3594 :
3595 : Node* receiver_data = __ LoadField(
3596 2236 : AccessBuilder::ForExternalStringResourceData(), receiver);
3597 :
3598 2236 : auto if_onebyte = __ MakeLabel();
3599 2236 : auto if_twobyte = __ MakeLabel();
3600 2236 : __ Branch(
3601 : __ Word32Equal(__ Word32And(receiver_instance_type,
3602 : __ Int32Constant(kStringEncodingMask)),
3603 : __ Int32Constant(kTwoByteStringTag)),
3604 2236 : &if_twobyte, &if_onebyte);
3605 :
3606 : __ Bind(&if_onebyte);
3607 : {
3608 2236 : Node* result = __ Load(MachineType::Uint8(), receiver_data, position);
3609 : __ Goto(&loop_done, result);
3610 : }
3611 :
3612 : __ Bind(&if_twobyte);
3613 : {
3614 2236 : Node* result = __ Load(MachineType::Uint16(), receiver_data,
3615 2236 : __ WordShl(position, __ IntPtrConstant(1)));
3616 : __ Goto(&loop_done, result);
3617 : }
3618 : }
3619 :
3620 : __ Bind(&if_slicedstring);
3621 : {
3622 : Node* receiver_offset =
3623 2236 : __ LoadField(AccessBuilder::ForSlicedStringOffset(), receiver);
3624 : Node* receiver_parent =
3625 2236 : __ LoadField(AccessBuilder::ForSlicedStringParent(), receiver);
3626 2236 : __ Goto(&loop_next, receiver_parent,
3627 : __ IntAdd(position, ChangeSmiToIntPtr(receiver_offset)));
3628 : }
3629 :
3630 : __ Bind(&if_runtime);
3631 : {
3632 2236 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3633 : Runtime::FunctionId id = Runtime::kStringCharCodeAt;
3634 2236 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3635 2236 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
3636 2236 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1),
3637 : receiver, ChangeIntPtrToSmi(position),
3638 : __ ExternalConstant(ExternalReference::Create(id)),
3639 2236 : __ Int32Constant(2), __ NoContextConstant());
3640 2236 : __ Goto(&loop_done, ChangeSmiToInt32(result));
3641 : }
3642 :
3643 : __ Bind(&loop_next);
3644 : __ Goto(&loop, loop_next.PhiAt(0), loop_next.PhiAt(1));
3645 : }
3646 : __ Bind(&loop_done);
3647 2236 : return loop_done.PhiAt(0);
3648 : }
3649 :
3650 223 : Node* EffectControlLinearizer::LowerStringCodePointAt(
3651 : Node* node, UnicodeEncoding encoding) {
3652 : Node* receiver = node->InputAt(0);
3653 : Node* position = node->InputAt(1);
3654 :
3655 : Builtins::Name builtin = encoding == UnicodeEncoding::UTF16
3656 : ? Builtins::kStringCodePointAtUTF16
3657 223 : : Builtins::kStringCodePointAtUTF32;
3658 :
3659 223 : Callable const callable = Builtins::CallableFor(isolate(), builtin);
3660 223 : Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite;
3661 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3662 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3663 223 : graph()->zone(), callable.descriptor(),
3664 223 : callable.descriptor().GetStackParameterCount(), flags, properties);
3665 669 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3666 446 : position, __ NoContextConstant());
3667 : }
3668 :
3669 2236 : Node* EffectControlLinearizer::LoadFromSeqString(Node* receiver, Node* position,
3670 : Node* is_one_byte) {
3671 2236 : auto one_byte_load = __ MakeLabel();
3672 4472 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
3673 2236 : __ GotoIf(is_one_byte, &one_byte_load);
3674 : Node* two_byte_result = __ LoadElement(
3675 2236 : AccessBuilder::ForSeqTwoByteStringCharacter(), receiver, position);
3676 : __ Goto(&done, two_byte_result);
3677 :
3678 : __ Bind(&one_byte_load);
3679 : Node* one_byte_element = __ LoadElement(
3680 2236 : AccessBuilder::ForSeqOneByteStringCharacter(), receiver, position);
3681 : __ Goto(&done, one_byte_element);
3682 :
3683 : __ Bind(&done);
3684 2236 : return done.PhiAt(0);
3685 : }
3686 :
3687 958 : Node* EffectControlLinearizer::LowerStringFromSingleCharCode(Node* node) {
3688 : Node* value = node->InputAt(0);
3689 958 : Node* code = __ Word32And(value, __ Uint32Constant(0xFFFF));
3690 :
3691 958 : auto if_not_one_byte = __ MakeDeferredLabel();
3692 958 : auto cache_miss = __ MakeDeferredLabel();
3693 1916 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3694 :
3695 : // Check if the {code} is a one byte character
3696 958 : Node* check1 = __ Uint32LessThanOrEqual(
3697 958 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3698 958 : __ GotoIfNot(check1, &if_not_one_byte);
3699 : {
3700 : // Load the isolate wide single character string cache.
3701 958 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3702 :
3703 : // Compute the {cache} index for {code}.
3704 958 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3705 :
3706 : // Check if we have an entry for the {code} in the single character string
3707 : // cache already.
3708 : Node* entry =
3709 958 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3710 :
3711 958 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3712 958 : __ GotoIf(check2, &cache_miss);
3713 :
3714 : // Use the {entry} from the {cache}.
3715 : __ Goto(&done, entry);
3716 :
3717 : __ Bind(&cache_miss);
3718 : {
3719 : // Allocate a new SeqOneByteString for {code}.
3720 : Node* vtrue2 =
3721 958 : __ Allocate(AllocationType::kYoung,
3722 958 : __ IntPtrConstant(SeqOneByteString::SizeFor(1)));
3723 1916 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3724 958 : __ HeapConstant(factory()->one_byte_string_map()));
3725 1916 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3726 958 : __ Int32Constant(Name::kEmptyHashField));
3727 1916 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3728 958 : __ Int32Constant(1));
3729 1916 : __ Store(
3730 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3731 : vtrue2,
3732 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3733 958 : code);
3734 :
3735 : // Remember it in the {cache}.
3736 1916 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3737 958 : vtrue2);
3738 : __ Goto(&done, vtrue2);
3739 : }
3740 : }
3741 :
3742 : __ Bind(&if_not_one_byte);
3743 : {
3744 : // Allocate a new SeqTwoByteString for {code}.
3745 : Node* vfalse1 =
3746 958 : __ Allocate(AllocationType::kYoung,
3747 958 : __ IntPtrConstant(SeqTwoByteString::SizeFor(1)));
3748 1916 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3749 958 : __ HeapConstant(factory()->string_map()));
3750 1916 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3751 958 : __ Int32Constant(Name::kEmptyHashField));
3752 1916 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3753 958 : __ Int32Constant(1));
3754 1916 : __ Store(
3755 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3756 : vfalse1,
3757 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3758 958 : code);
3759 : __ Goto(&done, vfalse1);
3760 : }
3761 :
3762 : __ Bind(&done);
3763 958 : return done.PhiAt(0);
3764 : }
3765 :
3766 : #ifdef V8_INTL_SUPPORT
3767 :
3768 105 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3769 : Node* receiver = node->InputAt(0);
3770 :
3771 : Callable callable =
3772 105 : Builtins::CallableFor(isolate(), Builtins::kStringToLowerCaseIntl);
3773 105 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3774 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3775 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3776 105 : graph()->zone(), callable.descriptor(),
3777 105 : callable.descriptor().GetStackParameterCount(), flags, properties);
3778 315 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3779 210 : __ NoContextConstant());
3780 : }
3781 :
3782 43 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3783 : Node* receiver = node->InputAt(0);
3784 43 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3785 : Runtime::FunctionId id = Runtime::kStringToUpperCaseIntl;
3786 43 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3787 43 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
3788 43 : return __ Call(call_descriptor, __ CEntryStubConstant(1), receiver,
3789 : __ ExternalConstant(ExternalReference::Create(id)),
3790 43 : __ Int32Constant(1), __ NoContextConstant());
3791 : }
3792 :
3793 : #else
3794 :
3795 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3796 : UNREACHABLE();
3797 : return nullptr;
3798 : }
3799 :
3800 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3801 : UNREACHABLE();
3802 : return nullptr;
3803 : }
3804 :
3805 : #endif // V8_INTL_SUPPORT
3806 :
3807 207 : Node* EffectControlLinearizer::LowerStringFromSingleCodePoint(Node* node) {
3808 : Node* value = node->InputAt(0);
3809 : Node* code = value;
3810 :
3811 207 : auto if_not_single_code = __ MakeDeferredLabel();
3812 207 : auto if_not_one_byte = __ MakeDeferredLabel();
3813 207 : auto cache_miss = __ MakeDeferredLabel();
3814 414 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3815 :
3816 : // Check if the {code} is a single code unit
3817 207 : Node* check0 = __ Uint32LessThanOrEqual(code, __ Uint32Constant(0xFFFF));
3818 207 : __ GotoIfNot(check0, &if_not_single_code);
3819 :
3820 : {
3821 : // Check if the {code} is a one byte character
3822 207 : Node* check1 = __ Uint32LessThanOrEqual(
3823 207 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3824 207 : __ GotoIfNot(check1, &if_not_one_byte);
3825 : {
3826 : // Load the isolate wide single character string cache.
3827 207 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3828 :
3829 : // Compute the {cache} index for {code}.
3830 207 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3831 :
3832 : // Check if we have an entry for the {code} in the single character string
3833 : // cache already.
3834 : Node* entry =
3835 207 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3836 :
3837 207 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3838 207 : __ GotoIf(check2, &cache_miss);
3839 :
3840 : // Use the {entry} from the {cache}.
3841 : __ Goto(&done, entry);
3842 :
3843 : __ Bind(&cache_miss);
3844 : {
3845 : // Allocate a new SeqOneByteString for {code}.
3846 : Node* vtrue2 =
3847 207 : __ Allocate(AllocationType::kYoung,
3848 207 : __ IntPtrConstant(SeqOneByteString::SizeFor(1)));
3849 414 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3850 207 : __ HeapConstant(factory()->one_byte_string_map()));
3851 414 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3852 207 : __ Int32Constant(Name::kEmptyHashField));
3853 414 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3854 207 : __ Int32Constant(1));
3855 414 : __ Store(
3856 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3857 : vtrue2,
3858 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3859 207 : code);
3860 :
3861 : // Remember it in the {cache}.
3862 414 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3863 207 : vtrue2);
3864 : __ Goto(&done, vtrue2);
3865 : }
3866 : }
3867 :
3868 : __ Bind(&if_not_one_byte);
3869 : {
3870 : // Allocate a new SeqTwoByteString for {code}.
3871 : Node* vfalse1 =
3872 207 : __ Allocate(AllocationType::kYoung,
3873 207 : __ IntPtrConstant(SeqTwoByteString::SizeFor(1)));
3874 414 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3875 207 : __ HeapConstant(factory()->string_map()));
3876 414 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3877 207 : __ IntPtrConstant(Name::kEmptyHashField));
3878 414 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3879 207 : __ Int32Constant(1));
3880 414 : __ Store(
3881 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3882 : vfalse1,
3883 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3884 207 : code);
3885 : __ Goto(&done, vfalse1);
3886 : }
3887 : }
3888 :
3889 : __ Bind(&if_not_single_code);
3890 : // Generate surrogate pair string
3891 : {
3892 207 : switch (UnicodeEncodingOf(node->op())) {
3893 : case UnicodeEncoding::UTF16:
3894 : break;
3895 :
3896 : case UnicodeEncoding::UTF32: {
3897 : // Convert UTF32 to UTF16 code units, and store as a 32 bit word.
3898 92 : Node* lead_offset = __ Int32Constant(0xD800 - (0x10000 >> 10));
3899 :
3900 : // lead = (codepoint >> 10) + LEAD_OFFSET
3901 : Node* lead =
3902 92 : __ Int32Add(__ Word32Shr(code, __ Int32Constant(10)), lead_offset);
3903 :
3904 : // trail = (codepoint & 0x3FF) + 0xDC00;
3905 92 : Node* trail = __ Int32Add(__ Word32And(code, __ Int32Constant(0x3FF)),
3906 92 : __ Int32Constant(0xDC00));
3907 :
3908 : // codpoint = (trail << 16) | lead;
3909 : #if V8_TARGET_BIG_ENDIAN
3910 : code = __ Word32Or(__ Word32Shl(lead, __ Int32Constant(16)), trail);
3911 : #else
3912 92 : code = __ Word32Or(__ Word32Shl(trail, __ Int32Constant(16)), lead);
3913 : #endif
3914 92 : break;
3915 : }
3916 : }
3917 :
3918 : // Allocate a new SeqTwoByteString for {code}.
3919 : Node* vfalse0 =
3920 207 : __ Allocate(AllocationType::kYoung,
3921 207 : __ IntPtrConstant(SeqTwoByteString::SizeFor(2)));
3922 414 : __ StoreField(AccessBuilder::ForMap(), vfalse0,
3923 207 : __ HeapConstant(factory()->string_map()));
3924 414 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse0,
3925 207 : __ Int32Constant(Name::kEmptyHashField));
3926 414 : __ StoreField(AccessBuilder::ForStringLength(), vfalse0,
3927 207 : __ Int32Constant(2));
3928 414 : __ Store(
3929 : StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier),
3930 : vfalse0,
3931 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3932 207 : code);
3933 : __ Goto(&done, vfalse0);
3934 : }
3935 :
3936 : __ Bind(&done);
3937 207 : return done.PhiAt(0);
3938 : }
3939 :
3940 266 : Node* EffectControlLinearizer::LowerStringIndexOf(Node* node) {
3941 : Node* subject = node->InputAt(0);
3942 : Node* search_string = node->InputAt(1);
3943 : Node* position = node->InputAt(2);
3944 :
3945 : Callable callable =
3946 266 : Builtins::CallableFor(isolate(), Builtins::kStringIndexOf);
3947 : Operator::Properties properties = Operator::kEliminatable;
3948 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3949 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3950 266 : graph()->zone(), callable.descriptor(),
3951 266 : callable.descriptor().GetStackParameterCount(), flags, properties);
3952 798 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), subject,
3953 532 : search_string, position, __ NoContextConstant());
3954 : }
3955 :
3956 23488 : Node* EffectControlLinearizer::LowerStringLength(Node* node) {
3957 : Node* subject = node->InputAt(0);
3958 :
3959 46976 : return __ LoadField(AccessBuilder::ForStringLength(), subject);
3960 : }
3961 :
3962 11227 : Node* EffectControlLinearizer::LowerStringComparison(Callable const& callable,
3963 : Node* node) {
3964 : Node* lhs = node->InputAt(0);
3965 : Node* rhs = node->InputAt(1);
3966 :
3967 : Operator::Properties properties = Operator::kEliminatable;
3968 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3969 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3970 11227 : graph()->zone(), callable.descriptor(),
3971 11227 : callable.descriptor().GetStackParameterCount(), flags, properties);
3972 33681 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3973 22454 : __ NoContextConstant());
3974 : }
3975 :
3976 1100 : Node* EffectControlLinearizer::LowerStringSubstring(Node* node) {
3977 : Node* receiver = node->InputAt(0);
3978 : Node* start = ChangeInt32ToIntPtr(node->InputAt(1));
3979 : Node* end = ChangeInt32ToIntPtr(node->InputAt(2));
3980 :
3981 : Callable callable =
3982 1100 : Builtins::CallableFor(isolate(), Builtins::kStringSubstring);
3983 : Operator::Properties properties = Operator::kEliminatable;
3984 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3985 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3986 1100 : graph()->zone(), callable.descriptor(),
3987 1100 : callable.descriptor().GetStackParameterCount(), flags, properties);
3988 3300 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3989 2200 : start, end, __ NoContextConstant());
3990 : }
3991 :
3992 10405 : Node* EffectControlLinearizer::LowerStringEqual(Node* node) {
3993 : return LowerStringComparison(
3994 20810 : Builtins::CallableFor(isolate(), Builtins::kStringEqual), node);
3995 : }
3996 :
3997 121 : Node* EffectControlLinearizer::LowerStringLessThan(Node* node) {
3998 : return LowerStringComparison(
3999 242 : Builtins::CallableFor(isolate(), Builtins::kStringLessThan), node);
4000 : }
4001 :
4002 701 : Node* EffectControlLinearizer::LowerStringLessThanOrEqual(Node* node) {
4003 : return LowerStringComparison(
4004 1402 : Builtins::CallableFor(isolate(), Builtins::kStringLessThanOrEqual), node);
4005 : }
4006 :
4007 474 : Node* EffectControlLinearizer::LowerCheckFloat64Hole(Node* node,
4008 : Node* frame_state) {
4009 : // If we reach this point w/o eliminating the {node} that's marked
4010 : // with allow-return-hole, we cannot do anything, so just deoptimize
4011 : // in case of the hole NaN.
4012 : CheckFloat64HoleParameters const& params =
4013 474 : CheckFloat64HoleParametersOf(node->op());
4014 : Node* value = node->InputAt(0);
4015 :
4016 474 : auto if_nan = __ MakeDeferredLabel();
4017 474 : auto done = __ MakeLabel();
4018 :
4019 : // First check whether {value} is a NaN at all...
4020 474 : __ Branch(__ Float64Equal(value, value), &done, &if_nan);
4021 :
4022 : __ Bind(&if_nan);
4023 : {
4024 : // ...and only if {value} is a NaN, perform the expensive bit
4025 : // check. See http://crbug.com/v8/8264 for details.
4026 474 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
4027 474 : __ Int32Constant(kHoleNanUpper32));
4028 : __ DeoptimizeIf(DeoptimizeReason::kHole, params.feedback(), check,
4029 474 : frame_state);
4030 : __ Goto(&done);
4031 : }
4032 :
4033 : __ Bind(&done);
4034 474 : return value;
4035 : }
4036 :
4037 102 : Node* EffectControlLinearizer::LowerCheckNotTaggedHole(Node* node,
4038 : Node* frame_state) {
4039 : Node* value = node->InputAt(0);
4040 102 : Node* check = __ WordEqual(value, __ TheHoleConstant());
4041 204 : __ DeoptimizeIf(DeoptimizeReason::kHole, VectorSlotPair(), check,
4042 102 : frame_state);
4043 102 : return value;
4044 : }
4045 :
4046 1681 : Node* EffectControlLinearizer::LowerConvertTaggedHoleToUndefined(Node* node) {
4047 : Node* value = node->InputAt(0);
4048 :
4049 1681 : auto if_is_hole = __ MakeDeferredLabel();
4050 3362 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4051 :
4052 1681 : Node* check = __ WordEqual(value, __ TheHoleConstant());
4053 1681 : __ GotoIf(check, &if_is_hole);
4054 : __ Goto(&done, value);
4055 :
4056 : __ Bind(&if_is_hole);
4057 1681 : __ Goto(&done, __ UndefinedConstant());
4058 :
4059 : __ Bind(&done);
4060 1681 : return done.PhiAt(0);
4061 : }
4062 :
4063 109 : void EffectControlLinearizer::LowerCheckEqualsInternalizedString(
4064 : Node* node, Node* frame_state) {
4065 : Node* exp = node->InputAt(0);
4066 : Node* val = node->InputAt(1);
4067 :
4068 109 : auto if_same = __ MakeLabel();
4069 109 : auto if_notsame = __ MakeDeferredLabel();
4070 109 : auto if_thinstring = __ MakeLabel();
4071 109 : auto if_notthinstring = __ MakeLabel();
4072 :
4073 : // Check if {exp} and {val} are the same, which is the likely case.
4074 109 : __ Branch(__ WordEqual(exp, val), &if_same, &if_notsame);
4075 :
4076 : __ Bind(&if_notsame);
4077 : {
4078 : // Now {val} could still be a non-internalized String that matches {exp}.
4079 218 : __ DeoptimizeIf(DeoptimizeReason::kWrongName, VectorSlotPair(),
4080 109 : ObjectIsSmi(val), frame_state);
4081 109 : Node* val_map = __ LoadField(AccessBuilder::ForMap(), val);
4082 : Node* val_instance_type =
4083 109 : __ LoadField(AccessBuilder::ForMapInstanceType(), val_map);
4084 :
4085 : // Check for the common case of ThinString first.
4086 109 : __ GotoIf(__ Word32Equal(val_instance_type,
4087 : __ Int32Constant(THIN_ONE_BYTE_STRING_TYPE)),
4088 109 : &if_thinstring);
4089 109 : __ Branch(
4090 : __ Word32Equal(val_instance_type, __ Int32Constant(THIN_STRING_TYPE)),
4091 109 : &if_thinstring, &if_notthinstring);
4092 :
4093 : __ Bind(&if_notthinstring);
4094 : {
4095 : // Check that the {val} is a non-internalized String, if it's anything
4096 : // else it cannot match the recorded feedback {exp} anyways.
4097 218 : __ DeoptimizeIfNot(
4098 : DeoptimizeReason::kWrongName, VectorSlotPair(),
4099 : __ Word32Equal(__ Word32And(val_instance_type,
4100 : __ Int32Constant(kIsNotStringMask |
4101 : kIsNotInternalizedMask)),
4102 : __ Int32Constant(kStringTag | kNotInternalizedTag)),
4103 109 : frame_state);
4104 :
4105 : // Try to find the {val} in the string table.
4106 : MachineSignature::Builder builder(graph()->zone(), 1, 2);
4107 : builder.AddReturn(MachineType::AnyTagged());
4108 : builder.AddParam(MachineType::Pointer());
4109 : builder.AddParam(MachineType::AnyTagged());
4110 109 : Node* try_internalize_string_function = __ ExternalConstant(
4111 109 : ExternalReference::try_internalize_string_function());
4112 : Node* const isolate_ptr =
4113 109 : __ ExternalConstant(ExternalReference::isolate_address(isolate()));
4114 : auto call_descriptor =
4115 109 : Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build());
4116 : Node* val_internalized =
4117 109 : __ Call(common()->Call(call_descriptor),
4118 109 : try_internalize_string_function, isolate_ptr, val);
4119 :
4120 : // Now see if the results match.
4121 218 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
4122 109 : __ WordEqual(exp, val_internalized), frame_state);
4123 : __ Goto(&if_same);
4124 : }
4125 :
4126 : __ Bind(&if_thinstring);
4127 : {
4128 : // The {val} is a ThinString, let's check the actual value.
4129 : Node* val_actual =
4130 109 : __ LoadField(AccessBuilder::ForThinStringActual(), val);
4131 218 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
4132 109 : __ WordEqual(exp, val_actual), frame_state);
4133 : __ Goto(&if_same);
4134 : }
4135 : }
4136 :
4137 : __ Bind(&if_same);
4138 109 : }
4139 :
4140 18 : void EffectControlLinearizer::LowerCheckEqualsSymbol(Node* node,
4141 : Node* frame_state) {
4142 : Node* exp = node->InputAt(0);
4143 : Node* val = node->InputAt(1);
4144 18 : Node* check = __ WordEqual(exp, val);
4145 36 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(), check,
4146 18 : frame_state);
4147 18 : }
4148 :
4149 54830 : Node* EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value) {
4150 : Node* result =
4151 54830 : __ Allocate(AllocationType::kYoung, __ IntPtrConstant(HeapNumber::kSize));
4152 54830 : __ StoreField(AccessBuilder::ForMap(), result, __ HeapNumberMapConstant());
4153 54831 : __ StoreField(AccessBuilder::ForHeapNumberValue(), result, value);
4154 54831 : return result;
4155 : }
4156 :
4157 135470 : Node* EffectControlLinearizer::ChangeIntPtrToSmi(Node* value) {
4158 : // Do shift on 32bit values if Smis are stored in the lower word.
4159 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4160 : return __ ChangeInt32ToInt64(
4161 : __ Word32Shl(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
4162 : }
4163 135470 : return __ WordShl(value, SmiShiftBitsConstant());
4164 : }
4165 :
4166 0 : Node* EffectControlLinearizer::ChangeInt32ToIntPtr(Node* value) {
4167 134685 : if (machine()->Is64()) {
4168 134685 : value = __ ChangeInt32ToInt64(value);
4169 : }
4170 0 : return value;
4171 : }
4172 :
4173 0 : Node* EffectControlLinearizer::ChangeIntPtrToInt32(Node* value) {
4174 0 : if (machine()->Is64()) {
4175 0 : value = __ TruncateInt64ToInt32(value);
4176 : }
4177 0 : return value;
4178 : }
4179 :
4180 132485 : Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) {
4181 : // Do shift on 32bit values if Smis are stored in the lower word.
4182 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4183 : return __ ChangeInt32ToInt64(__ Word32Shl(value, SmiShiftBitsConstant()));
4184 : }
4185 132486 : return ChangeIntPtrToSmi(ChangeInt32ToIntPtr(value));
4186 : }
4187 :
4188 0 : Node* EffectControlLinearizer::ChangeInt64ToSmi(Node* value) {
4189 : DCHECK(machine()->Is64());
4190 249 : return ChangeIntPtrToSmi(value);
4191 : }
4192 :
4193 0 : Node* EffectControlLinearizer::ChangeUint32ToUintPtr(Node* value) {
4194 675 : if (machine()->Is64()) {
4195 675 : value = __ ChangeUint32ToUint64(value);
4196 : }
4197 0 : return value;
4198 : }
4199 :
4200 658 : Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) {
4201 : // Do shift on 32bit values if Smis are stored in the lower word.
4202 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4203 : return __ ChangeUint32ToUint64(__ Word32Shl(value, SmiShiftBitsConstant()));
4204 : } else {
4205 658 : return __ WordShl(ChangeUint32ToUintPtr(value), SmiShiftBitsConstant());
4206 : }
4207 : }
4208 :
4209 150820 : Node* EffectControlLinearizer::ChangeSmiToIntPtr(Node* value) {
4210 : // Do shift on 32bit values if Smis are stored in the lower word.
4211 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4212 : return __ ChangeInt32ToInt64(
4213 : __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
4214 : }
4215 150821 : return __ WordSar(value, SmiShiftBitsConstant());
4216 : }
4217 :
4218 148147 : Node* EffectControlLinearizer::ChangeSmiToInt32(Node* value) {
4219 : // Do shift on 32bit values if Smis are stored in the lower word.
4220 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4221 : return __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant());
4222 : }
4223 148147 : if (machine()->Is64()) {
4224 296295 : return __ TruncateInt64ToInt32(ChangeSmiToIntPtr(value));
4225 : }
4226 0 : return ChangeSmiToIntPtr(value);
4227 : }
4228 :
4229 386 : Node* EffectControlLinearizer::ChangeSmiToInt64(Node* value) {
4230 386 : CHECK(machine()->Is64());
4231 386 : return ChangeSmiToIntPtr(value);
4232 : }
4233 :
4234 256750 : Node* EffectControlLinearizer::ObjectIsSmi(Node* value) {
4235 256750 : return __ WordEqual(__ WordAnd(value, __ IntPtrConstant(kSmiTagMask)),
4236 256750 : __ IntPtrConstant(kSmiTag));
4237 : }
4238 :
4239 0 : Node* EffectControlLinearizer::SmiMaxValueConstant() {
4240 658 : return __ Int32Constant(Smi::kMaxValue);
4241 : }
4242 :
4243 0 : Node* EffectControlLinearizer::SmiShiftBitsConstant() {
4244 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4245 : return __ Int32Constant(kSmiShiftSize + kSmiTagSize);
4246 : }
4247 286948 : return __ IntPtrConstant(kSmiShiftSize + kSmiTagSize);
4248 : }
4249 :
4250 20 : Node* EffectControlLinearizer::LowerPlainPrimitiveToNumber(Node* node) {
4251 : Node* value = node->InputAt(0);
4252 20 : return __ ToNumber(value);
4253 : }
4254 :
4255 9 : Node* EffectControlLinearizer::LowerPlainPrimitiveToWord32(Node* node) {
4256 : Node* value = node->InputAt(0);
4257 :
4258 9 : auto if_not_smi = __ MakeDeferredLabel();
4259 9 : auto if_to_number_smi = __ MakeLabel();
4260 18 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
4261 :
4262 9 : Node* check0 = ObjectIsSmi(value);
4263 9 : __ GotoIfNot(check0, &if_not_smi);
4264 9 : __ Goto(&done, ChangeSmiToInt32(value));
4265 :
4266 : __ Bind(&if_not_smi);
4267 9 : Node* to_number = __ ToNumber(value);
4268 :
4269 9 : Node* check1 = ObjectIsSmi(to_number);
4270 9 : __ GotoIf(check1, &if_to_number_smi);
4271 9 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
4272 9 : __ Goto(&done, __ TruncateFloat64ToWord32(number));
4273 :
4274 : __ Bind(&if_to_number_smi);
4275 9 : __ Goto(&done, ChangeSmiToInt32(to_number));
4276 :
4277 : __ Bind(&done);
4278 9 : return done.PhiAt(0);
4279 : }
4280 :
4281 0 : Node* EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node) {
4282 : Node* value = node->InputAt(0);
4283 :
4284 0 : auto if_not_smi = __ MakeDeferredLabel();
4285 0 : auto if_to_number_smi = __ MakeLabel();
4286 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
4287 :
4288 0 : Node* check0 = ObjectIsSmi(value);
4289 0 : __ GotoIfNot(check0, &if_not_smi);
4290 0 : Node* from_smi = ChangeSmiToInt32(value);
4291 0 : __ Goto(&done, __ ChangeInt32ToFloat64(from_smi));
4292 :
4293 : __ Bind(&if_not_smi);
4294 0 : Node* to_number = __ ToNumber(value);
4295 0 : Node* check1 = ObjectIsSmi(to_number);
4296 0 : __ GotoIf(check1, &if_to_number_smi);
4297 :
4298 0 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
4299 : __ Goto(&done, number);
4300 :
4301 : __ Bind(&if_to_number_smi);
4302 0 : Node* number_from_smi = ChangeSmiToInt32(to_number);
4303 0 : number_from_smi = __ ChangeInt32ToFloat64(number_from_smi);
4304 : __ Goto(&done, number_from_smi);
4305 :
4306 : __ Bind(&done);
4307 0 : return done.PhiAt(0);
4308 : }
4309 :
4310 1686 : Node* EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node) {
4311 : Node* object = node->InputAt(0);
4312 : Node* elements = node->InputAt(1);
4313 :
4314 1686 : auto if_not_fixed_array = __ MakeDeferredLabel();
4315 3372 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4316 :
4317 : // Load the current map of {elements}.
4318 3372 : Node* elements_map = __ LoadField(AccessBuilder::ForMap(), elements);
4319 :
4320 : // Check if {elements} is not a copy-on-write FixedArray.
4321 1686 : Node* check = __ WordEqual(elements_map, __ FixedArrayMapConstant());
4322 1686 : __ GotoIfNot(check, &if_not_fixed_array);
4323 : // Nothing to do if the {elements} are not copy-on-write.
4324 : __ Goto(&done, elements);
4325 :
4326 : __ Bind(&if_not_fixed_array);
4327 : // We need to take a copy of the {elements} and set them up for {object}.
4328 : Operator::Properties properties = Operator::kEliminatable;
4329 : Callable callable =
4330 1686 : Builtins::CallableFor(isolate(), Builtins::kCopyFastSmiOrObjectElements);
4331 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4332 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4333 1686 : graph()->zone(), callable.descriptor(),
4334 1686 : callable.descriptor().GetStackParameterCount(), flags, properties);
4335 5058 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4336 1686 : object, __ NoContextConstant());
4337 : __ Goto(&done, result);
4338 :
4339 : __ Bind(&done);
4340 1686 : return done.PhiAt(0);
4341 : }
4342 :
4343 3527 : Node* EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node,
4344 : Node* frame_state) {
4345 3527 : GrowFastElementsParameters params = GrowFastElementsParametersOf(node->op());
4346 : Node* object = node->InputAt(0);
4347 : Node* elements = node->InputAt(1);
4348 : Node* index = node->InputAt(2);
4349 : Node* elements_length = node->InputAt(3);
4350 :
4351 7054 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4352 3527 : auto if_grow = __ MakeDeferredLabel();
4353 : auto if_not_grow = __ MakeLabel();
4354 :
4355 : // Check if we need to grow the {elements} backing store.
4356 3527 : Node* check = __ Uint32LessThan(index, elements_length);
4357 3527 : __ GotoIfNot(check, &if_grow);
4358 : __ Goto(&done, elements);
4359 :
4360 : __ Bind(&if_grow);
4361 : // We need to grow the {elements} for {object}.
4362 : Operator::Properties properties = Operator::kEliminatable;
4363 : Callable callable =
4364 : (params.mode() == GrowFastElementsMode::kDoubleElements)
4365 : ? Builtins::CallableFor(isolate(), Builtins::kGrowFastDoubleElements)
4366 : : Builtins::CallableFor(isolate(),
4367 3527 : Builtins::kGrowFastSmiOrObjectElements);
4368 : CallDescriptor::Flags call_flags = CallDescriptor::kNoFlags;
4369 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4370 3527 : graph()->zone(), callable.descriptor(),
4371 3527 : callable.descriptor().GetStackParameterCount(), call_flags, properties);
4372 : Node* new_elements =
4373 10581 : __ Call(call_descriptor, __ HeapConstant(callable.code()), object,
4374 3527 : ChangeInt32ToSmi(index), __ NoContextConstant());
4375 :
4376 : // Ensure that we were able to grow the {elements}.
4377 3527 : __ DeoptimizeIf(DeoptimizeReason::kCouldNotGrowElements, params.feedback(),
4378 3527 : ObjectIsSmi(new_elements), frame_state);
4379 : __ Goto(&done, new_elements);
4380 :
4381 : __ Bind(&done);
4382 3527 : return done.PhiAt(0);
4383 : }
4384 :
4385 644 : void EffectControlLinearizer::LowerTransitionElementsKind(Node* node) {
4386 644 : ElementsTransition const transition = ElementsTransitionOf(node->op());
4387 : Node* object = node->InputAt(0);
4388 :
4389 644 : auto if_map_same = __ MakeDeferredLabel();
4390 644 : auto done = __ MakeLabel();
4391 :
4392 644 : Node* source_map = __ HeapConstant(transition.source());
4393 644 : Node* target_map = __ HeapConstant(transition.target());
4394 :
4395 : // Load the current map of {object}.
4396 644 : Node* object_map = __ LoadField(AccessBuilder::ForMap(), object);
4397 :
4398 : // Check if {object_map} is the same as {source_map}.
4399 644 : Node* check = __ WordEqual(object_map, source_map);
4400 644 : __ GotoIf(check, &if_map_same);
4401 : __ Goto(&done);
4402 :
4403 : __ Bind(&if_map_same);
4404 644 : switch (transition.mode()) {
4405 : case ElementsTransition::kFastTransition:
4406 : // In-place migration of {object}, just store the {target_map}.
4407 234 : __ StoreField(AccessBuilder::ForMap(), object, target_map);
4408 234 : break;
4409 : case ElementsTransition::kSlowTransition: {
4410 : // Instance migration, call out to the runtime for {object}.
4411 410 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4412 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4413 410 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4414 410 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4415 410 : __ Call(call_descriptor, __ CEntryStubConstant(1), object, target_map,
4416 : __ ExternalConstant(ExternalReference::Create(id)),
4417 410 : __ Int32Constant(2), __ NoContextConstant());
4418 : break;
4419 : }
4420 : }
4421 : __ Goto(&done);
4422 :
4423 : __ Bind(&done);
4424 644 : }
4425 :
4426 7226 : Node* EffectControlLinearizer::LowerLoadMessage(Node* node) {
4427 : Node* offset = node->InputAt(0);
4428 : Node* object_pattern =
4429 14452 : __ LoadField(AccessBuilder::ForExternalIntPtr(), offset);
4430 7226 : return __ BitcastWordToTagged(object_pattern);
4431 : }
4432 :
4433 29747 : void EffectControlLinearizer::LowerStoreMessage(Node* node) {
4434 : Node* offset = node->InputAt(0);
4435 : Node* object = node->InputAt(1);
4436 29747 : Node* object_pattern = __ BitcastTaggedToWord(object);
4437 29747 : __ StoreField(AccessBuilder::ForExternalIntPtr(), offset, object_pattern);
4438 29747 : }
4439 :
4440 582 : Node* EffectControlLinearizer::LowerLoadFieldByIndex(Node* node) {
4441 : Node* object = node->InputAt(0);
4442 : Node* index = node->InputAt(1);
4443 582 : Node* zero = __ IntPtrConstant(0);
4444 582 : Node* one = __ IntPtrConstant(1);
4445 :
4446 : // Sign-extend the {index} on 64-bit architectures.
4447 582 : if (machine()->Is64()) {
4448 582 : index = __ ChangeInt32ToInt64(index);
4449 : }
4450 :
4451 582 : auto if_double = __ MakeDeferredLabel();
4452 1164 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4453 :
4454 : // Check if field is a mutable double field.
4455 582 : __ GotoIfNot(__ WordEqual(__ WordAnd(index, one), zero), &if_double);
4456 :
4457 : // The field is a proper Tagged field on {object}. The {index} is shifted
4458 : // to the left by one in the code below.
4459 : {
4460 : // Check if field is in-object or out-of-object.
4461 582 : auto if_outofobject = __ MakeLabel();
4462 582 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4463 :
4464 : // The field is located in the {object} itself.
4465 : {
4466 : Node* offset =
4467 582 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4468 582 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4469 582 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4470 : __ Goto(&done, result);
4471 : }
4472 :
4473 : // The field is located in the properties backing store of {object}.
4474 : // The {index} is equal to the negated out of property index plus 1.
4475 : __ Bind(&if_outofobject);
4476 : {
4477 : Node* properties =
4478 582 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4479 : Node* offset =
4480 582 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4481 : __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4482 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4483 582 : kHeapObjectTag));
4484 582 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4485 : __ Goto(&done, result);
4486 : }
4487 : }
4488 :
4489 : // The field is a Double field, either unboxed in the object on 64-bit
4490 : // architectures, or as MutableHeapNumber.
4491 : __ Bind(&if_double);
4492 : {
4493 1164 : auto done_double = __ MakeLabel(MachineRepresentation::kFloat64);
4494 :
4495 582 : index = __ WordSar(index, one);
4496 :
4497 : // Check if field is in-object or out-of-object.
4498 582 : auto if_outofobject = __ MakeLabel();
4499 582 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4500 :
4501 : // The field is located in the {object} itself.
4502 : {
4503 : Node* offset =
4504 582 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)),
4505 582 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4506 : if (FLAG_unbox_double_fields) {
4507 582 : Node* result = __ Load(MachineType::Float64(), object, offset);
4508 : __ Goto(&done_double, result);
4509 : } else {
4510 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4511 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4512 : __ Goto(&done_double, result);
4513 : }
4514 : }
4515 :
4516 : __ Bind(&if_outofobject);
4517 : {
4518 : Node* properties =
4519 582 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4520 : Node* offset =
4521 582 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4522 : __ IntPtrConstant(kTaggedSizeLog2)),
4523 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4524 582 : kHeapObjectTag));
4525 582 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4526 582 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4527 : __ Goto(&done_double, result);
4528 : }
4529 :
4530 : __ Bind(&done_double);
4531 : {
4532 582 : Node* result = AllocateHeapNumberWithValue(done_double.PhiAt(0));
4533 : __ Goto(&done, result);
4534 : }
4535 : }
4536 :
4537 : __ Bind(&done);
4538 582 : return done.PhiAt(0);
4539 : }
4540 :
4541 464 : Node* EffectControlLinearizer::BuildReverseBytes(ExternalArrayType type,
4542 : Node* value) {
4543 464 : switch (type) {
4544 : case kExternalInt8Array:
4545 : case kExternalUint8Array:
4546 : case kExternalUint8ClampedArray:
4547 : return value;
4548 :
4549 : case kExternalInt16Array: {
4550 55 : Node* result = __ Word32ReverseBytes(value);
4551 55 : result = __ Word32Sar(result, __ Int32Constant(16));
4552 55 : return result;
4553 : }
4554 :
4555 : case kExternalUint16Array: {
4556 55 : Node* result = __ Word32ReverseBytes(value);
4557 55 : result = __ Word32Shr(result, __ Int32Constant(16));
4558 55 : return result;
4559 : }
4560 :
4561 : case kExternalInt32Array: // Fall through.
4562 : case kExternalUint32Array:
4563 96 : return __ Word32ReverseBytes(value);
4564 :
4565 : case kExternalFloat32Array: {
4566 56 : Node* result = __ BitcastFloat32ToInt32(value);
4567 56 : result = __ Word32ReverseBytes(result);
4568 56 : result = __ BitcastInt32ToFloat32(result);
4569 56 : return result;
4570 : }
4571 :
4572 : case kExternalFloat64Array: {
4573 56 : if (machine()->Is64()) {
4574 56 : Node* result = __ BitcastFloat64ToInt64(value);
4575 56 : result = __ Word64ReverseBytes(result);
4576 56 : result = __ BitcastInt64ToFloat64(result);
4577 56 : return result;
4578 : } else {
4579 0 : Node* lo = __ Word32ReverseBytes(__ Float64ExtractLowWord32(value));
4580 0 : Node* hi = __ Word32ReverseBytes(__ Float64ExtractHighWord32(value));
4581 0 : Node* result = __ Float64Constant(0.0);
4582 0 : result = __ Float64InsertLowWord32(result, hi);
4583 0 : result = __ Float64InsertHighWord32(result, lo);
4584 0 : return result;
4585 : }
4586 : }
4587 :
4588 : case kExternalBigInt64Array:
4589 : case kExternalBigUint64Array:
4590 0 : UNREACHABLE();
4591 : }
4592 0 : }
4593 :
4594 268 : Node* EffectControlLinearizer::LowerLoadDataViewElement(Node* node) {
4595 268 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4596 : Node* buffer = node->InputAt(0);
4597 : Node* storage = node->InputAt(1);
4598 : Node* byte_offset = node->InputAt(2);
4599 : Node* index = node->InputAt(3);
4600 : Node* is_little_endian = node->InputAt(4);
4601 :
4602 : // We need to keep the {buffer} alive so that the GC will not release the
4603 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4604 268 : __ Retain(buffer);
4605 :
4606 : // Compute the effective offset.
4607 268 : Node* offset = __ IntAdd(byte_offset, index);
4608 :
4609 : MachineType const machine_type =
4610 268 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4611 :
4612 268 : Node* value = __ LoadUnaligned(machine_type, storage, offset);
4613 268 : auto big_endian = __ MakeLabel();
4614 536 : auto done = __ MakeLabel(machine_type.representation());
4615 :
4616 268 : __ GotoIfNot(is_little_endian, &big_endian);
4617 : { // Little-endian load.
4618 : #if V8_TARGET_LITTLE_ENDIAN
4619 : __ Goto(&done, value);
4620 : #else
4621 : __ Goto(&done, BuildReverseBytes(element_type, value));
4622 : #endif // V8_TARGET_LITTLE_ENDIAN
4623 : }
4624 :
4625 : __ Bind(&big_endian);
4626 : { // Big-endian load.
4627 : #if V8_TARGET_LITTLE_ENDIAN
4628 268 : __ Goto(&done, BuildReverseBytes(element_type, value));
4629 : #else
4630 : __ Goto(&done, value);
4631 : #endif // V8_TARGET_LITTLE_ENDIAN
4632 : }
4633 :
4634 : // We're done, return {result}.
4635 : __ Bind(&done);
4636 268 : return done.PhiAt(0);
4637 : }
4638 :
4639 572 : Node* EffectControlLinearizer::LowerLoadStackArgument(Node* node) {
4640 : Node* base = node->InputAt(0);
4641 : Node* index = node->InputAt(1);
4642 :
4643 : Node* argument =
4644 1144 : __ LoadElement(AccessBuilder::ForStackArgument(), base, index);
4645 :
4646 572 : return __ BitcastWordToTagged(argument);
4647 : }
4648 :
4649 196 : void EffectControlLinearizer::LowerStoreDataViewElement(Node* node) {
4650 196 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4651 : Node* buffer = node->InputAt(0);
4652 : Node* storage = node->InputAt(1);
4653 : Node* byte_offset = node->InputAt(2);
4654 : Node* index = node->InputAt(3);
4655 : Node* value = node->InputAt(4);
4656 : Node* is_little_endian = node->InputAt(5);
4657 :
4658 : // We need to keep the {buffer} alive so that the GC will not release the
4659 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4660 196 : __ Retain(buffer);
4661 :
4662 : // Compute the effective offset.
4663 196 : Node* offset = __ IntAdd(byte_offset, index);
4664 :
4665 : MachineType const machine_type =
4666 196 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4667 :
4668 196 : auto big_endian = __ MakeLabel();
4669 392 : auto done = __ MakeLabel(machine_type.representation());
4670 :
4671 196 : __ GotoIfNot(is_little_endian, &big_endian);
4672 : { // Little-endian store.
4673 : #if V8_TARGET_LITTLE_ENDIAN
4674 : __ Goto(&done, value);
4675 : #else
4676 : __ Goto(&done, BuildReverseBytes(element_type, value));
4677 : #endif // V8_TARGET_LITTLE_ENDIAN
4678 : }
4679 :
4680 : __ Bind(&big_endian);
4681 : { // Big-endian store.
4682 : #if V8_TARGET_LITTLE_ENDIAN
4683 196 : __ Goto(&done, BuildReverseBytes(element_type, value));
4684 : #else
4685 : __ Goto(&done, value);
4686 : #endif // V8_TARGET_LITTLE_ENDIAN
4687 : }
4688 :
4689 : __ Bind(&done);
4690 : __ StoreUnaligned(machine_type.representation(), storage, offset,
4691 196 : done.PhiAt(0));
4692 196 : }
4693 :
4694 4546 : Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) {
4695 4546 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4696 : Node* buffer = node->InputAt(0);
4697 : Node* base = node->InputAt(1);
4698 : Node* external = node->InputAt(2);
4699 : Node* index = node->InputAt(3);
4700 :
4701 : // We need to keep the {buffer} alive so that the GC will not release the
4702 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4703 4546 : __ Retain(buffer);
4704 :
4705 : // Compute the effective storage pointer, handling the case where the
4706 : // {external} pointer is the effective storage pointer (i.e. the {base}
4707 : // is Smi zero).
4708 : Node* storage = IntPtrMatcher(base).Is(0)
4709 : ? external
4710 4546 : : __ UnsafePointerAdd(base, external);
4711 :
4712 : // Perform the actual typed element access.
4713 9092 : return __ LoadElement(AccessBuilder::ForTypedArrayElement(
4714 : array_type, true, LoadSensitivity::kCritical),
4715 9092 : storage, index);
4716 : }
4717 :
4718 4386 : void EffectControlLinearizer::LowerStoreTypedElement(Node* node) {
4719 4386 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4720 : Node* buffer = node->InputAt(0);
4721 : Node* base = node->InputAt(1);
4722 : Node* external = node->InputAt(2);
4723 : Node* index = node->InputAt(3);
4724 : Node* value = node->InputAt(4);
4725 :
4726 : // We need to keep the {buffer} alive so that the GC will not release the
4727 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4728 4386 : __ Retain(buffer);
4729 :
4730 : // Compute the effective storage pointer, handling the case where the
4731 : // {external} pointer is the effective storage pointer (i.e. the {base}
4732 : // is Smi zero).
4733 : Node* storage = IntPtrMatcher(base).Is(0)
4734 : ? external
4735 4386 : : __ UnsafePointerAdd(base, external);
4736 :
4737 : // Perform the actual typed element access.
4738 8772 : __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true),
4739 4386 : storage, index, value);
4740 4386 : }
4741 :
4742 326 : void EffectControlLinearizer::TransitionElementsTo(Node* node, Node* array,
4743 : ElementsKind from,
4744 : ElementsKind to) {
4745 : DCHECK(IsMoreGeneralElementsKindTransition(from, to));
4746 : DCHECK(to == HOLEY_ELEMENTS || to == HOLEY_DOUBLE_ELEMENTS);
4747 :
4748 : Handle<Map> target(to == HOLEY_ELEMENTS ? FastMapParameterOf(node->op())
4749 326 : : DoubleMapParameterOf(node->op()));
4750 326 : Node* target_map = __ HeapConstant(target);
4751 :
4752 326 : if (IsSimpleMapChangeTransition(from, to)) {
4753 106 : __ StoreField(AccessBuilder::ForMap(), array, target_map);
4754 : } else {
4755 : // Instance migration, call out to the runtime for {array}.
4756 220 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4757 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4758 220 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4759 220 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4760 220 : __ Call(call_descriptor, __ CEntryStubConstant(1), array, target_map,
4761 : __ ExternalConstant(ExternalReference::Create(id)),
4762 220 : __ Int32Constant(2), __ NoContextConstant());
4763 : }
4764 326 : }
4765 :
4766 0 : Node* EffectControlLinearizer::IsElementsKindGreaterThan(
4767 : Node* kind, ElementsKind reference_kind) {
4768 397 : Node* ref_kind = __ Int32Constant(reference_kind);
4769 397 : Node* ret = __ Int32LessThan(ref_kind, kind);
4770 0 : return ret;
4771 : }
4772 :
4773 86 : void EffectControlLinearizer::LowerTransitionAndStoreElement(Node* node) {
4774 : Node* array = node->InputAt(0);
4775 : Node* index = node->InputAt(1);
4776 : Node* value = node->InputAt(2);
4777 :
4778 : // Possibly transition array based on input and store.
4779 : //
4780 : // -- TRANSITION PHASE -----------------
4781 : // kind = ElementsKind(array)
4782 : // if value is not smi {
4783 : // if kind == HOLEY_SMI_ELEMENTS {
4784 : // if value is heap number {
4785 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4786 : // kind = HOLEY_DOUBLE_ELEMENTS
4787 : // } else {
4788 : // Transition array to HOLEY_ELEMENTS
4789 : // kind = HOLEY_ELEMENTS
4790 : // }
4791 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4792 : // if value is not heap number {
4793 : // Transition array to HOLEY_ELEMENTS
4794 : // kind = HOLEY_ELEMENTS
4795 : // }
4796 : // }
4797 : // }
4798 : //
4799 : // -- STORE PHASE ----------------------
4800 : // [make sure {kind} is up-to-date]
4801 : // if kind == HOLEY_DOUBLE_ELEMENTS {
4802 : // if value is smi {
4803 : // float_value = convert smi to float
4804 : // Store array[index] = float_value
4805 : // } else {
4806 : // float_value = value
4807 : // Store array[index] = float_value
4808 : // }
4809 : // } else {
4810 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
4811 : // Store array[index] = value
4812 : // }
4813 : //
4814 172 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4815 : Node* kind;
4816 : {
4817 86 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4818 86 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4819 86 : Node* andit = __ Word32And(bit_field2, mask);
4820 86 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4821 86 : kind = __ Word32Shr(andit, shift);
4822 : }
4823 :
4824 172 : auto do_store = __ MakeLabel(MachineRepresentation::kWord32);
4825 : // We can store a smi anywhere.
4826 86 : __ GotoIf(ObjectIsSmi(value), &do_store, kind);
4827 :
4828 : // {value} is a HeapObject.
4829 86 : auto transition_smi_array = __ MakeDeferredLabel();
4830 86 : auto transition_double_to_fast = __ MakeDeferredLabel();
4831 : {
4832 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4833 86 : &transition_smi_array);
4834 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), &do_store,
4835 86 : kind);
4836 :
4837 : // We have double elements kind. Only a HeapNumber can be stored
4838 : // without effecting a transition.
4839 86 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4840 86 : Node* heap_number_map = __ HeapNumberMapConstant();
4841 86 : Node* check = __ WordEqual(value_map, heap_number_map);
4842 86 : __ GotoIfNot(check, &transition_double_to_fast);
4843 : __ Goto(&do_store, kind);
4844 : }
4845 :
4846 : __ Bind(&transition_smi_array); // deferred code.
4847 : {
4848 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS or
4849 : // to HOLEY_ELEMENTS.
4850 86 : auto if_value_not_heap_number = __ MakeLabel();
4851 86 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4852 86 : Node* heap_number_map = __ HeapNumberMapConstant();
4853 86 : Node* check = __ WordEqual(value_map, heap_number_map);
4854 86 : __ GotoIfNot(check, &if_value_not_heap_number);
4855 : {
4856 : // {value} is a HeapNumber.
4857 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4858 86 : HOLEY_DOUBLE_ELEMENTS);
4859 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS));
4860 : }
4861 : __ Bind(&if_value_not_heap_number);
4862 : {
4863 86 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
4864 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4865 : }
4866 : }
4867 :
4868 : __ Bind(&transition_double_to_fast); // deferred code.
4869 : {
4870 86 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
4871 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4872 : }
4873 :
4874 : // Make sure kind is up-to-date.
4875 : __ Bind(&do_store);
4876 : kind = do_store.PhiAt(0);
4877 :
4878 86 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4879 86 : auto if_kind_is_double = __ MakeLabel();
4880 86 : auto done = __ MakeLabel();
4881 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4882 86 : &if_kind_is_double);
4883 : {
4884 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
4885 172 : __ StoreElement(AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS),
4886 86 : elements, index, value);
4887 : __ Goto(&done);
4888 : }
4889 : __ Bind(&if_kind_is_double);
4890 : {
4891 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
4892 86 : auto do_double_store = __ MakeLabel();
4893 86 : __ GotoIfNot(ObjectIsSmi(value), &do_double_store);
4894 : {
4895 86 : Node* int_value = ChangeSmiToInt32(value);
4896 86 : Node* float_value = __ ChangeInt32ToFloat64(int_value);
4897 172 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4898 86 : index, float_value);
4899 : __ Goto(&done);
4900 : }
4901 : __ Bind(&do_double_store);
4902 : {
4903 : Node* float_value =
4904 86 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
4905 172 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4906 86 : index, __ Float64SilenceNaN(float_value));
4907 : __ Goto(&done);
4908 : }
4909 : }
4910 :
4911 : __ Bind(&done);
4912 86 : }
4913 :
4914 28 : void EffectControlLinearizer::LowerTransitionAndStoreNumberElement(Node* node) {
4915 : Node* array = node->InputAt(0);
4916 : Node* index = node->InputAt(1);
4917 : Node* value = node->InputAt(2); // This is a Float64, not tagged.
4918 :
4919 : // Possibly transition array based on input and store.
4920 : //
4921 : // -- TRANSITION PHASE -----------------
4922 : // kind = ElementsKind(array)
4923 : // if kind == HOLEY_SMI_ELEMENTS {
4924 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4925 : // } else if kind != HOLEY_DOUBLE_ELEMENTS {
4926 : // This is UNREACHABLE, execute a debug break.
4927 : // }
4928 : //
4929 : // -- STORE PHASE ----------------------
4930 : // Store array[index] = value (it's a float)
4931 : //
4932 56 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4933 : Node* kind;
4934 : {
4935 28 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4936 28 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4937 28 : Node* andit = __ Word32And(bit_field2, mask);
4938 28 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4939 28 : kind = __ Word32Shr(andit, shift);
4940 : }
4941 :
4942 28 : auto do_store = __ MakeLabel();
4943 :
4944 : // {value} is a float64.
4945 28 : auto transition_smi_array = __ MakeDeferredLabel();
4946 : {
4947 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4948 28 : &transition_smi_array);
4949 : // We expect that our input array started at HOLEY_SMI_ELEMENTS, and
4950 : // climbs the lattice up to HOLEY_DOUBLE_ELEMENTS. Force a debug break
4951 : // if this assumption is broken. It also would be the case that
4952 : // loop peeling can break this assumption.
4953 28 : __ GotoIf(__ Word32Equal(kind, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)),
4954 28 : &do_store);
4955 : // TODO(turbofan): It would be good to have an "Unreachable()" node type.
4956 28 : __ DebugBreak();
4957 : __ Goto(&do_store);
4958 : }
4959 :
4960 : __ Bind(&transition_smi_array); // deferred code.
4961 : {
4962 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS.
4963 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4964 28 : HOLEY_DOUBLE_ELEMENTS);
4965 : __ Goto(&do_store);
4966 : }
4967 :
4968 : __ Bind(&do_store);
4969 :
4970 28 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4971 56 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, index,
4972 28 : __ Float64SilenceNaN(value));
4973 28 : }
4974 :
4975 20 : void EffectControlLinearizer::LowerTransitionAndStoreNonNumberElement(
4976 : Node* node) {
4977 : Node* array = node->InputAt(0);
4978 : Node* index = node->InputAt(1);
4979 : Node* value = node->InputAt(2);
4980 :
4981 : // Possibly transition array based on input and store.
4982 : //
4983 : // -- TRANSITION PHASE -----------------
4984 : // kind = ElementsKind(array)
4985 : // if kind == HOLEY_SMI_ELEMENTS {
4986 : // Transition array to HOLEY_ELEMENTS
4987 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4988 : // Transition array to HOLEY_ELEMENTS
4989 : // }
4990 : //
4991 : // -- STORE PHASE ----------------------
4992 : // // kind is HOLEY_ELEMENTS
4993 : // Store array[index] = value
4994 : //
4995 40 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4996 : Node* kind;
4997 : {
4998 20 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4999 20 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
5000 20 : Node* andit = __ Word32And(bit_field2, mask);
5001 20 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
5002 20 : kind = __ Word32Shr(andit, shift);
5003 : }
5004 :
5005 20 : auto do_store = __ MakeLabel();
5006 :
5007 20 : auto transition_smi_array = __ MakeDeferredLabel();
5008 20 : auto transition_double_to_fast = __ MakeDeferredLabel();
5009 : {
5010 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
5011 20 : &transition_smi_array);
5012 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
5013 20 : &transition_double_to_fast);
5014 : __ Goto(&do_store);
5015 : }
5016 :
5017 : __ Bind(&transition_smi_array); // deferred code.
5018 : {
5019 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_ELEMENTS.
5020 20 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
5021 : __ Goto(&do_store);
5022 : }
5023 :
5024 : __ Bind(&transition_double_to_fast); // deferred code.
5025 : {
5026 20 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
5027 : __ Goto(&do_store);
5028 : }
5029 :
5030 : __ Bind(&do_store);
5031 :
5032 20 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
5033 : // Our ElementsKind is HOLEY_ELEMENTS.
5034 20 : ElementAccess access = AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS);
5035 20 : Type value_type = ValueTypeParameterOf(node->op());
5036 20 : if (value_type.Is(Type::BooleanOrNullOrUndefined())) {
5037 16 : access.type = value_type;
5038 16 : access.write_barrier_kind = kNoWriteBarrier;
5039 : }
5040 20 : __ StoreElement(access, elements, index, value);
5041 20 : }
5042 :
5043 71 : void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
5044 : Node* array = node->InputAt(0);
5045 : Node* index = node->InputAt(1);
5046 : Node* value = node->InputAt(2); // int32
5047 :
5048 : // Store a signed small in an output array.
5049 : //
5050 : // kind = ElementsKind(array)
5051 : //
5052 : // -- STORE PHASE ----------------------
5053 : // if kind == HOLEY_DOUBLE_ELEMENTS {
5054 : // float_value = convert int32 to float
5055 : // Store array[index] = float_value
5056 : // } else {
5057 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
5058 : // smi_value = convert int32 to smi
5059 : // Store array[index] = smi_value
5060 : // }
5061 : //
5062 142 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
5063 : Node* kind;
5064 : {
5065 71 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
5066 71 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
5067 71 : Node* andit = __ Word32And(bit_field2, mask);
5068 71 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
5069 71 : kind = __ Word32Shr(andit, shift);
5070 : }
5071 :
5072 71 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
5073 71 : auto if_kind_is_double = __ MakeLabel();
5074 71 : auto done = __ MakeLabel();
5075 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
5076 71 : &if_kind_is_double);
5077 : {
5078 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
5079 : // In this case, we know our value is a signed small, and we can optimize
5080 : // the ElementAccess information.
5081 71 : ElementAccess access = AccessBuilder::ForFixedArrayElement();
5082 71 : access.type = Type::SignedSmall();
5083 71 : access.machine_type = MachineType::TaggedSigned();
5084 71 : access.write_barrier_kind = kNoWriteBarrier;
5085 71 : Node* smi_value = ChangeInt32ToSmi(value);
5086 71 : __ StoreElement(access, elements, index, smi_value);
5087 : __ Goto(&done);
5088 : }
5089 : __ Bind(&if_kind_is_double);
5090 : {
5091 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
5092 71 : Node* float_value = __ ChangeInt32ToFloat64(value);
5093 142 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
5094 71 : index, float_value);
5095 : __ Goto(&done);
5096 : }
5097 :
5098 : __ Bind(&done);
5099 71 : }
5100 :
5101 2539 : void EffectControlLinearizer::LowerRuntimeAbort(Node* node) {
5102 2539 : AbortReason reason = AbortReasonOf(node->op());
5103 2539 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
5104 : Runtime::FunctionId id = Runtime::kAbort;
5105 2539 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
5106 2539 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
5107 5078 : __ Call(call_descriptor, __ CEntryStubConstant(1),
5108 : jsgraph()->SmiConstant(static_cast<int>(reason)),
5109 : __ ExternalConstant(ExternalReference::Create(id)),
5110 2539 : __ Int32Constant(1), __ NoContextConstant());
5111 2539 : }
5112 :
5113 875 : Node* EffectControlLinearizer::LowerConvertReceiver(Node* node) {
5114 875 : ConvertReceiverMode const mode = ConvertReceiverModeOf(node->op());
5115 : Node* value = node->InputAt(0);
5116 : Node* global_proxy = node->InputAt(1);
5117 :
5118 875 : switch (mode) {
5119 : case ConvertReceiverMode::kNullOrUndefined: {
5120 : return global_proxy;
5121 : }
5122 : case ConvertReceiverMode::kNotNullOrUndefined: {
5123 809 : auto convert_to_object = __ MakeDeferredLabel();
5124 1618 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
5125 :
5126 : // Check if {value} is already a JSReceiver.
5127 1618 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
5128 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
5129 809 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
5130 : Node* value_instance_type =
5131 809 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
5132 809 : Node* check = __ Uint32LessThan(
5133 809 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
5134 809 : __ GotoIf(check, &convert_to_object);
5135 : __ Goto(&done_convert, value);
5136 :
5137 : // Wrap the primitive {value} into a JSValue.
5138 : __ Bind(&convert_to_object);
5139 : Operator::Properties properties = Operator::kEliminatable;
5140 809 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
5141 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5142 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5143 809 : graph()->zone(), callable.descriptor(),
5144 809 : callable.descriptor().GetStackParameterCount(), flags, properties);
5145 : Node* native_context = __ LoadField(
5146 809 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
5147 1618 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
5148 809 : value, native_context);
5149 : __ Goto(&done_convert, result);
5150 :
5151 : __ Bind(&done_convert);
5152 : return done_convert.PhiAt(0);
5153 : }
5154 : case ConvertReceiverMode::kAny: {
5155 66 : auto convert_to_object = __ MakeDeferredLabel();
5156 66 : auto convert_global_proxy = __ MakeDeferredLabel();
5157 132 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
5158 :
5159 : // Check if {value} is already a JSReceiver, or null/undefined.
5160 132 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
5161 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
5162 66 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
5163 : Node* value_instance_type =
5164 66 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
5165 66 : Node* check = __ Uint32LessThan(
5166 66 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
5167 66 : __ GotoIf(check, &convert_to_object);
5168 : __ Goto(&done_convert, value);
5169 :
5170 : // Wrap the primitive {value} into a JSValue.
5171 : __ Bind(&convert_to_object);
5172 66 : __ GotoIf(__ WordEqual(value, __ UndefinedConstant()),
5173 66 : &convert_global_proxy);
5174 66 : __ GotoIf(__ WordEqual(value, __ NullConstant()), &convert_global_proxy);
5175 : Operator::Properties properties = Operator::kEliminatable;
5176 66 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
5177 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5178 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5179 66 : graph()->zone(), callable.descriptor(),
5180 66 : callable.descriptor().GetStackParameterCount(), flags, properties);
5181 : Node* native_context = __ LoadField(
5182 66 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
5183 132 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
5184 66 : value, native_context);
5185 : __ Goto(&done_convert, result);
5186 :
5187 : // Replace the {value} with the {global_proxy}.
5188 : __ Bind(&convert_global_proxy);
5189 : __ Goto(&done_convert, global_proxy);
5190 :
5191 : __ Bind(&done_convert);
5192 : return done_convert.PhiAt(0);
5193 : }
5194 : }
5195 :
5196 0 : UNREACHABLE();
5197 : return nullptr;
5198 : }
5199 :
5200 8222 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundUp(Node* node) {
5201 : // Nothing to be done if a fast hardware instruction is available.
5202 8222 : if (machine()->Float64RoundUp().IsSupported()) {
5203 : return Nothing<Node*>();
5204 : }
5205 :
5206 : Node* const input = node->InputAt(0);
5207 :
5208 : // General case for ceil.
5209 : //
5210 : // if 0.0 < input then
5211 : // if 2^52 <= input then
5212 : // input
5213 : // else
5214 : // let temp1 = (2^52 + input) - 2^52 in
5215 : // if temp1 < input then
5216 : // temp1 + 1
5217 : // else
5218 : // temp1
5219 : // else
5220 : // if input == 0 then
5221 : // input
5222 : // else
5223 : // if input <= -2^52 then
5224 : // input
5225 : // else
5226 : // let temp1 = -0 - input in
5227 : // let temp2 = (2^52 + temp1) - 2^52 in
5228 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
5229 : // -0 - temp3
5230 :
5231 0 : auto if_not_positive = __ MakeDeferredLabel();
5232 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5233 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5234 0 : auto if_zero = __ MakeDeferredLabel();
5235 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
5236 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5237 :
5238 0 : Node* const zero = __ Float64Constant(0.0);
5239 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5240 0 : Node* const one = __ Float64Constant(1.0);
5241 :
5242 0 : Node* check0 = __ Float64LessThan(zero, input);
5243 0 : __ GotoIfNot(check0, &if_not_positive);
5244 : {
5245 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5246 0 : __ GotoIf(check1, &if_greater_than_two_52);
5247 : {
5248 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5249 0 : __ GotoIfNot(__ Float64LessThan(temp1, input), &done, temp1);
5250 0 : __ Goto(&done, __ Float64Add(temp1, one));
5251 : }
5252 :
5253 : __ Bind(&if_greater_than_two_52);
5254 : __ Goto(&done, input);
5255 : }
5256 :
5257 : __ Bind(&if_not_positive);
5258 : {
5259 0 : Node* check1 = __ Float64Equal(input, zero);
5260 0 : __ GotoIf(check1, &if_zero);
5261 :
5262 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5263 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5264 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5265 :
5266 : {
5267 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5268 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5269 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5270 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
5271 0 : __ GotoIfNot(check3, &done_temp3, temp2);
5272 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
5273 :
5274 : __ Bind(&done_temp3);
5275 : Node* temp3 = done_temp3.PhiAt(0);
5276 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
5277 : }
5278 : __ Bind(&if_less_than_minus_two_52);
5279 : __ Goto(&done, input);
5280 :
5281 : __ Bind(&if_zero);
5282 : __ Goto(&done, input);
5283 : }
5284 : __ Bind(&done);
5285 : return Just(done.PhiAt(0));
5286 : }
5287 :
5288 0 : Node* EffectControlLinearizer::BuildFloat64RoundDown(Node* value) {
5289 0 : if (machine()->Float64RoundDown().IsSupported()) {
5290 0 : return __ Float64RoundDown(value);
5291 : }
5292 :
5293 : Node* const input = value;
5294 :
5295 : // General case for floor.
5296 : //
5297 : // if 0.0 < input then
5298 : // if 2^52 <= input then
5299 : // input
5300 : // else
5301 : // let temp1 = (2^52 + input) - 2^52 in
5302 : // if input < temp1 then
5303 : // temp1 - 1
5304 : // else
5305 : // temp1
5306 : // else
5307 : // if input == 0 then
5308 : // input
5309 : // else
5310 : // if input <= -2^52 then
5311 : // input
5312 : // else
5313 : // let temp1 = -0 - input in
5314 : // let temp2 = (2^52 + temp1) - 2^52 in
5315 : // if temp2 < temp1 then
5316 : // -1 - temp2
5317 : // else
5318 : // -0 - temp2
5319 :
5320 0 : auto if_not_positive = __ MakeDeferredLabel();
5321 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5322 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5323 0 : auto if_temp2_lt_temp1 = __ MakeLabel();
5324 0 : auto if_zero = __ MakeDeferredLabel();
5325 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5326 :
5327 0 : Node* const zero = __ Float64Constant(0.0);
5328 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5329 :
5330 0 : Node* check0 = __ Float64LessThan(zero, input);
5331 0 : __ GotoIfNot(check0, &if_not_positive);
5332 : {
5333 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5334 0 : __ GotoIf(check1, &if_greater_than_two_52);
5335 : {
5336 0 : Node* const one = __ Float64Constant(1.0);
5337 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5338 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5339 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5340 : }
5341 :
5342 : __ Bind(&if_greater_than_two_52);
5343 : __ Goto(&done, input);
5344 : }
5345 :
5346 : __ Bind(&if_not_positive);
5347 : {
5348 0 : Node* check1 = __ Float64Equal(input, zero);
5349 0 : __ GotoIf(check1, &if_zero);
5350 :
5351 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5352 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5353 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5354 :
5355 : {
5356 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5357 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5358 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5359 0 : Node* check3 = __ Float64LessThan(temp2, temp1);
5360 0 : __ GotoIf(check3, &if_temp2_lt_temp1);
5361 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp2));
5362 :
5363 : __ Bind(&if_temp2_lt_temp1);
5364 0 : __ Goto(&done, __ Float64Sub(__ Float64Constant(-1.0), temp2));
5365 : }
5366 : __ Bind(&if_less_than_minus_two_52);
5367 : __ Goto(&done, input);
5368 :
5369 : __ Bind(&if_zero);
5370 : __ Goto(&done, input);
5371 : }
5372 : __ Bind(&done);
5373 : return done.PhiAt(0);
5374 : }
5375 :
5376 26794 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundDown(Node* node) {
5377 : // Nothing to be done if a fast hardware instruction is available.
5378 26794 : if (machine()->Float64RoundDown().IsSupported()) {
5379 : return Nothing<Node*>();
5380 : }
5381 :
5382 : Node* const input = node->InputAt(0);
5383 0 : return Just(BuildFloat64RoundDown(input));
5384 : }
5385 :
5386 266 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTiesEven(Node* node) {
5387 : // Nothing to be done if a fast hardware instruction is available.
5388 266 : if (machine()->Float64RoundTiesEven().IsSupported()) {
5389 : return Nothing<Node*>();
5390 : }
5391 :
5392 : Node* const input = node->InputAt(0);
5393 :
5394 : // Generate case for round ties to even:
5395 : //
5396 : // let value = floor(input) in
5397 : // let temp1 = input - value in
5398 : // if temp1 < 0.5 then
5399 : // value
5400 : // else if 0.5 < temp1 then
5401 : // value + 1.0
5402 : // else
5403 : // let temp2 = value % 2.0 in
5404 : // if temp2 == 0.0 then
5405 : // value
5406 : // else
5407 : // value + 1.0
5408 :
5409 0 : auto if_is_half = __ MakeLabel();
5410 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5411 :
5412 0 : Node* value = BuildFloat64RoundDown(input);
5413 0 : Node* temp1 = __ Float64Sub(input, value);
5414 :
5415 0 : Node* const half = __ Float64Constant(0.5);
5416 0 : Node* check0 = __ Float64LessThan(temp1, half);
5417 0 : __ GotoIf(check0, &done, value);
5418 :
5419 0 : Node* const one = __ Float64Constant(1.0);
5420 0 : Node* check1 = __ Float64LessThan(half, temp1);
5421 0 : __ GotoIfNot(check1, &if_is_half);
5422 0 : __ Goto(&done, __ Float64Add(value, one));
5423 :
5424 : __ Bind(&if_is_half);
5425 0 : Node* temp2 = __ Float64Mod(value, __ Float64Constant(2.0));
5426 0 : Node* check2 = __ Float64Equal(temp2, __ Float64Constant(0.0));
5427 0 : __ GotoIf(check2, &done, value);
5428 0 : __ Goto(&done, __ Float64Add(value, one));
5429 :
5430 : __ Bind(&done);
5431 : return Just(done.PhiAt(0));
5432 : }
5433 :
5434 140 : Node* EffectControlLinearizer::BuildFloat64RoundTruncate(Node* input) {
5435 140 : if (machine()->Float64RoundTruncate().IsSupported()) {
5436 140 : return __ Float64RoundTruncate(input);
5437 : }
5438 : // General case for trunc.
5439 : //
5440 : // if 0.0 < input then
5441 : // if 2^52 <= input then
5442 : // input
5443 : // else
5444 : // let temp1 = (2^52 + input) - 2^52 in
5445 : // if input < temp1 then
5446 : // temp1 - 1
5447 : // else
5448 : // temp1
5449 : // else
5450 : // if input == 0 then
5451 : // input
5452 : // else
5453 : // if input <= -2^52 then
5454 : // input
5455 : // else
5456 : // let temp1 = -0 - input in
5457 : // let temp2 = (2^52 + temp1) - 2^52 in
5458 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
5459 : // -0 - temp3
5460 : //
5461 : // Note: We do not use the Diamond helper class here, because it really hurts
5462 : // readability with nested diamonds.
5463 :
5464 0 : auto if_not_positive = __ MakeDeferredLabel();
5465 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5466 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5467 0 : auto if_zero = __ MakeDeferredLabel();
5468 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
5469 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5470 :
5471 0 : Node* const zero = __ Float64Constant(0.0);
5472 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5473 0 : Node* const one = __ Float64Constant(1.0);
5474 :
5475 0 : Node* check0 = __ Float64LessThan(zero, input);
5476 0 : __ GotoIfNot(check0, &if_not_positive);
5477 : {
5478 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5479 0 : __ GotoIf(check1, &if_greater_than_two_52);
5480 : {
5481 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5482 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5483 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5484 : }
5485 :
5486 : __ Bind(&if_greater_than_two_52);
5487 : __ Goto(&done, input);
5488 : }
5489 :
5490 : __ Bind(&if_not_positive);
5491 : {
5492 0 : Node* check1 = __ Float64Equal(input, zero);
5493 0 : __ GotoIf(check1, &if_zero);
5494 :
5495 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5496 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5497 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5498 :
5499 : {
5500 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5501 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5502 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5503 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
5504 0 : __ GotoIfNot(check3, &done_temp3, temp2);
5505 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
5506 :
5507 : __ Bind(&done_temp3);
5508 : Node* temp3 = done_temp3.PhiAt(0);
5509 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
5510 : }
5511 : __ Bind(&if_less_than_minus_two_52);
5512 : __ Goto(&done, input);
5513 :
5514 : __ Bind(&if_zero);
5515 : __ Goto(&done, input);
5516 : }
5517 : __ Bind(&done);
5518 : return done.PhiAt(0);
5519 : }
5520 :
5521 6914 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTruncate(Node* node) {
5522 : // Nothing to be done if a fast hardware instruction is available.
5523 6914 : if (machine()->Float64RoundTruncate().IsSupported()) {
5524 : return Nothing<Node*>();
5525 : }
5526 :
5527 : Node* const input = node->InputAt(0);
5528 0 : return Just(BuildFloat64RoundTruncate(input));
5529 : }
5530 :
5531 174 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntry(Node* node) {
5532 174 : Node* table = NodeProperties::GetValueInput(node, 0);
5533 174 : Node* key = NodeProperties::GetValueInput(node, 1);
5534 :
5535 : {
5536 : Callable const callable =
5537 174 : Builtins::CallableFor(isolate(), Builtins::kFindOrderedHashMapEntry);
5538 174 : Operator::Properties const properties = node->op()->properties();
5539 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
5540 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5541 174 : graph()->zone(), callable.descriptor(),
5542 174 : callable.descriptor().GetStackParameterCount(), flags, properties);
5543 522 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), table,
5544 348 : key, __ NoContextConstant());
5545 : }
5546 : }
5547 :
5548 17 : Node* EffectControlLinearizer::ComputeUnseededHash(Node* value) {
5549 : // See v8::internal::ComputeUnseededHash()
5550 17 : value = __ Int32Add(__ Word32Xor(value, __ Int32Constant(0xFFFFFFFF)),
5551 17 : __ Word32Shl(value, __ Int32Constant(15)));
5552 17 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(12)));
5553 17 : value = __ Int32Add(value, __ Word32Shl(value, __ Int32Constant(2)));
5554 17 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(4)));
5555 17 : value = __ Int32Mul(value, __ Int32Constant(2057));
5556 17 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(16)));
5557 17 : value = __ Word32And(value, __ Int32Constant(0x3FFFFFFF));
5558 17 : return value;
5559 : }
5560 :
5561 17 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntryForInt32Key(
5562 : Node* node) {
5563 17 : Node* table = NodeProperties::GetValueInput(node, 0);
5564 17 : Node* key = NodeProperties::GetValueInput(node, 1);
5565 :
5566 : // Compute the integer hash code.
5567 17 : Node* hash = ChangeUint32ToUintPtr(ComputeUnseededHash(key));
5568 :
5569 17 : Node* number_of_buckets = ChangeSmiToIntPtr(__ LoadField(
5570 34 : AccessBuilder::ForOrderedHashMapOrSetNumberOfBuckets(), table));
5571 17 : hash = __ WordAnd(hash, __ IntSub(number_of_buckets, __ IntPtrConstant(1)));
5572 17 : Node* first_entry = ChangeSmiToIntPtr(__ Load(
5573 : MachineType::TaggedSigned(), table,
5574 : __ IntAdd(__ WordShl(hash, __ IntPtrConstant(kTaggedSizeLog2)),
5575 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5576 17 : kHeapObjectTag))));
5577 :
5578 34 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
5579 34 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
5580 : __ Goto(&loop, first_entry);
5581 : __ Bind(&loop);
5582 : {
5583 : Node* entry = loop.PhiAt(0);
5584 : Node* check =
5585 17 : __ WordEqual(entry, __ IntPtrConstant(OrderedHashMap::kNotFound));
5586 17 : __ GotoIf(check, &done, entry);
5587 17 : entry = __ IntAdd(
5588 : __ IntMul(entry, __ IntPtrConstant(OrderedHashMap::kEntrySize)),
5589 17 : number_of_buckets);
5590 :
5591 17 : Node* candidate_key = __ Load(
5592 : MachineType::AnyTagged(), table,
5593 : __ IntAdd(__ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5594 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5595 17 : kHeapObjectTag)));
5596 :
5597 17 : auto if_match = __ MakeLabel();
5598 17 : auto if_notmatch = __ MakeLabel();
5599 17 : auto if_notsmi = __ MakeDeferredLabel();
5600 17 : __ GotoIfNot(ObjectIsSmi(candidate_key), &if_notsmi);
5601 17 : __ Branch(__ Word32Equal(ChangeSmiToInt32(candidate_key), key), &if_match,
5602 17 : &if_notmatch);
5603 :
5604 : __ Bind(&if_notsmi);
5605 34 : __ GotoIfNot(
5606 34 : __ WordEqual(__ LoadField(AccessBuilder::ForMap(), candidate_key),
5607 : __ HeapNumberMapConstant()),
5608 17 : &if_notmatch);
5609 34 : __ Branch(__ Float64Equal(__ LoadField(AccessBuilder::ForHeapNumberValue(),
5610 : candidate_key),
5611 : __ ChangeInt32ToFloat64(key)),
5612 17 : &if_match, &if_notmatch);
5613 :
5614 : __ Bind(&if_match);
5615 : __ Goto(&done, entry);
5616 :
5617 : __ Bind(&if_notmatch);
5618 : {
5619 17 : Node* next_entry = ChangeSmiToIntPtr(__ Load(
5620 : MachineType::TaggedSigned(), table,
5621 : __ IntAdd(
5622 : __ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5623 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() +
5624 : OrderedHashMap::kChainOffset * kTaggedSize -
5625 17 : kHeapObjectTag))));
5626 : __ Goto(&loop, next_entry);
5627 : }
5628 : }
5629 :
5630 : __ Bind(&done);
5631 17 : return done.PhiAt(0);
5632 : }
5633 :
5634 13 : Node* EffectControlLinearizer::LowerDateNow(Node* node) {
5635 13 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
5636 : Runtime::FunctionId id = Runtime::kDateCurrentTime;
5637 13 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
5638 13 : graph()->zone(), id, 0, properties, CallDescriptor::kNoFlags);
5639 13 : return __ Call(call_descriptor, __ CEntryStubConstant(1),
5640 : __ ExternalConstant(ExternalReference::Create(id)),
5641 13 : __ Int32Constant(0), __ NoContextConstant());
5642 : }
5643 :
5644 : #undef __
5645 :
5646 0 : Factory* EffectControlLinearizer::factory() const {
5647 0 : return isolate()->factory();
5648 : }
5649 :
5650 0 : Isolate* EffectControlLinearizer::isolate() const {
5651 0 : return jsgraph()->isolate();
5652 : }
5653 :
5654 : } // namespace compiler
5655 : } // namespace internal
5656 183042 : } // namespace v8
|
