Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/effect-control-linearizer.h"
6 :
7 : #include "src/code-factory.h"
8 : #include "src/compiler/access-builder.h"
9 : #include "src/compiler/compiler-source-position-table.h"
10 : #include "src/compiler/js-graph.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-properties.h"
14 : #include "src/compiler/node.h"
15 : #include "src/compiler/schedule.h"
16 : #include "src/objects-inl.h"
17 :
18 : namespace v8 {
19 : namespace internal {
20 : namespace compiler {
21 :
22 395305 : EffectControlLinearizer::EffectControlLinearizer(
23 : JSGraph* js_graph, Schedule* schedule, Zone* temp_zone,
24 : SourcePositionTable* source_positions)
25 : : js_graph_(js_graph),
26 : schedule_(schedule),
27 : temp_zone_(temp_zone),
28 : source_positions_(source_positions),
29 : graph_assembler_(js_graph, nullptr, nullptr, temp_zone),
30 395305 : frame_state_zapper_(nullptr) {}
31 :
32 1845045 : Graph* EffectControlLinearizer::graph() const { return js_graph_->graph(); }
33 0 : CommonOperatorBuilder* EffectControlLinearizer::common() const {
34 1202978 : return js_graph_->common();
35 : }
36 0 : SimplifiedOperatorBuilder* EffectControlLinearizer::simplified() const {
37 0 : return js_graph_->simplified();
38 : }
39 0 : MachineOperatorBuilder* EffectControlLinearizer::machine() const {
40 419386 : return js_graph_->machine();
41 : }
42 :
43 : namespace {
44 :
45 : struct BlockEffectControlData {
46 : Node* current_effect = nullptr; // New effect.
47 : Node* current_control = nullptr; // New control.
48 : Node* current_frame_state = nullptr; // New frame state.
49 : };
50 :
51 : class BlockEffectControlMap {
52 : public:
53 : explicit BlockEffectControlMap(Zone* temp_zone) : map_(temp_zone) {}
54 :
55 21393961 : BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) {
56 21393961 : return map_[std::make_pair(from->rpo_number(), to->rpo_number())];
57 : }
58 :
59 : const BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) const {
60 : return map_.at(std::make_pair(from->rpo_number(), to->rpo_number()));
61 : }
62 :
63 : private:
64 : typedef std::pair<int32_t, int32_t> Key;
65 : typedef ZoneMap<Key, BlockEffectControlData> Map;
66 :
67 : Map map_;
68 : };
69 :
70 : // Effect phis that need to be updated after the first pass.
71 : struct PendingEffectPhi {
72 : Node* effect_phi;
73 : BasicBlock* block;
74 :
75 : PendingEffectPhi(Node* effect_phi, BasicBlock* block)
76 43663 : : effect_phi(effect_phi), block(block) {}
77 : };
78 :
79 3928363 : void UpdateEffectPhi(Node* node, BasicBlock* block,
80 : BlockEffectControlMap* block_effects) {
81 : // Update all inputs to an effect phi with the effects from the given
82 : // block->effect map.
83 : DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
84 : DCHECK_EQ(static_cast<size_t>(node->op()->EffectInputCount()),
85 : block->PredecessorCount());
86 9022656 : for (int i = 0; i < node->op()->EffectInputCount(); i++) {
87 : Node* input = node->InputAt(i);
88 2086733 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
89 : const BlockEffectControlData& block_effect =
90 : block_effects->For(predecessor, block);
91 2086739 : if (input != block_effect.current_effect) {
92 717024 : node->ReplaceInput(i, block_effect.current_effect);
93 : }
94 : }
95 920819 : }
96 :
97 4627309 : void UpdateBlockControl(BasicBlock* block,
98 : BlockEffectControlMap* block_effects) {
99 13399526 : Node* control = block->NodeAt(0);
100 : DCHECK(NodeProperties::IsControl(control));
101 :
102 : // Do not rewire the end node.
103 4627309 : if (control->opcode() == IrOpcode::kEnd) return;
104 :
105 : // Update all inputs to the given control node with the correct control.
106 : DCHECK(control->opcode() == IrOpcode::kMerge ||
107 : static_cast<size_t>(control->op()->ControlInputCount()) ==
108 : block->PredecessorCount());
109 12701541 : if (static_cast<size_t>(control->op()->ControlInputCount()) !=
110 : block->PredecessorCount()) {
111 : return; // We already re-wired the control inputs of this node.
112 : }
113 22355562 : for (int i = 0; i < control->op()->ControlInputCount(); i++) {
114 4811137 : Node* input = NodeProperties::GetControlInput(control, i);
115 4811131 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
116 : const BlockEffectControlData& block_effect =
117 : block_effects->For(predecessor, block);
118 4811126 : if (input != block_effect.current_control) {
119 : NodeProperties::ReplaceControlInput(control, block_effect.current_control,
120 39683 : i);
121 : }
122 : }
123 : }
124 :
125 7711601 : bool HasIncomingBackEdges(BasicBlock* block) {
126 20869481 : for (BasicBlock* pred : block->predecessors()) {
127 7711601 : if (pred->rpo_number() >= block->rpo_number()) {
128 : return true;
129 : }
130 : }
131 : return false;
132 : }
133 :
134 188896 : void RemoveRegionNode(Node* node) {
135 : DCHECK(IrOpcode::kFinishRegion == node->opcode() ||
136 : IrOpcode::kBeginRegion == node->opcode());
137 : // Update the value/context uses to the value input of the finish node and
138 : // the effect uses to the effect input.
139 2939894 : for (Edge edge : node->use_edges()) {
140 : DCHECK(!edge.from()->IsDead());
141 1375499 : if (NodeProperties::IsEffectEdge(edge)) {
142 189716 : edge.UpdateTo(NodeProperties::GetEffectInput(node));
143 : } else {
144 : DCHECK(!NodeProperties::IsControlEdge(edge));
145 : DCHECK(!NodeProperties::IsFrameStateEdge(edge));
146 1185783 : edge.UpdateTo(node->InputAt(0));
147 : }
148 : }
149 188896 : node->Kill();
150 188896 : }
151 :
152 1324829 : void TryCloneBranch(Node* node, BasicBlock* block, Zone* temp_zone,
153 1277291 : Graph* graph, CommonOperatorBuilder* common,
154 : BlockEffectControlMap* block_effects,
155 : SourcePositionTable* source_positions) {
156 : DCHECK_EQ(IrOpcode::kBranch, node->opcode());
157 :
158 : // This optimization is a special case of (super)block cloning. It takes an
159 : // input graph as shown below and clones the Branch node for every predecessor
160 : // to the Merge, essentially removing the Merge completely. This avoids
161 : // materializing the bit for the Phi and may offer potential for further
162 : // branch folding optimizations (i.e. because one or more inputs to the Phi is
163 : // a constant). Note that there may be more Phi nodes hanging off the Merge,
164 : // but we can only a certain subset of them currently (actually only Phi and
165 : // EffectPhi nodes whose uses have either the IfTrue or IfFalse as control
166 : // input).
167 :
168 : // Control1 ... ControlN
169 : // ^ ^
170 : // | | Cond1 ... CondN
171 : // +----+ +----+ ^ ^
172 : // | | | |
173 : // | | +----+ |
174 : // Merge<--+ | +------------+
175 : // ^ \|/
176 : // | Phi
177 : // | |
178 : // Branch----+
179 : // ^
180 : // |
181 : // +-----+-----+
182 : // | |
183 : // IfTrue IfFalse
184 : // ^ ^
185 : // | |
186 :
187 : // The resulting graph (modulo the Phi and EffectPhi nodes) looks like this:
188 :
189 : // Control1 Cond1 ... ControlN CondN
190 : // ^ ^ ^ ^
191 : // \ / \ /
192 : // Branch ... Branch
193 : // ^ ^
194 : // | |
195 : // +---+---+ +---+----+
196 : // | | | |
197 : // IfTrue IfFalse ... IfTrue IfFalse
198 : // ^ ^ ^ ^
199 : // | | | |
200 : // +--+ +-------------+ |
201 : // | | +--------------+ +--+
202 : // | | | |
203 : // Merge Merge
204 : // ^ ^
205 : // | |
206 :
207 : SourcePositionTable::Scope scope(source_positions,
208 1188449 : source_positions->GetSourcePosition(node));
209 136381 : Node* branch = node;
210 2348941 : Node* cond = NodeProperties::GetValueInput(branch, 0);
211 2348940 : if (!cond->OwnedBy(branch) || cond->opcode() != IrOpcode::kPhi) return;
212 280776 : Node* merge = NodeProperties::GetControlInput(branch);
213 282450 : if (merge->opcode() != IrOpcode::kMerge ||
214 138055 : NodeProperties::GetControlInput(cond) != merge) {
215 : return;
216 : }
217 : // Grab the IfTrue/IfFalse projections of the Branch.
218 137985 : BranchMatcher matcher(branch);
219 : // Check/collect other Phi/EffectPhi nodes hanging off the Merge.
220 : NodeVector phis(temp_zone);
221 960805 : for (Node* const use : merge->uses()) {
222 412212 : if (use == branch || use == cond) continue;
223 : // We cannot currently deal with non-Phi/EffectPhi nodes hanging off the
224 : // Merge. Ideally, we would just clone the nodes (and everything that
225 : // depends on it to some distant join point), but that requires knowledge
226 : // about dominance/post-dominance.
227 139511 : if (!NodeProperties::IsPhi(use)) return;
228 138259 : for (Edge edge : use->use_edges()) {
229 : // Right now we can only handle Phi/EffectPhi nodes whose uses are
230 : // directly control-dependend on either the IfTrue or the IfFalse
231 : // successor, because we know exactly how to update those uses.
232 1956 : if (edge.from()->op()->ControlInputCount() != 1) return;
233 1923 : Node* control = NodeProperties::GetControlInput(edge.from());
234 1923 : if (NodeProperties::IsPhi(edge.from())) {
235 287 : control = NodeProperties::GetControlInput(control, edge.index());
236 : }
237 1923 : if (control != matcher.IfTrue() && control != matcher.IfFalse()) return;
238 : }
239 136303 : phis.push_back(use);
240 : }
241 136381 : BranchHint const hint = BranchHintOf(branch->op());
242 136381 : int const input_count = merge->op()->ControlInputCount();
243 : DCHECK_LE(1, input_count);
244 136381 : Node** const inputs = graph->zone()->NewArray<Node*>(2 * input_count);
245 : Node** const merge_true_inputs = &inputs[0];
246 136381 : Node** const merge_false_inputs = &inputs[input_count];
247 706836 : for (int index = 0; index < input_count; ++index) {
248 570455 : Node* cond1 = NodeProperties::GetValueInput(cond, index);
249 570455 : Node* control1 = NodeProperties::GetControlInput(merge, index);
250 570455 : Node* branch1 = graph->NewNode(common->Branch(hint), cond1, control1);
251 1140910 : merge_true_inputs[index] = graph->NewNode(common->IfTrue(), branch1);
252 1140910 : merge_false_inputs[index] = graph->NewNode(common->IfFalse(), branch1);
253 : }
254 136381 : Node* const merge_true = matcher.IfTrue();
255 136381 : Node* const merge_false = matcher.IfFalse();
256 136381 : merge_true->TrimInputCount(0);
257 136381 : merge_false->TrimInputCount(0);
258 706836 : for (int i = 0; i < input_count; ++i) {
259 1140910 : merge_true->AppendInput(graph->zone(), merge_true_inputs[i]);
260 1140910 : merge_false->AppendInput(graph->zone(), merge_false_inputs[i]);
261 : }
262 : DCHECK_EQ(2u, block->SuccessorCount());
263 136381 : NodeProperties::ChangeOp(matcher.IfTrue(), common->Merge(input_count));
264 136381 : NodeProperties::ChangeOp(matcher.IfFalse(), common->Merge(input_count));
265 : int const true_index =
266 136381 : block->SuccessorAt(0)->NodeAt(0) == matcher.IfTrue() ? 0 : 1;
267 : BlockEffectControlData* true_block_data =
268 136381 : &block_effects->For(block, block->SuccessorAt(true_index));
269 : BlockEffectControlData* false_block_data =
270 136381 : &block_effects->For(block, block->SuccessorAt(true_index ^ 1));
271 817963 : for (Node* const phi : phis) {
272 706550 : for (int index = 0; index < input_count; ++index) {
273 1140498 : inputs[index] = phi->InputAt(index);
274 : }
275 136301 : inputs[input_count] = merge_true;
276 272602 : Node* phi_true = graph->NewNode(phi->op(), input_count + 1, inputs);
277 136300 : inputs[input_count] = merge_false;
278 136300 : Node* phi_false = graph->NewNode(phi->op(), input_count + 1, inputs);
279 136300 : if (phi->UseCount() == 0) {
280 : DCHECK_EQ(phi->opcode(), IrOpcode::kEffectPhi);
281 : } else {
282 3280 : for (Edge edge : phi->use_edges()) {
283 1312 : Node* control = NodeProperties::GetControlInput(edge.from());
284 2624 : if (NodeProperties::IsPhi(edge.from())) {
285 272 : control = NodeProperties::GetControlInput(control, edge.index());
286 : }
287 : DCHECK(control == matcher.IfTrue() || control == matcher.IfFalse());
288 1312 : edge.UpdateTo((control == matcher.IfTrue()) ? phi_true : phi_false);
289 : }
290 : }
291 136300 : if (phi->opcode() == IrOpcode::kEffectPhi) {
292 136300 : true_block_data->current_effect = phi_true;
293 136300 : false_block_data->current_effect = phi_false;
294 : }
295 136300 : phi->Kill();
296 : }
297 : // Fix up IfTrue and IfFalse and kill all dead nodes.
298 136380 : if (branch == block->control_input()) {
299 136380 : true_block_data->current_control = merge_true;
300 136380 : false_block_data->current_control = merge_false;
301 : }
302 136380 : branch->Kill();
303 136380 : cond->Kill();
304 136380 : merge->Kill();
305 : }
306 : } // namespace
307 :
308 10341948 : void EffectControlLinearizer::Run() {
309 : BlockEffectControlMap block_effects(temp_zone());
310 : ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone());
311 : ZoneVector<BasicBlock*> pending_block_controls(temp_zone());
312 : NodeVector inputs_buffer(temp_zone());
313 :
314 5417916 : for (BasicBlock* block : *(schedule()->rpo_order())) {
315 : size_t instr = 0;
316 :
317 : // The control node should be the first.
318 12434888 : Node* control = block->NodeAt(instr);
319 : DCHECK(NodeProperties::IsControl(control));
320 : // Update the control inputs.
321 4627305 : if (HasIncomingBackEdges(block)) {
322 : // If there are back edges, we need to update later because we have not
323 : // computed the control yet. This should only happen for loops.
324 : DCHECK_EQ(IrOpcode::kLoop, control->opcode());
325 43660 : pending_block_controls.push_back(block);
326 : } else {
327 : // If there are no back edges, we can update now.
328 4583645 : UpdateBlockControl(block, &block_effects);
329 : }
330 : instr++;
331 :
332 : // Iterate over the phis and update the effect phis.
333 4627289 : Node* effect = nullptr;
334 : Node* terminate = nullptr;
335 12241494 : for (; instr < block->NodeCount(); instr++) {
336 4321602 : Node* node = block->NodeAt(instr);
337 : // Only go through the phis and effect phis.
338 4321602 : if (node->opcode() == IrOpcode::kEffectPhi) {
339 : // There should be at most one effect phi in a block.
340 : DCHECK_NULL(effect);
341 : // IfException blocks should not have effect phis.
342 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
343 906297 : effect = node;
344 :
345 : // Make sure we update the inputs to the incoming blocks' effects.
346 906297 : if (HasIncomingBackEdges(block)) {
347 : // In case of loops, we do not update the effect phi immediately
348 : // because the back predecessor has not been handled yet. We just
349 : // record the effect phi for later processing.
350 43663 : pending_effect_phis.push_back(PendingEffectPhi(node, block));
351 : } else {
352 862634 : UpdateEffectPhi(node, block, &block_effects);
353 : }
354 3415305 : } else if (node->opcode() == IrOpcode::kPhi) {
355 : // Just skip phis.
356 2870837 : } else if (node->opcode() == IrOpcode::kTerminate) {
357 : DCHECK(terminate == nullptr);
358 : terminate = node;
359 : } else {
360 : break;
361 : }
362 : }
363 :
364 4627353 : if (effect == nullptr) {
365 : // There was no effect phi.
366 : DCHECK(!HasIncomingBackEdges(block));
367 3721008 : if (block == schedule()->start()) {
368 : // Start block => effect is start.
369 : DCHECK_EQ(graph()->start(), control);
370 395306 : effect = graph()->start();
371 9583635 : } else if (control->opcode() == IrOpcode::kEnd) {
372 : // End block is just a dummy, no effect needed.
373 : DCHECK_EQ(BasicBlock::kNone, block->control());
374 : DCHECK_EQ(1u, block->size());
375 393466 : effect = nullptr;
376 : } else {
377 : // If all the predecessors have the same effect, we can use it as our
378 : // current effect.
379 : effect =
380 2932232 : block_effects.For(block->PredecessorAt(0), block).current_effect;
381 6023420 : for (size_t i = 1; i < block->PredecessorCount(); ++i) {
382 94002 : if (block_effects.For(block->PredecessorAt(i), block)
383 94002 : .current_effect != effect) {
384 14524 : effect = nullptr;
385 14524 : break;
386 : }
387 : }
388 2932231 : if (effect == nullptr) {
389 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
390 : // The input blocks do not have the same effect. We have
391 : // to create an effect phi node.
392 : inputs_buffer.clear();
393 29048 : inputs_buffer.resize(block->PredecessorCount(), jsgraph()->Dead());
394 14524 : inputs_buffer.push_back(control);
395 : effect = graph()->NewNode(
396 14524 : common()->EffectPhi(static_cast<int>(block->PredecessorCount())),
397 72620 : static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front()));
398 : // For loops, we update the effect phi node later to break cycles.
399 29048 : if (control->opcode() == IrOpcode::kLoop) {
400 0 : pending_effect_phis.push_back(PendingEffectPhi(effect, block));
401 : } else {
402 14524 : UpdateEffectPhi(effect, block, &block_effects);
403 : }
404 5835414 : } else if (control->opcode() == IrOpcode::kIfException) {
405 : // The IfException is connected into the effect chain, so we need
406 : // to update the effect here.
407 232513 : NodeProperties::ReplaceEffectInput(control, effect);
408 232513 : effect = control;
409 : }
410 : }
411 : }
412 :
413 : // Fixup the Terminate node.
414 4627348 : if (terminate != nullptr) {
415 42684 : NodeProperties::ReplaceEffectInput(terminate, effect);
416 : }
417 :
418 : // The frame state at block entry is determined by the frame states leaving
419 : // all predecessors. In case there is no frame state dominating this block,
420 : // we can rely on a checkpoint being present before the next deoptimization.
421 : // TODO(mstarzinger): Eventually we will need to go hunt for a frame state
422 : // once deoptimizing nodes roam freely through the schedule.
423 4627348 : Node* frame_state = nullptr;
424 4627348 : if (block != schedule()->start()) {
425 : // If all the predecessors have the same effect, we can use it
426 : // as our current effect.
427 : frame_state =
428 4231984 : block_effects.For(block->PredecessorAt(0), block).current_frame_state;
429 10610742 : for (size_t i = 1; i < block->PredecessorCount(); i++) {
430 1310364 : if (block_effects.For(block->PredecessorAt(i), block)
431 1310364 : .current_frame_state != frame_state) {
432 236977 : frame_state = nullptr;
433 236977 : frame_state_zapper_ = graph()->end();
434 236977 : break;
435 : }
436 : }
437 : }
438 :
439 : // Process the ordinary instructions.
440 61228288 : for (; instr < block->NodeCount(); instr++) {
441 : Node* node = block->NodeAt(instr);
442 28300549 : ProcessNode(node, &frame_state, &effect, &control);
443 : }
444 :
445 4627276 : switch (block->control()) {
446 : case BasicBlock::kGoto:
447 : case BasicBlock::kNone:
448 : break;
449 :
450 : case BasicBlock::kCall:
451 : case BasicBlock::kTailCall:
452 : case BasicBlock::kSwitch:
453 : case BasicBlock::kReturn:
454 : case BasicBlock::kDeoptimize:
455 : case BasicBlock::kThrow:
456 803403 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
457 803401 : break;
458 :
459 : case BasicBlock::kBranch:
460 1188450 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
461 : TryCloneBranch(block->control_input(), block, temp_zone(), graph(),
462 3565362 : common(), &block_effects, source_positions_);
463 1188449 : break;
464 : }
465 :
466 : // Store the effect, control and frame state for later use.
467 19536610 : for (BasicBlock* successor : block->successors()) {
468 5654762 : BlockEffectControlData* data = &block_effects.For(block, successor);
469 5654791 : if (data->current_effect == nullptr) {
470 5382192 : data->current_effect = effect;
471 : }
472 5654791 : if (data->current_control == nullptr) {
473 5382037 : data->current_control = control;
474 : }
475 5654791 : data->current_frame_state = frame_state;
476 : }
477 : }
478 :
479 : // Update the incoming edges of the effect phis that could not be processed
480 : // during the first pass (because they could have incoming back edges).
481 834271 : for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) {
482 : UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block,
483 43663 : &block_effects);
484 : }
485 834271 : for (BasicBlock* pending_block_control : pending_block_controls) {
486 43663 : UpdateBlockControl(pending_block_control, &block_effects);
487 : }
488 395303 : }
489 :
490 128381765 : void EffectControlLinearizer::ProcessNode(Node* node, Node** frame_state,
491 : Node** effect, Node** control) {
492 : SourcePositionTable::Scope scope(source_positions_,
493 30292249 : source_positions_->GetSourcePosition(node));
494 :
495 : // If the node needs to be wired into the effect/control chain, do this
496 : // here. Pass current frame state for lowering to eager deoptimization.
497 30292257 : if (TryWireInStateEffect(node, *frame_state, effect, control)) {
498 : return;
499 : }
500 :
501 : // If the node has a visible effect, then there must be a checkpoint in the
502 : // effect chain before we are allowed to place another eager deoptimization
503 : // point. We zap the frame state to ensure this invariant is maintained.
504 57979368 : if (region_observability_ == RegionObservability::kObservable &&
505 : !node->op()->HasProperty(Operator::kNoWrite)) {
506 3112768 : *frame_state = nullptr;
507 3112768 : frame_state_zapper_ = node;
508 : }
509 :
510 : // Remove the end markers of 'atomic' allocation region because the
511 : // region should be wired-in now.
512 29417881 : if (node->opcode() == IrOpcode::kFinishRegion) {
513 : // Reset the current region observability.
514 94448 : region_observability_ = RegionObservability::kObservable;
515 : // Update the value uses to the value input of the finish node and
516 : // the effect uses to the effect input.
517 94448 : return RemoveRegionNode(node);
518 : }
519 29323433 : if (node->opcode() == IrOpcode::kBeginRegion) {
520 : // Determine the observability for this region and use that for all
521 : // nodes inside the region (i.e. ignore the absence of kNoWrite on
522 : // StoreField and other operators).
523 : DCHECK_NE(RegionObservability::kNotObservable, region_observability_);
524 94448 : region_observability_ = RegionObservabilityOf(node->op());
525 : // Update the value uses to the value input of the finish node and
526 : // the effect uses to the effect input.
527 94448 : return RemoveRegionNode(node);
528 : }
529 :
530 : // Special treatment for checkpoint nodes.
531 29228985 : if (node->opcode() == IrOpcode::kCheckpoint) {
532 : // Unlink the check point; effect uses will be updated to the incoming
533 : // effect that is passed. The frame state is preserved for lowering.
534 : DCHECK_EQ(RegionObservability::kObservable, region_observability_);
535 2192055 : *frame_state = NodeProperties::GetFrameStateInput(node);
536 2192054 : return;
537 : }
538 :
539 27036930 : if (node->opcode() == IrOpcode::kIfSuccess) {
540 : // We always schedule IfSuccess with its call, so skip it here.
541 : DCHECK_EQ(IrOpcode::kCall, node->InputAt(0)->opcode());
542 : // The IfSuccess node should not belong to an exceptional call node
543 : // because such IfSuccess nodes should only start a basic block (and
544 : // basic block start nodes are not handled in the ProcessNode method).
545 : DCHECK(!NodeProperties::IsExceptionalCall(node->InputAt(0)));
546 : return;
547 : }
548 :
549 : // If the node takes an effect, replace with the current one.
550 54073874 : if (node->op()->EffectInputCount() > 0) {
551 : DCHECK_EQ(1, node->op()->EffectInputCount());
552 5990738 : Node* input_effect = NodeProperties::GetEffectInput(node);
553 :
554 5990737 : if (input_effect != *effect) {
555 2658059 : NodeProperties::ReplaceEffectInput(node, *effect);
556 : }
557 :
558 : // If the node produces an effect, update our current effect. (However,
559 : // ignore new effect chains started with ValueEffect.)
560 11981392 : if (node->op()->EffectOutputCount() > 0) {
561 : DCHECK_EQ(1, node->op()->EffectOutputCount());
562 5419844 : *effect = node;
563 : }
564 : } else {
565 : // New effect chain is only started with a Start or ValueEffect node.
566 : DCHECK(node->op()->EffectOutputCount() == 0 ||
567 : node->opcode() == IrOpcode::kStart);
568 : }
569 :
570 : // Rewire control inputs.
571 75321461 : for (int i = 0; i < node->op()->ControlInputCount(); i++) {
572 7082562 : NodeProperties::ReplaceControlInput(node, *control, i);
573 : }
574 : // Update the current control.
575 54073780 : if (node->op()->ControlOutputCount() > 0) {
576 4168069 : *control = node;
577 : }
578 : }
579 :
580 60584543 : bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
581 : Node* frame_state,
582 : Node** effect,
583 : Node** control) {
584 30292274 : gasm()->Reset(*effect, *control);
585 : Node* result = nullptr;
586 30292269 : switch (node->opcode()) {
587 : case IrOpcode::kChangeBitToTagged:
588 17544 : result = LowerChangeBitToTagged(node);
589 17544 : break;
590 : case IrOpcode::kChangeInt31ToTaggedSigned:
591 15150 : result = LowerChangeInt31ToTaggedSigned(node);
592 15150 : break;
593 : case IrOpcode::kChangeInt32ToTagged:
594 35722 : result = LowerChangeInt32ToTagged(node);
595 35722 : break;
596 : case IrOpcode::kChangeUint32ToTagged:
597 684 : result = LowerChangeUint32ToTagged(node);
598 684 : break;
599 : case IrOpcode::kChangeFloat64ToTagged:
600 46542 : result = LowerChangeFloat64ToTagged(node);
601 46543 : break;
602 : case IrOpcode::kChangeFloat64ToTaggedPointer:
603 62 : result = LowerChangeFloat64ToTaggedPointer(node);
604 62 : break;
605 : case IrOpcode::kChangeTaggedSignedToInt32:
606 59991 : result = LowerChangeTaggedSignedToInt32(node);
607 59992 : break;
608 : case IrOpcode::kChangeTaggedToBit:
609 182395 : result = LowerChangeTaggedToBit(node);
610 182394 : break;
611 : case IrOpcode::kChangeTaggedToInt32:
612 4769 : result = LowerChangeTaggedToInt32(node);
613 4769 : break;
614 : case IrOpcode::kChangeTaggedToUint32:
615 331 : result = LowerChangeTaggedToUint32(node);
616 331 : break;
617 : case IrOpcode::kChangeTaggedToFloat64:
618 : result = LowerChangeTaggedToFloat64(node);
619 32644 : break;
620 : case IrOpcode::kChangeTaggedToTaggedSigned:
621 8 : result = LowerChangeTaggedToTaggedSigned(node);
622 8 : break;
623 : case IrOpcode::kTruncateTaggedToBit:
624 79122 : result = LowerTruncateTaggedToBit(node);
625 79122 : break;
626 : case IrOpcode::kTruncateTaggedPointerToBit:
627 142 : result = LowerTruncateTaggedPointerToBit(node);
628 142 : break;
629 : case IrOpcode::kTruncateTaggedToFloat64:
630 231 : result = LowerTruncateTaggedToFloat64(node);
631 231 : break;
632 : case IrOpcode::kCheckBounds:
633 24906 : result = LowerCheckBounds(node, frame_state);
634 24906 : break;
635 : case IrOpcode::kCheckMaps:
636 47324 : result = LowerCheckMaps(node, frame_state);
637 47324 : break;
638 : case IrOpcode::kCheckNumber:
639 451 : result = LowerCheckNumber(node, frame_state);
640 451 : break;
641 : case IrOpcode::kCheckReceiver:
642 473 : result = LowerCheckReceiver(node, frame_state);
643 473 : break;
644 : case IrOpcode::kCheckString:
645 1196 : result = LowerCheckString(node, frame_state);
646 1196 : break;
647 : case IrOpcode::kCheckInternalizedString:
648 1757 : result = LowerCheckInternalizedString(node, frame_state);
649 1757 : break;
650 : case IrOpcode::kCheckIf:
651 29523 : result = LowerCheckIf(node, frame_state);
652 29523 : break;
653 : case IrOpcode::kCheckedInt32Add:
654 42749 : result = LowerCheckedInt32Add(node, frame_state);
655 42748 : break;
656 : case IrOpcode::kCheckedInt32Sub:
657 18699 : result = LowerCheckedInt32Sub(node, frame_state);
658 18699 : break;
659 : case IrOpcode::kCheckedInt32Div:
660 228 : result = LowerCheckedInt32Div(node, frame_state);
661 228 : break;
662 : case IrOpcode::kCheckedInt32Mod:
663 1017 : result = LowerCheckedInt32Mod(node, frame_state);
664 1017 : break;
665 : case IrOpcode::kCheckedUint32Div:
666 100 : result = LowerCheckedUint32Div(node, frame_state);
667 100 : break;
668 : case IrOpcode::kCheckedUint32Mod:
669 69 : result = LowerCheckedUint32Mod(node, frame_state);
670 69 : break;
671 : case IrOpcode::kCheckedInt32Mul:
672 6747 : result = LowerCheckedInt32Mul(node, frame_state);
673 6747 : break;
674 : case IrOpcode::kCheckedInt32ToTaggedSigned:
675 0 : result = LowerCheckedInt32ToTaggedSigned(node, frame_state);
676 0 : break;
677 : case IrOpcode::kCheckedUint32ToInt32:
678 219 : result = LowerCheckedUint32ToInt32(node, frame_state);
679 219 : break;
680 : case IrOpcode::kCheckedUint32ToTaggedSigned:
681 24 : result = LowerCheckedUint32ToTaggedSigned(node, frame_state);
682 24 : break;
683 : case IrOpcode::kCheckedFloat64ToInt32:
684 6193 : result = LowerCheckedFloat64ToInt32(node, frame_state);
685 6193 : break;
686 : case IrOpcode::kCheckedTaggedSignedToInt32:
687 27194 : if (frame_state == nullptr) {
688 : V8_Fatal(__FILE__, __LINE__, "No frame state (zapped by #%d: %s)",
689 : frame_state_zapper_->id(),
690 0 : frame_state_zapper_->op()->mnemonic());
691 : }
692 27194 : result = LowerCheckedTaggedSignedToInt32(node, frame_state);
693 27194 : break;
694 : case IrOpcode::kCheckedTaggedToInt32:
695 2490 : result = LowerCheckedTaggedToInt32(node, frame_state);
696 2490 : break;
697 : case IrOpcode::kCheckedTaggedToFloat64:
698 20767 : result = LowerCheckedTaggedToFloat64(node, frame_state);
699 20767 : break;
700 : case IrOpcode::kCheckedTaggedToTaggedSigned:
701 23022 : result = LowerCheckedTaggedToTaggedSigned(node, frame_state);
702 23022 : break;
703 : case IrOpcode::kCheckedTaggedToTaggedPointer:
704 19690 : result = LowerCheckedTaggedToTaggedPointer(node, frame_state);
705 19690 : break;
706 : case IrOpcode::kTruncateTaggedToWord32:
707 1155 : result = LowerTruncateTaggedToWord32(node);
708 1155 : break;
709 : case IrOpcode::kCheckedTruncateTaggedToWord32:
710 2075 : result = LowerCheckedTruncateTaggedToWord32(node, frame_state);
711 2076 : break;
712 : case IrOpcode::kObjectIsDetectableCallable:
713 17020 : result = LowerObjectIsDetectableCallable(node);
714 17020 : break;
715 : case IrOpcode::kObjectIsNaN:
716 2976 : result = LowerObjectIsNaN(node);
717 2976 : break;
718 : case IrOpcode::kObjectIsNonCallable:
719 8189 : result = LowerObjectIsNonCallable(node);
720 8189 : break;
721 : case IrOpcode::kObjectIsNumber:
722 11024 : result = LowerObjectIsNumber(node);
723 11024 : break;
724 : case IrOpcode::kObjectIsReceiver:
725 21909 : result = LowerObjectIsReceiver(node);
726 21909 : break;
727 : case IrOpcode::kObjectIsSmi:
728 8555 : result = LowerObjectIsSmi(node);
729 8556 : break;
730 : case IrOpcode::kObjectIsString:
731 4846 : result = LowerObjectIsString(node);
732 4846 : break;
733 : case IrOpcode::kObjectIsSymbol:
734 665 : result = LowerObjectIsSymbol(node);
735 665 : break;
736 : case IrOpcode::kObjectIsUndetectable:
737 8923 : result = LowerObjectIsUndetectable(node);
738 8923 : break;
739 : case IrOpcode::kArgumentsFrame:
740 3951 : result = LowerArgumentsFrame(node);
741 3951 : break;
742 : case IrOpcode::kArgumentsLength:
743 3998 : result = LowerArgumentsLength(node);
744 3998 : break;
745 : case IrOpcode::kNewUnmappedArgumentsElements:
746 3798 : result = LowerNewUnmappedArgumentsElements(node);
747 3798 : break;
748 : case IrOpcode::kArrayBufferWasNeutered:
749 325 : result = LowerArrayBufferWasNeutered(node);
750 325 : break;
751 : case IrOpcode::kStringFromCharCode:
752 752 : result = LowerStringFromCharCode(node);
753 752 : break;
754 : case IrOpcode::kStringFromCodePoint:
755 57 : result = LowerStringFromCodePoint(node);
756 57 : break;
757 : case IrOpcode::kStringIndexOf:
758 168 : result = LowerStringIndexOf(node);
759 168 : break;
760 : case IrOpcode::kStringCharAt:
761 287 : result = LowerStringCharAt(node);
762 287 : break;
763 : case IrOpcode::kStringCharCodeAt:
764 379 : result = LowerStringCharCodeAt(node);
765 379 : break;
766 : case IrOpcode::kStringEqual:
767 2205 : result = LowerStringEqual(node);
768 2205 : break;
769 : case IrOpcode::kStringLessThan:
770 379 : result = LowerStringLessThan(node);
771 379 : break;
772 : case IrOpcode::kStringLessThanOrEqual:
773 105 : result = LowerStringLessThanOrEqual(node);
774 105 : break;
775 : case IrOpcode::kCheckFloat64Hole:
776 651 : result = LowerCheckFloat64Hole(node, frame_state);
777 651 : break;
778 : case IrOpcode::kCheckTaggedHole:
779 917 : result = LowerCheckTaggedHole(node, frame_state);
780 917 : break;
781 : case IrOpcode::kConvertTaggedHoleToUndefined:
782 1529 : result = LowerConvertTaggedHoleToUndefined(node);
783 1529 : break;
784 : case IrOpcode::kPlainPrimitiveToNumber:
785 16 : result = LowerPlainPrimitiveToNumber(node);
786 16 : break;
787 : case IrOpcode::kPlainPrimitiveToWord32:
788 6372 : result = LowerPlainPrimitiveToWord32(node);
789 6370 : break;
790 : case IrOpcode::kPlainPrimitiveToFloat64:
791 1891 : result = LowerPlainPrimitiveToFloat64(node);
792 1891 : break;
793 : case IrOpcode::kEnsureWritableFastElements:
794 181 : result = LowerEnsureWritableFastElements(node);
795 181 : break;
796 : case IrOpcode::kMaybeGrowFastElements:
797 2054 : result = LowerMaybeGrowFastElements(node, frame_state);
798 2054 : break;
799 : case IrOpcode::kTransitionElementsKind:
800 462 : LowerTransitionElementsKind(node);
801 462 : break;
802 : case IrOpcode::kLoadTypedElement:
803 3452 : result = LowerLoadTypedElement(node);
804 3452 : break;
805 : case IrOpcode::kStoreTypedElement:
806 3062 : LowerStoreTypedElement(node);
807 3062 : break;
808 : case IrOpcode::kFloat64RoundUp:
809 11544 : if (!LowerFloat64RoundUp(node).To(&result)) {
810 : return false;
811 : }
812 : break;
813 : case IrOpcode::kFloat64RoundDown:
814 17750 : if (!LowerFloat64RoundDown(node).To(&result)) {
815 : return false;
816 : }
817 : break;
818 : case IrOpcode::kFloat64RoundTruncate:
819 2778 : if (!LowerFloat64RoundTruncate(node).To(&result)) {
820 : return false;
821 : }
822 : break;
823 : case IrOpcode::kFloat64RoundTiesEven:
824 600 : if (!LowerFloat64RoundTiesEven(node).To(&result)) {
825 : return false;
826 : }
827 : break;
828 : default:
829 : return false;
830 : }
831 874503 : *effect = gasm()->ExtractCurrentEffect();
832 874498 : *control = gasm()->ExtractCurrentControl();
833 874499 : NodeProperties::ReplaceUses(node, result, *effect, *control);
834 874505 : return true;
835 : }
836 :
837 : #define __ gasm()->
838 :
839 46542 : Node* EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node) {
840 46542 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
841 : Node* value = node->InputAt(0);
842 :
843 46543 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
844 : auto if_heapnumber =
845 : __ MakeLabelFor(GraphAssemblerLabelType::kNonDeferred,
846 93086 : 1 + (mode == CheckForMinusZeroMode::kCheckForMinusZero) +
847 139629 : !machine()->Is64());
848 46542 : auto if_int32 = __ MakeLabel<1>();
849 :
850 46542 : Node* value32 = __ RoundFloat64ToInt32(value);
851 : __ GotoIf(__ Float64Equal(value, __ ChangeInt32ToFloat64(value32)),
852 46543 : &if_int32);
853 : __ Goto(&if_heapnumber);
854 :
855 : __ Bind(&if_int32);
856 : {
857 46542 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
858 26068 : Node* zero = __ Int32Constant(0);
859 26069 : auto if_zero = __ MakeDeferredLabel<1>();
860 26069 : auto if_smi = __ MakeLabel<2>();
861 :
862 26069 : __ GotoIf(__ Word32Equal(value32, zero), &if_zero);
863 : __ Goto(&if_smi);
864 :
865 : __ Bind(&if_zero);
866 : {
867 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
868 : __ GotoIf(__ Int32LessThan(__ Float64ExtractHighWord32(value), zero),
869 26069 : &if_heapnumber);
870 : __ Goto(&if_smi);
871 : }
872 :
873 26069 : __ Bind(&if_smi);
874 : }
875 :
876 46543 : if (machine()->Is64()) {
877 46543 : Node* value_smi = ChangeInt32ToSmi(value32);
878 : __ Goto(&done, value_smi);
879 : } else {
880 0 : Node* add = __ Int32AddWithOverflow(value32, value32);
881 0 : Node* ovf = __ Projection(1, add);
882 0 : __ GotoIf(ovf, &if_heapnumber);
883 0 : Node* value_smi = __ Projection(0, add);
884 : __ Goto(&done, value_smi);
885 : }
886 : }
887 :
888 46542 : __ Bind(&if_heapnumber);
889 : {
890 46542 : Node* value_number = AllocateHeapNumberWithValue(value);
891 : __ Goto(&done, value_number);
892 : }
893 :
894 46543 : __ Bind(&done);
895 46543 : return done.PhiAt(0);
896 : }
897 :
898 62 : Node* EffectControlLinearizer::LowerChangeFloat64ToTaggedPointer(Node* node) {
899 : Node* value = node->InputAt(0);
900 62 : return AllocateHeapNumberWithValue(value);
901 : }
902 :
903 17544 : Node* EffectControlLinearizer::LowerChangeBitToTagged(Node* node) {
904 : Node* value = node->InputAt(0);
905 :
906 17544 : auto if_true = __ MakeLabel<1>();
907 17544 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
908 :
909 17544 : __ GotoIf(value, &if_true);
910 17544 : __ Goto(&done, __ FalseConstant());
911 :
912 : __ Bind(&if_true);
913 17544 : __ Goto(&done, __ TrueConstant());
914 :
915 17544 : __ Bind(&done);
916 17544 : return done.PhiAt(0);
917 : }
918 :
919 15150 : Node* EffectControlLinearizer::LowerChangeInt31ToTaggedSigned(Node* node) {
920 : Node* value = node->InputAt(0);
921 15150 : return ChangeInt32ToSmi(value);
922 : }
923 :
924 35721 : Node* EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node) {
925 : Node* value = node->InputAt(0);
926 :
927 35721 : if (machine()->Is64()) {
928 35721 : return ChangeInt32ToSmi(value);
929 : }
930 :
931 0 : auto if_overflow = __ MakeDeferredLabel<1>();
932 0 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
933 :
934 0 : Node* add = __ Int32AddWithOverflow(value, value);
935 0 : Node* ovf = __ Projection(1, add);
936 0 : __ GotoIf(ovf, &if_overflow);
937 0 : __ Goto(&done, __ Projection(0, add));
938 :
939 : __ Bind(&if_overflow);
940 0 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt32ToFloat64(value));
941 : __ Goto(&done, number);
942 :
943 0 : __ Bind(&done);
944 : return done.PhiAt(0);
945 : }
946 :
947 684 : Node* EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node) {
948 : Node* value = node->InputAt(0);
949 :
950 684 : auto if_not_in_smi_range = __ MakeDeferredLabel<1>();
951 684 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
952 :
953 684 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
954 684 : __ GotoUnless(check, &if_not_in_smi_range);
955 684 : __ Goto(&done, ChangeUint32ToSmi(value));
956 :
957 : __ Bind(&if_not_in_smi_range);
958 684 : Node* number = AllocateHeapNumberWithValue(__ ChangeUint32ToFloat64(value));
959 :
960 : __ Goto(&done, number);
961 684 : __ Bind(&done);
962 :
963 684 : return done.PhiAt(0);
964 : }
965 :
966 59991 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt32(Node* node) {
967 : Node* value = node->InputAt(0);
968 59991 : return ChangeSmiToInt32(value);
969 : }
970 :
971 182395 : Node* EffectControlLinearizer::LowerChangeTaggedToBit(Node* node) {
972 : Node* value = node->InputAt(0);
973 182395 : return __ WordEqual(value, __ TrueConstant());
974 : }
975 :
976 79122 : Node* EffectControlLinearizer::LowerTruncateTaggedToBit(Node* node) {
977 : Node* value = node->InputAt(0);
978 :
979 79122 : auto if_smi = __ MakeDeferredLabel<1>();
980 79122 : auto if_heapnumber = __ MakeDeferredLabel<1>();
981 79122 : auto done = __ MakeLabel<6>(MachineRepresentation::kBit);
982 :
983 79122 : Node* zero = __ Int32Constant(0);
984 79122 : Node* fzero = __ Float64Constant(0.0);
985 :
986 : // Check if {value} is false.
987 79122 : __ GotoIf(__ WordEqual(value, __ FalseConstant()), &done, zero);
988 :
989 : // Check if {value} is a Smi.
990 79122 : Node* check_smi = ObjectIsSmi(value);
991 79122 : __ GotoIf(check_smi, &if_smi);
992 :
993 : // Check if {value} is the empty string.
994 79122 : __ GotoIf(__ WordEqual(value, __ EmptyStringConstant()), &done, zero);
995 :
996 : // Load the map of {value}.
997 79122 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
998 :
999 : // Check if the {value} is undetectable and immediately return false.
1000 : Node* value_map_bitfield =
1001 79122 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1002 : __ GotoUnless(
1003 : __ Word32Equal(__ Word32And(value_map_bitfield,
1004 : __ Int32Constant(1 << Map::kIsUndetectable)),
1005 : zero),
1006 79122 : &done, zero);
1007 :
1008 : // Check if {value} is a HeapNumber.
1009 : __ GotoIf(__ WordEqual(value_map, __ HeapNumberMapConstant()),
1010 79122 : &if_heapnumber);
1011 :
1012 : // All other values that reach here are true.
1013 79122 : __ Goto(&done, __ Int32Constant(1));
1014 :
1015 : __ Bind(&if_heapnumber);
1016 : {
1017 : // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or
1018 : // NaN.
1019 : Node* value_value =
1020 79122 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1021 79122 : __ Goto(&done, __ Float64LessThan(fzero, __ Float64Abs(value_value)));
1022 : }
1023 :
1024 : __ Bind(&if_smi);
1025 : {
1026 : // If {value} is a Smi, then we only need to check that it's not zero.
1027 : __ Goto(&done,
1028 79122 : __ Word32Equal(__ WordEqual(value, __ IntPtrConstant(0)), zero));
1029 : }
1030 :
1031 79122 : __ Bind(&done);
1032 79122 : return done.PhiAt(0);
1033 : }
1034 :
1035 142 : Node* EffectControlLinearizer::LowerTruncateTaggedPointerToBit(Node* node) {
1036 : Node* value = node->InputAt(0);
1037 :
1038 142 : auto if_heapnumber = __ MakeDeferredLabel<1>();
1039 142 : auto done = __ MakeLabel<5>(MachineRepresentation::kBit);
1040 :
1041 142 : Node* zero = __ Int32Constant(0);
1042 142 : Node* fzero = __ Float64Constant(0.0);
1043 :
1044 : // Check if {value} is false.
1045 142 : __ GotoIf(__ WordEqual(value, __ FalseConstant()), &done, zero);
1046 :
1047 : // Check if {value} is the empty string.
1048 142 : __ GotoIf(__ WordEqual(value, __ EmptyStringConstant()), &done, zero);
1049 :
1050 : // Load the map of {value}.
1051 142 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1052 :
1053 : // Check if the {value} is undetectable and immediately return false.
1054 : Node* value_map_bitfield =
1055 142 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1056 : __ GotoUnless(
1057 : __ Word32Equal(__ Word32And(value_map_bitfield,
1058 : __ Int32Constant(1 << Map::kIsUndetectable)),
1059 : zero),
1060 142 : &done, zero);
1061 :
1062 : // Check if {value} is a HeapNumber.
1063 : __ GotoIf(__ WordEqual(value_map, __ HeapNumberMapConstant()),
1064 142 : &if_heapnumber);
1065 :
1066 : // All other values that reach here are true.
1067 142 : __ Goto(&done, __ Int32Constant(1));
1068 :
1069 : __ Bind(&if_heapnumber);
1070 : {
1071 : // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or
1072 : // NaN.
1073 : Node* value_value =
1074 142 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1075 142 : __ Goto(&done, __ Float64LessThan(fzero, __ Float64Abs(value_value)));
1076 : }
1077 :
1078 142 : __ Bind(&done);
1079 142 : return done.PhiAt(0);
1080 : }
1081 :
1082 4769 : Node* EffectControlLinearizer::LowerChangeTaggedToInt32(Node* node) {
1083 : Node* value = node->InputAt(0);
1084 :
1085 4769 : auto if_not_smi = __ MakeDeferredLabel<1>();
1086 4769 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1087 :
1088 4769 : Node* check = ObjectIsSmi(value);
1089 4769 : __ GotoUnless(check, &if_not_smi);
1090 4769 : __ Goto(&done, ChangeSmiToInt32(value));
1091 :
1092 : __ Bind(&if_not_smi);
1093 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1094 4769 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1095 4769 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1096 : __ Goto(&done, vfalse);
1097 :
1098 4769 : __ Bind(&done);
1099 4769 : return done.PhiAt(0);
1100 : }
1101 :
1102 331 : Node* EffectControlLinearizer::LowerChangeTaggedToUint32(Node* node) {
1103 : Node* value = node->InputAt(0);
1104 :
1105 331 : auto if_not_smi = __ MakeDeferredLabel<1>();
1106 331 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1107 :
1108 331 : Node* check = ObjectIsSmi(value);
1109 331 : __ GotoUnless(check, &if_not_smi);
1110 331 : __ Goto(&done, ChangeSmiToInt32(value));
1111 :
1112 : __ Bind(&if_not_smi);
1113 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1114 331 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1115 331 : vfalse = __ ChangeFloat64ToUint32(vfalse);
1116 : __ Goto(&done, vfalse);
1117 :
1118 331 : __ Bind(&done);
1119 331 : return done.PhiAt(0);
1120 : }
1121 :
1122 0 : Node* EffectControlLinearizer::LowerChangeTaggedToFloat64(Node* node) {
1123 32644 : return LowerTruncateTaggedToFloat64(node);
1124 : }
1125 :
1126 8 : Node* EffectControlLinearizer::LowerChangeTaggedToTaggedSigned(Node* node) {
1127 : Node* value = node->InputAt(0);
1128 :
1129 8 : auto if_not_smi = __ MakeDeferredLabel<1>();
1130 8 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1131 :
1132 8 : Node* check = ObjectIsSmi(value);
1133 8 : __ GotoUnless(check, &if_not_smi);
1134 : __ Goto(&done, value);
1135 :
1136 : __ Bind(&if_not_smi);
1137 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1138 8 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1139 8 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1140 8 : vfalse = ChangeInt32ToSmi(vfalse);
1141 : __ Goto(&done, vfalse);
1142 :
1143 8 : __ Bind(&done);
1144 8 : return done.PhiAt(0);
1145 : }
1146 :
1147 32875 : Node* EffectControlLinearizer::LowerTruncateTaggedToFloat64(Node* node) {
1148 : Node* value = node->InputAt(0);
1149 :
1150 32875 : auto if_not_smi = __ MakeDeferredLabel<1>();
1151 32875 : auto done = __ MakeLabel<2>(MachineRepresentation::kFloat64);
1152 :
1153 32875 : Node* check = ObjectIsSmi(value);
1154 32875 : __ GotoUnless(check, &if_not_smi);
1155 32875 : Node* vtrue = ChangeSmiToInt32(value);
1156 32875 : vtrue = __ ChangeInt32ToFloat64(vtrue);
1157 : __ Goto(&done, vtrue);
1158 :
1159 : __ Bind(&if_not_smi);
1160 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1161 32875 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1162 : __ Goto(&done, vfalse);
1163 :
1164 32875 : __ Bind(&done);
1165 32875 : return done.PhiAt(0);
1166 : }
1167 :
1168 24906 : Node* EffectControlLinearizer::LowerCheckBounds(Node* node, Node* frame_state) {
1169 : Node* index = node->InputAt(0);
1170 : Node* limit = node->InputAt(1);
1171 :
1172 24906 : Node* check = __ Uint32LessThan(index, limit);
1173 24906 : __ DeoptimizeUnless(DeoptimizeReason::kOutOfBounds, check, frame_state);
1174 24906 : return index;
1175 : }
1176 :
1177 47324 : Node* EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
1178 47324 : CheckMapsParameters const& p = CheckMapsParametersOf(node->op());
1179 : Node* value = node->InputAt(0);
1180 :
1181 : ZoneHandleSet<Map> const& maps = p.maps();
1182 : size_t const map_count = maps.size();
1183 :
1184 47324 : if (p.flags() & CheckMapsFlag::kTryMigrateInstance) {
1185 : auto done =
1186 697 : __ MakeLabelFor(GraphAssemblerLabelType::kNonDeferred, map_count * 2);
1187 697 : auto migrate = __ MakeDeferredLabel<1>();
1188 :
1189 : // Load the current map of the {value}.
1190 697 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1191 :
1192 : // Perform the map checks.
1193 1500 : for (size_t i = 0; i < map_count; ++i) {
1194 803 : Node* map = __ HeapConstant(maps[i]);
1195 803 : Node* check = __ WordEqual(value_map, map);
1196 803 : if (i == map_count - 1) {
1197 697 : __ GotoUnless(check, &migrate);
1198 : __ Goto(&done);
1199 : } else {
1200 106 : __ GotoIf(check, &done);
1201 : }
1202 : }
1203 :
1204 : // Perform the (deferred) instance migration.
1205 : __ Bind(&migrate);
1206 : {
1207 : // If map is not deprecated the migration attempt does not make sense.
1208 : Node* bitfield3 =
1209 697 : __ LoadField(AccessBuilder::ForMapBitField3(), value_map);
1210 : Node* if_not_deprecated = __ WordEqual(
1211 : __ Word32And(bitfield3, __ Int32Constant(Map::Deprecated::kMask)),
1212 697 : __ Int32Constant(0));
1213 : __ DeoptimizeIf(DeoptimizeReason::kWrongMap, if_not_deprecated,
1214 697 : frame_state);
1215 :
1216 697 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
1217 : Runtime::FunctionId id = Runtime::kTryMigrateInstance;
1218 : CallDescriptor const* desc = Linkage::GetRuntimeCallDescriptor(
1219 697 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
1220 : Node* result =
1221 : __ Call(desc, __ CEntryStubConstant(1), value,
1222 : __ ExternalConstant(ExternalReference(id, isolate())),
1223 1394 : __ Int32Constant(1), __ NoContextConstant());
1224 697 : Node* check = ObjectIsSmi(result);
1225 : __ DeoptimizeIf(DeoptimizeReason::kInstanceMigrationFailed, check,
1226 697 : frame_state);
1227 : }
1228 :
1229 : // Reload the current map of the {value}.
1230 697 : value_map = __ LoadField(AccessBuilder::ForMap(), value);
1231 :
1232 : // Perform the map checks again.
1233 1500 : for (size_t i = 0; i < map_count; ++i) {
1234 803 : Node* map = __ HeapConstant(maps[i]);
1235 803 : Node* check = __ WordEqual(value_map, map);
1236 803 : if (i == map_count - 1) {
1237 697 : __ DeoptimizeUnless(DeoptimizeReason::kWrongMap, check, frame_state);
1238 : } else {
1239 106 : __ GotoIf(check, &done);
1240 : }
1241 : }
1242 :
1243 : __ Goto(&done);
1244 697 : __ Bind(&done);
1245 : } else {
1246 : auto done =
1247 : __ MakeLabelFor(GraphAssemblerLabelType::kNonDeferred, map_count);
1248 :
1249 : // Load the current map of the {value}.
1250 46627 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1251 :
1252 95617 : for (size_t i = 0; i < map_count; ++i) {
1253 48990 : Node* map = __ HeapConstant(maps[i]);
1254 48990 : Node* check = __ WordEqual(value_map, map);
1255 48990 : if (i == map_count - 1) {
1256 46627 : __ DeoptimizeUnless(DeoptimizeReason::kWrongMap, check, frame_state);
1257 : } else {
1258 2363 : __ GotoIf(check, &done);
1259 : }
1260 : }
1261 : __ Goto(&done);
1262 46627 : __ Bind(&done);
1263 : }
1264 47324 : return value;
1265 : }
1266 :
1267 451 : Node* EffectControlLinearizer::LowerCheckNumber(Node* node, Node* frame_state) {
1268 : Node* value = node->InputAt(0);
1269 :
1270 451 : auto if_not_smi = __ MakeDeferredLabel<1>();
1271 451 : auto done = __ MakeLabel<2>();
1272 :
1273 451 : Node* check0 = ObjectIsSmi(value);
1274 451 : __ GotoUnless(check0, &if_not_smi);
1275 : __ Goto(&done);
1276 :
1277 : __ Bind(&if_not_smi);
1278 451 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1279 451 : Node* check1 = __ WordEqual(value_map, __ HeapNumberMapConstant());
1280 451 : __ DeoptimizeUnless(DeoptimizeReason::kNotAHeapNumber, check1, frame_state);
1281 : __ Goto(&done);
1282 :
1283 451 : __ Bind(&done);
1284 451 : return value;
1285 : }
1286 :
1287 473 : Node* EffectControlLinearizer::LowerCheckReceiver(Node* node,
1288 : Node* frame_state) {
1289 : Node* value = node->InputAt(0);
1290 :
1291 473 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1292 : Node* value_instance_type =
1293 473 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1294 :
1295 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1296 : Node* check = __ Uint32LessThanOrEqual(
1297 473 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1298 : __ DeoptimizeUnless(DeoptimizeReason::kNotAJavaScriptObject, check,
1299 473 : frame_state);
1300 473 : return value;
1301 : }
1302 :
1303 1196 : Node* EffectControlLinearizer::LowerCheckString(Node* node, Node* frame_state) {
1304 : Node* value = node->InputAt(0);
1305 :
1306 1196 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1307 : Node* value_instance_type =
1308 1196 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1309 :
1310 : Node* check = __ Uint32LessThan(value_instance_type,
1311 1196 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
1312 1196 : __ DeoptimizeUnless(DeoptimizeReason::kWrongInstanceType, check, frame_state);
1313 1196 : return value;
1314 : }
1315 :
1316 1757 : Node* EffectControlLinearizer::LowerCheckInternalizedString(Node* node,
1317 : Node* frame_state) {
1318 : Node* value = node->InputAt(0);
1319 :
1320 1757 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1321 : Node* value_instance_type =
1322 1757 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1323 :
1324 : Node* check = __ Word32Equal(
1325 : __ Word32And(value_instance_type,
1326 : __ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
1327 1757 : __ Int32Constant(kInternalizedTag));
1328 1757 : __ DeoptimizeUnless(DeoptimizeReason::kWrongInstanceType, check, frame_state);
1329 :
1330 1757 : return value;
1331 : }
1332 :
1333 29523 : Node* EffectControlLinearizer::LowerCheckIf(Node* node, Node* frame_state) {
1334 : Node* value = node->InputAt(0);
1335 : __ DeoptimizeUnless(DeoptimizeKind::kEager, DeoptimizeReason::kNoReason,
1336 29523 : value, frame_state);
1337 29523 : return value;
1338 : }
1339 :
1340 42749 : Node* EffectControlLinearizer::LowerCheckedInt32Add(Node* node,
1341 : Node* frame_state) {
1342 : Node* lhs = node->InputAt(0);
1343 : Node* rhs = node->InputAt(1);
1344 :
1345 42749 : Node* value = __ Int32AddWithOverflow(lhs, rhs);
1346 42749 : Node* check = __ Projection(1, value);
1347 42748 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, check, frame_state);
1348 42746 : return __ Projection(0, value);
1349 : }
1350 :
1351 18699 : Node* EffectControlLinearizer::LowerCheckedInt32Sub(Node* node,
1352 : Node* frame_state) {
1353 : Node* lhs = node->InputAt(0);
1354 : Node* rhs = node->InputAt(1);
1355 :
1356 18699 : Node* value = __ Int32SubWithOverflow(lhs, rhs);
1357 18699 : Node* check = __ Projection(1, value);
1358 18699 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, check, frame_state);
1359 18699 : return __ Projection(0, value);
1360 : }
1361 :
1362 228 : Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node,
1363 : Node* frame_state) {
1364 : Node* lhs = node->InputAt(0);
1365 : Node* rhs = node->InputAt(1);
1366 :
1367 228 : auto if_not_positive = __ MakeDeferredLabel<1>();
1368 228 : auto if_is_minint = __ MakeDeferredLabel<1>();
1369 228 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1370 228 : auto minint_check_done = __ MakeLabel<2>();
1371 :
1372 228 : Node* zero = __ Int32Constant(0);
1373 :
1374 : // Check if {rhs} is positive (and not zero).
1375 228 : Node* check0 = __ Int32LessThan(zero, rhs);
1376 228 : __ GotoUnless(check0, &if_not_positive);
1377 :
1378 : // Fast case, no additional checking required.
1379 228 : __ Goto(&done, __ Int32Div(lhs, rhs));
1380 :
1381 : {
1382 : __ Bind(&if_not_positive);
1383 :
1384 : // Check if {rhs} is zero.
1385 228 : Node* check = __ Word32Equal(rhs, zero);
1386 228 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, check, frame_state);
1387 :
1388 : // Check if {lhs} is zero, as that would produce minus zero.
1389 228 : check = __ Word32Equal(lhs, zero);
1390 228 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, check, frame_state);
1391 :
1392 : // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
1393 : // to return -kMinInt, which is not representable.
1394 228 : Node* minint = __ Int32Constant(std::numeric_limits<int32_t>::min());
1395 228 : Node* check1 = graph()->NewNode(machine()->Word32Equal(), lhs, minint);
1396 228 : __ GotoIf(check1, &if_is_minint);
1397 : __ Goto(&minint_check_done);
1398 :
1399 : __ Bind(&if_is_minint);
1400 : // Check if {rhs} is -1.
1401 228 : Node* minusone = __ Int32Constant(-1);
1402 228 : Node* is_minus_one = __ Word32Equal(rhs, minusone);
1403 228 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, is_minus_one, frame_state);
1404 : __ Goto(&minint_check_done);
1405 :
1406 228 : __ Bind(&minint_check_done);
1407 : // Perform the actual integer division.
1408 228 : __ Goto(&done, __ Int32Div(lhs, rhs));
1409 : }
1410 :
1411 228 : __ Bind(&done);
1412 : Node* value = done.PhiAt(0);
1413 :
1414 : // Check if the remainder is non-zero.
1415 228 : Node* check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
1416 228 : __ DeoptimizeUnless(DeoptimizeReason::kLostPrecision, check, frame_state);
1417 :
1418 228 : return value;
1419 : }
1420 :
1421 1017 : Node* EffectControlLinearizer::LowerCheckedInt32Mod(Node* node,
1422 : Node* frame_state) {
1423 : // General case for signed integer modulus, with optimization for (unknown)
1424 : // power of 2 right hand side.
1425 : //
1426 : // if rhs <= 0 then
1427 : // rhs = -rhs
1428 : // deopt if rhs == 0
1429 : // if lhs < 0 then
1430 : // let res = lhs % rhs in
1431 : // deopt if res == 0
1432 : // res
1433 : // else
1434 : // let msk = rhs - 1 in
1435 : // if rhs & msk == 0 then
1436 : // lhs & msk
1437 : // else
1438 : // lhs % rhs
1439 : //
1440 : Node* lhs = node->InputAt(0);
1441 : Node* rhs = node->InputAt(1);
1442 :
1443 1017 : auto if_rhs_not_positive = __ MakeDeferredLabel<1>();
1444 1017 : auto if_lhs_negative = __ MakeDeferredLabel<1>();
1445 1017 : auto if_power_of_two = __ MakeLabel<1>();
1446 1017 : auto rhs_checked = __ MakeLabel<2>(MachineRepresentation::kWord32);
1447 1017 : auto done = __ MakeLabel<3>(MachineRepresentation::kWord32);
1448 :
1449 1017 : Node* zero = __ Int32Constant(0);
1450 :
1451 : // Check if {rhs} is not strictly positive.
1452 1017 : Node* check0 = __ Int32LessThanOrEqual(rhs, zero);
1453 1017 : __ GotoIf(check0, &if_rhs_not_positive);
1454 : __ Goto(&rhs_checked, rhs);
1455 :
1456 : __ Bind(&if_rhs_not_positive);
1457 : {
1458 : // Negate {rhs}, might still produce a negative result in case of
1459 : // -2^31, but that is handled safely below.
1460 1017 : Node* vtrue0 = __ Int32Sub(zero, rhs);
1461 :
1462 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1463 1017 : Node* check = __ Word32Equal(vtrue0, zero);
1464 1017 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, check, frame_state);
1465 : __ Goto(&rhs_checked, vtrue0);
1466 : }
1467 :
1468 1017 : __ Bind(&rhs_checked);
1469 : rhs = rhs_checked.PhiAt(0);
1470 :
1471 : // Check if {lhs} is negative.
1472 1017 : Node* check1 = __ Int32LessThan(lhs, zero);
1473 1017 : __ GotoIf(check1, &if_lhs_negative);
1474 :
1475 : // {lhs} non-negative.
1476 : {
1477 1017 : Node* one = __ Int32Constant(1);
1478 1017 : Node* msk = __ Int32Sub(rhs, one);
1479 :
1480 : // Check if {rhs} minus one is a valid mask.
1481 1017 : Node* check2 = __ Word32Equal(__ Word32And(rhs, msk), zero);
1482 1017 : __ GotoIf(check2, &if_power_of_two);
1483 : // Compute the remainder using the generic {lhs % rhs}.
1484 1017 : __ Goto(&done, __ Int32Mod(lhs, rhs));
1485 :
1486 : __ Bind(&if_power_of_two);
1487 : // Compute the remainder using {lhs & msk}.
1488 1017 : __ Goto(&done, __ Word32And(lhs, msk));
1489 : }
1490 :
1491 : __ Bind(&if_lhs_negative);
1492 : {
1493 : // Compute the remainder using {lhs % msk}.
1494 1017 : Node* vtrue1 = __ Int32Mod(lhs, rhs);
1495 :
1496 : // Check if we would have to return -0.
1497 1017 : Node* check = __ Word32Equal(vtrue1, zero);
1498 1017 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, check, frame_state);
1499 : __ Goto(&done, vtrue1);
1500 : }
1501 :
1502 1017 : __ Bind(&done);
1503 1017 : return done.PhiAt(0);
1504 : }
1505 :
1506 100 : Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node,
1507 : Node* frame_state) {
1508 : Node* lhs = node->InputAt(0);
1509 : Node* rhs = node->InputAt(1);
1510 :
1511 100 : Node* zero = __ Int32Constant(0);
1512 :
1513 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1514 100 : Node* check = __ Word32Equal(rhs, zero);
1515 100 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, check, frame_state);
1516 :
1517 : // Perform the actual unsigned integer division.
1518 100 : Node* value = __ Uint32Div(lhs, rhs);
1519 :
1520 : // Check if the remainder is non-zero.
1521 100 : check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
1522 100 : __ DeoptimizeUnless(DeoptimizeReason::kLostPrecision, check, frame_state);
1523 100 : return value;
1524 : }
1525 :
1526 69 : Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node,
1527 : Node* frame_state) {
1528 : Node* lhs = node->InputAt(0);
1529 : Node* rhs = node->InputAt(1);
1530 :
1531 69 : Node* zero = __ Int32Constant(0);
1532 :
1533 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1534 69 : Node* check = __ Word32Equal(rhs, zero);
1535 69 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, check, frame_state);
1536 :
1537 : // Perform the actual unsigned integer modulus.
1538 69 : return __ Uint32Mod(lhs, rhs);
1539 : }
1540 :
1541 6747 : Node* EffectControlLinearizer::LowerCheckedInt32Mul(Node* node,
1542 : Node* frame_state) {
1543 6747 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1544 : Node* lhs = node->InputAt(0);
1545 : Node* rhs = node->InputAt(1);
1546 :
1547 6747 : Node* projection = __ Int32MulWithOverflow(lhs, rhs);
1548 6747 : Node* check = __ Projection(1, projection);
1549 6747 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, check, frame_state);
1550 :
1551 6747 : Node* value = __ Projection(0, projection);
1552 :
1553 6747 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1554 4216 : auto if_zero = __ MakeDeferredLabel<1>();
1555 4216 : auto check_done = __ MakeLabel<2>();
1556 4216 : Node* zero = __ Int32Constant(0);
1557 4216 : Node* check_zero = __ Word32Equal(value, zero);
1558 4216 : __ GotoIf(check_zero, &if_zero);
1559 : __ Goto(&check_done);
1560 :
1561 : __ Bind(&if_zero);
1562 : // We may need to return negative zero.
1563 4216 : Node* check_or = __ Int32LessThan(__ Word32Or(lhs, rhs), zero);
1564 4216 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, check_or, frame_state);
1565 : __ Goto(&check_done);
1566 :
1567 4216 : __ Bind(&check_done);
1568 : }
1569 :
1570 6747 : return value;
1571 : }
1572 :
1573 0 : Node* EffectControlLinearizer::LowerCheckedInt32ToTaggedSigned(
1574 : Node* node, Node* frame_state) {
1575 : DCHECK(SmiValuesAre31Bits());
1576 : Node* value = node->InputAt(0);
1577 :
1578 0 : Node* add = __ Int32AddWithOverflow(value, value);
1579 0 : Node* check = __ Projection(1, add);
1580 0 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, check, frame_state);
1581 0 : return __ Projection(0, add);
1582 : }
1583 :
1584 219 : Node* EffectControlLinearizer::LowerCheckedUint32ToInt32(Node* node,
1585 : Node* frame_state) {
1586 : Node* value = node->InputAt(0);
1587 219 : Node* unsafe = __ Int32LessThan(value, __ Int32Constant(0));
1588 219 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, unsafe, frame_state);
1589 219 : return value;
1590 : }
1591 :
1592 24 : Node* EffectControlLinearizer::LowerCheckedUint32ToTaggedSigned(
1593 : Node* node, Node* frame_state) {
1594 : Node* value = node->InputAt(0);
1595 24 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
1596 24 : __ DeoptimizeUnless(DeoptimizeReason::kLostPrecision, check, frame_state);
1597 24 : return ChangeUint32ToSmi(value);
1598 : }
1599 :
1600 8683 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt32(
1601 : CheckForMinusZeroMode mode, Node* value, Node* frame_state) {
1602 8683 : Node* value32 = __ RoundFloat64ToInt32(value);
1603 8683 : Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32));
1604 : __ DeoptimizeUnless(DeoptimizeReason::kLostPrecisionOrNaN, check_same,
1605 8683 : frame_state);
1606 :
1607 8683 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1608 : // Check if {value} is -0.
1609 2277 : auto if_zero = __ MakeDeferredLabel<1>();
1610 2277 : auto check_done = __ MakeLabel<2>();
1611 :
1612 2277 : Node* check_zero = __ Word32Equal(value32, __ Int32Constant(0));
1613 2277 : __ GotoIf(check_zero, &if_zero);
1614 : __ Goto(&check_done);
1615 :
1616 : __ Bind(&if_zero);
1617 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
1618 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
1619 2277 : __ Int32Constant(0));
1620 2277 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, check_negative, frame_state);
1621 : __ Goto(&check_done);
1622 :
1623 2277 : __ Bind(&check_done);
1624 : }
1625 8683 : return value32;
1626 : }
1627 :
1628 6193 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt32(Node* node,
1629 : Node* frame_state) {
1630 6193 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1631 : Node* value = node->InputAt(0);
1632 6193 : return BuildCheckedFloat64ToInt32(mode, value, frame_state);
1633 : }
1634 :
1635 27194 : Node* EffectControlLinearizer::LowerCheckedTaggedSignedToInt32(
1636 : Node* node, Node* frame_state) {
1637 : Node* value = node->InputAt(0);
1638 27194 : Node* check = ObjectIsSmi(value);
1639 27194 : __ DeoptimizeUnless(DeoptimizeReason::kNotASmi, check, frame_state);
1640 27194 : return ChangeSmiToInt32(value);
1641 : }
1642 :
1643 2490 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt32(Node* node,
1644 : Node* frame_state) {
1645 2490 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1646 : Node* value = node->InputAt(0);
1647 :
1648 2490 : auto if_not_smi = __ MakeDeferredLabel<1>();
1649 2490 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1650 :
1651 2490 : Node* check = ObjectIsSmi(value);
1652 2490 : __ GotoUnless(check, &if_not_smi);
1653 : // In the Smi case, just convert to int32.
1654 2490 : __ Goto(&done, ChangeSmiToInt32(value));
1655 :
1656 : // In the non-Smi case, check the heap numberness, load the number and convert
1657 : // to int32.
1658 : __ Bind(&if_not_smi);
1659 2490 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1660 2490 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
1661 : __ DeoptimizeUnless(DeoptimizeReason::kNotAHeapNumber, check_map,
1662 2490 : frame_state);
1663 2490 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1664 2490 : vfalse = BuildCheckedFloat64ToInt32(mode, vfalse, frame_state);
1665 : __ Goto(&done, vfalse);
1666 :
1667 2490 : __ Bind(&done);
1668 2490 : return done.PhiAt(0);
1669 : }
1670 :
1671 22842 : Node* EffectControlLinearizer::BuildCheckedHeapNumberOrOddballToFloat64(
1672 : CheckTaggedInputMode mode, Node* value, Node* frame_state) {
1673 22842 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1674 22843 : Node* check_number = __ WordEqual(value_map, __ HeapNumberMapConstant());
1675 22843 : switch (mode) {
1676 : case CheckTaggedInputMode::kNumber: {
1677 : __ DeoptimizeUnless(DeoptimizeReason::kNotAHeapNumber, check_number,
1678 1473 : frame_state);
1679 1473 : break;
1680 : }
1681 : case CheckTaggedInputMode::kNumberOrOddball: {
1682 21370 : auto check_done = __ MakeLabel<2>();
1683 :
1684 21370 : __ GotoIf(check_number, &check_done);
1685 : // For oddballs also contain the numeric value, let us just check that
1686 : // we have an oddball here.
1687 : Node* instance_type =
1688 21370 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1689 : Node* check_oddball =
1690 21370 : __ Word32Equal(instance_type, __ Int32Constant(ODDBALL_TYPE));
1691 : __ DeoptimizeUnless(DeoptimizeReason::kNotANumberOrOddball, check_oddball,
1692 21369 : frame_state);
1693 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1694 : __ Goto(&check_done);
1695 :
1696 21370 : __ Bind(&check_done);
1697 : break;
1698 : }
1699 : }
1700 22843 : return __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1701 : }
1702 :
1703 20767 : Node* EffectControlLinearizer::LowerCheckedTaggedToFloat64(Node* node,
1704 : Node* frame_state) {
1705 20767 : CheckTaggedInputMode mode = CheckTaggedInputModeOf(node->op());
1706 : Node* value = node->InputAt(0);
1707 :
1708 20767 : auto if_smi = __ MakeLabel<1>();
1709 20767 : auto done = __ MakeLabel<2>(MachineRepresentation::kFloat64);
1710 :
1711 20767 : Node* check = ObjectIsSmi(value);
1712 20767 : __ GotoIf(check, &if_smi);
1713 :
1714 : // In the Smi case, just convert to int32 and then float64.
1715 : // Otherwise, check heap numberness and load the number.
1716 : Node* number =
1717 20767 : BuildCheckedHeapNumberOrOddballToFloat64(mode, value, frame_state);
1718 : __ Goto(&done, number);
1719 :
1720 : __ Bind(&if_smi);
1721 20767 : Node* from_smi = ChangeSmiToInt32(value);
1722 20767 : from_smi = __ ChangeInt32ToFloat64(from_smi);
1723 : __ Goto(&done, from_smi);
1724 :
1725 20767 : __ Bind(&done);
1726 20767 : return done.PhiAt(0);
1727 : }
1728 :
1729 23022 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedSigned(
1730 : Node* node, Node* frame_state) {
1731 : Node* value = node->InputAt(0);
1732 :
1733 23022 : Node* check = ObjectIsSmi(value);
1734 23022 : __ DeoptimizeUnless(DeoptimizeReason::kNotASmi, check, frame_state);
1735 :
1736 23022 : return value;
1737 : }
1738 :
1739 19690 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedPointer(
1740 : Node* node, Node* frame_state) {
1741 : Node* value = node->InputAt(0);
1742 :
1743 19690 : Node* check = ObjectIsSmi(value);
1744 19690 : __ DeoptimizeIf(DeoptimizeReason::kSmi, check, frame_state);
1745 19690 : return value;
1746 : }
1747 :
1748 1155 : Node* EffectControlLinearizer::LowerTruncateTaggedToWord32(Node* node) {
1749 : Node* value = node->InputAt(0);
1750 :
1751 1155 : auto if_not_smi = __ MakeDeferredLabel<1>();
1752 1155 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1753 :
1754 1155 : Node* check = ObjectIsSmi(value);
1755 1155 : __ GotoUnless(check, &if_not_smi);
1756 1155 : __ Goto(&done, ChangeSmiToInt32(value));
1757 :
1758 : __ Bind(&if_not_smi);
1759 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1760 1155 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1761 1155 : vfalse = __ TruncateFloat64ToWord32(vfalse);
1762 : __ Goto(&done, vfalse);
1763 :
1764 1155 : __ Bind(&done);
1765 1155 : return done.PhiAt(0);
1766 : }
1767 :
1768 2076 : Node* EffectControlLinearizer::LowerCheckedTruncateTaggedToWord32(
1769 : Node* node, Node* frame_state) {
1770 : Node* value = node->InputAt(0);
1771 :
1772 2076 : auto if_not_smi = __ MakeLabel<1>();
1773 2076 : auto done = __ MakeLabel<2>(MachineRepresentation::kWord32);
1774 :
1775 2076 : Node* check = ObjectIsSmi(value);
1776 2076 : __ GotoUnless(check, &if_not_smi);
1777 : // In the Smi case, just convert to int32.
1778 2076 : __ Goto(&done, ChangeSmiToInt32(value));
1779 :
1780 : // Otherwise, check that it's a heap number or oddball and truncate the value
1781 : // to int32.
1782 : __ Bind(&if_not_smi);
1783 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
1784 2076 : CheckTaggedInputMode::kNumberOrOddball, value, frame_state);
1785 2076 : number = __ TruncateFloat64ToWord32(number);
1786 : __ Goto(&done, number);
1787 :
1788 2076 : __ Bind(&done);
1789 2076 : return done.PhiAt(0);
1790 : }
1791 :
1792 17020 : Node* EffectControlLinearizer::LowerObjectIsDetectableCallable(Node* node) {
1793 : Node* value = node->InputAt(0);
1794 :
1795 17020 : auto if_smi = __ MakeDeferredLabel<1>();
1796 17020 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1797 :
1798 17020 : Node* check = ObjectIsSmi(value);
1799 17020 : __ GotoIf(check, &if_smi);
1800 :
1801 17020 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1802 : Node* value_bit_field =
1803 17020 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1804 : Node* vfalse = __ Word32Equal(
1805 : __ Int32Constant(1 << Map::kIsCallable),
1806 : __ Word32And(value_bit_field,
1807 : __ Int32Constant((1 << Map::kIsCallable) |
1808 17020 : (1 << Map::kIsUndetectable))));
1809 : __ Goto(&done, vfalse);
1810 :
1811 : __ Bind(&if_smi);
1812 17020 : __ Goto(&done, __ Int32Constant(0));
1813 :
1814 17020 : __ Bind(&done);
1815 17020 : return done.PhiAt(0);
1816 : }
1817 :
1818 2976 : Node* EffectControlLinearizer::LowerObjectIsNaN(Node* node) {
1819 : Node* value = node->InputAt(0);
1820 2976 : Node* zero = __ Int32Constant(0);
1821 :
1822 2976 : auto done = __ MakeLabel<3>(MachineRepresentation::kBit);
1823 :
1824 : // Check if {value} is a Smi.
1825 2976 : __ GotoIf(ObjectIsSmi(value), &done, zero);
1826 :
1827 : // Check if {value} is a HeapNumber.
1828 2976 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1829 : __ GotoUnless(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
1830 2976 : zero);
1831 :
1832 : // Check if {value} contains a NaN.
1833 2976 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1834 : __ Goto(&done,
1835 2976 : __ Word32Equal(__ Float64Equal(value_value, value_value), zero));
1836 :
1837 2976 : __ Bind(&done);
1838 2976 : return done.PhiAt(0);
1839 : }
1840 :
1841 8189 : Node* EffectControlLinearizer::LowerObjectIsNonCallable(Node* node) {
1842 : Node* value = node->InputAt(0);
1843 :
1844 8189 : auto if_primitive = __ MakeDeferredLabel<2>();
1845 8189 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1846 :
1847 8189 : Node* check0 = ObjectIsSmi(value);
1848 8189 : __ GotoIf(check0, &if_primitive);
1849 :
1850 8189 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1851 : Node* value_instance_type =
1852 8189 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1853 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1854 : Node* check1 = __ Uint32LessThanOrEqual(
1855 8189 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1856 8189 : __ GotoUnless(check1, &if_primitive);
1857 :
1858 : Node* value_bit_field =
1859 8189 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1860 : Node* check2 = __ Word32Equal(
1861 : __ Int32Constant(0),
1862 8189 : __ Word32And(value_bit_field, __ Int32Constant(1 << Map::kIsCallable)));
1863 : __ Goto(&done, check2);
1864 :
1865 8189 : __ Bind(&if_primitive);
1866 8189 : __ Goto(&done, __ Int32Constant(0));
1867 :
1868 8189 : __ Bind(&done);
1869 8189 : return done.PhiAt(0);
1870 : }
1871 :
1872 11024 : Node* EffectControlLinearizer::LowerObjectIsNumber(Node* node) {
1873 : Node* value = node->InputAt(0);
1874 :
1875 11024 : auto if_smi = __ MakeLabel<1>();
1876 11024 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1877 :
1878 11024 : __ GotoIf(ObjectIsSmi(value), &if_smi);
1879 11024 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1880 11024 : __ Goto(&done, __ WordEqual(value_map, __ HeapNumberMapConstant()));
1881 :
1882 : __ Bind(&if_smi);
1883 11024 : __ Goto(&done, __ Int32Constant(1));
1884 :
1885 11024 : __ Bind(&done);
1886 11024 : return done.PhiAt(0);
1887 : }
1888 :
1889 21909 : Node* EffectControlLinearizer::LowerObjectIsReceiver(Node* node) {
1890 : Node* value = node->InputAt(0);
1891 :
1892 21909 : auto if_smi = __ MakeDeferredLabel<1>();
1893 21909 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1894 :
1895 21909 : __ GotoIf(ObjectIsSmi(value), &if_smi);
1896 :
1897 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1898 21909 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1899 : Node* value_instance_type =
1900 21909 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1901 : Node* result = __ Uint32LessThanOrEqual(
1902 21909 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1903 : __ Goto(&done, result);
1904 :
1905 : __ Bind(&if_smi);
1906 21909 : __ Goto(&done, __ Int32Constant(0));
1907 :
1908 21909 : __ Bind(&done);
1909 21909 : return done.PhiAt(0);
1910 : }
1911 :
1912 8556 : Node* EffectControlLinearizer::LowerObjectIsSmi(Node* node) {
1913 : Node* value = node->InputAt(0);
1914 8556 : return ObjectIsSmi(value);
1915 : }
1916 :
1917 4846 : Node* EffectControlLinearizer::LowerObjectIsString(Node* node) {
1918 : Node* value = node->InputAt(0);
1919 :
1920 4846 : auto if_smi = __ MakeDeferredLabel<1>();
1921 4846 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1922 :
1923 4846 : Node* check = ObjectIsSmi(value);
1924 4846 : __ GotoIf(check, &if_smi);
1925 4846 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1926 : Node* value_instance_type =
1927 4846 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1928 : Node* vfalse = __ Uint32LessThan(value_instance_type,
1929 4846 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
1930 : __ Goto(&done, vfalse);
1931 :
1932 : __ Bind(&if_smi);
1933 4846 : __ Goto(&done, __ Int32Constant(0));
1934 :
1935 4846 : __ Bind(&done);
1936 4846 : return done.PhiAt(0);
1937 : }
1938 :
1939 665 : Node* EffectControlLinearizer::LowerObjectIsSymbol(Node* node) {
1940 : Node* value = node->InputAt(0);
1941 :
1942 665 : auto if_smi = __ MakeDeferredLabel<1>();
1943 665 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1944 :
1945 665 : Node* check = ObjectIsSmi(value);
1946 665 : __ GotoIf(check, &if_smi);
1947 665 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1948 : Node* value_instance_type =
1949 665 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1950 : Node* vfalse =
1951 665 : __ Word32Equal(value_instance_type, __ Uint32Constant(SYMBOL_TYPE));
1952 : __ Goto(&done, vfalse);
1953 :
1954 : __ Bind(&if_smi);
1955 665 : __ Goto(&done, __ Int32Constant(0));
1956 :
1957 665 : __ Bind(&done);
1958 665 : return done.PhiAt(0);
1959 : }
1960 :
1961 8923 : Node* EffectControlLinearizer::LowerObjectIsUndetectable(Node* node) {
1962 : Node* value = node->InputAt(0);
1963 :
1964 8923 : auto if_smi = __ MakeDeferredLabel<1>();
1965 8923 : auto done = __ MakeLabel<2>(MachineRepresentation::kBit);
1966 :
1967 8923 : Node* check = ObjectIsSmi(value);
1968 8923 : __ GotoIf(check, &if_smi);
1969 :
1970 8923 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1971 : Node* value_bit_field =
1972 8923 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1973 : Node* vfalse = __ Word32Equal(
1974 : __ Word32Equal(__ Int32Constant(0),
1975 : __ Word32And(value_bit_field,
1976 : __ Int32Constant(1 << Map::kIsUndetectable))),
1977 8923 : __ Int32Constant(0));
1978 : __ Goto(&done, vfalse);
1979 :
1980 : __ Bind(&if_smi);
1981 8923 : __ Goto(&done, __ Int32Constant(0));
1982 :
1983 8923 : __ Bind(&done);
1984 8923 : return done.PhiAt(0);
1985 : }
1986 :
1987 11994 : Node* EffectControlLinearizer::LowerArgumentsLength(Node* node) {
1988 3998 : Node* arguments_frame = NodeProperties::GetValueInput(node, 0);
1989 3998 : int formal_parameter_count = FormalParameterCountOf(node->op());
1990 3998 : bool is_rest_length = IsRestLengthOf(node->op());
1991 : DCHECK(formal_parameter_count >= 0);
1992 :
1993 3998 : if (is_rest_length) {
1994 : // The ArgumentsLength node is computing the number of rest parameters,
1995 : // which is max(0, actual_parameter_count - formal_parameter_count).
1996 : // We have to distinguish the case, when there is an arguments adaptor frame
1997 : // (i.e., arguments_frame != LoadFramePointer()).
1998 551 : auto if_adaptor_frame = __ MakeLabel<1>();
1999 551 : auto done = __ MakeLabel<3>(MachineRepresentation::kTaggedSigned);
2000 :
2001 551 : Node* frame = __ LoadFramePointer();
2002 551 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done, __ SmiConstant(0));
2003 : __ Goto(&if_adaptor_frame);
2004 :
2005 : __ Bind(&if_adaptor_frame);
2006 : Node* arguments_length = __ Load(
2007 : MachineType::TaggedSigned(), arguments_frame,
2008 551 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
2009 :
2010 : Node* rest_length =
2011 551 : __ IntSub(arguments_length, __ SmiConstant(formal_parameter_count));
2012 : __ GotoIf(__ IntLessThan(rest_length, __ SmiConstant(0)), &done,
2013 551 : __ SmiConstant(0));
2014 : __ Goto(&done, rest_length);
2015 :
2016 551 : __ Bind(&done);
2017 : return done.PhiAt(0);
2018 : } else {
2019 : // The ArgumentsLength node is computing the actual number of arguments.
2020 : // We have to distinguish the case when there is an arguments adaptor frame
2021 : // (i.e., arguments_frame != LoadFramePointer()).
2022 3447 : auto if_adaptor_frame = __ MakeLabel<1>();
2023 3447 : auto done = __ MakeLabel<2>(MachineRepresentation::kTaggedSigned);
2024 :
2025 3447 : Node* frame = __ LoadFramePointer();
2026 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done,
2027 3447 : __ SmiConstant(formal_parameter_count));
2028 : __ Goto(&if_adaptor_frame);
2029 :
2030 : __ Bind(&if_adaptor_frame);
2031 : Node* arguments_length = __ Load(
2032 : MachineType::TaggedSigned(), arguments_frame,
2033 3447 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
2034 : __ Goto(&done, arguments_length);
2035 :
2036 3447 : __ Bind(&done);
2037 : return done.PhiAt(0);
2038 : }
2039 : }
2040 :
2041 3951 : Node* EffectControlLinearizer::LowerArgumentsFrame(Node* node) {
2042 3951 : auto done = __ MakeLabel<2>(MachineType::PointerRepresentation());
2043 :
2044 3951 : Node* frame = __ LoadFramePointer();
2045 : Node* parent_frame =
2046 : __ Load(MachineType::AnyTagged(), frame,
2047 3951 : __ IntPtrConstant(StandardFrameConstants::kCallerFPOffset));
2048 : Node* parent_frame_type = __ Load(
2049 : MachineType::AnyTagged(), parent_frame,
2050 3951 : __ IntPtrConstant(CommonFrameConstants::kContextOrFrameTypeOffset));
2051 : __ GotoIf(__ WordEqual(parent_frame_type,
2052 : __ IntPtrConstant(StackFrame::TypeToMarker(
2053 : StackFrame::ARGUMENTS_ADAPTOR))),
2054 3951 : &done, parent_frame);
2055 : __ Goto(&done, frame);
2056 :
2057 3951 : __ Bind(&done);
2058 3951 : return done.PhiAt(0);
2059 : }
2060 :
2061 7596 : Node* EffectControlLinearizer::LowerNewUnmappedArgumentsElements(Node* node) {
2062 3798 : Node* frame = NodeProperties::GetValueInput(node, 0);
2063 3798 : Node* length = NodeProperties::GetValueInput(node, 1);
2064 :
2065 : Callable const callable =
2066 3798 : CodeFactory::NewUnmappedArgumentsElements(isolate());
2067 3798 : Operator::Properties const properties = node->op()->properties();
2068 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
2069 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
2070 11394 : isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
2071 : return __ Call(desc, __ HeapConstant(callable.code()), frame, length,
2072 7596 : __ NoContextConstant());
2073 : }
2074 :
2075 325 : Node* EffectControlLinearizer::LowerArrayBufferWasNeutered(Node* node) {
2076 : Node* value = node->InputAt(0);
2077 :
2078 : Node* value_bit_field =
2079 325 : __ LoadField(AccessBuilder::ForJSArrayBufferBitField(), value);
2080 : return __ Word32Equal(
2081 : __ Word32Equal(
2082 : __ Word32And(value_bit_field,
2083 : __ Int32Constant(JSArrayBuffer::WasNeutered::kMask)),
2084 : __ Int32Constant(0)),
2085 325 : __ Int32Constant(0));
2086 : }
2087 :
2088 287 : Node* EffectControlLinearizer::LowerStringCharAt(Node* node) {
2089 : Node* receiver = node->InputAt(0);
2090 : Node* position = node->InputAt(1);
2091 :
2092 287 : Callable const callable = CodeFactory::StringCharAt(isolate());
2093 287 : Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite;
2094 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2095 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
2096 861 : isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
2097 : return __ Call(desc, __ HeapConstant(callable.code()), receiver, position,
2098 574 : __ NoContextConstant());
2099 : }
2100 :
2101 379 : Node* EffectControlLinearizer::LowerStringCharCodeAt(Node* node) {
2102 : Node* receiver = node->InputAt(0);
2103 : Node* position = node->InputAt(1);
2104 :
2105 379 : Callable const callable = CodeFactory::StringCharCodeAt(isolate());
2106 379 : Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite;
2107 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2108 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
2109 : isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties,
2110 1137 : MachineType::TaggedSigned());
2111 : return __ Call(desc, __ HeapConstant(callable.code()), receiver, position,
2112 758 : __ NoContextConstant());
2113 : }
2114 :
2115 752 : Node* EffectControlLinearizer::LowerStringFromCharCode(Node* node) {
2116 : Node* value = node->InputAt(0);
2117 :
2118 752 : auto runtime_call = __ MakeDeferredLabel<2>();
2119 : auto if_undefined = __ MakeDeferredLabel<1>();
2120 752 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
2121 :
2122 : // Compute the character code.
2123 752 : Node* code = __ Word32And(value, __ Int32Constant(String::kMaxUtf16CodeUnit));
2124 :
2125 : // Check if the {code} is a one-byte char code.
2126 : Node* check0 = __ Int32LessThanOrEqual(
2127 752 : code, __ Int32Constant(String::kMaxOneByteCharCode));
2128 752 : __ GotoUnless(check0, &runtime_call);
2129 :
2130 : // Load the isolate wide single character string cache.
2131 752 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
2132 :
2133 : // Compute the {cache} index for {code}.
2134 752 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
2135 :
2136 : // Check if we have an entry for the {code} in the single character string
2137 : // cache already.
2138 : Node* entry =
2139 752 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
2140 :
2141 752 : Node* check1 = __ WordEqual(entry, __ UndefinedConstant());
2142 752 : __ GotoIf(check1, &runtime_call);
2143 : __ Goto(&done, entry);
2144 :
2145 : // Let %StringFromCharCode handle this case.
2146 : // TODO(turbofan): At some point we may consider adding a stub for this
2147 : // deferred case, so that we don't need to call to C++ here.
2148 752 : __ Bind(&runtime_call);
2149 : {
2150 752 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
2151 : Runtime::FunctionId id = Runtime::kStringCharFromCode;
2152 : CallDescriptor const* desc = Linkage::GetRuntimeCallDescriptor(
2153 752 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
2154 : Node* vtrue1 =
2155 : __ Call(desc, __ CEntryStubConstant(1), ChangeInt32ToSmi(code),
2156 : __ ExternalConstant(ExternalReference(id, isolate())),
2157 1504 : __ Int32Constant(1), __ NoContextConstant());
2158 : __ Goto(&done, vtrue1);
2159 : }
2160 752 : __ Bind(&done);
2161 752 : return done.PhiAt(0);
2162 : }
2163 :
2164 114 : Node* EffectControlLinearizer::LowerStringFromCodePoint(Node* node) {
2165 : Node* value = node->InputAt(0);
2166 : Node* code = value;
2167 :
2168 57 : auto if_not_single_code = __ MakeDeferredLabel<1>();
2169 57 : auto if_not_one_byte = __ MakeDeferredLabel<1>();
2170 57 : auto cache_miss = __ MakeDeferredLabel<1>();
2171 57 : auto done = __ MakeLabel<4>(MachineRepresentation::kTagged);
2172 :
2173 : // Check if the {code} is a single code unit
2174 57 : Node* check0 = __ Uint32LessThanOrEqual(code, __ Uint32Constant(0xFFFF));
2175 57 : __ GotoUnless(check0, &if_not_single_code);
2176 :
2177 : {
2178 : // Check if the {code} is a one byte character
2179 : Node* check1 = __ Uint32LessThanOrEqual(
2180 57 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
2181 57 : __ GotoUnless(check1, &if_not_one_byte);
2182 : {
2183 : // Load the isolate wide single character string cache.
2184 57 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
2185 :
2186 : // Compute the {cache} index for {code}.
2187 57 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
2188 :
2189 : // Check if we have an entry for the {code} in the single character string
2190 : // cache already.
2191 : Node* entry =
2192 57 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
2193 :
2194 57 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
2195 57 : __ GotoIf(check2, &cache_miss);
2196 :
2197 : // Use the {entry} from the {cache}.
2198 : __ Goto(&done, entry);
2199 :
2200 : __ Bind(&cache_miss);
2201 : {
2202 : // Allocate a new SeqOneByteString for {code}.
2203 : Node* vtrue2 = __ Allocate(
2204 57 : NOT_TENURED, __ Int32Constant(SeqOneByteString::SizeFor(1)));
2205 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
2206 57 : __ HeapConstant(factory()->one_byte_string_map()));
2207 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
2208 57 : __ IntPtrConstant(Name::kEmptyHashField));
2209 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
2210 57 : __ SmiConstant(1));
2211 : __ Store(
2212 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
2213 : vtrue2,
2214 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
2215 114 : code);
2216 :
2217 : // Remember it in the {cache}.
2218 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
2219 57 : vtrue2);
2220 : __ Goto(&done, vtrue2);
2221 : }
2222 : }
2223 :
2224 : __ Bind(&if_not_one_byte);
2225 : {
2226 : // Allocate a new SeqTwoByteString for {code}.
2227 : Node* vfalse1 = __ Allocate(
2228 57 : NOT_TENURED, __ Int32Constant(SeqTwoByteString::SizeFor(1)));
2229 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
2230 57 : __ HeapConstant(factory()->string_map()));
2231 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
2232 57 : __ IntPtrConstant(Name::kEmptyHashField));
2233 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
2234 57 : __ SmiConstant(1));
2235 : __ Store(
2236 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
2237 : vfalse1,
2238 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
2239 114 : code);
2240 : __ Goto(&done, vfalse1);
2241 : }
2242 : }
2243 :
2244 : __ Bind(&if_not_single_code);
2245 : // Generate surrogate pair string
2246 : {
2247 57 : switch (UnicodeEncodingOf(node->op())) {
2248 : case UnicodeEncoding::UTF16:
2249 : break;
2250 :
2251 : case UnicodeEncoding::UTF32: {
2252 : // Convert UTF32 to UTF16 code units, and store as a 32 bit word.
2253 0 : Node* lead_offset = __ Int32Constant(0xD800 - (0x10000 >> 10));
2254 :
2255 : // lead = (codepoint >> 10) + LEAD_OFFSET
2256 : Node* lead =
2257 0 : __ Int32Add(__ Word32Shr(code, __ Int32Constant(10)), lead_offset);
2258 :
2259 : // trail = (codepoint & 0x3FF) + 0xDC00;
2260 : Node* trail = __ Int32Add(__ Word32And(code, __ Int32Constant(0x3FF)),
2261 0 : __ Int32Constant(0xDC00));
2262 :
2263 : // codpoint = (trail << 16) | lead;
2264 0 : code = __ Word32Or(__ Word32Shl(trail, __ Int32Constant(16)), lead);
2265 0 : break;
2266 : }
2267 : }
2268 :
2269 : // Allocate a new SeqTwoByteString for {code}.
2270 : Node* vfalse0 = __ Allocate(NOT_TENURED,
2271 57 : __ Int32Constant(SeqTwoByteString::SizeFor(2)));
2272 : __ StoreField(AccessBuilder::ForMap(), vfalse0,
2273 57 : __ HeapConstant(factory()->string_map()));
2274 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse0,
2275 57 : __ IntPtrConstant(Name::kEmptyHashField));
2276 57 : __ StoreField(AccessBuilder::ForStringLength(), vfalse0, __ SmiConstant(2));
2277 : __ Store(
2278 : StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier),
2279 : vfalse0,
2280 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
2281 114 : code);
2282 : __ Goto(&done, vfalse0);
2283 : }
2284 :
2285 57 : __ Bind(&done);
2286 57 : return done.PhiAt(0);
2287 : }
2288 :
2289 168 : Node* EffectControlLinearizer::LowerStringIndexOf(Node* node) {
2290 : Node* subject = node->InputAt(0);
2291 : Node* search_string = node->InputAt(1);
2292 : Node* position = node->InputAt(2);
2293 :
2294 168 : Callable callable = CodeFactory::StringIndexOf(isolate());
2295 : Operator::Properties properties = Operator::kEliminatable;
2296 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2297 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
2298 504 : isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
2299 : return __ Call(desc, __ HeapConstant(callable.code()), subject, search_string,
2300 336 : position, __ NoContextConstant());
2301 : }
2302 :
2303 2689 : Node* EffectControlLinearizer::LowerStringComparison(Callable const& callable,
2304 : Node* node) {
2305 : Node* lhs = node->InputAt(0);
2306 : Node* rhs = node->InputAt(1);
2307 :
2308 : Operator::Properties properties = Operator::kEliminatable;
2309 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2310 : CallDescriptor* desc = Linkage::GetStubCallDescriptor(
2311 8067 : isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
2312 : return __ Call(desc, __ HeapConstant(callable.code()), lhs, rhs,
2313 5378 : __ NoContextConstant());
2314 : }
2315 :
2316 2205 : Node* EffectControlLinearizer::LowerStringEqual(Node* node) {
2317 2205 : return LowerStringComparison(CodeFactory::StringEqual(isolate()), node);
2318 : }
2319 :
2320 379 : Node* EffectControlLinearizer::LowerStringLessThan(Node* node) {
2321 379 : return LowerStringComparison(CodeFactory::StringLessThan(isolate()), node);
2322 : }
2323 :
2324 105 : Node* EffectControlLinearizer::LowerStringLessThanOrEqual(Node* node) {
2325 : return LowerStringComparison(CodeFactory::StringLessThanOrEqual(isolate()),
2326 105 : node);
2327 : }
2328 :
2329 651 : Node* EffectControlLinearizer::LowerCheckFloat64Hole(Node* node,
2330 : Node* frame_state) {
2331 : // If we reach this point w/o eliminating the {node} that's marked
2332 : // with allow-return-hole, we cannot do anything, so just deoptimize
2333 : // in case of the hole NaN (similar to Crankshaft).
2334 : Node* value = node->InputAt(0);
2335 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
2336 651 : __ Int32Constant(kHoleNanUpper32));
2337 651 : __ DeoptimizeIf(DeoptimizeReason::kHole, check, frame_state);
2338 651 : return value;
2339 : }
2340 :
2341 917 : Node* EffectControlLinearizer::LowerCheckTaggedHole(Node* node,
2342 : Node* frame_state) {
2343 : Node* value = node->InputAt(0);
2344 917 : Node* check = __ WordEqual(value, __ TheHoleConstant());
2345 917 : __ DeoptimizeIf(DeoptimizeReason::kHole, check, frame_state);
2346 917 : return value;
2347 : }
2348 :
2349 1529 : Node* EffectControlLinearizer::LowerConvertTaggedHoleToUndefined(Node* node) {
2350 : Node* value = node->InputAt(0);
2351 :
2352 1529 : auto if_is_hole = __ MakeDeferredLabel<1>();
2353 1529 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
2354 :
2355 1529 : Node* check = __ WordEqual(value, __ TheHoleConstant());
2356 1529 : __ GotoIf(check, &if_is_hole);
2357 : __ Goto(&done, value);
2358 :
2359 : __ Bind(&if_is_hole);
2360 1529 : __ Goto(&done, __ UndefinedConstant());
2361 :
2362 1529 : __ Bind(&done);
2363 1529 : return done.PhiAt(0);
2364 : }
2365 :
2366 47288 : Node* EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value) {
2367 47288 : Node* result = __ Allocate(NOT_TENURED, __ Int32Constant(HeapNumber::kSize));
2368 47289 : __ StoreField(AccessBuilder::ForMap(), result, __ HeapNumberMapConstant());
2369 47289 : __ StoreField(AccessBuilder::ForHeapNumberValue(), result, value);
2370 47289 : return result;
2371 : }
2372 :
2373 102269 : Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) {
2374 102269 : if (machine()->Is64()) {
2375 102269 : value = __ ChangeInt32ToInt64(value);
2376 : }
2377 102270 : return __ WordShl(value, SmiShiftBitsConstant());
2378 : }
2379 :
2380 708 : Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) {
2381 708 : if (machine()->Is64()) {
2382 708 : value = __ ChangeUint32ToUint64(value);
2383 : }
2384 708 : return __ WordShl(value, SmiShiftBitsConstant());
2385 : }
2386 :
2387 170228 : Node* EffectControlLinearizer::ChangeSmiToInt32(Node* value) {
2388 170229 : value = __ WordSar(value, SmiShiftBitsConstant());
2389 170229 : if (machine()->Is64()) {
2390 170229 : value = __ TruncateInt64ToInt32(value);
2391 : }
2392 170228 : return value;
2393 : }
2394 :
2395 317335 : Node* EffectControlLinearizer::ObjectIsSmi(Node* value) {
2396 : return __ WordEqual(__ WordAnd(value, __ IntPtrConstant(kSmiTagMask)),
2397 317335 : __ IntPtrConstant(kSmiTag));
2398 : }
2399 :
2400 0 : Node* EffectControlLinearizer::SmiMaxValueConstant() {
2401 708 : return __ Int32Constant(Smi::kMaxValue);
2402 : }
2403 :
2404 0 : Node* EffectControlLinearizer::SmiShiftBitsConstant() {
2405 273206 : return __ IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2406 : }
2407 :
2408 16 : Node* EffectControlLinearizer::LowerPlainPrimitiveToNumber(Node* node) {
2409 : Node* value = node->InputAt(0);
2410 16 : return __ ToNumber(value);
2411 : }
2412 :
2413 6372 : Node* EffectControlLinearizer::LowerPlainPrimitiveToWord32(Node* node) {
2414 : Node* value = node->InputAt(0);
2415 :
2416 6372 : auto if_not_smi = __ MakeDeferredLabel<1>();
2417 6372 : auto if_to_number_smi = __ MakeLabel<1>();
2418 6372 : auto done = __ MakeLabel<3>(MachineRepresentation::kWord32);
2419 :
2420 6372 : Node* check0 = ObjectIsSmi(value);
2421 6372 : __ GotoUnless(check0, &if_not_smi);
2422 6372 : __ Goto(&done, ChangeSmiToInt32(value));
2423 :
2424 : __ Bind(&if_not_smi);
2425 6371 : Node* to_number = __ ToNumber(value);
2426 :
2427 6372 : Node* check1 = ObjectIsSmi(to_number);
2428 6372 : __ GotoIf(check1, &if_to_number_smi);
2429 6372 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
2430 6372 : __ Goto(&done, __ TruncateFloat64ToWord32(number));
2431 :
2432 : __ Bind(&if_to_number_smi);
2433 6372 : __ Goto(&done, ChangeSmiToInt32(to_number));
2434 :
2435 6372 : __ Bind(&done);
2436 6370 : return done.PhiAt(0);
2437 : }
2438 :
2439 1891 : Node* EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node) {
2440 : Node* value = node->InputAt(0);
2441 :
2442 1891 : auto if_not_smi = __ MakeDeferredLabel<1>();
2443 1891 : auto if_to_number_smi = __ MakeLabel<1>();
2444 1891 : auto done = __ MakeLabel<3>(MachineRepresentation::kFloat64);
2445 :
2446 1891 : Node* check0 = ObjectIsSmi(value);
2447 1891 : __ GotoUnless(check0, &if_not_smi);
2448 1891 : Node* from_smi = ChangeSmiToInt32(value);
2449 1891 : __ Goto(&done, __ ChangeInt32ToFloat64(from_smi));
2450 :
2451 : __ Bind(&if_not_smi);
2452 1891 : Node* to_number = __ ToNumber(value);
2453 1891 : Node* check1 = ObjectIsSmi(to_number);
2454 1891 : __ GotoIf(check1, &if_to_number_smi);
2455 :
2456 1891 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
2457 : __ Goto(&done, number);
2458 :
2459 : __ Bind(&if_to_number_smi);
2460 1891 : Node* number_from_smi = ChangeSmiToInt32(to_number);
2461 1891 : number_from_smi = __ ChangeInt32ToFloat64(number_from_smi);
2462 : __ Goto(&done, number_from_smi);
2463 :
2464 1891 : __ Bind(&done);
2465 1891 : return done.PhiAt(0);
2466 : }
2467 :
2468 181 : Node* EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node) {
2469 : Node* object = node->InputAt(0);
2470 : Node* elements = node->InputAt(1);
2471 :
2472 181 : auto if_not_fixed_array = __ MakeDeferredLabel<1>();
2473 181 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
2474 :
2475 : // Load the current map of {elements}.
2476 181 : Node* elements_map = __ LoadField(AccessBuilder::ForMap(), elements);
2477 :
2478 : // Check if {elements} is not a copy-on-write FixedArray.
2479 181 : Node* check = __ WordEqual(elements_map, __ FixedArrayMapConstant());
2480 181 : __ GotoUnless(check, &if_not_fixed_array);
2481 : // Nothing to do if the {elements} are not copy-on-write.
2482 : __ Goto(&done, elements);
2483 :
2484 : __ Bind(&if_not_fixed_array);
2485 : // We need to take a copy of the {elements} and set them up for {object}.
2486 : Operator::Properties properties = Operator::kEliminatable;
2487 181 : Callable callable = CodeFactory::CopyFastSmiOrObjectElements(isolate());
2488 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2489 : CallDescriptor const* const desc = Linkage::GetStubCallDescriptor(
2490 543 : isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
2491 : Node* result = __ Call(desc, __ HeapConstant(callable.code()), object,
2492 362 : __ NoContextConstant());
2493 : __ Goto(&done, result);
2494 :
2495 181 : __ Bind(&done);
2496 181 : return done.PhiAt(0);
2497 : }
2498 :
2499 2054 : Node* EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node,
2500 : Node* frame_state) {
2501 2054 : GrowFastElementsFlags flags = GrowFastElementsFlagsOf(node->op());
2502 : Node* object = node->InputAt(0);
2503 : Node* elements = node->InputAt(1);
2504 : Node* index = node->InputAt(2);
2505 : Node* length = node->InputAt(3);
2506 :
2507 2054 : auto done = __ MakeLabel<2>(MachineRepresentation::kTagged);
2508 2054 : auto done_grow = __ MakeLabel<2>(MachineRepresentation::kTagged);
2509 2054 : auto if_grow = __ MakeDeferredLabel<1>();
2510 2054 : auto if_not_grow = __ MakeLabel<1>();
2511 :
2512 : Node* check0 = (flags & GrowFastElementsFlag::kHoleyElements)
2513 190 : ? __ Uint32LessThanOrEqual(length, index)
2514 2244 : : __ Word32Equal(length, index);
2515 2054 : __ GotoUnless(check0, &if_not_grow);
2516 : {
2517 : // Load the length of the {elements} backing store.
2518 : Node* elements_length =
2519 2054 : __ LoadField(AccessBuilder::ForFixedArrayLength(), elements);
2520 2054 : elements_length = ChangeSmiToInt32(elements_length);
2521 :
2522 : // Check if we need to grow the {elements} backing store.
2523 2054 : Node* check1 = __ Uint32LessThan(index, elements_length);
2524 2054 : __ GotoUnless(check1, &if_grow);
2525 : __ Goto(&done_grow, elements);
2526 :
2527 : __ Bind(&if_grow);
2528 : // We need to grow the {elements} for {object}.
2529 : Operator::Properties properties = Operator::kEliminatable;
2530 : Callable callable =
2531 : (flags & GrowFastElementsFlag::kDoubleElements)
2532 : ? CodeFactory::GrowFastDoubleElements(isolate())
2533 4108 : : CodeFactory::GrowFastSmiOrObjectElements(isolate());
2534 : CallDescriptor::Flags call_flags = CallDescriptor::kNoFlags;
2535 : CallDescriptor const* const desc = Linkage::GetStubCallDescriptor(
2536 : isolate(), graph()->zone(), callable.descriptor(), 0, call_flags,
2537 6162 : properties);
2538 : Node* new_object = __ Call(desc, __ HeapConstant(callable.code()), object,
2539 4108 : ChangeInt32ToSmi(index), __ NoContextConstant());
2540 :
2541 : // Ensure that we were able to grow the {elements}.
2542 : // TODO(turbofan): We use kSmi as reason here similar to Crankshaft,
2543 : // but maybe we should just introduce a reason that makes sense.
2544 : __ DeoptimizeIf(DeoptimizeReason::kSmi, ObjectIsSmi(new_object),
2545 2054 : frame_state);
2546 : __ Goto(&done_grow, new_object);
2547 :
2548 2054 : __ Bind(&done_grow);
2549 :
2550 : // For JSArray {object}s we also need to update the "length".
2551 2054 : if (flags & GrowFastElementsFlag::kArrayObject) {
2552 : // Compute the new {length}.
2553 : Node* object_length =
2554 2041 : ChangeInt32ToSmi(__ Int32Add(index, __ Int32Constant(1)));
2555 :
2556 : // Update the "length" property of the {object}.
2557 : __ StoreField(AccessBuilder::ForJSArrayLength(FAST_ELEMENTS), object,
2558 2041 : object_length);
2559 : }
2560 : __ Goto(&done, done_grow.PhiAt(0));
2561 : }
2562 :
2563 : __ Bind(&if_not_grow);
2564 : {
2565 : // In case of non-holey {elements}, we need to verify that the {index} is
2566 : // in-bounds, otherwise for holey {elements}, the check above already
2567 : // guards the index (and the operator forces {index} to be unsigned).
2568 2054 : if (!(flags & GrowFastElementsFlag::kHoleyElements)) {
2569 1864 : Node* check1 = __ Uint32LessThan(index, length);
2570 1864 : __ DeoptimizeUnless(DeoptimizeReason::kOutOfBounds, check1, frame_state);
2571 : }
2572 : __ Goto(&done, elements);
2573 : }
2574 2054 : __ Bind(&done);
2575 2054 : return done.PhiAt(0);
2576 : }
2577 :
2578 462 : void EffectControlLinearizer::LowerTransitionElementsKind(Node* node) {
2579 462 : ElementsTransition const transition = ElementsTransitionOf(node->op());
2580 : Node* object = node->InputAt(0);
2581 :
2582 462 : auto if_map_same = __ MakeDeferredLabel<1>();
2583 462 : auto done = __ MakeLabel<2>();
2584 :
2585 462 : Node* source_map = __ HeapConstant(transition.source());
2586 462 : Node* target_map = __ HeapConstant(transition.target());
2587 :
2588 : // Load the current map of {object}.
2589 462 : Node* object_map = __ LoadField(AccessBuilder::ForMap(), object);
2590 :
2591 : // Check if {object_map} is the same as {source_map}.
2592 462 : Node* check = __ WordEqual(object_map, source_map);
2593 462 : __ GotoIf(check, &if_map_same);
2594 : __ Goto(&done);
2595 :
2596 : __ Bind(&if_map_same);
2597 462 : switch (transition.mode()) {
2598 : case ElementsTransition::kFastTransition:
2599 : // In-place migration of {object}, just store the {target_map}.
2600 202 : __ StoreField(AccessBuilder::ForMap(), object, target_map);
2601 202 : break;
2602 : case ElementsTransition::kSlowTransition: {
2603 : // Instance migration, call out to the runtime for {object}.
2604 260 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
2605 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
2606 : CallDescriptor const* desc = Linkage::GetRuntimeCallDescriptor(
2607 260 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
2608 : __ Call(desc, __ CEntryStubConstant(1), object, target_map,
2609 : __ ExternalConstant(ExternalReference(id, isolate())),
2610 520 : __ Int32Constant(2), __ NoContextConstant());
2611 : break;
2612 : }
2613 : }
2614 : __ Goto(&done);
2615 :
2616 462 : __ Bind(&done);
2617 462 : }
2618 :
2619 3452 : Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) {
2620 3452 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
2621 : Node* buffer = node->InputAt(0);
2622 : Node* base = node->InputAt(1);
2623 : Node* external = node->InputAt(2);
2624 : Node* index = node->InputAt(3);
2625 :
2626 : // We need to keep the {buffer} alive so that the GC will not release the
2627 : // ArrayBuffer (if there's any) as long as we are still operating on it.
2628 3452 : __ Retain(buffer);
2629 :
2630 : // Compute the effective storage pointer, handling the case where the
2631 : // {external} pointer is the effective storage pointer (i.e. the {base}
2632 : // is Smi zero).
2633 : Node* storage = NumberMatcher(base).Is(0) ? external : __ UnsafePointerAdd(
2634 3452 : base, external);
2635 :
2636 : // Perform the actual typed element access.
2637 : return __ LoadElement(AccessBuilder::ForTypedArrayElement(array_type, true),
2638 3452 : storage, index);
2639 : }
2640 :
2641 3062 : void EffectControlLinearizer::LowerStoreTypedElement(Node* node) {
2642 3062 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
2643 : Node* buffer = node->InputAt(0);
2644 : Node* base = node->InputAt(1);
2645 : Node* external = node->InputAt(2);
2646 : Node* index = node->InputAt(3);
2647 : Node* value = node->InputAt(4);
2648 :
2649 : // We need to keep the {buffer} alive so that the GC will not release the
2650 : // ArrayBuffer (if there's any) as long as we are still operating on it.
2651 3062 : __ Retain(buffer);
2652 :
2653 : // Compute the effective storage pointer, handling the case where the
2654 : // {external} pointer is the effective storage pointer (i.e. the {base}
2655 : // is Smi zero).
2656 : Node* storage = NumberMatcher(base).Is(0) ? external : __ UnsafePointerAdd(
2657 3062 : base, external);
2658 :
2659 : // Perform the actual typed element access.
2660 : __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true),
2661 3062 : storage, index, value);
2662 3062 : }
2663 :
2664 5772 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundUp(Node* node) {
2665 : // Nothing to be done if a fast hardware instruction is available.
2666 5772 : if (machine()->Float64RoundUp().IsSupported()) {
2667 : return Nothing<Node*>();
2668 : }
2669 :
2670 : Node* const input = node->InputAt(0);
2671 :
2672 : // General case for ceil.
2673 : //
2674 : // if 0.0 < input then
2675 : // if 2^52 <= input then
2676 : // input
2677 : // else
2678 : // let temp1 = (2^52 + input) - 2^52 in
2679 : // if temp1 < input then
2680 : // temp1 + 1
2681 : // else
2682 : // temp1
2683 : // else
2684 : // if input == 0 then
2685 : // input
2686 : // else
2687 : // if input <= -2^52 then
2688 : // input
2689 : // else
2690 : // let temp1 = -0 - input in
2691 : // let temp2 = (2^52 + temp1) - 2^52 in
2692 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
2693 : // -0 - temp3
2694 :
2695 0 : auto if_not_positive = __ MakeDeferredLabel<1>();
2696 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel<1>();
2697 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel<1>();
2698 0 : auto if_zero = __ MakeDeferredLabel<1>();
2699 0 : auto done_temp3 = __ MakeLabel<2>(MachineRepresentation::kFloat64);
2700 0 : auto done = __ MakeLabel<6>(MachineRepresentation::kFloat64);
2701 :
2702 0 : Node* const zero = __ Float64Constant(0.0);
2703 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
2704 0 : Node* const one = __ Float64Constant(1.0);
2705 :
2706 0 : Node* check0 = __ Float64LessThan(zero, input);
2707 0 : __ GotoUnless(check0, &if_not_positive);
2708 : {
2709 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
2710 0 : __ GotoIf(check1, &if_greater_than_two_52);
2711 : {
2712 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
2713 0 : __ GotoUnless(__ Float64LessThan(temp1, input), &done, temp1);
2714 0 : __ Goto(&done, __ Float64Add(temp1, one));
2715 : }
2716 :
2717 : __ Bind(&if_greater_than_two_52);
2718 : __ Goto(&done, input);
2719 : }
2720 :
2721 : __ Bind(&if_not_positive);
2722 : {
2723 0 : Node* check1 = __ Float64Equal(input, zero);
2724 0 : __ GotoIf(check1, &if_zero);
2725 :
2726 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
2727 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
2728 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
2729 :
2730 : {
2731 0 : Node* const minus_zero = __ Float64Constant(-0.0);
2732 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
2733 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
2734 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
2735 0 : __ GotoUnless(check3, &done_temp3, temp2);
2736 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
2737 :
2738 0 : __ Bind(&done_temp3);
2739 : Node* temp3 = done_temp3.PhiAt(0);
2740 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
2741 : }
2742 : __ Bind(&if_less_than_minus_two_52);
2743 : __ Goto(&done, input);
2744 :
2745 : __ Bind(&if_zero);
2746 : __ Goto(&done, input);
2747 : }
2748 0 : __ Bind(&done);
2749 : return Just(done.PhiAt(0));
2750 : }
2751 :
2752 0 : Node* EffectControlLinearizer::BuildFloat64RoundDown(Node* value) {
2753 0 : Node* round_down = __ Float64RoundDown(value);
2754 0 : if (round_down != nullptr) {
2755 : return round_down;
2756 : }
2757 :
2758 : Node* const input = value;
2759 :
2760 : // General case for floor.
2761 : //
2762 : // if 0.0 < input then
2763 : // if 2^52 <= input then
2764 : // input
2765 : // else
2766 : // let temp1 = (2^52 + input) - 2^52 in
2767 : // if input < temp1 then
2768 : // temp1 - 1
2769 : // else
2770 : // temp1
2771 : // else
2772 : // if input == 0 then
2773 : // input
2774 : // else
2775 : // if input <= -2^52 then
2776 : // input
2777 : // else
2778 : // let temp1 = -0 - input in
2779 : // let temp2 = (2^52 + temp1) - 2^52 in
2780 : // if temp2 < temp1 then
2781 : // -1 - temp2
2782 : // else
2783 : // -0 - temp2
2784 :
2785 0 : auto if_not_positive = __ MakeDeferredLabel<1>();
2786 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel<1>();
2787 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel<1>();
2788 0 : auto if_temp2_lt_temp1 = __ MakeLabel<1>();
2789 0 : auto if_zero = __ MakeDeferredLabel<1>();
2790 0 : auto done = __ MakeLabel<7>(MachineRepresentation::kFloat64);
2791 :
2792 0 : Node* const zero = __ Float64Constant(0.0);
2793 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
2794 :
2795 0 : Node* check0 = __ Float64LessThan(zero, input);
2796 0 : __ GotoUnless(check0, &if_not_positive);
2797 : {
2798 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
2799 0 : __ GotoIf(check1, &if_greater_than_two_52);
2800 : {
2801 0 : Node* const one = __ Float64Constant(1.0);
2802 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
2803 0 : __ GotoUnless(__ Float64LessThan(input, temp1), &done, temp1);
2804 0 : __ Goto(&done, __ Float64Sub(temp1, one));
2805 : }
2806 :
2807 : __ Bind(&if_greater_than_two_52);
2808 : __ Goto(&done, input);
2809 : }
2810 :
2811 : __ Bind(&if_not_positive);
2812 : {
2813 0 : Node* check1 = __ Float64Equal(input, zero);
2814 0 : __ GotoIf(check1, &if_zero);
2815 :
2816 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
2817 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
2818 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
2819 :
2820 : {
2821 0 : Node* const minus_zero = __ Float64Constant(-0.0);
2822 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
2823 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
2824 0 : Node* check3 = __ Float64LessThan(temp2, temp1);
2825 0 : __ GotoIf(check3, &if_temp2_lt_temp1);
2826 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp2));
2827 :
2828 : __ Bind(&if_temp2_lt_temp1);
2829 0 : __ Goto(&done, __ Float64Sub(__ Float64Constant(-1.0), temp2));
2830 : }
2831 : __ Bind(&if_less_than_minus_two_52);
2832 : __ Goto(&done, input);
2833 :
2834 : __ Bind(&if_zero);
2835 : __ Goto(&done, input);
2836 : }
2837 0 : __ Bind(&done);
2838 : return done.PhiAt(0);
2839 : }
2840 :
2841 8875 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundDown(Node* node) {
2842 : // Nothing to be done if a fast hardware instruction is available.
2843 8875 : if (machine()->Float64RoundDown().IsSupported()) {
2844 : return Nothing<Node*>();
2845 : }
2846 :
2847 : Node* const input = node->InputAt(0);
2848 0 : return Just(BuildFloat64RoundDown(input));
2849 : }
2850 :
2851 300 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTiesEven(Node* node) {
2852 : // Nothing to be done if a fast hardware instruction is available.
2853 300 : if (machine()->Float64RoundTiesEven().IsSupported()) {
2854 : return Nothing<Node*>();
2855 : }
2856 :
2857 : Node* const input = node->InputAt(0);
2858 :
2859 : // Generate case for round ties to even:
2860 : //
2861 : // let value = floor(input) in
2862 : // let temp1 = input - value in
2863 : // if temp1 < 0.5 then
2864 : // value
2865 : // else if 0.5 < temp1 then
2866 : // value + 1.0
2867 : // else
2868 : // let temp2 = value % 2.0 in
2869 : // if temp2 == 0.0 then
2870 : // value
2871 : // else
2872 : // value + 1.0
2873 :
2874 0 : auto if_is_half = __ MakeLabel<1>();
2875 0 : auto done = __ MakeLabel<4>(MachineRepresentation::kFloat64);
2876 :
2877 0 : Node* value = BuildFloat64RoundDown(input);
2878 0 : Node* temp1 = __ Float64Sub(input, value);
2879 :
2880 0 : Node* const half = __ Float64Constant(0.5);
2881 0 : Node* check0 = __ Float64LessThan(temp1, half);
2882 0 : __ GotoIf(check0, &done, value);
2883 :
2884 0 : Node* const one = __ Float64Constant(1.0);
2885 0 : Node* check1 = __ Float64LessThan(half, temp1);
2886 0 : __ GotoUnless(check1, &if_is_half);
2887 0 : __ Goto(&done, __ Float64Add(value, one));
2888 :
2889 : __ Bind(&if_is_half);
2890 0 : Node* temp2 = __ Float64Mod(value, __ Float64Constant(2.0));
2891 0 : Node* check2 = __ Float64Equal(temp2, __ Float64Constant(0.0));
2892 0 : __ GotoIf(check2, &done, value);
2893 0 : __ Goto(&done, __ Float64Add(value, one));
2894 :
2895 0 : __ Bind(&done);
2896 : return Just(done.PhiAt(0));
2897 : }
2898 :
2899 1389 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTruncate(Node* node) {
2900 : // Nothing to be done if a fast hardware instruction is available.
2901 1389 : if (machine()->Float64RoundTruncate().IsSupported()) {
2902 : return Nothing<Node*>();
2903 : }
2904 :
2905 : Node* const input = node->InputAt(0);
2906 :
2907 : // General case for trunc.
2908 : //
2909 : // if 0.0 < input then
2910 : // if 2^52 <= input then
2911 : // input
2912 : // else
2913 : // let temp1 = (2^52 + input) - 2^52 in
2914 : // if input < temp1 then
2915 : // temp1 - 1
2916 : // else
2917 : // temp1
2918 : // else
2919 : // if input == 0 then
2920 : // input
2921 : // else
2922 : // if input <= -2^52 then
2923 : // input
2924 : // else
2925 : // let temp1 = -0 - input in
2926 : // let temp2 = (2^52 + temp1) - 2^52 in
2927 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
2928 : // -0 - temp3
2929 : //
2930 : // Note: We do not use the Diamond helper class here, because it really hurts
2931 : // readability with nested diamonds.
2932 :
2933 0 : auto if_not_positive = __ MakeDeferredLabel<1>();
2934 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel<1>();
2935 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel<1>();
2936 0 : auto if_zero = __ MakeDeferredLabel<1>();
2937 0 : auto done_temp3 = __ MakeLabel<2>(MachineRepresentation::kFloat64);
2938 0 : auto done = __ MakeLabel<6>(MachineRepresentation::kFloat64);
2939 :
2940 0 : Node* const zero = __ Float64Constant(0.0);
2941 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
2942 0 : Node* const one = __ Float64Constant(1.0);
2943 :
2944 0 : Node* check0 = __ Float64LessThan(zero, input);
2945 0 : __ GotoUnless(check0, &if_not_positive);
2946 : {
2947 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
2948 0 : __ GotoIf(check1, &if_greater_than_two_52);
2949 : {
2950 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
2951 0 : __ GotoUnless(__ Float64LessThan(input, temp1), &done, temp1);
2952 0 : __ Goto(&done, __ Float64Sub(temp1, one));
2953 : }
2954 :
2955 : __ Bind(&if_greater_than_two_52);
2956 : __ Goto(&done, input);
2957 : }
2958 :
2959 : __ Bind(&if_not_positive);
2960 : {
2961 0 : Node* check1 = __ Float64Equal(input, zero);
2962 0 : __ GotoIf(check1, &if_zero);
2963 :
2964 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
2965 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
2966 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
2967 :
2968 : {
2969 0 : Node* const minus_zero = __ Float64Constant(-0.0);
2970 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
2971 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
2972 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
2973 0 : __ GotoUnless(check3, &done_temp3, temp2);
2974 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
2975 :
2976 0 : __ Bind(&done_temp3);
2977 : Node* temp3 = done_temp3.PhiAt(0);
2978 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
2979 : }
2980 : __ Bind(&if_less_than_minus_two_52);
2981 : __ Goto(&done, input);
2982 :
2983 : __ Bind(&if_zero);
2984 : __ Goto(&done, input);
2985 : }
2986 0 : __ Bind(&done);
2987 : return Just(done.PhiAt(0));
2988 : }
2989 :
2990 : #undef __
2991 :
2992 0 : Factory* EffectControlLinearizer::factory() const {
2993 0 : return isolate()->factory();
2994 : }
2995 :
2996 12245 : Isolate* EffectControlLinearizer::isolate() const {
2997 21801 : return jsgraph()->isolate();
2998 : }
2999 :
3000 : } // namespace compiler
3001 : } // namespace internal
3002 : } // namespace v8
|
