Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/effect-control-linearizer.h"
6 :
7 : #include "src/code-factory.h"
8 : #include "src/compiler/access-builder.h"
9 : #include "src/compiler/compiler-source-position-table.h"
10 : #include "src/compiler/js-graph.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-origin-table.h"
14 : #include "src/compiler/node-properties.h"
15 : #include "src/compiler/node.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/heap/factory-inl.h"
18 : #include "src/objects/heap-number.h"
19 : #include "src/objects/oddball.h"
20 :
21 : namespace v8 {
22 : namespace internal {
23 : namespace compiler {
24 :
25 456086 : EffectControlLinearizer::EffectControlLinearizer(
26 : JSGraph* js_graph, Schedule* schedule, Zone* temp_zone,
27 : SourcePositionTable* source_positions, NodeOriginTable* node_origins,
28 : MaskArrayIndexEnable mask_array_index)
29 : : js_graph_(js_graph),
30 : schedule_(schedule),
31 : temp_zone_(temp_zone),
32 : mask_array_index_(mask_array_index),
33 : source_positions_(source_positions),
34 : node_origins_(node_origins),
35 : graph_assembler_(js_graph, nullptr, nullptr, temp_zone),
36 456086 : frame_state_zapper_(nullptr) {}
37 :
38 1750214 : Graph* EffectControlLinearizer::graph() const { return js_graph_->graph(); }
39 0 : CommonOperatorBuilder* EffectControlLinearizer::common() const {
40 1069451 : return js_graph_->common();
41 : }
42 0 : SimplifiedOperatorBuilder* EffectControlLinearizer::simplified() const {
43 0 : return js_graph_->simplified();
44 : }
45 0 : MachineOperatorBuilder* EffectControlLinearizer::machine() const {
46 336528 : return js_graph_->machine();
47 : }
48 :
49 : namespace {
50 :
51 : struct BlockEffectControlData {
52 : Node* current_effect = nullptr; // New effect.
53 : Node* current_control = nullptr; // New control.
54 : Node* current_frame_state = nullptr; // New frame state.
55 : };
56 :
57 : class BlockEffectControlMap {
58 : public:
59 : explicit BlockEffectControlMap(Zone* temp_zone) : map_(temp_zone) {}
60 :
61 18620640 : BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) {
62 18620640 : return map_[std::make_pair(from->rpo_number(), to->rpo_number())];
63 : }
64 :
65 : const BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) const {
66 : return map_.at(std::make_pair(from->rpo_number(), to->rpo_number()));
67 : }
68 :
69 : private:
70 : typedef std::pair<int32_t, int32_t> Key;
71 : typedef ZoneMap<Key, BlockEffectControlData> Map;
72 :
73 : Map map_;
74 : };
75 :
76 : // Effect phis that need to be updated after the first pass.
77 : struct PendingEffectPhi {
78 : Node* effect_phi;
79 : BasicBlock* block;
80 :
81 : PendingEffectPhi(Node* effect_phi, BasicBlock* block)
82 40683 : : effect_phi(effect_phi), block(block) {}
83 : };
84 :
85 2082 : void ConnectUnreachableToEnd(Node* effect, Node* control, JSGraph* jsgraph) {
86 1041 : Graph* graph = jsgraph->graph();
87 : CommonOperatorBuilder* common = jsgraph->common();
88 2082 : if (effect->opcode() == IrOpcode::kDead) return;
89 1041 : if (effect->opcode() != IrOpcode::kUnreachable) {
90 0 : effect = graph->NewNode(common->Unreachable(), effect, control);
91 : }
92 1041 : Node* throw_node = graph->NewNode(common->Throw(), effect, control);
93 1041 : NodeProperties::MergeControlToEnd(graph, common, throw_node);
94 : }
95 :
96 3305554 : void UpdateEffectPhi(Node* node, BasicBlock* block,
97 : BlockEffectControlMap* block_effects, JSGraph* jsgraph) {
98 : // Update all inputs to an effect phi with the effects from the given
99 : // block->effect map.
100 : DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
101 : DCHECK_EQ(static_cast<size_t>(node->op()->EffectInputCount()),
102 : block->PredecessorCount());
103 6785434 : for (int i = 0; i < node->op()->EffectInputCount(); i++) {
104 : Node* input = node->InputAt(i);
105 1739922 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
106 : const BlockEffectControlData& block_effect =
107 : block_effects->For(predecessor, block);
108 1739965 : Node* effect = block_effect.current_effect;
109 1739965 : if (input != effect) {
110 153833 : node->ReplaceInput(i, effect);
111 : }
112 : }
113 782825 : }
114 :
115 4227938 : void UpdateBlockControl(BasicBlock* block,
116 : BlockEffectControlMap* block_effects) {
117 11991266 : Node* control = block->NodeAt(0);
118 : DCHECK(NodeProperties::IsControl(control));
119 :
120 : // Do not rewire the end node.
121 4227938 : if (control->opcode() == IrOpcode::kEnd) return;
122 :
123 : // Update all inputs to the given control node with the correct control.
124 : DCHECK(control->opcode() == IrOpcode::kMerge ||
125 : static_cast<size_t>(control->op()->ControlInputCount()) ==
126 : block->PredecessorCount());
127 11318793 : if (static_cast<size_t>(control->op()->ControlInputCount()) !=
128 : block->PredecessorCount()) {
129 : return; // We already re-wired the control inputs of this node.
130 : }
131 19692815 : for (int i = 0; i < control->op()->ControlInputCount(); i++) {
132 4166191 : Node* input = NodeProperties::GetControlInput(control, i);
133 4166172 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
134 : const BlockEffectControlData& block_effect =
135 : block_effects->For(predecessor, block);
136 4166157 : if (input != block_effect.current_control) {
137 : NodeProperties::ReplaceControlInput(control, block_effect.current_control,
138 35328 : i);
139 : }
140 : }
141 : }
142 :
143 6725501 : bool HasIncomingBackEdges(BasicBlock* block) {
144 18369585 : for (BasicBlock* pred : block->predecessors()) {
145 6725501 : if (pred->rpo_number() >= block->rpo_number()) {
146 : return true;
147 : }
148 : }
149 : return false;
150 : }
151 :
152 276494 : void RemoveRenameNode(Node* node) {
153 : DCHECK(IrOpcode::kFinishRegion == node->opcode() ||
154 : IrOpcode::kBeginRegion == node->opcode() ||
155 : IrOpcode::kTypeGuard == node->opcode());
156 : // Update the value/context uses to the value input of the finish node and
157 : // the effect uses to the effect input.
158 3420229 : for (Edge edge : node->use_edges()) {
159 : DCHECK(!edge.from()->IsDead());
160 1433620 : if (NodeProperties::IsEffectEdge(edge)) {
161 277932 : edge.UpdateTo(NodeProperties::GetEffectInput(node));
162 : } else {
163 : DCHECK(!NodeProperties::IsControlEdge(edge));
164 : DCHECK(!NodeProperties::IsFrameStateEdge(edge));
165 1155689 : edge.UpdateTo(node->InputAt(0));
166 : }
167 : }
168 276495 : node->Kill();
169 276495 : }
170 :
171 1146403 : void TryCloneBranch(Node* node, BasicBlock* block, Zone* temp_zone,
172 828033 : Graph* graph, CommonOperatorBuilder* common,
173 : BlockEffectControlMap* block_effects,
174 : SourcePositionTable* source_positions,
175 : NodeOriginTable* node_origins) {
176 : DCHECK_EQ(IrOpcode::kBranch, node->opcode());
177 :
178 : // This optimization is a special case of (super)block cloning. It takes an
179 : // input graph as shown below and clones the Branch node for every predecessor
180 : // to the Merge, essentially removing the Merge completely. This avoids
181 : // materializing the bit for the Phi and may offer potential for further
182 : // branch folding optimizations (i.e. because one or more inputs to the Phi is
183 : // a constant). Note that there may be more Phi nodes hanging off the Merge,
184 : // but we can only a certain subset of them currently (actually only Phi and
185 : // EffectPhi nodes whose uses have either the IfTrue or IfFalse as control
186 : // input).
187 :
188 : // Control1 ... ControlN
189 : // ^ ^
190 : // | | Cond1 ... CondN
191 : // +----+ +----+ ^ ^
192 : // | | | |
193 : // | | +----+ |
194 : // Merge<--+ | +------------+
195 : // ^ \|/
196 : // | Phi
197 : // | |
198 : // Branch----+
199 : // ^
200 : // |
201 : // +-----+-----+
202 : // | |
203 : // IfTrue IfFalse
204 : // ^ ^
205 : // | |
206 :
207 : // The resulting graph (modulo the Phi and EffectPhi nodes) looks like this:
208 :
209 : // Control1 Cond1 ... ControlN CondN
210 : // ^ ^ ^ ^
211 : // \ / \ /
212 : // Branch ... Branch
213 : // ^ ^
214 : // | |
215 : // +---+---+ +---+----+
216 : // | | | |
217 : // IfTrue IfFalse ... IfTrue IfFalse
218 : // ^ ^ ^ ^
219 : // | | | |
220 : // +--+ +-------------+ |
221 : // | | +--------------+ +--+
222 : // | | | |
223 : // Merge Merge
224 : // ^ ^
225 : // | |
226 :
227 : SourcePositionTable::Scope scope(source_positions,
228 1058512 : source_positions->GetSourcePosition(node));
229 : NodeOriginTable::Scope origin_scope(node_origins, "clone branch", node);
230 87890 : Node* branch = node;
231 2091527 : Node* cond = NodeProperties::GetValueInput(branch, 0);
232 2091529 : if (!cond->OwnedBy(branch) || cond->opcode() != IrOpcode::kPhi) return;
233 187749 : Node* merge = NodeProperties::GetControlInput(branch);
234 197557 : if (merge->opcode() != IrOpcode::kMerge ||
235 97699 : NodeProperties::GetControlInput(cond) != merge) {
236 : return;
237 : }
238 : // Grab the IfTrue/IfFalse projections of the Branch.
239 97624 : BranchMatcher matcher(branch);
240 : // Check/collect other Phi/EffectPhi nodes hanging off the Merge.
241 : NodeVector phis(temp_zone);
242 748500 : for (Node* const use : merge->uses()) {
243 281493 : if (use == branch || use == cond) continue;
244 : // We cannot currently deal with non-Phi/EffectPhi nodes hanging off the
245 : // Merge. Ideally, we would just clone the nodes (and everything that
246 : // depends on it to some distant join point), but that requires knowledge
247 : // about dominance/post-dominance.
248 104790 : if (!NodeProperties::IsPhi(use)) return;
249 184824 : for (Edge edge : use->use_edges()) {
250 : // Right now we can only handle Phi/EffectPhi nodes whose uses are
251 : // directly control-dependend on either the IfTrue or the IfFalse
252 : // successor, because we know exactly how to update those uses.
253 10487 : if (edge.from()->op()->ControlInputCount() != 1) return;
254 9215 : Node* control = NodeProperties::GetControlInput(edge.from());
255 9215 : if (NodeProperties::IsPhi(edge.from())) {
256 675 : control = NodeProperties::GetControlInput(control, edge.index());
257 : }
258 9215 : if (control != matcher.IfTrue() && control != matcher.IfFalse()) return;
259 : }
260 85324 : phis.push_back(use);
261 : }
262 87890 : BranchHint const hint = BranchHintOf(branch->op());
263 87891 : int const input_count = merge->op()->ControlInputCount();
264 : DCHECK_LE(1, input_count);
265 87891 : Node** const inputs = graph->zone()->NewArray<Node*>(2 * input_count);
266 : Node** const merge_true_inputs = &inputs[0];
267 87891 : Node** const merge_false_inputs = &inputs[input_count];
268 457962 : for (int index = 0; index < input_count; ++index) {
269 370071 : Node* cond1 = NodeProperties::GetValueInput(cond, index);
270 370071 : Node* control1 = NodeProperties::GetControlInput(merge, index);
271 370071 : Node* branch1 = graph->NewNode(common->Branch(hint), cond1, control1);
272 740142 : merge_true_inputs[index] = graph->NewNode(common->IfTrue(), branch1);
273 740142 : merge_false_inputs[index] = graph->NewNode(common->IfFalse(), branch1);
274 : }
275 87891 : Node* const merge_true = matcher.IfTrue();
276 87891 : Node* const merge_false = matcher.IfFalse();
277 87891 : merge_true->TrimInputCount(0);
278 87891 : merge_false->TrimInputCount(0);
279 457962 : for (int i = 0; i < input_count; ++i) {
280 740142 : merge_true->AppendInput(graph->zone(), merge_true_inputs[i]);
281 740142 : merge_false->AppendInput(graph->zone(), merge_false_inputs[i]);
282 : }
283 : DCHECK_EQ(2u, block->SuccessorCount());
284 87891 : NodeProperties::ChangeOp(matcher.IfTrue(), common->Merge(input_count));
285 87891 : NodeProperties::ChangeOp(matcher.IfFalse(), common->Merge(input_count));
286 : int const true_index =
287 87891 : block->SuccessorAt(0)->NodeAt(0) == matcher.IfTrue() ? 0 : 1;
288 : BlockEffectControlData* true_block_data =
289 87891 : &block_effects->For(block, block->SuccessorAt(true_index));
290 : BlockEffectControlData* false_block_data =
291 87891 : &block_effects->For(block, block->SuccessorAt(true_index ^ 1));
292 517078 : for (Node* const phi : phis) {
293 449931 : for (int index = 0; index < input_count; ++index) {
294 729214 : inputs[index] = phi->InputAt(index);
295 : }
296 85324 : inputs[input_count] = merge_true;
297 170648 : Node* phi_true = graph->NewNode(phi->op(), input_count + 1, inputs);
298 85324 : inputs[input_count] = merge_false;
299 85324 : Node* phi_false = graph->NewNode(phi->op(), input_count + 1, inputs);
300 85324 : if (phi->UseCount() == 0) {
301 : DCHECK_EQ(phi->opcode(), IrOpcode::kEffectPhi);
302 : } else {
303 24442 : for (Edge edge : phi->use_edges()) {
304 6766 : Node* control = NodeProperties::GetControlInput(edge.from());
305 13532 : if (NodeProperties::IsPhi(edge.from())) {
306 670 : control = NodeProperties::GetControlInput(control, edge.index());
307 : }
308 : DCHECK(control == matcher.IfTrue() || control == matcher.IfFalse());
309 6766 : edge.UpdateTo((control == matcher.IfTrue()) ? phi_true : phi_false);
310 : }
311 : }
312 85324 : if (phi->opcode() == IrOpcode::kEffectPhi) {
313 85324 : true_block_data->current_effect = phi_true;
314 85324 : false_block_data->current_effect = phi_false;
315 : }
316 85324 : phi->Kill();
317 : }
318 : // Fix up IfTrue and IfFalse and kill all dead nodes.
319 87891 : if (branch == block->control_input()) {
320 87891 : true_block_data->current_control = merge_true;
321 87891 : false_block_data->current_control = merge_false;
322 : }
323 87891 : branch->Kill();
324 87891 : cond->Kill();
325 87891 : merge->Kill();
326 : }
327 :
328 : } // namespace
329 :
330 9665442 : void EffectControlLinearizer::Run() {
331 : BlockEffectControlMap block_effects(temp_zone());
332 : ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone());
333 : ZoneVector<BasicBlock*> pending_block_controls(temp_zone());
334 : NodeVector inputs_buffer(temp_zone());
335 :
336 5140144 : for (BasicBlock* block : *(schedule()->rpo_order())) {
337 : size_t instr = 0;
338 :
339 : // The control node should be the first.
340 11412830 : Node* control = block->NodeAt(instr);
341 : DCHECK(NodeProperties::IsControl(control));
342 : // Update the control inputs.
343 4227943 : if (HasIncomingBackEdges(block)) {
344 : // If there are back edges, we need to update later because we have not
345 : // computed the control yet. This should only happen for loops.
346 : DCHECK_EQ(IrOpcode::kLoop, control->opcode());
347 40679 : pending_block_controls.push_back(block);
348 : } else {
349 : // If there are no back edges, we can update now.
350 4187264 : UpdateBlockControl(block, &block_effects);
351 : }
352 : instr++;
353 :
354 : // Iterate over the phis and update the effect phis.
355 : Node* effect_phi = nullptr;
356 : Node* terminate = nullptr;
357 10722926 : for (; instr < block->NodeCount(); instr++) {
358 3758130 : Node* node = block->NodeAt(instr);
359 : // Only go through the phis and effect phis.
360 3758130 : if (node->opcode() == IrOpcode::kEffectPhi) {
361 : // There should be at most one effect phi in a block.
362 : DCHECK_NULL(effect_phi);
363 : // IfException blocks should not have effect phis.
364 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
365 : effect_phi = node;
366 2986200 : } else if (node->opcode() == IrOpcode::kPhi) {
367 : // Just skip phis.
368 2665344 : } else if (node->opcode() == IrOpcode::kTerminate) {
369 : DCHECK_NULL(terminate);
370 : terminate = node;
371 : } else {
372 : break;
373 : }
374 : }
375 :
376 4227940 : if (effect_phi) {
377 : // Make sure we update the inputs to the incoming blocks' effects.
378 771975 : if (HasIncomingBackEdges(block)) {
379 : // In case of loops, we do not update the effect phi immediately
380 : // because the back predecessor has not been handled yet. We just
381 : // record the effect phi for later processing.
382 40676 : pending_effect_phis.push_back(PendingEffectPhi(effect_phi, block));
383 : } else {
384 731299 : UpdateEffectPhi(effect_phi, block, &block_effects, jsgraph());
385 : }
386 : }
387 :
388 4227962 : Node* effect = effect_phi;
389 4227962 : if (effect == nullptr) {
390 : // There was no effect phi.
391 3455960 : if (block == schedule()->start()) {
392 : // Start block => effect is start.
393 : DCHECK_EQ(graph()->start(), control);
394 456089 : effect = graph()->start();
395 8544546 : } else if (control->opcode() == IrOpcode::kEnd) {
396 : // End block is just a dummy, no effect needed.
397 : DCHECK_EQ(BasicBlock::kNone, block->control());
398 : DCHECK_EQ(1u, block->size());
399 455013 : effect = nullptr;
400 : } else {
401 : // If all the predecessors have the same effect, we can use it as our
402 : // current effect.
403 7770928 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
404 : const BlockEffectControlData& data =
405 : block_effects.For(block->PredecessorAt(i), block);
406 2623879 : if (!effect) effect = data.current_effect;
407 2623879 : if (data.current_effect != effect) {
408 10844 : effect = nullptr;
409 10844 : break;
410 : }
411 : }
412 2544804 : if (effect == nullptr) {
413 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
414 : // The input blocks do not have the same effect. We have
415 : // to create an effect phi node.
416 : inputs_buffer.clear();
417 21688 : inputs_buffer.resize(block->PredecessorCount(), jsgraph()->Dead());
418 10844 : inputs_buffer.push_back(control);
419 : effect = graph()->NewNode(
420 10844 : common()->EffectPhi(static_cast<int>(block->PredecessorCount())),
421 54220 : static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front()));
422 : // For loops, we update the effect phi node later to break cycles.
423 21688 : if (control->opcode() == IrOpcode::kLoop) {
424 14 : pending_effect_phis.push_back(PendingEffectPhi(effect, block));
425 : } else {
426 10837 : UpdateEffectPhi(effect, block, &block_effects, jsgraph());
427 : }
428 5067920 : } else if (control->opcode() == IrOpcode::kIfException) {
429 : // The IfException is connected into the effect chain, so we need
430 : // to update the effect here.
431 164924 : NodeProperties::ReplaceEffectInput(control, effect);
432 164924 : effect = control;
433 : }
434 : }
435 : }
436 :
437 : // Fixup the Terminate node.
438 4227908 : if (terminate != nullptr) {
439 40682 : NodeProperties::ReplaceEffectInput(terminate, effect);
440 : }
441 :
442 : // The frame state at block entry is determined by the frame states leaving
443 : // all predecessors. In case there is no frame state dominating this block,
444 : // we can rely on a checkpoint being present before the next deoptimization.
445 : // TODO(mstarzinger): Eventually we will need to go hunt for a frame state
446 : // once deoptimizing nodes roam freely through the schedule.
447 4227908 : Node* frame_state = nullptr;
448 4227908 : if (block != schedule()->start()) {
449 : // If all the predecessors have the same effect, we can use it
450 : // as our current effect.
451 : frame_state =
452 3771823 : block_effects.For(block->PredecessorAt(0), block).current_frame_state;
453 9521998 : for (size_t i = 1; i < block->PredecessorCount(); i++) {
454 1132202 : if (block_effects.For(block->PredecessorAt(i), block)
455 1132202 : .current_frame_state != frame_state) {
456 143026 : frame_state = nullptr;
457 143026 : frame_state_zapper_ = graph()->end();
458 143026 : break;
459 : }
460 : }
461 : }
462 :
463 : // Process the ordinary instructions.
464 64203261 : for (; instr < block->NodeCount(); instr++) {
465 : Node* node = block->NodeAt(instr);
466 29987693 : ProcessNode(node, &frame_state, &effect, &control);
467 : }
468 :
469 4227904 : switch (block->control()) {
470 : case BasicBlock::kGoto:
471 : case BasicBlock::kNone:
472 : break;
473 :
474 : case BasicBlock::kCall:
475 : case BasicBlock::kTailCall:
476 : case BasicBlock::kSwitch:
477 : case BasicBlock::kReturn:
478 : case BasicBlock::kDeoptimize:
479 : case BasicBlock::kThrow:
480 839914 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
481 839915 : break;
482 :
483 : case BasicBlock::kBranch:
484 1058519 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
485 : TryCloneBranch(block->control_input(), block, temp_zone(), graph(),
486 : common(), &block_effects, source_positions_,
487 3175650 : node_origins_);
488 1058544 : break;
489 : }
490 :
491 : // Store the effect, control and frame state for later use.
492 17694655 : for (BasicBlock* successor : block->successors()) {
493 5010831 : BlockEffectControlData* data = &block_effects.For(block, successor);
494 5010865 : if (data->current_effect == nullptr) {
495 4840185 : data->current_effect = effect;
496 : }
497 5010865 : if (data->current_control == nullptr) {
498 4835063 : data->current_control = control;
499 : }
500 5010865 : data->current_frame_state = frame_state;
501 : }
502 : }
503 :
504 952905 : for (BasicBlock* pending_block_control : pending_block_controls) {
505 40683 : UpdateBlockControl(pending_block_control, &block_effects);
506 : }
507 : // Update the incoming edges of the effect phis that could not be processed
508 : // during the first pass (because they could have incoming back edges).
509 952905 : for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) {
510 : UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block,
511 40683 : &block_effects, jsgraph());
512 : }
513 456116 : }
514 :
515 162310626 : void EffectControlLinearizer::ProcessNode(Node* node, Node** frame_state,
516 2082 : Node** effect, Node** control) {
517 : SourcePositionTable::Scope scope(source_positions_,
518 31886012 : source_positions_->GetSourcePosition(node));
519 31886062 : NodeOriginTable::Scope origin_scope(node_origins_, "process node", node);
520 :
521 : // If the node needs to be wired into the effect/control chain, do this
522 : // here. Pass current frame state for lowering to eager deoptimization.
523 31886062 : if (TryWireInStateEffect(node, *frame_state, effect, control)) {
524 : return;
525 : }
526 :
527 : // If the node has a visible effect, then there must be a checkpoint in the
528 : // effect chain before we are allowed to place another eager deoptimization
529 : // point. We zap the frame state to ensure this invariant is maintained.
530 60693423 : if (region_observability_ == RegionObservability::kObservable &&
531 : !node->op()->HasProperty(Operator::kNoWrite)) {
532 3280489 : *frame_state = nullptr;
533 3280489 : frame_state_zapper_ = node;
534 : }
535 :
536 : // Remove the end markers of 'atomic' allocation region because the
537 : // region should be wired-in now.
538 30824514 : if (node->opcode() == IrOpcode::kFinishRegion) {
539 : // Reset the current region observability.
540 128061 : region_observability_ = RegionObservability::kObservable;
541 : // Update the value uses to the value input of the finish node and
542 : // the effect uses to the effect input.
543 128061 : return RemoveRenameNode(node);
544 : }
545 30696453 : if (node->opcode() == IrOpcode::kBeginRegion) {
546 : // Determine the observability for this region and use that for all
547 : // nodes inside the region (i.e. ignore the absence of kNoWrite on
548 : // StoreField and other operators).
549 : DCHECK_NE(RegionObservability::kNotObservable, region_observability_);
550 128061 : region_observability_ = RegionObservabilityOf(node->op());
551 : // Update the value uses to the value input of the finish node and
552 : // the effect uses to the effect input.
553 128061 : return RemoveRenameNode(node);
554 : }
555 30568392 : if (node->opcode() == IrOpcode::kTypeGuard) {
556 20373 : return RemoveRenameNode(node);
557 : }
558 :
559 : // Special treatment for checkpoint nodes.
560 30548019 : if (node->opcode() == IrOpcode::kCheckpoint) {
561 : // Unlink the check point; effect uses will be updated to the incoming
562 : // effect that is passed. The frame state is preserved for lowering.
563 : DCHECK_EQ(RegionObservability::kObservable, region_observability_);
564 2493666 : *frame_state = NodeProperties::GetFrameStateInput(node);
565 2493668 : return;
566 : }
567 :
568 : // The IfSuccess nodes should always start a basic block (and basic block
569 : // start nodes are not handled in the ProcessNode method).
570 : DCHECK_NE(IrOpcode::kIfSuccess, node->opcode());
571 :
572 : // If the node takes an effect, replace with the current one.
573 56108706 : if (node->op()->EffectInputCount() > 0) {
574 : DCHECK_EQ(1, node->op()->EffectInputCount());
575 6512084 : Node* input_effect = NodeProperties::GetEffectInput(node);
576 :
577 6512084 : if (input_effect != *effect) {
578 2478916 : NodeProperties::ReplaceEffectInput(node, *effect);
579 : }
580 :
581 : // If the node produces an effect, update our current effect. (However,
582 : // ignore new effect chains started with ValueEffect.)
583 13024184 : if (node->op()->EffectOutputCount() > 0) {
584 : DCHECK_EQ(1, node->op()->EffectOutputCount());
585 5843179 : *effect = node;
586 : }
587 : } else {
588 : // New effect chain is only started with a Start or ValueEffect node.
589 : DCHECK(node->op()->EffectOutputCount() == 0 ||
590 : node->opcode() == IrOpcode::kStart);
591 : }
592 :
593 : // Rewire control inputs.
594 77439856 : for (int i = 0; i < node->op()->ControlInputCount(); i++) {
595 7110379 : NodeProperties::ReplaceControlInput(node, *control, i);
596 : }
597 : // Update the current control.
598 56108720 : if (node->op()->ControlOutputCount() > 0) {
599 4364394 : *control = node;
600 : }
601 :
602 : // Break the effect chain on {Unreachable} and reconnect to the graph end.
603 : // Mark the following code for deletion by connecting to the {Dead} node.
604 28054360 : if (node->opcode() == IrOpcode::kUnreachable) {
605 1041 : ConnectUnreachableToEnd(*effect, *control, jsgraph());
606 1041 : *effect = *control = jsgraph()->Dead();
607 : }
608 : }
609 :
610 64833703 : bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
611 : Node* frame_state,
612 : Node** effect,
613 : Node** control) {
614 31886050 : gasm()->Reset(*effect, *control);
615 : Node* result = nullptr;
616 31886067 : switch (node->opcode()) {
617 : case IrOpcode::kChangeBitToTagged:
618 16462 : result = LowerChangeBitToTagged(node);
619 16462 : break;
620 : case IrOpcode::kChangeInt31ToTaggedSigned:
621 38566 : result = LowerChangeInt31ToTaggedSigned(node);
622 38566 : break;
623 : case IrOpcode::kChangeInt32ToTagged:
624 35400 : result = LowerChangeInt32ToTagged(node);
625 35400 : break;
626 : case IrOpcode::kChangeInt64ToTagged:
627 260 : result = LowerChangeInt64ToTagged(node);
628 260 : break;
629 : case IrOpcode::kChangeUint32ToTagged:
630 542 : result = LowerChangeUint32ToTagged(node);
631 542 : break;
632 : case IrOpcode::kChangeUint64ToTagged:
633 81 : result = LowerChangeUint64ToTagged(node);
634 81 : break;
635 : case IrOpcode::kChangeFloat64ToTagged:
636 52985 : result = LowerChangeFloat64ToTagged(node);
637 52986 : break;
638 : case IrOpcode::kChangeFloat64ToTaggedPointer:
639 91 : result = LowerChangeFloat64ToTaggedPointer(node);
640 91 : break;
641 : case IrOpcode::kChangeTaggedSignedToInt32:
642 52640 : result = LowerChangeTaggedSignedToInt32(node);
643 52640 : break;
644 : case IrOpcode::kChangeTaggedSignedToInt64:
645 1893 : result = LowerChangeTaggedSignedToInt64(node);
646 1893 : break;
647 : case IrOpcode::kChangeTaggedToBit:
648 127535 : result = LowerChangeTaggedToBit(node);
649 127535 : break;
650 : case IrOpcode::kChangeTaggedToInt32:
651 1882 : result = LowerChangeTaggedToInt32(node);
652 1882 : break;
653 : case IrOpcode::kChangeTaggedToUint32:
654 564 : result = LowerChangeTaggedToUint32(node);
655 564 : break;
656 : case IrOpcode::kChangeTaggedToInt64:
657 12 : result = LowerChangeTaggedToInt64(node);
658 12 : break;
659 : case IrOpcode::kChangeTaggedToFloat64:
660 : result = LowerChangeTaggedToFloat64(node);
661 10774 : break;
662 : case IrOpcode::kChangeTaggedToTaggedSigned:
663 15 : result = LowerChangeTaggedToTaggedSigned(node);
664 15 : break;
665 : case IrOpcode::kTruncateTaggedToBit:
666 40938 : result = LowerTruncateTaggedToBit(node);
667 40939 : break;
668 : case IrOpcode::kTruncateTaggedPointerToBit:
669 473 : result = LowerTruncateTaggedPointerToBit(node);
670 473 : break;
671 : case IrOpcode::kTruncateTaggedToFloat64:
672 743 : result = LowerTruncateTaggedToFloat64(node);
673 743 : break;
674 : case IrOpcode::kPoisonIndex:
675 1167 : result = LowerPoisonIndex(node);
676 1167 : break;
677 : case IrOpcode::kCheckMaps:
678 55458 : LowerCheckMaps(node, frame_state);
679 55459 : break;
680 : case IrOpcode::kCompareMaps:
681 5713 : result = LowerCompareMaps(node);
682 5713 : break;
683 : case IrOpcode::kCheckNumber:
684 519 : result = LowerCheckNumber(node, frame_state);
685 519 : break;
686 : case IrOpcode::kCheckReceiver:
687 815 : result = LowerCheckReceiver(node, frame_state);
688 815 : break;
689 : case IrOpcode::kCheckReceiverOrNullOrUndefined:
690 87 : result = LowerCheckReceiverOrNullOrUndefined(node, frame_state);
691 87 : break;
692 : case IrOpcode::kCheckSymbol:
693 37 : result = LowerCheckSymbol(node, frame_state);
694 37 : break;
695 : case IrOpcode::kCheckString:
696 6323 : result = LowerCheckString(node, frame_state);
697 6323 : break;
698 : case IrOpcode::kCheckInternalizedString:
699 1430 : result = LowerCheckInternalizedString(node, frame_state);
700 1430 : break;
701 : case IrOpcode::kCheckIf:
702 15244 : LowerCheckIf(node, frame_state);
703 15244 : break;
704 : case IrOpcode::kCheckedInt32Add:
705 56282 : result = LowerCheckedInt32Add(node, frame_state);
706 56284 : break;
707 : case IrOpcode::kCheckedInt32Sub:
708 7527 : result = LowerCheckedInt32Sub(node, frame_state);
709 7527 : break;
710 : case IrOpcode::kCheckedInt32Div:
711 243 : result = LowerCheckedInt32Div(node, frame_state);
712 243 : break;
713 : case IrOpcode::kCheckedInt32Mod:
714 856 : result = LowerCheckedInt32Mod(node, frame_state);
715 856 : break;
716 : case IrOpcode::kCheckedUint32Div:
717 49 : result = LowerCheckedUint32Div(node, frame_state);
718 49 : break;
719 : case IrOpcode::kCheckedUint32Mod:
720 37 : result = LowerCheckedUint32Mod(node, frame_state);
721 37 : break;
722 : case IrOpcode::kCheckedInt32Mul:
723 5673 : result = LowerCheckedInt32Mul(node, frame_state);
724 5673 : break;
725 : case IrOpcode::kCheckedInt32ToTaggedSigned:
726 0 : result = LowerCheckedInt32ToTaggedSigned(node, frame_state);
727 0 : break;
728 : case IrOpcode::kCheckedInt64ToInt32:
729 7 : result = LowerCheckedInt64ToInt32(node, frame_state);
730 7 : break;
731 : case IrOpcode::kCheckedInt64ToTaggedSigned:
732 1725 : result = LowerCheckedInt64ToTaggedSigned(node, frame_state);
733 1725 : break;
734 : case IrOpcode::kCheckedUint32Bounds:
735 44198 : result = LowerCheckedUint32Bounds(node, frame_state);
736 44198 : break;
737 : case IrOpcode::kCheckedUint32ToInt32:
738 253 : result = LowerCheckedUint32ToInt32(node, frame_state);
739 253 : break;
740 : case IrOpcode::kCheckedUint32ToTaggedSigned:
741 14 : result = LowerCheckedUint32ToTaggedSigned(node, frame_state);
742 14 : break;
743 : case IrOpcode::kCheckedUint64Bounds:
744 192 : result = LowerCheckedUint64Bounds(node, frame_state);
745 192 : break;
746 : case IrOpcode::kCheckedUint64ToInt32:
747 0 : result = LowerCheckedUint64ToInt32(node, frame_state);
748 0 : break;
749 : case IrOpcode::kCheckedUint64ToTaggedSigned:
750 0 : result = LowerCheckedUint64ToTaggedSigned(node, frame_state);
751 0 : break;
752 : case IrOpcode::kCheckedFloat64ToInt32:
753 2350 : result = LowerCheckedFloat64ToInt32(node, frame_state);
754 2350 : break;
755 : case IrOpcode::kCheckedFloat64ToInt64:
756 14 : result = LowerCheckedFloat64ToInt64(node, frame_state);
757 14 : break;
758 : case IrOpcode::kCheckedTaggedSignedToInt32:
759 24835 : if (frame_state == nullptr) {
760 0 : FATAL("No frame state (zapped by #%d: %s)", frame_state_zapper_->id(),
761 0 : frame_state_zapper_->op()->mnemonic());
762 : }
763 24835 : result = LowerCheckedTaggedSignedToInt32(node, frame_state);
764 24836 : break;
765 : case IrOpcode::kCheckedTaggedToInt32:
766 2400 : result = LowerCheckedTaggedToInt32(node, frame_state);
767 2400 : break;
768 : case IrOpcode::kCheckedTaggedToInt64:
769 70 : result = LowerCheckedTaggedToInt64(node, frame_state);
770 70 : break;
771 : case IrOpcode::kCheckedTaggedToFloat64:
772 58011 : result = LowerCheckedTaggedToFloat64(node, frame_state);
773 58011 : break;
774 : case IrOpcode::kCheckedTaggedToTaggedSigned:
775 19906 : result = LowerCheckedTaggedToTaggedSigned(node, frame_state);
776 19906 : break;
777 : case IrOpcode::kCheckedTaggedToTaggedPointer:
778 35936 : result = LowerCheckedTaggedToTaggedPointer(node, frame_state);
779 35937 : break;
780 : case IrOpcode::kTruncateTaggedToWord32:
781 423 : result = LowerTruncateTaggedToWord32(node);
782 423 : break;
783 : case IrOpcode::kCheckedTruncateTaggedToWord32:
784 1311 : result = LowerCheckedTruncateTaggedToWord32(node, frame_state);
785 1311 : break;
786 : case IrOpcode::kNumberToString:
787 1092 : result = LowerNumberToString(node);
788 1092 : break;
789 : case IrOpcode::kObjectIsArrayBufferView:
790 16 : result = LowerObjectIsArrayBufferView(node);
791 16 : break;
792 : case IrOpcode::kObjectIsBigInt:
793 25 : result = LowerObjectIsBigInt(node);
794 25 : break;
795 : case IrOpcode::kObjectIsCallable:
796 98 : result = LowerObjectIsCallable(node);
797 98 : break;
798 : case IrOpcode::kObjectIsConstructor:
799 206 : result = LowerObjectIsConstructor(node);
800 206 : break;
801 : case IrOpcode::kObjectIsDetectableCallable:
802 13424 : result = LowerObjectIsDetectableCallable(node);
803 13424 : break;
804 : case IrOpcode::kObjectIsMinusZero:
805 21 : result = LowerObjectIsMinusZero(node);
806 21 : break;
807 : case IrOpcode::kNumberIsMinusZero:
808 92 : result = LowerNumberIsMinusZero(node);
809 92 : break;
810 : case IrOpcode::kObjectIsNaN:
811 685 : result = LowerObjectIsNaN(node);
812 685 : break;
813 : case IrOpcode::kNumberIsNaN:
814 10327 : result = LowerNumberIsNaN(node);
815 10327 : break;
816 : case IrOpcode::kObjectIsNonCallable:
817 5753 : result = LowerObjectIsNonCallable(node);
818 5753 : break;
819 : case IrOpcode::kObjectIsNumber:
820 7213 : result = LowerObjectIsNumber(node);
821 7213 : break;
822 : case IrOpcode::kObjectIsReceiver:
823 18237 : result = LowerObjectIsReceiver(node);
824 18237 : break;
825 : case IrOpcode::kObjectIsSmi:
826 4720 : result = LowerObjectIsSmi(node);
827 4720 : break;
828 : case IrOpcode::kObjectIsString:
829 1921 : result = LowerObjectIsString(node);
830 1921 : break;
831 : case IrOpcode::kObjectIsSymbol:
832 20 : result = LowerObjectIsSymbol(node);
833 20 : break;
834 : case IrOpcode::kObjectIsUndetectable:
835 1303 : result = LowerObjectIsUndetectable(node);
836 1303 : break;
837 : case IrOpcode::kArgumentsFrame:
838 16640 : result = LowerArgumentsFrame(node);
839 16640 : break;
840 : case IrOpcode::kArgumentsLength:
841 16668 : result = LowerArgumentsLength(node);
842 16668 : break;
843 : case IrOpcode::kToBoolean:
844 218 : result = LowerToBoolean(node);
845 218 : break;
846 : case IrOpcode::kTypeOf:
847 18996 : result = LowerTypeOf(node);
848 18996 : break;
849 : case IrOpcode::kNewDoubleElements:
850 28 : result = LowerNewDoubleElements(node);
851 28 : break;
852 : case IrOpcode::kNewSmiOrObjectElements:
853 507 : result = LowerNewSmiOrObjectElements(node);
854 507 : break;
855 : case IrOpcode::kNewArgumentsElements:
856 16155 : result = LowerNewArgumentsElements(node);
857 16155 : break;
858 : case IrOpcode::kNewConsString:
859 3328 : result = LowerNewConsString(node);
860 3328 : break;
861 : case IrOpcode::kSameValue:
862 12 : result = LowerSameValue(node);
863 12 : break;
864 : case IrOpcode::kDeadValue:
865 1243 : result = LowerDeadValue(node);
866 1243 : break;
867 : case IrOpcode::kStringConcat:
868 18931 : result = LowerStringConcat(node);
869 18931 : break;
870 : case IrOpcode::kStringFromSingleCharCode:
871 966 : result = LowerStringFromSingleCharCode(node);
872 966 : break;
873 : case IrOpcode::kStringFromSingleCodePoint:
874 222 : result = LowerStringFromSingleCodePoint(node);
875 222 : break;
876 : case IrOpcode::kStringIndexOf:
877 289 : result = LowerStringIndexOf(node);
878 289 : break;
879 : case IrOpcode::kStringLength:
880 21117 : result = LowerStringLength(node);
881 21117 : break;
882 : case IrOpcode::kStringToNumber:
883 344 : result = LowerStringToNumber(node);
884 344 : break;
885 : case IrOpcode::kStringCharCodeAt:
886 2398 : result = LowerStringCharCodeAt(node);
887 2398 : break;
888 : case IrOpcode::kStringCodePointAt:
889 249 : result = LowerStringCodePointAt(node, UnicodeEncodingOf(node->op()));
890 249 : break;
891 : case IrOpcode::kStringToLowerCaseIntl:
892 88 : result = LowerStringToLowerCaseIntl(node);
893 88 : break;
894 : case IrOpcode::kStringToUpperCaseIntl:
895 28 : result = LowerStringToUpperCaseIntl(node);
896 28 : break;
897 : case IrOpcode::kStringSubstring:
898 1081 : result = LowerStringSubstring(node);
899 1081 : break;
900 : case IrOpcode::kStringEqual:
901 10236 : result = LowerStringEqual(node);
902 10236 : break;
903 : case IrOpcode::kStringLessThan:
904 251 : result = LowerStringLessThan(node);
905 251 : break;
906 : case IrOpcode::kStringLessThanOrEqual:
907 865 : result = LowerStringLessThanOrEqual(node);
908 865 : break;
909 : case IrOpcode::kNumberIsFloat64Hole:
910 66 : result = LowerNumberIsFloat64Hole(node);
911 66 : break;
912 : case IrOpcode::kNumberIsFinite:
913 141 : result = LowerNumberIsFinite(node);
914 141 : break;
915 : case IrOpcode::kObjectIsFiniteNumber:
916 7 : result = LowerObjectIsFiniteNumber(node);
917 7 : break;
918 : case IrOpcode::kNumberIsInteger:
919 119 : result = LowerNumberIsInteger(node);
920 119 : break;
921 : case IrOpcode::kObjectIsInteger:
922 7 : result = LowerObjectIsInteger(node);
923 7 : break;
924 : case IrOpcode::kNumberIsSafeInteger:
925 7 : result = LowerNumberIsSafeInteger(node);
926 7 : break;
927 : case IrOpcode::kObjectIsSafeInteger:
928 7 : result = LowerObjectIsSafeInteger(node);
929 7 : break;
930 : case IrOpcode::kCheckFloat64Hole:
931 471 : result = LowerCheckFloat64Hole(node, frame_state);
932 471 : break;
933 : case IrOpcode::kCheckNotTaggedHole:
934 93 : result = LowerCheckNotTaggedHole(node, frame_state);
935 93 : break;
936 : case IrOpcode::kConvertTaggedHoleToUndefined:
937 1507 : result = LowerConvertTaggedHoleToUndefined(node);
938 1507 : break;
939 : case IrOpcode::kCheckEqualsInternalizedString:
940 57 : LowerCheckEqualsInternalizedString(node, frame_state);
941 57 : break;
942 : case IrOpcode::kAllocate:
943 112045 : result = LowerAllocate(node);
944 112045 : break;
945 : case IrOpcode::kCheckEqualsSymbol:
946 19 : LowerCheckEqualsSymbol(node, frame_state);
947 19 : break;
948 : case IrOpcode::kPlainPrimitiveToNumber:
949 23 : result = LowerPlainPrimitiveToNumber(node);
950 23 : break;
951 : case IrOpcode::kPlainPrimitiveToWord32:
952 9 : result = LowerPlainPrimitiveToWord32(node);
953 9 : break;
954 : case IrOpcode::kPlainPrimitiveToFloat64:
955 0 : result = LowerPlainPrimitiveToFloat64(node);
956 0 : break;
957 : case IrOpcode::kEnsureWritableFastElements:
958 1746 : result = LowerEnsureWritableFastElements(node);
959 1746 : break;
960 : case IrOpcode::kMaybeGrowFastElements:
961 3665 : result = LowerMaybeGrowFastElements(node, frame_state);
962 3665 : break;
963 : case IrOpcode::kTransitionElementsKind:
964 592 : LowerTransitionElementsKind(node);
965 592 : break;
966 : case IrOpcode::kLoadFieldByIndex:
967 1036 : result = LowerLoadFieldByIndex(node);
968 1036 : break;
969 : case IrOpcode::kLoadTypedElement:
970 4594 : result = LowerLoadTypedElement(node);
971 4594 : break;
972 : case IrOpcode::kLoadDataViewElement:
973 268 : result = LowerLoadDataViewElement(node);
974 268 : break;
975 : case IrOpcode::kStoreTypedElement:
976 4162 : LowerStoreTypedElement(node);
977 4162 : break;
978 : case IrOpcode::kStoreDataViewElement:
979 197 : LowerStoreDataViewElement(node);
980 197 : break;
981 : case IrOpcode::kStoreSignedSmallElement:
982 78 : LowerStoreSignedSmallElement(node);
983 78 : break;
984 : case IrOpcode::kFindOrderedHashMapEntry:
985 190 : result = LowerFindOrderedHashMapEntry(node);
986 190 : break;
987 : case IrOpcode::kFindOrderedHashMapEntryForInt32Key:
988 15 : result = LowerFindOrderedHashMapEntryForInt32Key(node);
989 15 : break;
990 : case IrOpcode::kTransitionAndStoreNumberElement:
991 22 : LowerTransitionAndStoreNumberElement(node);
992 22 : break;
993 : case IrOpcode::kTransitionAndStoreNonNumberElement:
994 21 : LowerTransitionAndStoreNonNumberElement(node);
995 21 : break;
996 : case IrOpcode::kTransitionAndStoreElement:
997 81 : LowerTransitionAndStoreElement(node);
998 81 : break;
999 : case IrOpcode::kRuntimeAbort:
1000 2463 : LowerRuntimeAbort(node);
1001 2463 : break;
1002 : case IrOpcode::kConvertReceiver:
1003 892 : result = LowerConvertReceiver(node);
1004 892 : break;
1005 : case IrOpcode::kFloat64RoundUp:
1006 16446 : if (!LowerFloat64RoundUp(node).To(&result)) {
1007 : return false;
1008 : }
1009 : break;
1010 : case IrOpcode::kFloat64RoundDown:
1011 51816 : if (!LowerFloat64RoundDown(node).To(&result)) {
1012 : return false;
1013 : }
1014 : break;
1015 : case IrOpcode::kFloat64RoundTruncate:
1016 13824 : if (!LowerFloat64RoundTruncate(node).To(&result)) {
1017 : return false;
1018 : }
1019 : break;
1020 : case IrOpcode::kFloat64RoundTiesEven:
1021 532 : if (!LowerFloat64RoundTiesEven(node).To(&result)) {
1022 : return false;
1023 : }
1024 : break;
1025 : case IrOpcode::kDateNow:
1026 7 : result = LowerDateNow(node);
1027 7 : break;
1028 : default:
1029 : return false;
1030 : }
1031 :
1032 3184758 : if ((result ? 1 : 0) != node->op()->ValueOutputCount()) {
1033 0 : FATAL(
1034 : "Effect control linearizer lowering of '%s':"
1035 : " value output count does not agree.",
1036 0 : node->op()->mnemonic());
1037 : }
1038 :
1039 1061586 : *effect = gasm()->ExtractCurrentEffect();
1040 1061587 : *control = gasm()->ExtractCurrentControl();
1041 1061587 : NodeProperties::ReplaceUses(node, result, *effect, *control);
1042 1061587 : return true;
1043 : }
1044 :
1045 : #define __ gasm()->
1046 :
1047 52985 : Node* EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node) {
1048 52985 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1049 : Node* value = node->InputAt(0);
1050 :
1051 52986 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1052 52986 : auto if_heapnumber = __ MakeDeferredLabel();
1053 52986 : auto if_int32 = __ MakeLabel();
1054 :
1055 52986 : Node* value32 = __ RoundFloat64ToInt32(value);
1056 : __ GotoIf(__ Float64Equal(value, __ ChangeInt32ToFloat64(value32)),
1057 52985 : &if_int32);
1058 : __ Goto(&if_heapnumber);
1059 :
1060 : __ Bind(&if_int32);
1061 : {
1062 52986 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1063 40325 : Node* zero = __ Int32Constant(0);
1064 40325 : auto if_zero = __ MakeDeferredLabel();
1065 40325 : auto if_smi = __ MakeLabel();
1066 :
1067 40325 : __ GotoIf(__ Word32Equal(value32, zero), &if_zero);
1068 : __ Goto(&if_smi);
1069 :
1070 : __ Bind(&if_zero);
1071 : {
1072 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
1073 : __ GotoIf(__ Int32LessThan(__ Float64ExtractHighWord32(value), zero),
1074 40325 : &if_heapnumber);
1075 : __ Goto(&if_smi);
1076 : }
1077 :
1078 : __ Bind(&if_smi);
1079 : }
1080 :
1081 : if (SmiValuesAre32Bits()) {
1082 52986 : Node* value_smi = ChangeInt32ToSmi(value32);
1083 : __ Goto(&done, value_smi);
1084 : } else {
1085 : DCHECK(SmiValuesAre31Bits());
1086 : Node* add = __ Int32AddWithOverflow(value32, value32);
1087 : Node* ovf = __ Projection(1, add);
1088 : __ GotoIf(ovf, &if_heapnumber);
1089 : Node* value_smi = __ Projection(0, add);
1090 : value_smi = ChangeInt32ToIntPtr(value_smi);
1091 : __ Goto(&done, value_smi);
1092 : }
1093 : }
1094 :
1095 : __ Bind(&if_heapnumber);
1096 : {
1097 52985 : Node* value_number = AllocateHeapNumberWithValue(value);
1098 : __ Goto(&done, value_number);
1099 : }
1100 :
1101 : __ Bind(&done);
1102 52986 : return done.PhiAt(0);
1103 : }
1104 :
1105 91 : Node* EffectControlLinearizer::LowerChangeFloat64ToTaggedPointer(Node* node) {
1106 : Node* value = node->InputAt(0);
1107 91 : return AllocateHeapNumberWithValue(value);
1108 : }
1109 :
1110 16462 : Node* EffectControlLinearizer::LowerChangeBitToTagged(Node* node) {
1111 : Node* value = node->InputAt(0);
1112 :
1113 16462 : auto if_true = __ MakeLabel();
1114 16462 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1115 :
1116 16462 : __ GotoIf(value, &if_true);
1117 16462 : __ Goto(&done, __ FalseConstant());
1118 :
1119 : __ Bind(&if_true);
1120 16462 : __ Goto(&done, __ TrueConstant());
1121 :
1122 : __ Bind(&done);
1123 16462 : return done.PhiAt(0);
1124 : }
1125 :
1126 38566 : Node* EffectControlLinearizer::LowerChangeInt31ToTaggedSigned(Node* node) {
1127 : Node* value = node->InputAt(0);
1128 38566 : return ChangeInt32ToSmi(value);
1129 : }
1130 :
1131 35400 : Node* EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node) {
1132 : Node* value = node->InputAt(0);
1133 :
1134 : if (SmiValuesAre32Bits()) {
1135 35400 : return ChangeInt32ToSmi(value);
1136 : }
1137 : DCHECK(SmiValuesAre31Bits());
1138 :
1139 : auto if_overflow = __ MakeDeferredLabel();
1140 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1141 :
1142 : Node* add = __ Int32AddWithOverflow(value, value);
1143 : Node* ovf = __ Projection(1, add);
1144 : __ GotoIf(ovf, &if_overflow);
1145 : Node* value_smi = __ Projection(0, add);
1146 : value_smi = ChangeInt32ToIntPtr(value_smi);
1147 : __ Goto(&done, value_smi);
1148 :
1149 : __ Bind(&if_overflow);
1150 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt32ToFloat64(value));
1151 : __ Goto(&done, number);
1152 :
1153 : __ Bind(&done);
1154 : return done.PhiAt(0);
1155 : }
1156 :
1157 260 : Node* EffectControlLinearizer::LowerChangeInt64ToTagged(Node* node) {
1158 : Node* value = node->InputAt(0);
1159 :
1160 260 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1161 260 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1162 :
1163 260 : Node* value32 = __ TruncateInt64ToInt32(value);
1164 : __ GotoIfNot(__ Word64Equal(__ ChangeInt32ToInt64(value32), value),
1165 260 : &if_not_in_smi_range);
1166 :
1167 : if (SmiValuesAre32Bits()) {
1168 : Node* value_smi = ChangeInt64ToSmi(value);
1169 : __ Goto(&done, value_smi);
1170 : } else {
1171 : Node* add = __ Int32AddWithOverflow(value32, value32);
1172 : Node* ovf = __ Projection(1, add);
1173 : __ GotoIf(ovf, &if_not_in_smi_range);
1174 : Node* value_smi = ChangeInt32ToIntPtr(__ Projection(0, add));
1175 : __ Goto(&done, value_smi);
1176 : }
1177 :
1178 : __ Bind(&if_not_in_smi_range);
1179 260 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1180 : __ Goto(&done, number);
1181 :
1182 : __ Bind(&done);
1183 260 : return done.PhiAt(0);
1184 : }
1185 :
1186 542 : Node* EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node) {
1187 : Node* value = node->InputAt(0);
1188 :
1189 542 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1190 542 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1191 :
1192 542 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
1193 542 : __ GotoIfNot(check, &if_not_in_smi_range);
1194 542 : __ Goto(&done, ChangeUint32ToSmi(value));
1195 :
1196 : __ Bind(&if_not_in_smi_range);
1197 542 : Node* number = AllocateHeapNumberWithValue(__ ChangeUint32ToFloat64(value));
1198 :
1199 : __ Goto(&done, number);
1200 : __ Bind(&done);
1201 :
1202 542 : return done.PhiAt(0);
1203 : }
1204 :
1205 81 : Node* EffectControlLinearizer::LowerChangeUint64ToTagged(Node* node) {
1206 : Node* value = node->InputAt(0);
1207 :
1208 81 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1209 81 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1210 :
1211 : Node* check =
1212 81 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
1213 81 : __ GotoIfNot(check, &if_not_in_smi_range);
1214 : __ Goto(&done, ChangeInt64ToSmi(value));
1215 :
1216 : __ Bind(&if_not_in_smi_range);
1217 81 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1218 :
1219 : __ Goto(&done, number);
1220 : __ Bind(&done);
1221 :
1222 81 : return done.PhiAt(0);
1223 : }
1224 :
1225 52640 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt32(Node* node) {
1226 : Node* value = node->InputAt(0);
1227 52640 : return ChangeSmiToInt32(value);
1228 : }
1229 :
1230 1893 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt64(Node* node) {
1231 : Node* value = node->InputAt(0);
1232 1893 : return ChangeSmiToInt64(value);
1233 : }
1234 :
1235 127535 : Node* EffectControlLinearizer::LowerChangeTaggedToBit(Node* node) {
1236 : Node* value = node->InputAt(0);
1237 127535 : return __ WordEqual(value, __ TrueConstant());
1238 : }
1239 :
1240 41411 : void EffectControlLinearizer::TruncateTaggedPointerToBit(
1241 : Node* node, GraphAssemblerLabel<1>* done) {
1242 : Node* value = node->InputAt(0);
1243 :
1244 41411 : auto if_heapnumber = __ MakeDeferredLabel();
1245 41411 : auto if_bigint = __ MakeDeferredLabel();
1246 :
1247 41411 : Node* zero = __ Int32Constant(0);
1248 41411 : Node* fzero = __ Float64Constant(0.0);
1249 :
1250 : // Check if {value} is false.
1251 41412 : __ GotoIf(__ WordEqual(value, __ FalseConstant()), done, zero);
1252 :
1253 : // Check if {value} is the empty string.
1254 41412 : __ GotoIf(__ WordEqual(value, __ EmptyStringConstant()), done, zero);
1255 :
1256 : // Load the map of {value}.
1257 41412 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1258 :
1259 : // Check if the {value} is undetectable and immediately return false.
1260 : // This includes undefined and null.
1261 : Node* value_map_bitfield =
1262 41412 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1263 : __ GotoIfNot(
1264 : __ Word32Equal(
1265 : __ Word32And(value_map_bitfield,
1266 : __ Int32Constant(Map::IsUndetectableBit::kMask)),
1267 : zero),
1268 41412 : done, zero);
1269 :
1270 : // Check if {value} is a HeapNumber.
1271 : __ GotoIf(__ WordEqual(value_map, __ HeapNumberMapConstant()),
1272 41412 : &if_heapnumber);
1273 :
1274 : // Check if {value} is a BigInt.
1275 : Node* value_instance_type =
1276 41412 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1277 : __ GotoIf(__ Word32Equal(value_instance_type, __ Int32Constant(BIGINT_TYPE)),
1278 41412 : &if_bigint);
1279 :
1280 : // All other values that reach here are true.
1281 41412 : __ Goto(done, __ Int32Constant(1));
1282 :
1283 : __ Bind(&if_heapnumber);
1284 : {
1285 : // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or
1286 : // NaN.
1287 : Node* value_value =
1288 41412 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1289 41412 : __ Goto(done, __ Float64LessThan(fzero, __ Float64Abs(value_value)));
1290 : }
1291 :
1292 : __ Bind(&if_bigint);
1293 : {
1294 41412 : Node* bitfield = __ LoadField(AccessBuilder::ForBigIntBitfield(), value);
1295 : Node* length_is_zero = __ Word32Equal(
1296 : __ Word32And(bitfield, __ Int32Constant(BigInt::LengthBits::kMask)),
1297 41412 : __ Int32Constant(0));
1298 41412 : __ Goto(done, __ Word32Equal(length_is_zero, zero));
1299 : }
1300 41412 : }
1301 :
1302 40938 : Node* EffectControlLinearizer::LowerTruncateTaggedToBit(Node* node) {
1303 40938 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1304 40938 : auto if_smi = __ MakeDeferredLabel();
1305 :
1306 : Node* value = node->InputAt(0);
1307 40938 : __ GotoIf(ObjectIsSmi(value), &if_smi);
1308 :
1309 40939 : TruncateTaggedPointerToBit(node, &done);
1310 :
1311 : __ Bind(&if_smi);
1312 : {
1313 : // If {value} is a Smi, then we only need to check that it's not zero.
1314 : __ Goto(&done, __ Word32Equal(__ WordEqual(value, __ IntPtrConstant(0)),
1315 40939 : __ Int32Constant(0)));
1316 : }
1317 :
1318 : __ Bind(&done);
1319 40939 : return done.PhiAt(0);
1320 : }
1321 :
1322 473 : Node* EffectControlLinearizer::LowerTruncateTaggedPointerToBit(Node* node) {
1323 473 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1324 :
1325 473 : TruncateTaggedPointerToBit(node, &done);
1326 :
1327 : __ Bind(&done);
1328 473 : return done.PhiAt(0);
1329 : }
1330 :
1331 1881 : Node* EffectControlLinearizer::LowerChangeTaggedToInt32(Node* node) {
1332 : Node* value = node->InputAt(0);
1333 :
1334 1881 : auto if_not_smi = __ MakeDeferredLabel();
1335 1881 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1336 :
1337 1881 : Node* check = ObjectIsSmi(value);
1338 1882 : __ GotoIfNot(check, &if_not_smi);
1339 1882 : __ Goto(&done, ChangeSmiToInt32(value));
1340 :
1341 : __ Bind(&if_not_smi);
1342 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1343 1882 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1344 1882 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1345 : __ Goto(&done, vfalse);
1346 :
1347 : __ Bind(&done);
1348 1882 : return done.PhiAt(0);
1349 : }
1350 :
1351 564 : Node* EffectControlLinearizer::LowerChangeTaggedToUint32(Node* node) {
1352 : Node* value = node->InputAt(0);
1353 :
1354 564 : auto if_not_smi = __ MakeDeferredLabel();
1355 564 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1356 :
1357 564 : Node* check = ObjectIsSmi(value);
1358 564 : __ GotoIfNot(check, &if_not_smi);
1359 564 : __ Goto(&done, ChangeSmiToInt32(value));
1360 :
1361 : __ Bind(&if_not_smi);
1362 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1363 564 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1364 564 : vfalse = __ ChangeFloat64ToUint32(vfalse);
1365 : __ Goto(&done, vfalse);
1366 :
1367 : __ Bind(&done);
1368 564 : return done.PhiAt(0);
1369 : }
1370 :
1371 12 : Node* EffectControlLinearizer::LowerChangeTaggedToInt64(Node* node) {
1372 : Node* value = node->InputAt(0);
1373 :
1374 12 : auto if_not_smi = __ MakeDeferredLabel();
1375 12 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
1376 :
1377 12 : Node* check = ObjectIsSmi(value);
1378 12 : __ GotoIfNot(check, &if_not_smi);
1379 12 : __ Goto(&done, ChangeSmiToInt64(value));
1380 :
1381 : __ Bind(&if_not_smi);
1382 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1383 12 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1384 12 : vfalse = __ ChangeFloat64ToInt64(vfalse);
1385 : __ Goto(&done, vfalse);
1386 :
1387 : __ Bind(&done);
1388 12 : return done.PhiAt(0);
1389 : }
1390 :
1391 0 : Node* EffectControlLinearizer::LowerChangeTaggedToFloat64(Node* node) {
1392 10774 : return LowerTruncateTaggedToFloat64(node);
1393 : }
1394 :
1395 15 : Node* EffectControlLinearizer::LowerChangeTaggedToTaggedSigned(Node* node) {
1396 : Node* value = node->InputAt(0);
1397 :
1398 15 : auto if_not_smi = __ MakeDeferredLabel();
1399 15 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1400 :
1401 15 : Node* check = ObjectIsSmi(value);
1402 15 : __ GotoIfNot(check, &if_not_smi);
1403 : __ Goto(&done, value);
1404 :
1405 : __ Bind(&if_not_smi);
1406 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1407 15 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1408 15 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1409 15 : vfalse = ChangeInt32ToSmi(vfalse);
1410 : __ Goto(&done, vfalse);
1411 :
1412 : __ Bind(&done);
1413 15 : return done.PhiAt(0);
1414 : }
1415 :
1416 11517 : Node* EffectControlLinearizer::LowerTruncateTaggedToFloat64(Node* node) {
1417 : Node* value = node->InputAt(0);
1418 :
1419 11517 : auto if_not_smi = __ MakeDeferredLabel();
1420 11517 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
1421 :
1422 11517 : Node* check = ObjectIsSmi(value);
1423 11517 : __ GotoIfNot(check, &if_not_smi);
1424 11517 : Node* vtrue = ChangeSmiToInt32(value);
1425 11517 : vtrue = __ ChangeInt32ToFloat64(vtrue);
1426 : __ Goto(&done, vtrue);
1427 :
1428 : __ Bind(&if_not_smi);
1429 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1430 11517 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1431 : __ Goto(&done, vfalse);
1432 :
1433 : __ Bind(&done);
1434 11517 : return done.PhiAt(0);
1435 : }
1436 :
1437 1167 : Node* EffectControlLinearizer::LowerPoisonIndex(Node* node) {
1438 : Node* index = node->InputAt(0);
1439 1167 : if (mask_array_index_ == kMaskArrayIndex) {
1440 0 : index = __ Word32PoisonOnSpeculation(index);
1441 : }
1442 1167 : return index;
1443 : }
1444 :
1445 55458 : void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
1446 55458 : CheckMapsParameters const& p = CheckMapsParametersOf(node->op());
1447 : Node* value = node->InputAt(0);
1448 :
1449 55458 : ZoneHandleSet<Map> const& maps = p.maps();
1450 : size_t const map_count = maps.size();
1451 :
1452 55458 : if (p.flags() & CheckMapsFlag::kTryMigrateInstance) {
1453 707 : auto done = __ MakeLabel();
1454 707 : auto migrate = __ MakeDeferredLabel();
1455 :
1456 : // Load the current map of the {value}.
1457 707 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1458 :
1459 : // Perform the map checks.
1460 1578 : for (size_t i = 0; i < map_count; ++i) {
1461 871 : Node* map = __ HeapConstant(maps[i]);
1462 871 : Node* check = __ WordEqual(value_map, map);
1463 871 : if (i == map_count - 1) {
1464 707 : __ Branch(check, &done, &migrate, IsSafetyCheck::kCriticalSafetyCheck);
1465 : } else {
1466 164 : auto next_map = __ MakeLabel();
1467 164 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1468 : __ Bind(&next_map);
1469 : }
1470 : }
1471 :
1472 : // Perform the (deferred) instance migration.
1473 : __ Bind(&migrate);
1474 : {
1475 : // If map is not deprecated the migration attempt does not make sense.
1476 : Node* bitfield3 =
1477 707 : __ LoadField(AccessBuilder::ForMapBitField3(), value_map);
1478 : Node* if_not_deprecated = __ WordEqual(
1479 : __ Word32And(bitfield3,
1480 : __ Int32Constant(Map::IsDeprecatedBit::kMask)),
1481 707 : __ Int32Constant(0));
1482 707 : __ DeoptimizeIf(DeoptimizeReason::kWrongMap, p.feedback(),
1483 : if_not_deprecated, frame_state,
1484 707 : IsSafetyCheck::kCriticalSafetyCheck);
1485 :
1486 707 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
1487 : Runtime::FunctionId id = Runtime::kTryMigrateInstance;
1488 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1489 707 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
1490 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1), value,
1491 : __ ExternalConstant(ExternalReference::Create(id)),
1492 707 : __ Int32Constant(1), __ NoContextConstant());
1493 707 : Node* check = ObjectIsSmi(result);
1494 : __ DeoptimizeIf(DeoptimizeReason::kInstanceMigrationFailed, p.feedback(),
1495 707 : check, frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1496 : }
1497 :
1498 : // Reload the current map of the {value}.
1499 707 : value_map = __ LoadField(AccessBuilder::ForMap(), value);
1500 :
1501 : // Perform the map checks again.
1502 1578 : for (size_t i = 0; i < map_count; ++i) {
1503 871 : Node* map = __ HeapConstant(maps[i]);
1504 871 : Node* check = __ WordEqual(value_map, map);
1505 871 : if (i == map_count - 1) {
1506 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1507 707 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1508 : } else {
1509 164 : auto next_map = __ MakeLabel();
1510 164 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1511 : __ Bind(&next_map);
1512 : }
1513 : }
1514 :
1515 : __ Goto(&done);
1516 : __ Bind(&done);
1517 : } else {
1518 54751 : auto done = __ MakeLabel();
1519 :
1520 : // Load the current map of the {value}.
1521 54751 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1522 :
1523 115254 : for (size_t i = 0; i < map_count; ++i) {
1524 60502 : Node* map = __ HeapConstant(maps[i]);
1525 60503 : Node* check = __ WordEqual(value_map, map);
1526 60502 : if (i == map_count - 1) {
1527 54751 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1528 54751 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1529 : } else {
1530 5751 : auto next_map = __ MakeLabel();
1531 5751 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1532 : __ Bind(&next_map);
1533 : }
1534 : }
1535 : __ Goto(&done);
1536 : __ Bind(&done);
1537 : }
1538 55459 : }
1539 :
1540 5713 : Node* EffectControlLinearizer::LowerCompareMaps(Node* node) {
1541 5713 : ZoneHandleSet<Map> const& maps = CompareMapsParametersOf(node->op()).maps();
1542 : size_t const map_count = maps.size();
1543 : Node* value = node->InputAt(0);
1544 :
1545 5713 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1546 :
1547 : // Load the current map of the {value}.
1548 5713 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1549 :
1550 11482 : for (size_t i = 0; i < map_count; ++i) {
1551 5769 : Node* map = __ HeapConstant(maps[i]);
1552 5769 : Node* check = __ WordEqual(value_map, map);
1553 5769 : auto next_map = __ MakeLabel();
1554 5769 : auto passed = __ MakeLabel();
1555 5769 : __ Branch(check, &passed, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1556 :
1557 : __ Bind(&passed);
1558 5769 : __ Goto(&done, __ Int32Constant(1));
1559 :
1560 : __ Bind(&next_map);
1561 : }
1562 5713 : __ Goto(&done, __ Int32Constant(0));
1563 :
1564 : __ Bind(&done);
1565 5713 : return done.PhiAt(0);
1566 : }
1567 :
1568 1038 : Node* EffectControlLinearizer::LowerCheckNumber(Node* node, Node* frame_state) {
1569 : Node* value = node->InputAt(0);
1570 519 : const CheckParameters& params = CheckParametersOf(node->op());
1571 :
1572 519 : auto if_not_smi = __ MakeDeferredLabel();
1573 519 : auto done = __ MakeLabel();
1574 :
1575 519 : Node* check0 = ObjectIsSmi(value);
1576 519 : __ GotoIfNot(check0, &if_not_smi);
1577 : __ Goto(&done);
1578 :
1579 : __ Bind(&if_not_smi);
1580 519 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1581 519 : Node* check1 = __ WordEqual(value_map, __ HeapNumberMapConstant());
1582 519 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
1583 519 : check1, frame_state);
1584 : __ Goto(&done);
1585 :
1586 : __ Bind(&done);
1587 519 : return value;
1588 : }
1589 :
1590 815 : Node* EffectControlLinearizer::LowerCheckReceiver(Node* node,
1591 : Node* frame_state) {
1592 : Node* value = node->InputAt(0);
1593 :
1594 815 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1595 : Node* value_instance_type =
1596 815 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1597 :
1598 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1599 : Node* check = __ Uint32LessThanOrEqual(
1600 815 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1601 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObject, VectorSlotPair(),
1602 815 : check, frame_state);
1603 815 : return value;
1604 : }
1605 :
1606 87 : Node* EffectControlLinearizer::LowerCheckReceiverOrNullOrUndefined(
1607 : Node* node, Node* frame_state) {
1608 : Node* value = node->InputAt(0);
1609 :
1610 87 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1611 : Node* value_instance_type =
1612 87 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1613 :
1614 : // Rule out all primitives except oddballs (true, false, undefined, null).
1615 : STATIC_ASSERT(LAST_PRIMITIVE_TYPE == ODDBALL_TYPE);
1616 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1617 : Node* check0 = __ Uint32LessThanOrEqual(__ Uint32Constant(ODDBALL_TYPE),
1618 87 : value_instance_type);
1619 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1620 87 : VectorSlotPair(), check0, frame_state);
1621 :
1622 : // Rule out booleans.
1623 87 : Node* check1 = __ WordEqual(value_map, __ BooleanMapConstant());
1624 : __ DeoptimizeIf(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1625 87 : VectorSlotPair(), check1, frame_state);
1626 87 : return value;
1627 : }
1628 :
1629 37 : Node* EffectControlLinearizer::LowerCheckSymbol(Node* node, Node* frame_state) {
1630 : Node* value = node->InputAt(0);
1631 :
1632 37 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1633 :
1634 : Node* check =
1635 37 : __ WordEqual(value_map, __ HeapConstant(factory()->symbol_map()));
1636 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASymbol, VectorSlotPair(), check,
1637 37 : frame_state);
1638 37 : return value;
1639 : }
1640 :
1641 12646 : Node* EffectControlLinearizer::LowerCheckString(Node* node, Node* frame_state) {
1642 : Node* value = node->InputAt(0);
1643 6323 : const CheckParameters& params = CheckParametersOf(node->op());
1644 :
1645 6323 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1646 : Node* value_instance_type =
1647 6323 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1648 :
1649 : Node* check = __ Uint32LessThan(value_instance_type,
1650 6323 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
1651 6323 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAString, params.feedback(), check,
1652 6323 : frame_state);
1653 6323 : return value;
1654 : }
1655 :
1656 1430 : Node* EffectControlLinearizer::LowerCheckInternalizedString(Node* node,
1657 : Node* frame_state) {
1658 : Node* value = node->InputAt(0);
1659 :
1660 1430 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1661 : Node* value_instance_type =
1662 1430 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1663 :
1664 : Node* check = __ Word32Equal(
1665 : __ Word32And(value_instance_type,
1666 : __ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
1667 1430 : __ Int32Constant(kInternalizedTag));
1668 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1669 1430 : check, frame_state);
1670 :
1671 1430 : return value;
1672 : }
1673 :
1674 30488 : void EffectControlLinearizer::LowerCheckIf(Node* node, Node* frame_state) {
1675 : Node* value = node->InputAt(0);
1676 30488 : const CheckIfParameters& p = CheckIfParametersOf(node->op());
1677 30488 : __ DeoptimizeIfNot(p.reason(), p.feedback(), value, frame_state);
1678 15244 : }
1679 :
1680 37862 : Node* EffectControlLinearizer::LowerStringConcat(Node* node) {
1681 : Node* lhs = node->InputAt(1);
1682 : Node* rhs = node->InputAt(2);
1683 :
1684 : Callable const callable =
1685 18931 : CodeFactory::StringAdd(isolate(), STRING_ADD_CHECK_NONE);
1686 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1687 : graph()->zone(), callable.descriptor(),
1688 : callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags,
1689 37862 : Operator::kNoDeopt | Operator::kNoWrite | Operator::kNoThrow);
1690 :
1691 : Node* value =
1692 : __ Call(call_descriptor, jsgraph()->HeapConstant(callable.code()), lhs,
1693 37862 : rhs, __ NoContextConstant());
1694 :
1695 18931 : return value;
1696 : }
1697 :
1698 56282 : Node* EffectControlLinearizer::LowerCheckedInt32Add(Node* node,
1699 : Node* frame_state) {
1700 : Node* lhs = node->InputAt(0);
1701 : Node* rhs = node->InputAt(1);
1702 :
1703 56282 : Node* value = __ Int32AddWithOverflow(lhs, rhs);
1704 56284 : Node* check = __ Projection(1, value);
1705 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1706 56283 : frame_state);
1707 56285 : return __ Projection(0, value);
1708 : }
1709 :
1710 7527 : Node* EffectControlLinearizer::LowerCheckedInt32Sub(Node* node,
1711 : Node* frame_state) {
1712 : Node* lhs = node->InputAt(0);
1713 : Node* rhs = node->InputAt(1);
1714 :
1715 7527 : Node* value = __ Int32SubWithOverflow(lhs, rhs);
1716 7527 : Node* check = __ Projection(1, value);
1717 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1718 7527 : frame_state);
1719 7527 : return __ Projection(0, value);
1720 : }
1721 :
1722 243 : Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node,
1723 : Node* frame_state) {
1724 : Node* lhs = node->InputAt(0);
1725 : Node* rhs = node->InputAt(1);
1726 243 : Node* zero = __ Int32Constant(0);
1727 :
1728 : // Check if the {rhs} is a known power of two.
1729 : Int32Matcher m(rhs);
1730 243 : if (m.IsPowerOf2()) {
1731 : // Since we know that {rhs} is a power of two, we can perform a fast
1732 : // check to see if the relevant least significant bits of the {lhs}
1733 : // are all zero, and if so we know that we can perform a division
1734 : // safely (and fast by doing an arithmetic - aka sign preserving -
1735 : // right shift on {lhs}).
1736 : int32_t divisor = m.Value();
1737 138 : Node* mask = __ Int32Constant(divisor - 1);
1738 138 : Node* shift = __ Int32Constant(WhichPowerOf2(divisor));
1739 138 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
1740 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1741 138 : check, frame_state);
1742 138 : return __ Word32Sar(lhs, shift);
1743 : } else {
1744 105 : auto if_rhs_positive = __ MakeLabel();
1745 105 : auto if_rhs_negative = __ MakeDeferredLabel();
1746 105 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1747 :
1748 : // Check if {rhs} is positive (and not zero).
1749 105 : Node* check_rhs_positive = __ Int32LessThan(zero, rhs);
1750 105 : __ Branch(check_rhs_positive, &if_rhs_positive, &if_rhs_negative);
1751 :
1752 : __ Bind(&if_rhs_positive);
1753 : {
1754 : // Fast case, no additional checking required.
1755 105 : __ Goto(&done, __ Int32Div(lhs, rhs));
1756 : }
1757 :
1758 : __ Bind(&if_rhs_negative);
1759 : {
1760 105 : auto if_lhs_minint = __ MakeDeferredLabel();
1761 105 : auto if_lhs_notminint = __ MakeLabel();
1762 :
1763 : // Check if {rhs} is zero.
1764 105 : Node* check_rhs_zero = __ Word32Equal(rhs, zero);
1765 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
1766 105 : check_rhs_zero, frame_state);
1767 :
1768 : // Check if {lhs} is zero, as that would produce minus zero.
1769 105 : Node* check_lhs_zero = __ Word32Equal(lhs, zero);
1770 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
1771 105 : check_lhs_zero, frame_state);
1772 :
1773 : // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
1774 : // to return -kMinInt, which is not representable as Word32.
1775 : Node* check_lhs_minint = graph()->NewNode(machine()->Word32Equal(), lhs,
1776 210 : __ Int32Constant(kMinInt));
1777 105 : __ Branch(check_lhs_minint, &if_lhs_minint, &if_lhs_notminint);
1778 :
1779 : __ Bind(&if_lhs_minint);
1780 : {
1781 : // Check that {rhs} is not -1, otherwise result would be -kMinInt.
1782 105 : Node* check_rhs_minusone = __ Word32Equal(rhs, __ Int32Constant(-1));
1783 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(),
1784 105 : check_rhs_minusone, frame_state);
1785 :
1786 : // Perform the actual integer division.
1787 105 : __ Goto(&done, __ Int32Div(lhs, rhs));
1788 : }
1789 :
1790 : __ Bind(&if_lhs_notminint);
1791 : {
1792 : // Perform the actual integer division.
1793 105 : __ Goto(&done, __ Int32Div(lhs, rhs));
1794 : }
1795 : }
1796 :
1797 : __ Bind(&done);
1798 : Node* value = done.PhiAt(0);
1799 :
1800 : // Check if the remainder is non-zero.
1801 105 : Node* check = __ Word32Equal(lhs, __ Int32Mul(value, rhs));
1802 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1803 105 : check, frame_state);
1804 :
1805 : return value;
1806 : }
1807 : }
1808 :
1809 893 : Node* EffectControlLinearizer::BuildUint32Mod(Node* lhs, Node* rhs) {
1810 893 : auto if_rhs_power_of_two = __ MakeLabel();
1811 893 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1812 :
1813 : // Compute the mask for the {rhs}.
1814 893 : Node* one = __ Int32Constant(1);
1815 893 : Node* msk = __ Int32Sub(rhs, one);
1816 :
1817 : // Check if the {rhs} is a power of two.
1818 : __ GotoIf(__ Word32Equal(__ Word32And(rhs, msk), __ Int32Constant(0)),
1819 893 : &if_rhs_power_of_two);
1820 : {
1821 : // The {rhs} is not a power of two, do a generic Uint32Mod.
1822 893 : __ Goto(&done, __ Uint32Mod(lhs, rhs));
1823 : }
1824 :
1825 : __ Bind(&if_rhs_power_of_two);
1826 : {
1827 : // The {rhs} is a power of two, just do a fast bit masking.
1828 893 : __ Goto(&done, __ Word32And(lhs, msk));
1829 : }
1830 :
1831 : __ Bind(&done);
1832 893 : return done.PhiAt(0);
1833 : }
1834 :
1835 856 : Node* EffectControlLinearizer::LowerCheckedInt32Mod(Node* node,
1836 : Node* frame_state) {
1837 : // General case for signed integer modulus, with optimization for (unknown)
1838 : // power of 2 right hand side.
1839 : //
1840 : // if rhs <= 0 then
1841 : // rhs = -rhs
1842 : // deopt if rhs == 0
1843 : // let msk = rhs - 1 in
1844 : // if lhs < 0 then
1845 : // let lhs_abs = -lsh in
1846 : // let res = if rhs & msk == 0 then
1847 : // lhs_abs & msk
1848 : // else
1849 : // lhs_abs % rhs in
1850 : // if lhs < 0 then
1851 : // deopt if res == 0
1852 : // -res
1853 : // else
1854 : // res
1855 : // else
1856 : // if rhs & msk == 0 then
1857 : // lhs & msk
1858 : // else
1859 : // lhs % rhs
1860 : //
1861 : Node* lhs = node->InputAt(0);
1862 : Node* rhs = node->InputAt(1);
1863 :
1864 856 : auto if_rhs_not_positive = __ MakeDeferredLabel();
1865 856 : auto if_lhs_negative = __ MakeDeferredLabel();
1866 : auto if_rhs_power_of_two = __ MakeLabel();
1867 856 : auto rhs_checked = __ MakeLabel(MachineRepresentation::kWord32);
1868 856 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1869 :
1870 856 : Node* zero = __ Int32Constant(0);
1871 :
1872 : // Check if {rhs} is not strictly positive.
1873 856 : Node* check0 = __ Int32LessThanOrEqual(rhs, zero);
1874 856 : __ GotoIf(check0, &if_rhs_not_positive);
1875 : __ Goto(&rhs_checked, rhs);
1876 :
1877 : __ Bind(&if_rhs_not_positive);
1878 : {
1879 : // Negate {rhs}, might still produce a negative result in case of
1880 : // -2^31, but that is handled safely below.
1881 856 : Node* vtrue0 = __ Int32Sub(zero, rhs);
1882 :
1883 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1884 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
1885 856 : __ Word32Equal(vtrue0, zero), frame_state);
1886 : __ Goto(&rhs_checked, vtrue0);
1887 : }
1888 :
1889 : __ Bind(&rhs_checked);
1890 : rhs = rhs_checked.PhiAt(0);
1891 :
1892 856 : __ GotoIf(__ Int32LessThan(lhs, zero), &if_lhs_negative);
1893 : {
1894 : // The {lhs} is a non-negative integer.
1895 856 : __ Goto(&done, BuildUint32Mod(lhs, rhs));
1896 : }
1897 :
1898 : __ Bind(&if_lhs_negative);
1899 : {
1900 : // The {lhs} is a negative integer. This is very unlikely and
1901 : // we intentionally don't use the BuildUint32Mod() here, which
1902 : // would try to figure out whether {rhs} is a power of two,
1903 : // since this is intended to be a slow-path.
1904 856 : Node* res = __ Uint32Mod(__ Int32Sub(zero, lhs), rhs);
1905 :
1906 : // Check if we would have to return -0.
1907 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
1908 856 : __ Word32Equal(res, zero), frame_state);
1909 856 : __ Goto(&done, __ Int32Sub(zero, res));
1910 : }
1911 :
1912 : __ Bind(&done);
1913 856 : return done.PhiAt(0);
1914 : }
1915 :
1916 49 : Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node,
1917 : Node* frame_state) {
1918 : Node* lhs = node->InputAt(0);
1919 : Node* rhs = node->InputAt(1);
1920 49 : Node* zero = __ Int32Constant(0);
1921 :
1922 : // Check if the {rhs} is a known power of two.
1923 : Uint32Matcher m(rhs);
1924 49 : if (m.IsPowerOf2()) {
1925 : // Since we know that {rhs} is a power of two, we can perform a fast
1926 : // check to see if the relevant least significant bits of the {lhs}
1927 : // are all zero, and if so we know that we can perform a division
1928 : // safely (and fast by doing a logical - aka zero extending - right
1929 : // shift on {lhs}).
1930 : uint32_t divisor = m.Value();
1931 37 : Node* mask = __ Uint32Constant(divisor - 1);
1932 37 : Node* shift = __ Uint32Constant(WhichPowerOf2(divisor));
1933 37 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
1934 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1935 37 : check, frame_state);
1936 37 : return __ Word32Shr(lhs, shift);
1937 : } else {
1938 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1939 12 : Node* check = __ Word32Equal(rhs, zero);
1940 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
1941 12 : frame_state);
1942 :
1943 : // Perform the actual unsigned integer division.
1944 12 : Node* value = __ Uint32Div(lhs, rhs);
1945 :
1946 : // Check if the remainder is non-zero.
1947 12 : check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
1948 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1949 12 : check, frame_state);
1950 12 : return value;
1951 : }
1952 : }
1953 :
1954 37 : Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node,
1955 : Node* frame_state) {
1956 : Node* lhs = node->InputAt(0);
1957 : Node* rhs = node->InputAt(1);
1958 :
1959 37 : Node* zero = __ Int32Constant(0);
1960 :
1961 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1962 37 : Node* check = __ Word32Equal(rhs, zero);
1963 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
1964 37 : frame_state);
1965 :
1966 : // Perform the actual unsigned integer modulus.
1967 37 : return BuildUint32Mod(lhs, rhs);
1968 : }
1969 :
1970 5673 : Node* EffectControlLinearizer::LowerCheckedInt32Mul(Node* node,
1971 : Node* frame_state) {
1972 5673 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1973 : Node* lhs = node->InputAt(0);
1974 : Node* rhs = node->InputAt(1);
1975 :
1976 5673 : Node* projection = __ Int32MulWithOverflow(lhs, rhs);
1977 5673 : Node* check = __ Projection(1, projection);
1978 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1979 5673 : frame_state);
1980 :
1981 5673 : Node* value = __ Projection(0, projection);
1982 :
1983 5673 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1984 3931 : auto if_zero = __ MakeDeferredLabel();
1985 3931 : auto check_done = __ MakeLabel();
1986 3931 : Node* zero = __ Int32Constant(0);
1987 3931 : Node* check_zero = __ Word32Equal(value, zero);
1988 3931 : __ GotoIf(check_zero, &if_zero);
1989 : __ Goto(&check_done);
1990 :
1991 : __ Bind(&if_zero);
1992 : // We may need to return negative zero.
1993 3931 : Node* check_or = __ Int32LessThan(__ Word32Or(lhs, rhs), zero);
1994 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(), check_or,
1995 3931 : frame_state);
1996 : __ Goto(&check_done);
1997 :
1998 : __ Bind(&check_done);
1999 : }
2000 :
2001 5673 : return value;
2002 : }
2003 :
2004 0 : Node* EffectControlLinearizer::LowerCheckedInt32ToTaggedSigned(
2005 0 : Node* node, Node* frame_state) {
2006 : DCHECK(SmiValuesAre31Bits());
2007 : Node* value = node->InputAt(0);
2008 0 : const CheckParameters& params = CheckParametersOf(node->op());
2009 :
2010 0 : Node* add = __ Int32AddWithOverflow(value, value);
2011 0 : Node* check = __ Projection(1, add);
2012 0 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2013 0 : frame_state);
2014 0 : Node* result = __ Projection(0, add);
2015 : result = ChangeInt32ToIntPtr(result);
2016 0 : return result;
2017 : }
2018 :
2019 14 : Node* EffectControlLinearizer::LowerCheckedInt64ToInt32(Node* node,
2020 : Node* frame_state) {
2021 : Node* value = node->InputAt(0);
2022 7 : const CheckParameters& params = CheckParametersOf(node->op());
2023 :
2024 7 : Node* value32 = __ TruncateInt64ToInt32(value);
2025 7 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2026 7 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2027 7 : frame_state);
2028 7 : return value32;
2029 : }
2030 :
2031 1725 : Node* EffectControlLinearizer::LowerCheckedInt64ToTaggedSigned(
2032 1725 : Node* node, Node* frame_state) {
2033 : Node* value = node->InputAt(0);
2034 1725 : const CheckParameters& params = CheckParametersOf(node->op());
2035 :
2036 1725 : Node* value32 = __ TruncateInt64ToInt32(value);
2037 1725 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2038 1725 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2039 1725 : frame_state);
2040 :
2041 : if (SmiValuesAre32Bits()) {
2042 : return ChangeInt64ToSmi(value);
2043 : } else {
2044 : Node* add = __ Int32AddWithOverflow(value32, value32);
2045 : Node* check = __ Projection(1, add);
2046 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2047 : frame_state);
2048 : Node* result = __ Projection(0, add);
2049 : result = ChangeInt32ToIntPtr(result);
2050 : return result;
2051 : }
2052 : }
2053 :
2054 88396 : Node* EffectControlLinearizer::LowerCheckedUint32Bounds(Node* node,
2055 : Node* frame_state) {
2056 : Node* index = node->InputAt(0);
2057 : Node* limit = node->InputAt(1);
2058 44198 : const CheckParameters& params = CheckParametersOf(node->op());
2059 :
2060 44198 : Node* check = __ Uint32LessThan(index, limit);
2061 44198 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, params.feedback(), check,
2062 44198 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2063 44198 : return index;
2064 : }
2065 :
2066 506 : Node* EffectControlLinearizer::LowerCheckedUint32ToInt32(Node* node,
2067 : Node* frame_state) {
2068 : Node* value = node->InputAt(0);
2069 253 : const CheckParameters& params = CheckParametersOf(node->op());
2070 253 : Node* unsafe = __ Int32LessThan(value, __ Int32Constant(0));
2071 253 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), unsafe,
2072 253 : frame_state);
2073 253 : return value;
2074 : }
2075 :
2076 14 : Node* EffectControlLinearizer::LowerCheckedUint32ToTaggedSigned(
2077 14 : Node* node, Node* frame_state) {
2078 : Node* value = node->InputAt(0);
2079 14 : const CheckParameters& params = CheckParametersOf(node->op());
2080 14 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
2081 14 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2082 14 : frame_state);
2083 14 : return ChangeUint32ToSmi(value);
2084 : }
2085 :
2086 192 : Node* EffectControlLinearizer::LowerCheckedUint64Bounds(Node* node,
2087 : Node* frame_state) {
2088 192 : CheckParameters const& params = CheckParametersOf(node->op());
2089 : Node* const index = node->InputAt(0);
2090 : Node* const limit = node->InputAt(1);
2091 :
2092 192 : Node* check = __ Uint64LessThan(index, limit);
2093 192 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, params.feedback(), check,
2094 192 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2095 192 : return index;
2096 : }
2097 :
2098 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToInt32(Node* node,
2099 : Node* frame_state) {
2100 : Node* value = node->InputAt(0);
2101 0 : const CheckParameters& params = CheckParametersOf(node->op());
2102 :
2103 0 : Node* check = __ Uint64LessThanOrEqual(value, __ Int64Constant(kMaxInt));
2104 0 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2105 0 : frame_state);
2106 0 : return __ TruncateInt64ToInt32(value);
2107 : }
2108 :
2109 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToTaggedSigned(
2110 0 : Node* node, Node* frame_state) {
2111 : Node* value = node->InputAt(0);
2112 0 : const CheckParameters& params = CheckParametersOf(node->op());
2113 :
2114 : Node* check =
2115 0 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
2116 0 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2117 0 : frame_state);
2118 0 : return ChangeInt64ToSmi(value);
2119 : }
2120 :
2121 4750 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt32(
2122 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2123 : Node* frame_state) {
2124 4750 : Node* value32 = __ RoundFloat64ToInt32(value);
2125 4750 : Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32));
2126 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2127 4750 : check_same, frame_state);
2128 :
2129 4750 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2130 : // Check if {value} is -0.
2131 2158 : auto if_zero = __ MakeDeferredLabel();
2132 2158 : auto check_done = __ MakeLabel();
2133 :
2134 2158 : Node* check_zero = __ Word32Equal(value32, __ Int32Constant(0));
2135 2158 : __ GotoIf(check_zero, &if_zero);
2136 : __ Goto(&check_done);
2137 :
2138 : __ Bind(&if_zero);
2139 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2140 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2141 2158 : __ Int32Constant(0));
2142 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2143 2158 : frame_state);
2144 : __ Goto(&check_done);
2145 :
2146 : __ Bind(&check_done);
2147 : }
2148 4750 : return value32;
2149 : }
2150 :
2151 2350 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt32(Node* node,
2152 : Node* frame_state) {
2153 2350 : const CheckMinusZeroParameters& params =
2154 2350 : CheckMinusZeroParametersOf(node->op());
2155 : Node* value = node->InputAt(0);
2156 2350 : return BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), value,
2157 2350 : frame_state);
2158 : }
2159 :
2160 84 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt64(
2161 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2162 : Node* frame_state) {
2163 84 : Node* value64 = __ TruncateFloat64ToInt64(value);
2164 84 : Node* check_same = __ Float64Equal(value, __ ChangeInt64ToFloat64(value64));
2165 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2166 84 : check_same, frame_state);
2167 :
2168 84 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2169 : // Check if {value} is -0.
2170 0 : auto if_zero = __ MakeDeferredLabel();
2171 0 : auto check_done = __ MakeLabel();
2172 :
2173 0 : Node* check_zero = __ Word64Equal(value64, __ Int64Constant(0));
2174 0 : __ GotoIf(check_zero, &if_zero);
2175 : __ Goto(&check_done);
2176 :
2177 : __ Bind(&if_zero);
2178 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2179 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2180 0 : __ Int32Constant(0));
2181 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2182 0 : frame_state);
2183 : __ Goto(&check_done);
2184 :
2185 : __ Bind(&check_done);
2186 : }
2187 84 : return value64;
2188 : }
2189 :
2190 14 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt64(Node* node,
2191 : Node* frame_state) {
2192 14 : const CheckMinusZeroParameters& params =
2193 14 : CheckMinusZeroParametersOf(node->op());
2194 : Node* value = node->InputAt(0);
2195 14 : return BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), value,
2196 14 : frame_state);
2197 : }
2198 :
2199 24835 : Node* EffectControlLinearizer::LowerCheckedTaggedSignedToInt32(
2200 24835 : Node* node, Node* frame_state) {
2201 : Node* value = node->InputAt(0);
2202 24835 : const CheckParameters& params = CheckParametersOf(node->op());
2203 24836 : Node* check = ObjectIsSmi(value);
2204 24836 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2205 24836 : frame_state);
2206 24836 : return ChangeSmiToInt32(value);
2207 : }
2208 :
2209 2400 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt32(Node* node,
2210 : Node* frame_state) {
2211 2400 : const CheckMinusZeroParameters& params =
2212 2400 : CheckMinusZeroParametersOf(node->op());
2213 : Node* value = node->InputAt(0);
2214 :
2215 2400 : auto if_not_smi = __ MakeDeferredLabel();
2216 2400 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2217 :
2218 2400 : Node* check = ObjectIsSmi(value);
2219 2400 : __ GotoIfNot(check, &if_not_smi);
2220 : // In the Smi case, just convert to int32.
2221 2400 : __ Goto(&done, ChangeSmiToInt32(value));
2222 :
2223 : // In the non-Smi case, check the heap numberness, load the number and convert
2224 : // to int32.
2225 : __ Bind(&if_not_smi);
2226 2400 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2227 2400 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2228 2400 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2229 2400 : check_map, frame_state);
2230 2400 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2231 : vfalse = BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), vfalse,
2232 2400 : frame_state);
2233 : __ Goto(&done, vfalse);
2234 :
2235 : __ Bind(&done);
2236 2400 : return done.PhiAt(0);
2237 : }
2238 :
2239 70 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt64(Node* node,
2240 : Node* frame_state) {
2241 70 : const CheckMinusZeroParameters& params =
2242 70 : CheckMinusZeroParametersOf(node->op());
2243 : Node* value = node->InputAt(0);
2244 :
2245 70 : auto if_not_smi = __ MakeDeferredLabel();
2246 70 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
2247 :
2248 70 : Node* check = ObjectIsSmi(value);
2249 70 : __ GotoIfNot(check, &if_not_smi);
2250 : // In the Smi case, just convert to int64.
2251 70 : __ Goto(&done, ChangeSmiToInt64(value));
2252 :
2253 : // In the non-Smi case, check the heap numberness, load the number and convert
2254 : // to int64.
2255 : __ Bind(&if_not_smi);
2256 70 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2257 70 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2258 70 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2259 70 : check_map, frame_state);
2260 70 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2261 : vfalse = BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), vfalse,
2262 70 : frame_state);
2263 : __ Goto(&done, vfalse);
2264 :
2265 : __ Bind(&done);
2266 70 : return done.PhiAt(0);
2267 : }
2268 :
2269 59322 : Node* EffectControlLinearizer::BuildCheckedHeapNumberOrOddballToFloat64(
2270 : CheckTaggedInputMode mode, const VectorSlotPair& feedback, Node* value,
2271 : Node* frame_state) {
2272 59322 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2273 59322 : Node* check_number = __ WordEqual(value_map, __ HeapNumberMapConstant());
2274 59322 : switch (mode) {
2275 : case CheckTaggedInputMode::kNumber: {
2276 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, feedback,
2277 8728 : check_number, frame_state);
2278 8728 : break;
2279 : }
2280 : case CheckTaggedInputMode::kNumberOrOddball: {
2281 50594 : auto check_done = __ MakeLabel();
2282 :
2283 50594 : __ GotoIf(check_number, &check_done);
2284 : // For oddballs also contain the numeric value, let us just check that
2285 : // we have an oddball here.
2286 : Node* instance_type =
2287 50594 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2288 : Node* check_oddball =
2289 50594 : __ Word32Equal(instance_type, __ Int32Constant(ODDBALL_TYPE));
2290 : __ DeoptimizeIfNot(DeoptimizeReason::kNotANumberOrOddball, feedback,
2291 50594 : check_oddball, frame_state);
2292 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
2293 : __ Goto(&check_done);
2294 :
2295 : __ Bind(&check_done);
2296 : break;
2297 : }
2298 : }
2299 59322 : return __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2300 : }
2301 :
2302 58011 : Node* EffectControlLinearizer::LowerCheckedTaggedToFloat64(Node* node,
2303 : Node* frame_state) {
2304 58011 : CheckTaggedInputParameters const& p =
2305 58011 : CheckTaggedInputParametersOf(node->op());
2306 : Node* value = node->InputAt(0);
2307 :
2308 58011 : auto if_smi = __ MakeLabel();
2309 58011 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
2310 :
2311 58011 : Node* check = ObjectIsSmi(value);
2312 58011 : __ GotoIf(check, &if_smi);
2313 :
2314 : // In the Smi case, just convert to int32 and then float64.
2315 : // Otherwise, check heap numberness and load the number.
2316 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2317 116022 : p.mode(), p.feedback(), value, frame_state);
2318 : __ Goto(&done, number);
2319 :
2320 : __ Bind(&if_smi);
2321 58011 : Node* from_smi = ChangeSmiToInt32(value);
2322 58011 : from_smi = __ ChangeInt32ToFloat64(from_smi);
2323 : __ Goto(&done, from_smi);
2324 :
2325 : __ Bind(&done);
2326 58011 : return done.PhiAt(0);
2327 : }
2328 :
2329 19906 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedSigned(
2330 19906 : Node* node, Node* frame_state) {
2331 : Node* value = node->InputAt(0);
2332 19906 : const CheckParameters& params = CheckParametersOf(node->op());
2333 :
2334 19906 : Node* check = ObjectIsSmi(value);
2335 19906 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2336 19906 : frame_state);
2337 :
2338 19906 : return value;
2339 : }
2340 :
2341 35937 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedPointer(
2342 35937 : Node* node, Node* frame_state) {
2343 : Node* value = node->InputAt(0);
2344 35937 : const CheckParameters& params = CheckParametersOf(node->op());
2345 :
2346 35938 : Node* check = ObjectIsSmi(value);
2347 35938 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2348 35938 : frame_state);
2349 35938 : return value;
2350 : }
2351 :
2352 423 : Node* EffectControlLinearizer::LowerTruncateTaggedToWord32(Node* node) {
2353 : Node* value = node->InputAt(0);
2354 :
2355 423 : auto if_not_smi = __ MakeDeferredLabel();
2356 423 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2357 :
2358 423 : Node* check = ObjectIsSmi(value);
2359 423 : __ GotoIfNot(check, &if_not_smi);
2360 423 : __ Goto(&done, ChangeSmiToInt32(value));
2361 :
2362 : __ Bind(&if_not_smi);
2363 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
2364 423 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2365 423 : vfalse = __ TruncateFloat64ToWord32(vfalse);
2366 : __ Goto(&done, vfalse);
2367 :
2368 : __ Bind(&done);
2369 423 : return done.PhiAt(0);
2370 : }
2371 :
2372 1311 : Node* EffectControlLinearizer::LowerCheckedTruncateTaggedToWord32(
2373 1311 : Node* node, Node* frame_state) {
2374 1311 : const CheckTaggedInputParameters& params =
2375 1311 : CheckTaggedInputParametersOf(node->op());
2376 : Node* value = node->InputAt(0);
2377 :
2378 1311 : auto if_not_smi = __ MakeLabel();
2379 1311 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2380 :
2381 1311 : Node* check = ObjectIsSmi(value);
2382 1311 : __ GotoIfNot(check, &if_not_smi);
2383 : // In the Smi case, just convert to int32.
2384 1311 : __ Goto(&done, ChangeSmiToInt32(value));
2385 :
2386 : // Otherwise, check that it's a heap number or oddball and truncate the value
2387 : // to int32.
2388 : __ Bind(&if_not_smi);
2389 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2390 2622 : params.mode(), params.feedback(), value, frame_state);
2391 1311 : number = __ TruncateFloat64ToWord32(number);
2392 : __ Goto(&done, number);
2393 :
2394 : __ Bind(&done);
2395 1311 : return done.PhiAt(0);
2396 : }
2397 :
2398 224090 : Node* EffectControlLinearizer::LowerAllocate(Node* node) {
2399 : Node* size = node->InputAt(0);
2400 112045 : PretenureFlag pretenure = PretenureFlagOf(node->op());
2401 112045 : Node* new_node = __ Allocate(pretenure, size);
2402 112045 : return new_node;
2403 : }
2404 :
2405 1092 : Node* EffectControlLinearizer::LowerNumberToString(Node* node) {
2406 : Node* argument = node->InputAt(0);
2407 :
2408 : Callable const callable =
2409 1092 : Builtins::CallableFor(isolate(), Builtins::kNumberToString);
2410 : Operator::Properties properties = Operator::kEliminatable;
2411 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2412 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2413 : graph()->zone(), callable.descriptor(),
2414 2184 : callable.descriptor().GetStackParameterCount(), flags, properties);
2415 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), argument,
2416 2184 : __ NoContextConstant());
2417 : }
2418 :
2419 16 : Node* EffectControlLinearizer::LowerObjectIsArrayBufferView(Node* node) {
2420 : Node* value = node->InputAt(0);
2421 :
2422 16 : auto if_smi = __ MakeDeferredLabel();
2423 16 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2424 :
2425 16 : Node* check = ObjectIsSmi(value);
2426 16 : __ GotoIf(check, &if_smi);
2427 :
2428 16 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2429 : Node* value_instance_type =
2430 16 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2431 : STATIC_ASSERT(JS_TYPED_ARRAY_TYPE + 1 == JS_DATA_VIEW_TYPE);
2432 : Node* vfalse = __ Uint32LessThan(
2433 : __ Int32Sub(value_instance_type, __ Int32Constant(JS_TYPED_ARRAY_TYPE)),
2434 16 : __ Int32Constant(2));
2435 : __ Goto(&done, vfalse);
2436 :
2437 : __ Bind(&if_smi);
2438 16 : __ Goto(&done, __ Int32Constant(0));
2439 :
2440 : __ Bind(&done);
2441 16 : return done.PhiAt(0);
2442 : }
2443 :
2444 25 : Node* EffectControlLinearizer::LowerObjectIsBigInt(Node* node) {
2445 : Node* value = node->InputAt(0);
2446 :
2447 25 : auto if_smi = __ MakeDeferredLabel();
2448 25 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2449 :
2450 25 : Node* check = ObjectIsSmi(value);
2451 25 : __ GotoIf(check, &if_smi);
2452 25 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2453 : Node* value_instance_type =
2454 25 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2455 : Node* vfalse =
2456 25 : __ Word32Equal(value_instance_type, __ Uint32Constant(BIGINT_TYPE));
2457 : __ Goto(&done, vfalse);
2458 :
2459 : __ Bind(&if_smi);
2460 25 : __ Goto(&done, __ Int32Constant(0));
2461 :
2462 : __ Bind(&done);
2463 25 : return done.PhiAt(0);
2464 : }
2465 :
2466 98 : Node* EffectControlLinearizer::LowerObjectIsCallable(Node* node) {
2467 : Node* value = node->InputAt(0);
2468 :
2469 98 : auto if_smi = __ MakeDeferredLabel();
2470 98 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2471 :
2472 98 : Node* check = ObjectIsSmi(value);
2473 98 : __ GotoIf(check, &if_smi);
2474 :
2475 98 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2476 : Node* value_bit_field =
2477 98 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2478 : Node* vfalse =
2479 : __ Word32Equal(__ Int32Constant(Map::IsCallableBit::kMask),
2480 : __ Word32And(value_bit_field,
2481 98 : __ Int32Constant(Map::IsCallableBit::kMask)));
2482 : __ Goto(&done, vfalse);
2483 :
2484 : __ Bind(&if_smi);
2485 98 : __ Goto(&done, __ Int32Constant(0));
2486 :
2487 : __ Bind(&done);
2488 98 : return done.PhiAt(0);
2489 : }
2490 :
2491 206 : Node* EffectControlLinearizer::LowerObjectIsConstructor(Node* node) {
2492 : Node* value = node->InputAt(0);
2493 :
2494 206 : auto if_smi = __ MakeDeferredLabel();
2495 206 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2496 :
2497 206 : Node* check = ObjectIsSmi(value);
2498 206 : __ GotoIf(check, &if_smi);
2499 :
2500 206 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2501 : Node* value_bit_field =
2502 206 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2503 : Node* vfalse = __ Word32Equal(
2504 : __ Int32Constant(Map::IsConstructorBit::kMask),
2505 : __ Word32And(value_bit_field,
2506 206 : __ Int32Constant(Map::IsConstructorBit::kMask)));
2507 : __ Goto(&done, vfalse);
2508 :
2509 : __ Bind(&if_smi);
2510 206 : __ Goto(&done, __ Int32Constant(0));
2511 :
2512 : __ Bind(&done);
2513 206 : return done.PhiAt(0);
2514 : }
2515 :
2516 13423 : Node* EffectControlLinearizer::LowerObjectIsDetectableCallable(Node* node) {
2517 : Node* value = node->InputAt(0);
2518 :
2519 13423 : auto if_smi = __ MakeDeferredLabel();
2520 13423 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2521 :
2522 13423 : Node* check = ObjectIsSmi(value);
2523 13424 : __ GotoIf(check, &if_smi);
2524 :
2525 13423 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2526 : Node* value_bit_field =
2527 13424 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2528 : Node* vfalse = __ Word32Equal(
2529 : __ Int32Constant(Map::IsCallableBit::kMask),
2530 : __ Word32And(value_bit_field,
2531 : __ Int32Constant((Map::IsCallableBit::kMask) |
2532 13424 : (Map::IsUndetectableBit::kMask))));
2533 : __ Goto(&done, vfalse);
2534 :
2535 : __ Bind(&if_smi);
2536 13424 : __ Goto(&done, __ Int32Constant(0));
2537 :
2538 : __ Bind(&done);
2539 13424 : return done.PhiAt(0);
2540 : }
2541 :
2542 66 : Node* EffectControlLinearizer::LowerNumberIsFloat64Hole(Node* node) {
2543 : Node* value = node->InputAt(0);
2544 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
2545 66 : __ Int32Constant(kHoleNanUpper32));
2546 66 : return check;
2547 : }
2548 :
2549 141 : Node* EffectControlLinearizer::LowerNumberIsFinite(Node* node) {
2550 : Node* number = node->InputAt(0);
2551 141 : Node* diff = __ Float64Sub(number, number);
2552 141 : Node* check = __ Float64Equal(diff, diff);
2553 141 : return check;
2554 : }
2555 :
2556 7 : Node* EffectControlLinearizer::LowerObjectIsFiniteNumber(Node* node) {
2557 : Node* object = node->InputAt(0);
2558 7 : Node* zero = __ Int32Constant(0);
2559 7 : Node* one = __ Int32Constant(1);
2560 :
2561 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2562 :
2563 : // Check if {object} is a Smi.
2564 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2565 :
2566 : // Check if {object} is a HeapNumber.
2567 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2568 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2569 7 : zero);
2570 :
2571 : // {object} is a HeapNumber.
2572 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2573 7 : Node* diff = __ Float64Sub(value, value);
2574 7 : Node* check = __ Float64Equal(diff, diff);
2575 : __ Goto(&done, check);
2576 :
2577 : __ Bind(&done);
2578 7 : return done.PhiAt(0);
2579 : }
2580 :
2581 119 : Node* EffectControlLinearizer::LowerNumberIsInteger(Node* node) {
2582 : Node* number = node->InputAt(0);
2583 119 : Node* trunc = BuildFloat64RoundTruncate(number);
2584 119 : Node* diff = __ Float64Sub(number, trunc);
2585 119 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2586 119 : return check;
2587 : }
2588 :
2589 7 : Node* EffectControlLinearizer::LowerObjectIsInteger(Node* node) {
2590 : Node* object = node->InputAt(0);
2591 7 : Node* zero = __ Int32Constant(0);
2592 7 : Node* one = __ Int32Constant(1);
2593 :
2594 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2595 :
2596 : // Check if {object} is a Smi.
2597 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2598 :
2599 : // Check if {object} is a HeapNumber.
2600 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2601 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2602 7 : zero);
2603 :
2604 : // {object} is a HeapNumber.
2605 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2606 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2607 7 : Node* diff = __ Float64Sub(value, trunc);
2608 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2609 : __ Goto(&done, check);
2610 :
2611 : __ Bind(&done);
2612 7 : return done.PhiAt(0);
2613 : }
2614 :
2615 7 : Node* EffectControlLinearizer::LowerNumberIsSafeInteger(Node* node) {
2616 : Node* number = node->InputAt(0);
2617 7 : Node* zero = __ Int32Constant(0);
2618 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2619 :
2620 7 : Node* trunc = BuildFloat64RoundTruncate(number);
2621 7 : Node* diff = __ Float64Sub(number, trunc);
2622 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2623 7 : __ GotoIfNot(check, &done, zero);
2624 : Node* in_range = __ Float64LessThanOrEqual(
2625 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2626 : __ Goto(&done, in_range);
2627 :
2628 : __ Bind(&done);
2629 7 : return done.PhiAt(0);
2630 : }
2631 :
2632 7 : Node* EffectControlLinearizer::LowerObjectIsSafeInteger(Node* node) {
2633 : Node* object = node->InputAt(0);
2634 7 : Node* zero = __ Int32Constant(0);
2635 7 : Node* one = __ Int32Constant(1);
2636 :
2637 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2638 :
2639 : // Check if {object} is a Smi.
2640 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2641 :
2642 : // Check if {object} is a HeapNumber.
2643 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2644 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2645 7 : zero);
2646 :
2647 : // {object} is a HeapNumber.
2648 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2649 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2650 7 : Node* diff = __ Float64Sub(value, trunc);
2651 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2652 7 : __ GotoIfNot(check, &done, zero);
2653 : Node* in_range = __ Float64LessThanOrEqual(
2654 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2655 : __ Goto(&done, in_range);
2656 :
2657 : __ Bind(&done);
2658 7 : return done.PhiAt(0);
2659 : }
2660 :
2661 : namespace {
2662 :
2663 61289 : const int64_t kMinusZeroBits = bit_cast<int64_t>(-0.0);
2664 61289 : const int32_t kMinusZeroLoBits = static_cast<int32_t>(kMinusZeroBits);
2665 61289 : const int32_t kMinusZeroHiBits = static_cast<int32_t>(kMinusZeroBits >> 32);
2666 :
2667 : } // namespace
2668 :
2669 21 : Node* EffectControlLinearizer::LowerObjectIsMinusZero(Node* node) {
2670 : Node* value = node->InputAt(0);
2671 21 : Node* zero = __ Int32Constant(0);
2672 :
2673 21 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2674 :
2675 : // Check if {value} is a Smi.
2676 21 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2677 :
2678 : // Check if {value} is a HeapNumber.
2679 21 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2680 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2681 21 : zero);
2682 :
2683 : // Check if {value} contains -0.
2684 21 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2685 21 : if (machine()->Is64()) {
2686 21 : Node* value64 = __ BitcastFloat64ToInt64(value_value);
2687 21 : __ Goto(&done, __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits)));
2688 : } else {
2689 0 : Node* value_lo = __ Float64ExtractLowWord32(value_value);
2690 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2691 0 : &done, zero);
2692 0 : Node* value_hi = __ Float64ExtractHighWord32(value_value);
2693 : __ Goto(&done,
2694 0 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2695 : }
2696 :
2697 : __ Bind(&done);
2698 21 : return done.PhiAt(0);
2699 : }
2700 :
2701 92 : Node* EffectControlLinearizer::LowerNumberIsMinusZero(Node* node) {
2702 : Node* value = node->InputAt(0);
2703 :
2704 92 : if (machine()->Is64()) {
2705 92 : Node* value64 = __ BitcastFloat64ToInt64(value);
2706 92 : return __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits));
2707 : } else {
2708 0 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2709 :
2710 0 : Node* value_lo = __ Float64ExtractLowWord32(value);
2711 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2712 0 : &done, __ Int32Constant(0));
2713 0 : Node* value_hi = __ Float64ExtractHighWord32(value);
2714 : __ Goto(&done,
2715 0 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2716 :
2717 : __ Bind(&done);
2718 : return done.PhiAt(0);
2719 : }
2720 : }
2721 :
2722 685 : Node* EffectControlLinearizer::LowerObjectIsNaN(Node* node) {
2723 : Node* value = node->InputAt(0);
2724 685 : Node* zero = __ Int32Constant(0);
2725 :
2726 685 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2727 :
2728 : // Check if {value} is a Smi.
2729 685 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2730 :
2731 : // Check if {value} is a HeapNumber.
2732 685 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2733 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2734 685 : zero);
2735 :
2736 : // Check if {value} contains a NaN.
2737 685 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2738 : __ Goto(&done,
2739 685 : __ Word32Equal(__ Float64Equal(value_value, value_value), zero));
2740 :
2741 : __ Bind(&done);
2742 685 : return done.PhiAt(0);
2743 : }
2744 :
2745 10327 : Node* EffectControlLinearizer::LowerNumberIsNaN(Node* node) {
2746 : Node* number = node->InputAt(0);
2747 10327 : Node* diff = __ Float64Equal(number, number);
2748 10327 : Node* check = __ Word32Equal(diff, __ Int32Constant(0));
2749 10327 : return check;
2750 : }
2751 :
2752 5753 : Node* EffectControlLinearizer::LowerObjectIsNonCallable(Node* node) {
2753 : Node* value = node->InputAt(0);
2754 :
2755 5753 : auto if_primitive = __ MakeDeferredLabel();
2756 5753 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2757 :
2758 5753 : Node* check0 = ObjectIsSmi(value);
2759 5753 : __ GotoIf(check0, &if_primitive);
2760 :
2761 5753 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2762 : Node* value_instance_type =
2763 5753 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2764 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
2765 : Node* check1 = __ Uint32LessThanOrEqual(
2766 5753 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
2767 5753 : __ GotoIfNot(check1, &if_primitive);
2768 :
2769 : Node* value_bit_field =
2770 5753 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2771 : Node* check2 =
2772 : __ Word32Equal(__ Int32Constant(0),
2773 : __ Word32And(value_bit_field,
2774 5753 : __ Int32Constant(Map::IsCallableBit::kMask)));
2775 : __ Goto(&done, check2);
2776 :
2777 : __ Bind(&if_primitive);
2778 5753 : __ Goto(&done, __ Int32Constant(0));
2779 :
2780 : __ Bind(&done);
2781 5753 : return done.PhiAt(0);
2782 : }
2783 :
2784 7213 : Node* EffectControlLinearizer::LowerObjectIsNumber(Node* node) {
2785 : Node* value = node->InputAt(0);
2786 :
2787 7213 : auto if_smi = __ MakeLabel();
2788 7213 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2789 :
2790 7213 : __ GotoIf(ObjectIsSmi(value), &if_smi);
2791 7213 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2792 7213 : __ Goto(&done, __ WordEqual(value_map, __ HeapNumberMapConstant()));
2793 :
2794 : __ Bind(&if_smi);
2795 7213 : __ Goto(&done, __ Int32Constant(1));
2796 :
2797 : __ Bind(&done);
2798 7213 : return done.PhiAt(0);
2799 : }
2800 :
2801 18235 : Node* EffectControlLinearizer::LowerObjectIsReceiver(Node* node) {
2802 : Node* value = node->InputAt(0);
2803 :
2804 18235 : auto if_smi = __ MakeDeferredLabel();
2805 18235 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2806 :
2807 18235 : __ GotoIf(ObjectIsSmi(value), &if_smi);
2808 :
2809 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
2810 18237 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2811 : Node* value_instance_type =
2812 18237 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2813 : Node* result = __ Uint32LessThanOrEqual(
2814 18236 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
2815 : __ Goto(&done, result);
2816 :
2817 : __ Bind(&if_smi);
2818 18237 : __ Goto(&done, __ Int32Constant(0));
2819 :
2820 : __ Bind(&done);
2821 18237 : return done.PhiAt(0);
2822 : }
2823 :
2824 4720 : Node* EffectControlLinearizer::LowerObjectIsSmi(Node* node) {
2825 : Node* value = node->InputAt(0);
2826 4720 : return ObjectIsSmi(value);
2827 : }
2828 :
2829 1921 : Node* EffectControlLinearizer::LowerObjectIsString(Node* node) {
2830 : Node* value = node->InputAt(0);
2831 :
2832 1921 : auto if_smi = __ MakeDeferredLabel();
2833 1921 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2834 :
2835 1921 : Node* check = ObjectIsSmi(value);
2836 1921 : __ GotoIf(check, &if_smi);
2837 1921 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2838 : Node* value_instance_type =
2839 1921 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2840 : Node* vfalse = __ Uint32LessThan(value_instance_type,
2841 1921 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
2842 : __ Goto(&done, vfalse);
2843 :
2844 : __ Bind(&if_smi);
2845 1921 : __ Goto(&done, __ Int32Constant(0));
2846 :
2847 : __ Bind(&done);
2848 1921 : return done.PhiAt(0);
2849 : }
2850 :
2851 20 : Node* EffectControlLinearizer::LowerObjectIsSymbol(Node* node) {
2852 : Node* value = node->InputAt(0);
2853 :
2854 20 : auto if_smi = __ MakeDeferredLabel();
2855 20 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2856 :
2857 20 : Node* check = ObjectIsSmi(value);
2858 20 : __ GotoIf(check, &if_smi);
2859 20 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2860 : Node* value_instance_type =
2861 20 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2862 : Node* vfalse =
2863 20 : __ Word32Equal(value_instance_type, __ Uint32Constant(SYMBOL_TYPE));
2864 : __ Goto(&done, vfalse);
2865 :
2866 : __ Bind(&if_smi);
2867 20 : __ Goto(&done, __ Int32Constant(0));
2868 :
2869 : __ Bind(&done);
2870 20 : return done.PhiAt(0);
2871 : }
2872 :
2873 1303 : Node* EffectControlLinearizer::LowerObjectIsUndetectable(Node* node) {
2874 : Node* value = node->InputAt(0);
2875 :
2876 1303 : auto if_smi = __ MakeDeferredLabel();
2877 1303 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2878 :
2879 1303 : Node* check = ObjectIsSmi(value);
2880 1303 : __ GotoIf(check, &if_smi);
2881 :
2882 1303 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2883 : Node* value_bit_field =
2884 1303 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2885 : Node* vfalse = __ Word32Equal(
2886 : __ Word32Equal(
2887 : __ Int32Constant(0),
2888 : __ Word32And(value_bit_field,
2889 : __ Int32Constant(Map::IsUndetectableBit::kMask))),
2890 1303 : __ Int32Constant(0));
2891 : __ Goto(&done, vfalse);
2892 :
2893 : __ Bind(&if_smi);
2894 1303 : __ Goto(&done, __ Int32Constant(0));
2895 :
2896 : __ Bind(&done);
2897 1303 : return done.PhiAt(0);
2898 : }
2899 :
2900 18996 : Node* EffectControlLinearizer::LowerTypeOf(Node* node) {
2901 : Node* obj = node->InputAt(0);
2902 18996 : Callable const callable = Builtins::CallableFor(isolate(), Builtins::kTypeof);
2903 : Operator::Properties const properties = Operator::kEliminatable;
2904 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
2905 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2906 : graph()->zone(), callable.descriptor(),
2907 37992 : callable.descriptor().GetStackParameterCount(), flags, properties);
2908 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
2909 37992 : __ NoContextConstant());
2910 : }
2911 :
2912 218 : Node* EffectControlLinearizer::LowerToBoolean(Node* node) {
2913 : Node* obj = node->InputAt(0);
2914 : Callable const callable =
2915 218 : Builtins::CallableFor(isolate(), Builtins::kToBoolean);
2916 : Operator::Properties const properties = Operator::kEliminatable;
2917 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
2918 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2919 : graph()->zone(), callable.descriptor(),
2920 436 : callable.descriptor().GetStackParameterCount(), flags, properties);
2921 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
2922 436 : __ NoContextConstant());
2923 : }
2924 :
2925 50004 : Node* EffectControlLinearizer::LowerArgumentsLength(Node* node) {
2926 16668 : Node* arguments_frame = NodeProperties::GetValueInput(node, 0);
2927 16668 : int formal_parameter_count = FormalParameterCountOf(node->op());
2928 16668 : bool is_rest_length = IsRestLengthOf(node->op());
2929 : DCHECK_LE(0, formal_parameter_count);
2930 :
2931 16668 : if (is_rest_length) {
2932 : // The ArgumentsLength node is computing the number of rest parameters,
2933 : // which is max(0, actual_parameter_count - formal_parameter_count).
2934 : // We have to distinguish the case, when there is an arguments adaptor frame
2935 : // (i.e., arguments_frame != LoadFramePointer()).
2936 233 : auto if_adaptor_frame = __ MakeLabel();
2937 233 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
2938 :
2939 233 : Node* frame = __ LoadFramePointer();
2940 233 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done, __ SmiConstant(0));
2941 : __ Goto(&if_adaptor_frame);
2942 :
2943 : __ Bind(&if_adaptor_frame);
2944 : Node* arguments_length = __ Load(
2945 : MachineType::TaggedSigned(), arguments_frame,
2946 233 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
2947 :
2948 : Node* rest_length =
2949 233 : __ IntSub(arguments_length, __ SmiConstant(formal_parameter_count));
2950 : __ GotoIf(__ IntLessThan(rest_length, __ SmiConstant(0)), &done,
2951 233 : __ SmiConstant(0));
2952 : __ Goto(&done, rest_length);
2953 :
2954 : __ Bind(&done);
2955 : return done.PhiAt(0);
2956 : } else {
2957 : // The ArgumentsLength node is computing the actual number of arguments.
2958 : // We have to distinguish the case when there is an arguments adaptor frame
2959 : // (i.e., arguments_frame != LoadFramePointer()).
2960 16435 : auto if_adaptor_frame = __ MakeLabel();
2961 16435 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
2962 :
2963 16435 : Node* frame = __ LoadFramePointer();
2964 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done,
2965 16435 : __ SmiConstant(formal_parameter_count));
2966 : __ Goto(&if_adaptor_frame);
2967 :
2968 : __ Bind(&if_adaptor_frame);
2969 : Node* arguments_length = __ Load(
2970 : MachineType::TaggedSigned(), arguments_frame,
2971 16435 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
2972 : __ Goto(&done, arguments_length);
2973 :
2974 : __ Bind(&done);
2975 : return done.PhiAt(0);
2976 : }
2977 : }
2978 :
2979 16640 : Node* EffectControlLinearizer::LowerArgumentsFrame(Node* node) {
2980 16640 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
2981 :
2982 16640 : Node* frame = __ LoadFramePointer();
2983 : Node* parent_frame =
2984 : __ Load(MachineType::Pointer(), frame,
2985 16640 : __ IntPtrConstant(StandardFrameConstants::kCallerFPOffset));
2986 : Node* parent_frame_type = __ Load(
2987 : MachineType::AnyTagged(), parent_frame,
2988 16640 : __ IntPtrConstant(CommonFrameConstants::kContextOrFrameTypeOffset));
2989 : __ GotoIf(__ WordEqual(parent_frame_type,
2990 : __ IntPtrConstant(StackFrame::TypeToMarker(
2991 : StackFrame::ARGUMENTS_ADAPTOR))),
2992 16640 : &done, parent_frame);
2993 : __ Goto(&done, frame);
2994 :
2995 : __ Bind(&done);
2996 16640 : return done.PhiAt(0);
2997 : }
2998 :
2999 28 : Node* EffectControlLinearizer::LowerNewDoubleElements(Node* node) {
3000 28 : PretenureFlag const pretenure = PretenureFlagOf(node->op());
3001 : Node* length = node->InputAt(0);
3002 :
3003 28 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3004 28 : Node* zero_length = __ Word32Equal(length, __ Int32Constant(0));
3005 : __ GotoIf(zero_length, &done,
3006 28 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3007 :
3008 : // Compute the effective size of the backing store.
3009 : Node* size =
3010 : __ Int32Add(__ Word32Shl(length, __ Int32Constant(kDoubleSizeLog2)),
3011 28 : __ Int32Constant(FixedDoubleArray::kHeaderSize));
3012 :
3013 : // Allocate the result and initialize the header.
3014 28 : Node* result = __ Allocate(pretenure, size);
3015 : __ StoreField(AccessBuilder::ForMap(), result,
3016 28 : __ FixedDoubleArrayMapConstant());
3017 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3018 28 : ChangeInt32ToSmi(length));
3019 :
3020 : // Initialize the backing store with holes.
3021 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
3022 : Node* limit = ChangeUint32ToUintPtr(length);
3023 : Node* the_hole =
3024 28 : __ LoadField(AccessBuilder::ForHeapNumberValue(), __ TheHoleConstant());
3025 28 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3026 28 : __ Goto(&loop, __ IntPtrConstant(0));
3027 : __ Bind(&loop);
3028 : {
3029 : // Check if we've initialized everything.
3030 : Node* index = loop.PhiAt(0);
3031 28 : Node* check = __ UintLessThan(index, limit);
3032 28 : __ GotoIfNot(check, &done, result);
3033 :
3034 : // Storing "the_hole" doesn't need a write barrier.
3035 : StoreRepresentation rep(MachineRepresentation::kFloat64, kNoWriteBarrier);
3036 : Node* offset = __ IntAdd(
3037 : __ WordShl(index, __ IntPtrConstant(kDoubleSizeLog2)),
3038 28 : __ IntPtrConstant(FixedDoubleArray::kHeaderSize - kHeapObjectTag));
3039 28 : __ Store(rep, result, offset, the_hole);
3040 :
3041 : // Advance the {index}.
3042 28 : index = __ IntAdd(index, __ IntPtrConstant(1));
3043 : __ Goto(&loop, index);
3044 : }
3045 :
3046 : __ Bind(&done);
3047 28 : return done.PhiAt(0);
3048 : }
3049 :
3050 507 : Node* EffectControlLinearizer::LowerNewSmiOrObjectElements(Node* node) {
3051 507 : PretenureFlag const pretenure = PretenureFlagOf(node->op());
3052 : Node* length = node->InputAt(0);
3053 :
3054 507 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3055 507 : Node* zero_length = __ Word32Equal(length, __ Int32Constant(0));
3056 : __ GotoIf(zero_length, &done,
3057 507 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3058 :
3059 : // Compute the effective size of the backing store.
3060 : Node* size =
3061 : __ Int32Add(__ Word32Shl(length, __ Int32Constant(kTaggedSizeLog2)),
3062 507 : __ Int32Constant(FixedArray::kHeaderSize));
3063 :
3064 : // Allocate the result and initialize the header.
3065 507 : Node* result = __ Allocate(pretenure, size);
3066 507 : __ StoreField(AccessBuilder::ForMap(), result, __ FixedArrayMapConstant());
3067 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3068 507 : ChangeInt32ToSmi(length));
3069 :
3070 : // Initialize the backing store with holes.
3071 : Node* limit = ChangeUint32ToUintPtr(length);
3072 507 : Node* the_hole = __ TheHoleConstant();
3073 507 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3074 507 : __ Goto(&loop, __ IntPtrConstant(0));
3075 : __ Bind(&loop);
3076 : {
3077 : // Check if we've initialized everything.
3078 : Node* index = loop.PhiAt(0);
3079 507 : Node* check = __ UintLessThan(index, limit);
3080 507 : __ GotoIfNot(check, &done, result);
3081 :
3082 : // Storing "the_hole" doesn't need a write barrier.
3083 : StoreRepresentation rep(MachineRepresentation::kTagged, kNoWriteBarrier);
3084 : Node* offset =
3085 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)),
3086 507 : __ IntPtrConstant(FixedArray::kHeaderSize - kHeapObjectTag));
3087 507 : __ Store(rep, result, offset, the_hole);
3088 :
3089 : // Advance the {index}.
3090 507 : index = __ IntAdd(index, __ IntPtrConstant(1));
3091 : __ Goto(&loop, index);
3092 : }
3093 :
3094 : __ Bind(&done);
3095 507 : return done.PhiAt(0);
3096 : }
3097 :
3098 48465 : Node* EffectControlLinearizer::LowerNewArgumentsElements(Node* node) {
3099 16155 : Node* frame = NodeProperties::GetValueInput(node, 0);
3100 16155 : Node* length = NodeProperties::GetValueInput(node, 1);
3101 16155 : int mapped_count = NewArgumentsElementsMappedCountOf(node->op());
3102 :
3103 : Callable const callable =
3104 16155 : Builtins::CallableFor(isolate(), Builtins::kNewArgumentsElements);
3105 16155 : Operator::Properties const properties = node->op()->properties();
3106 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
3107 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3108 : graph()->zone(), callable.descriptor(),
3109 32310 : callable.descriptor().GetStackParameterCount(), flags, properties);
3110 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), frame,
3111 32310 : length, __ SmiConstant(mapped_count), __ NoContextConstant());
3112 : }
3113 :
3114 3328 : Node* EffectControlLinearizer::LowerNewConsString(Node* node) {
3115 : Node* length = node->InputAt(0);
3116 : Node* first = node->InputAt(1);
3117 : Node* second = node->InputAt(2);
3118 :
3119 : // Determine the instance types of {first} and {second}.
3120 3328 : Node* first_map = __ LoadField(AccessBuilder::ForMap(), first);
3121 : Node* first_instance_type =
3122 3328 : __ LoadField(AccessBuilder::ForMapInstanceType(), first_map);
3123 3328 : Node* second_map = __ LoadField(AccessBuilder::ForMap(), second);
3124 : Node* second_instance_type =
3125 3328 : __ LoadField(AccessBuilder::ForMapInstanceType(), second_map);
3126 :
3127 : // Determine the proper map for the resulting ConsString.
3128 : // If both {first} and {second} are one-byte strings, we
3129 : // create a new ConsOneByteString, otherwise we create a
3130 : // new ConsString instead.
3131 3328 : auto if_onebyte = __ MakeLabel();
3132 3328 : auto if_twobyte = __ MakeLabel();
3133 3328 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3134 : STATIC_ASSERT(kOneByteStringTag != 0);
3135 : STATIC_ASSERT(kTwoByteStringTag == 0);
3136 3328 : Node* instance_type = __ Word32And(first_instance_type, second_instance_type);
3137 : Node* encoding =
3138 3328 : __ Word32And(instance_type, __ Int32Constant(kStringEncodingMask));
3139 : __ Branch(__ Word32Equal(encoding, __ Int32Constant(kTwoByteStringTag)),
3140 3328 : &if_twobyte, &if_onebyte);
3141 : __ Bind(&if_onebyte);
3142 : __ Goto(&done,
3143 6656 : jsgraph()->HeapConstant(factory()->cons_one_byte_string_map()));
3144 : __ Bind(&if_twobyte);
3145 6656 : __ Goto(&done, jsgraph()->HeapConstant(factory()->cons_string_map()));
3146 : __ Bind(&done);
3147 : Node* result_map = done.PhiAt(0);
3148 :
3149 : // Allocate the resulting ConsString.
3150 3328 : Node* result = __ Allocate(NOT_TENURED, __ Int32Constant(ConsString::kSize));
3151 3328 : __ StoreField(AccessBuilder::ForMap(), result, result_map);
3152 : __ StoreField(AccessBuilder::ForNameHashField(), result,
3153 3328 : __ Int32Constant(Name::kEmptyHashField));
3154 3328 : __ StoreField(AccessBuilder::ForStringLength(), result, length);
3155 3328 : __ StoreField(AccessBuilder::ForConsStringFirst(), result, first);
3156 3328 : __ StoreField(AccessBuilder::ForConsStringSecond(), result, second);
3157 3328 : return result;
3158 : }
3159 :
3160 12 : Node* EffectControlLinearizer::LowerSameValue(Node* node) {
3161 : Node* lhs = node->InputAt(0);
3162 : Node* rhs = node->InputAt(1);
3163 :
3164 : Callable const callable =
3165 12 : Builtins::CallableFor(isolate(), Builtins::kSameValue);
3166 : Operator::Properties properties = Operator::kEliminatable;
3167 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3168 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3169 : graph()->zone(), callable.descriptor(),
3170 24 : callable.descriptor().GetStackParameterCount(), flags, properties);
3171 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3172 24 : __ NoContextConstant());
3173 : }
3174 :
3175 1243 : Node* EffectControlLinearizer::LowerDeadValue(Node* node) {
3176 1243 : Node* input = NodeProperties::GetValueInput(node, 0);
3177 1243 : if (input->opcode() != IrOpcode::kUnreachable) {
3178 959 : Node* unreachable = __ Unreachable();
3179 959 : NodeProperties::ReplaceValueInput(node, unreachable, 0);
3180 : }
3181 1243 : return node;
3182 : }
3183 :
3184 344 : Node* EffectControlLinearizer::LowerStringToNumber(Node* node) {
3185 : Node* string = node->InputAt(0);
3186 :
3187 : Callable const callable =
3188 344 : Builtins::CallableFor(isolate(), Builtins::kStringToNumber);
3189 : Operator::Properties properties = Operator::kEliminatable;
3190 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3191 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3192 : graph()->zone(), callable.descriptor(),
3193 688 : callable.descriptor().GetStackParameterCount(), flags, properties);
3194 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), string,
3195 688 : __ NoContextConstant());
3196 : }
3197 :
3198 2398 : Node* EffectControlLinearizer::LowerStringCharCodeAt(Node* node) {
3199 : Node* receiver = node->InputAt(0);
3200 : Node* position = node->InputAt(1);
3201 :
3202 : // We need a loop here to properly deal with indirect strings
3203 : // (SlicedString, ConsString and ThinString).
3204 : auto loop = __ MakeLoopLabel(MachineRepresentation::kTagged,
3205 2398 : MachineType::PointerRepresentation());
3206 : auto loop_next = __ MakeLabel(MachineRepresentation::kTagged,
3207 2398 : MachineType::PointerRepresentation());
3208 2398 : auto loop_done = __ MakeLabel(MachineRepresentation::kWord32);
3209 2398 : __ Goto(&loop, receiver, position);
3210 : __ Bind(&loop);
3211 : {
3212 : Node* receiver = loop.PhiAt(0);
3213 : Node* position = loop.PhiAt(1);
3214 2398 : Node* receiver_map = __ LoadField(AccessBuilder::ForMap(), receiver);
3215 : Node* receiver_instance_type =
3216 2398 : __ LoadField(AccessBuilder::ForMapInstanceType(), receiver_map);
3217 : Node* receiver_representation = __ Word32And(
3218 2398 : receiver_instance_type, __ Int32Constant(kStringRepresentationMask));
3219 :
3220 : // Dispatch on the current {receiver}s string representation.
3221 2398 : auto if_seqstring = __ MakeLabel();
3222 2398 : auto if_consstring = __ MakeLabel();
3223 2398 : auto if_thinstring = __ MakeLabel();
3224 2398 : auto if_externalstring = __ MakeLabel();
3225 2398 : auto if_slicedstring = __ MakeLabel();
3226 2398 : auto if_runtime = __ MakeDeferredLabel();
3227 : __ GotoIf(__ Word32Equal(receiver_representation,
3228 : __ Int32Constant(kSeqStringTag)),
3229 2398 : &if_seqstring);
3230 : __ GotoIf(__ Word32Equal(receiver_representation,
3231 : __ Int32Constant(kConsStringTag)),
3232 2398 : &if_consstring);
3233 : __ GotoIf(__ Word32Equal(receiver_representation,
3234 : __ Int32Constant(kThinStringTag)),
3235 2398 : &if_thinstring);
3236 : __ GotoIf(__ Word32Equal(receiver_representation,
3237 : __ Int32Constant(kExternalStringTag)),
3238 2398 : &if_externalstring);
3239 : __ Branch(__ Word32Equal(receiver_representation,
3240 : __ Int32Constant(kSlicedStringTag)),
3241 2398 : &if_slicedstring, &if_runtime);
3242 :
3243 : __ Bind(&if_seqstring);
3244 : {
3245 : Node* receiver_is_onebyte = __ Word32Equal(
3246 : __ Word32Equal(__ Word32And(receiver_instance_type,
3247 : __ Int32Constant(kStringEncodingMask)),
3248 : __ Int32Constant(kTwoByteStringTag)),
3249 2398 : __ Int32Constant(0));
3250 2398 : Node* result = LoadFromSeqString(receiver, position, receiver_is_onebyte);
3251 : __ Goto(&loop_done, result);
3252 : }
3253 :
3254 : __ Bind(&if_thinstring);
3255 : {
3256 : Node* receiver_actual =
3257 2398 : __ LoadField(AccessBuilder::ForThinStringActual(), receiver);
3258 : __ Goto(&loop_next, receiver_actual, position);
3259 : }
3260 :
3261 : __ Bind(&if_consstring);
3262 : {
3263 : Node* receiver_second =
3264 2398 : __ LoadField(AccessBuilder::ForConsStringSecond(), receiver);
3265 : __ GotoIfNot(__ WordEqual(receiver_second, __ EmptyStringConstant()),
3266 2398 : &if_runtime);
3267 : Node* receiver_first =
3268 2398 : __ LoadField(AccessBuilder::ForConsStringFirst(), receiver);
3269 : __ Goto(&loop_next, receiver_first, position);
3270 : }
3271 :
3272 : __ Bind(&if_externalstring);
3273 : {
3274 : // We need to bailout to the runtime for uncached external strings.
3275 : __ GotoIf(__ Word32Equal(
3276 : __ Word32And(receiver_instance_type,
3277 : __ Int32Constant(kUncachedExternalStringMask)),
3278 : __ Int32Constant(kUncachedExternalStringTag)),
3279 2398 : &if_runtime);
3280 :
3281 : Node* receiver_data = __ LoadField(
3282 2398 : AccessBuilder::ForExternalStringResourceData(), receiver);
3283 :
3284 2398 : auto if_onebyte = __ MakeLabel();
3285 2398 : auto if_twobyte = __ MakeLabel();
3286 : __ Branch(
3287 : __ Word32Equal(__ Word32And(receiver_instance_type,
3288 : __ Int32Constant(kStringEncodingMask)),
3289 : __ Int32Constant(kTwoByteStringTag)),
3290 2398 : &if_twobyte, &if_onebyte);
3291 :
3292 : __ Bind(&if_onebyte);
3293 : {
3294 2398 : Node* result = __ Load(MachineType::Uint8(), receiver_data, position);
3295 : __ Goto(&loop_done, result);
3296 : }
3297 :
3298 : __ Bind(&if_twobyte);
3299 : {
3300 : Node* result = __ Load(MachineType::Uint16(), receiver_data,
3301 2398 : __ WordShl(position, __ IntPtrConstant(1)));
3302 : __ Goto(&loop_done, result);
3303 : }
3304 : }
3305 :
3306 : __ Bind(&if_slicedstring);
3307 : {
3308 : Node* receiver_offset =
3309 2398 : __ LoadField(AccessBuilder::ForSlicedStringOffset(), receiver);
3310 : Node* receiver_parent =
3311 2398 : __ LoadField(AccessBuilder::ForSlicedStringParent(), receiver);
3312 : __ Goto(&loop_next, receiver_parent,
3313 2398 : __ IntAdd(position, ChangeSmiToIntPtr(receiver_offset)));
3314 : }
3315 :
3316 : __ Bind(&if_runtime);
3317 : {
3318 2398 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3319 : Runtime::FunctionId id = Runtime::kStringCharCodeAt;
3320 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3321 2398 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
3322 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1),
3323 : receiver, ChangeIntPtrToSmi(position),
3324 : __ ExternalConstant(ExternalReference::Create(id)),
3325 2398 : __ Int32Constant(2), __ NoContextConstant());
3326 2398 : __ Goto(&loop_done, ChangeSmiToInt32(result));
3327 : }
3328 :
3329 : __ Bind(&loop_next);
3330 : __ Goto(&loop, loop_next.PhiAt(0), loop_next.PhiAt(1));
3331 : }
3332 : __ Bind(&loop_done);
3333 2398 : return loop_done.PhiAt(0);
3334 : }
3335 :
3336 249 : Node* EffectControlLinearizer::LowerStringCodePointAt(
3337 : Node* node, UnicodeEncoding encoding) {
3338 : Node* receiver = node->InputAt(0);
3339 : Node* position = node->InputAt(1);
3340 :
3341 : Builtins::Name builtin = encoding == UnicodeEncoding::UTF16
3342 : ? Builtins::kStringCodePointAtUTF16
3343 249 : : Builtins::kStringCodePointAtUTF32;
3344 :
3345 249 : Callable const callable = Builtins::CallableFor(isolate(), builtin);
3346 249 : Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite;
3347 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3348 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3349 : graph()->zone(), callable.descriptor(),
3350 498 : callable.descriptor().GetStackParameterCount(), flags, properties);
3351 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3352 498 : position, __ NoContextConstant());
3353 : }
3354 :
3355 2398 : Node* EffectControlLinearizer::LoadFromSeqString(Node* receiver, Node* position,
3356 : Node* is_one_byte) {
3357 2398 : auto one_byte_load = __ MakeLabel();
3358 2398 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
3359 2398 : __ GotoIf(is_one_byte, &one_byte_load);
3360 : Node* two_byte_result = __ LoadElement(
3361 2398 : AccessBuilder::ForSeqTwoByteStringCharacter(), receiver, position);
3362 : __ Goto(&done, two_byte_result);
3363 :
3364 : __ Bind(&one_byte_load);
3365 : Node* one_byte_element = __ LoadElement(
3366 2398 : AccessBuilder::ForSeqOneByteStringCharacter(), receiver, position);
3367 : __ Goto(&done, one_byte_element);
3368 :
3369 : __ Bind(&done);
3370 2398 : return done.PhiAt(0);
3371 : }
3372 :
3373 966 : Node* EffectControlLinearizer::LowerStringFromSingleCharCode(Node* node) {
3374 : Node* value = node->InputAt(0);
3375 966 : Node* code = __ Word32And(value, __ Uint32Constant(0xFFFF));
3376 :
3377 966 : auto if_not_one_byte = __ MakeDeferredLabel();
3378 966 : auto cache_miss = __ MakeDeferredLabel();
3379 966 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3380 :
3381 : // Check if the {code} is a one byte character
3382 : Node* check1 = __ Uint32LessThanOrEqual(
3383 966 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3384 966 : __ GotoIfNot(check1, &if_not_one_byte);
3385 : {
3386 : // Load the isolate wide single character string cache.
3387 966 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3388 :
3389 : // Compute the {cache} index for {code}.
3390 966 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3391 :
3392 : // Check if we have an entry for the {code} in the single character string
3393 : // cache already.
3394 : Node* entry =
3395 966 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3396 :
3397 966 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3398 966 : __ GotoIf(check2, &cache_miss);
3399 :
3400 : // Use the {entry} from the {cache}.
3401 : __ Goto(&done, entry);
3402 :
3403 : __ Bind(&cache_miss);
3404 : {
3405 : // Allocate a new SeqOneByteString for {code}.
3406 : Node* vtrue2 = __ Allocate(
3407 966 : NOT_TENURED, __ Int32Constant(SeqOneByteString::SizeFor(1)));
3408 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3409 966 : __ HeapConstant(factory()->one_byte_string_map()));
3410 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3411 966 : __ Int32Constant(Name::kEmptyHashField));
3412 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3413 966 : __ Int32Constant(1));
3414 : __ Store(
3415 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3416 : vtrue2,
3417 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3418 1932 : code);
3419 :
3420 : // Remember it in the {cache}.
3421 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3422 966 : vtrue2);
3423 : __ Goto(&done, vtrue2);
3424 : }
3425 : }
3426 :
3427 : __ Bind(&if_not_one_byte);
3428 : {
3429 : // Allocate a new SeqTwoByteString for {code}.
3430 : Node* vfalse1 = __ Allocate(NOT_TENURED,
3431 966 : __ Int32Constant(SeqTwoByteString::SizeFor(1)));
3432 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3433 966 : __ HeapConstant(factory()->string_map()));
3434 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3435 966 : __ Int32Constant(Name::kEmptyHashField));
3436 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3437 966 : __ Int32Constant(1));
3438 : __ Store(
3439 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3440 : vfalse1,
3441 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3442 1932 : code);
3443 : __ Goto(&done, vfalse1);
3444 : }
3445 :
3446 : __ Bind(&done);
3447 966 : return done.PhiAt(0);
3448 : }
3449 :
3450 : #ifdef V8_INTL_SUPPORT
3451 :
3452 88 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3453 : Node* receiver = node->InputAt(0);
3454 :
3455 : Callable callable =
3456 88 : Builtins::CallableFor(isolate(), Builtins::kStringToLowerCaseIntl);
3457 88 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3458 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3459 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3460 : graph()->zone(), callable.descriptor(),
3461 176 : callable.descriptor().GetStackParameterCount(), flags, properties);
3462 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3463 176 : __ NoContextConstant());
3464 : }
3465 :
3466 28 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3467 : Node* receiver = node->InputAt(0);
3468 28 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3469 : Runtime::FunctionId id = Runtime::kStringToUpperCaseIntl;
3470 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3471 28 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
3472 : return __ Call(call_descriptor, __ CEntryStubConstant(1), receiver,
3473 : __ ExternalConstant(ExternalReference::Create(id)),
3474 28 : __ Int32Constant(1), __ NoContextConstant());
3475 : }
3476 :
3477 : #else
3478 :
3479 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3480 : UNREACHABLE();
3481 : return nullptr;
3482 : }
3483 :
3484 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3485 : UNREACHABLE();
3486 : return nullptr;
3487 : }
3488 :
3489 : #endif // V8_INTL_SUPPORT
3490 :
3491 444 : Node* EffectControlLinearizer::LowerStringFromSingleCodePoint(Node* node) {
3492 : Node* value = node->InputAt(0);
3493 : Node* code = value;
3494 :
3495 222 : auto if_not_single_code = __ MakeDeferredLabel();
3496 222 : auto if_not_one_byte = __ MakeDeferredLabel();
3497 222 : auto cache_miss = __ MakeDeferredLabel();
3498 222 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3499 :
3500 : // Check if the {code} is a single code unit
3501 222 : Node* check0 = __ Uint32LessThanOrEqual(code, __ Uint32Constant(0xFFFF));
3502 222 : __ GotoIfNot(check0, &if_not_single_code);
3503 :
3504 : {
3505 : // Check if the {code} is a one byte character
3506 : Node* check1 = __ Uint32LessThanOrEqual(
3507 222 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3508 222 : __ GotoIfNot(check1, &if_not_one_byte);
3509 : {
3510 : // Load the isolate wide single character string cache.
3511 222 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3512 :
3513 : // Compute the {cache} index for {code}.
3514 222 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3515 :
3516 : // Check if we have an entry for the {code} in the single character string
3517 : // cache already.
3518 : Node* entry =
3519 222 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3520 :
3521 222 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3522 222 : __ GotoIf(check2, &cache_miss);
3523 :
3524 : // Use the {entry} from the {cache}.
3525 : __ Goto(&done, entry);
3526 :
3527 : __ Bind(&cache_miss);
3528 : {
3529 : // Allocate a new SeqOneByteString for {code}.
3530 : Node* vtrue2 = __ Allocate(
3531 222 : NOT_TENURED, __ Int32Constant(SeqOneByteString::SizeFor(1)));
3532 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3533 222 : __ HeapConstant(factory()->one_byte_string_map()));
3534 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3535 222 : __ Int32Constant(Name::kEmptyHashField));
3536 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3537 222 : __ Int32Constant(1));
3538 : __ Store(
3539 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3540 : vtrue2,
3541 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3542 444 : code);
3543 :
3544 : // Remember it in the {cache}.
3545 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3546 222 : vtrue2);
3547 : __ Goto(&done, vtrue2);
3548 : }
3549 : }
3550 :
3551 : __ Bind(&if_not_one_byte);
3552 : {
3553 : // Allocate a new SeqTwoByteString for {code}.
3554 : Node* vfalse1 = __ Allocate(
3555 222 : NOT_TENURED, __ Int32Constant(SeqTwoByteString::SizeFor(1)));
3556 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3557 222 : __ HeapConstant(factory()->string_map()));
3558 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3559 222 : __ IntPtrConstant(Name::kEmptyHashField));
3560 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3561 222 : __ Int32Constant(1));
3562 : __ Store(
3563 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3564 : vfalse1,
3565 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3566 444 : code);
3567 : __ Goto(&done, vfalse1);
3568 : }
3569 : }
3570 :
3571 : __ Bind(&if_not_single_code);
3572 : // Generate surrogate pair string
3573 : {
3574 222 : switch (UnicodeEncodingOf(node->op())) {
3575 : case UnicodeEncoding::UTF16:
3576 : break;
3577 :
3578 : case UnicodeEncoding::UTF32: {
3579 : // Convert UTF32 to UTF16 code units, and store as a 32 bit word.
3580 99 : Node* lead_offset = __ Int32Constant(0xD800 - (0x10000 >> 10));
3581 :
3582 : // lead = (codepoint >> 10) + LEAD_OFFSET
3583 : Node* lead =
3584 99 : __ Int32Add(__ Word32Shr(code, __ Int32Constant(10)), lead_offset);
3585 :
3586 : // trail = (codepoint & 0x3FF) + 0xDC00;
3587 : Node* trail = __ Int32Add(__ Word32And(code, __ Int32Constant(0x3FF)),
3588 99 : __ Int32Constant(0xDC00));
3589 :
3590 : // codpoint = (trail << 16) | lead;
3591 : #if V8_TARGET_BIG_ENDIAN
3592 : code = __ Word32Or(__ Word32Shl(lead, __ Int32Constant(16)), trail);
3593 : #else
3594 99 : code = __ Word32Or(__ Word32Shl(trail, __ Int32Constant(16)), lead);
3595 : #endif
3596 99 : break;
3597 : }
3598 : }
3599 :
3600 : // Allocate a new SeqTwoByteString for {code}.
3601 : Node* vfalse0 = __ Allocate(NOT_TENURED,
3602 222 : __ Int32Constant(SeqTwoByteString::SizeFor(2)));
3603 : __ StoreField(AccessBuilder::ForMap(), vfalse0,
3604 222 : __ HeapConstant(factory()->string_map()));
3605 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse0,
3606 222 : __ Int32Constant(Name::kEmptyHashField));
3607 : __ StoreField(AccessBuilder::ForStringLength(), vfalse0,
3608 222 : __ Int32Constant(2));
3609 : __ Store(
3610 : StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier),
3611 : vfalse0,
3612 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3613 444 : code);
3614 : __ Goto(&done, vfalse0);
3615 : }
3616 :
3617 : __ Bind(&done);
3618 222 : return done.PhiAt(0);
3619 : }
3620 :
3621 289 : Node* EffectControlLinearizer::LowerStringIndexOf(Node* node) {
3622 : Node* subject = node->InputAt(0);
3623 : Node* search_string = node->InputAt(1);
3624 : Node* position = node->InputAt(2);
3625 :
3626 : Callable callable =
3627 289 : Builtins::CallableFor(isolate(), Builtins::kStringIndexOf);
3628 : Operator::Properties properties = Operator::kEliminatable;
3629 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3630 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3631 : graph()->zone(), callable.descriptor(),
3632 578 : callable.descriptor().GetStackParameterCount(), flags, properties);
3633 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), subject,
3634 578 : search_string, position, __ NoContextConstant());
3635 : }
3636 :
3637 21117 : Node* EffectControlLinearizer::LowerStringLength(Node* node) {
3638 : Node* subject = node->InputAt(0);
3639 :
3640 21117 : return __ LoadField(AccessBuilder::ForStringLength(), subject);
3641 : }
3642 :
3643 11352 : Node* EffectControlLinearizer::LowerStringComparison(Callable const& callable,
3644 : Node* node) {
3645 : Node* lhs = node->InputAt(0);
3646 : Node* rhs = node->InputAt(1);
3647 :
3648 : Operator::Properties properties = Operator::kEliminatable;
3649 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3650 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3651 : graph()->zone(), callable.descriptor(),
3652 22704 : callable.descriptor().GetStackParameterCount(), flags, properties);
3653 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3654 22704 : __ NoContextConstant());
3655 : }
3656 :
3657 1081 : Node* EffectControlLinearizer::LowerStringSubstring(Node* node) {
3658 : Node* receiver = node->InputAt(0);
3659 : Node* start = ChangeInt32ToIntPtr(node->InputAt(1));
3660 : Node* end = ChangeInt32ToIntPtr(node->InputAt(2));
3661 :
3662 : Callable callable =
3663 1081 : Builtins::CallableFor(isolate(), Builtins::kStringSubstring);
3664 : Operator::Properties properties = Operator::kEliminatable;
3665 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3666 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3667 : graph()->zone(), callable.descriptor(),
3668 2162 : callable.descriptor().GetStackParameterCount(), flags, properties);
3669 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3670 2162 : start, end, __ NoContextConstant());
3671 : }
3672 :
3673 10236 : Node* EffectControlLinearizer::LowerStringEqual(Node* node) {
3674 : return LowerStringComparison(
3675 10236 : Builtins::CallableFor(isolate(), Builtins::kStringEqual), node);
3676 : }
3677 :
3678 251 : Node* EffectControlLinearizer::LowerStringLessThan(Node* node) {
3679 : return LowerStringComparison(
3680 251 : Builtins::CallableFor(isolate(), Builtins::kStringLessThan), node);
3681 : }
3682 :
3683 865 : Node* EffectControlLinearizer::LowerStringLessThanOrEqual(Node* node) {
3684 : return LowerStringComparison(
3685 865 : Builtins::CallableFor(isolate(), Builtins::kStringLessThanOrEqual), node);
3686 : }
3687 :
3688 471 : Node* EffectControlLinearizer::LowerCheckFloat64Hole(Node* node,
3689 : Node* frame_state) {
3690 : // If we reach this point w/o eliminating the {node} that's marked
3691 : // with allow-return-hole, we cannot do anything, so just deoptimize
3692 : // in case of the hole NaN.
3693 : CheckFloat64HoleParameters const& params =
3694 471 : CheckFloat64HoleParametersOf(node->op());
3695 : Node* value = node->InputAt(0);
3696 :
3697 471 : auto if_nan = __ MakeDeferredLabel();
3698 471 : auto done = __ MakeLabel();
3699 :
3700 : // First check whether {value} is a NaN at all...
3701 471 : __ Branch(__ Float64Equal(value, value), &done, &if_nan);
3702 :
3703 : __ Bind(&if_nan);
3704 : {
3705 : // ...and only if {value} is a NaN, perform the expensive bit
3706 : // check. See http://crbug.com/v8/8264 for details.
3707 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
3708 471 : __ Int32Constant(kHoleNanUpper32));
3709 471 : __ DeoptimizeIf(DeoptimizeReason::kHole, params.feedback(), check,
3710 471 : frame_state);
3711 : __ Goto(&done);
3712 : }
3713 :
3714 : __ Bind(&done);
3715 471 : return value;
3716 : }
3717 :
3718 93 : Node* EffectControlLinearizer::LowerCheckNotTaggedHole(Node* node,
3719 : Node* frame_state) {
3720 : Node* value = node->InputAt(0);
3721 93 : Node* check = __ WordEqual(value, __ TheHoleConstant());
3722 : __ DeoptimizeIf(DeoptimizeReason::kHole, VectorSlotPair(), check,
3723 93 : frame_state);
3724 93 : return value;
3725 : }
3726 :
3727 1507 : Node* EffectControlLinearizer::LowerConvertTaggedHoleToUndefined(Node* node) {
3728 : Node* value = node->InputAt(0);
3729 :
3730 1507 : auto if_is_hole = __ MakeDeferredLabel();
3731 1507 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3732 :
3733 1507 : Node* check = __ WordEqual(value, __ TheHoleConstant());
3734 1507 : __ GotoIf(check, &if_is_hole);
3735 : __ Goto(&done, value);
3736 :
3737 : __ Bind(&if_is_hole);
3738 1507 : __ Goto(&done, __ UndefinedConstant());
3739 :
3740 : __ Bind(&done);
3741 1507 : return done.PhiAt(0);
3742 : }
3743 :
3744 57 : void EffectControlLinearizer::LowerCheckEqualsInternalizedString(
3745 : Node* node, Node* frame_state) {
3746 : Node* exp = node->InputAt(0);
3747 : Node* val = node->InputAt(1);
3748 :
3749 57 : auto if_same = __ MakeLabel();
3750 57 : auto if_notsame = __ MakeDeferredLabel();
3751 57 : auto if_thinstring = __ MakeLabel();
3752 57 : auto if_notthinstring = __ MakeLabel();
3753 :
3754 : // Check if {exp} and {val} are the same, which is the likely case.
3755 57 : __ Branch(__ WordEqual(exp, val), &if_same, &if_notsame);
3756 :
3757 : __ Bind(&if_notsame);
3758 : {
3759 : // Now {val} could still be a non-internalized String that matches {exp}.
3760 : __ DeoptimizeIf(DeoptimizeReason::kWrongName, VectorSlotPair(),
3761 57 : ObjectIsSmi(val), frame_state);
3762 57 : Node* val_map = __ LoadField(AccessBuilder::ForMap(), val);
3763 : Node* val_instance_type =
3764 57 : __ LoadField(AccessBuilder::ForMapInstanceType(), val_map);
3765 :
3766 : // Check for the common case of ThinString first.
3767 : __ GotoIf(__ Word32Equal(val_instance_type,
3768 : __ Int32Constant(THIN_ONE_BYTE_STRING_TYPE)),
3769 57 : &if_thinstring);
3770 : __ Branch(
3771 : __ Word32Equal(val_instance_type, __ Int32Constant(THIN_STRING_TYPE)),
3772 57 : &if_thinstring, &if_notthinstring);
3773 :
3774 : __ Bind(&if_notthinstring);
3775 : {
3776 : // Check that the {val} is a non-internalized String, if it's anything
3777 : // else it cannot match the recorded feedback {exp} anyways.
3778 : __ DeoptimizeIfNot(
3779 : DeoptimizeReason::kWrongName, VectorSlotPair(),
3780 : __ Word32Equal(__ Word32And(val_instance_type,
3781 : __ Int32Constant(kIsNotStringMask |
3782 : kIsNotInternalizedMask)),
3783 : __ Int32Constant(kStringTag | kNotInternalizedTag)),
3784 57 : frame_state);
3785 :
3786 : // Try to find the {val} in the string table.
3787 57 : MachineSignature::Builder builder(graph()->zone(), 1, 2);
3788 : builder.AddReturn(MachineType::AnyTagged());
3789 : builder.AddParam(MachineType::Pointer());
3790 : builder.AddParam(MachineType::AnyTagged());
3791 : Node* try_internalize_string_function = __ ExternalConstant(
3792 57 : ExternalReference::try_internalize_string_function());
3793 : Node* const isolate_ptr =
3794 57 : __ ExternalConstant(ExternalReference::isolate_address(isolate()));
3795 : auto call_descriptor =
3796 114 : Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build());
3797 : Node* val_internalized =
3798 : __ Call(common()->Call(call_descriptor),
3799 57 : try_internalize_string_function, isolate_ptr, val);
3800 :
3801 : // Now see if the results match.
3802 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
3803 57 : __ WordEqual(exp, val_internalized), frame_state);
3804 : __ Goto(&if_same);
3805 : }
3806 :
3807 : __ Bind(&if_thinstring);
3808 : {
3809 : // The {val} is a ThinString, let's check the actual value.
3810 : Node* val_actual =
3811 57 : __ LoadField(AccessBuilder::ForThinStringActual(), val);
3812 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
3813 57 : __ WordEqual(exp, val_actual), frame_state);
3814 : __ Goto(&if_same);
3815 : }
3816 : }
3817 :
3818 : __ Bind(&if_same);
3819 57 : }
3820 :
3821 19 : void EffectControlLinearizer::LowerCheckEqualsSymbol(Node* node,
3822 : Node* frame_state) {
3823 : Node* exp = node->InputAt(0);
3824 : Node* val = node->InputAt(1);
3825 19 : Node* check = __ WordEqual(exp, val);
3826 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(), check,
3827 19 : frame_state);
3828 19 : }
3829 :
3830 54995 : Node* EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value) {
3831 54995 : Node* result = __ Allocate(NOT_TENURED, __ Int32Constant(HeapNumber::kSize));
3832 54996 : __ StoreField(AccessBuilder::ForMap(), result, __ HeapNumberMapConstant());
3833 54996 : __ StoreField(AccessBuilder::ForHeapNumberValue(), result, value);
3834 54996 : return result;
3835 : }
3836 :
3837 135708 : Node* EffectControlLinearizer::ChangeIntPtrToSmi(Node* value) {
3838 : // Do shift on 32bit values if Smis are stored in the lower word.
3839 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3840 : return __ ChangeInt32ToInt64(
3841 : __ Word32Shl(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
3842 : }
3843 135708 : return __ WordShl(value, SmiShiftBitsConstant());
3844 : }
3845 :
3846 0 : Node* EffectControlLinearizer::ChangeInt32ToIntPtr(Node* value) {
3847 133406 : if (machine()->Is64()) {
3848 133406 : value = __ ChangeInt32ToInt64(value);
3849 : }
3850 0 : return value;
3851 : }
3852 :
3853 0 : Node* EffectControlLinearizer::ChangeIntPtrToInt32(Node* value) {
3854 0 : if (machine()->Is64()) {
3855 0 : value = __ TruncateInt64ToInt32(value);
3856 : }
3857 0 : return value;
3858 : }
3859 :
3860 131244 : Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) {
3861 : // Do shift on 32bit values if Smis are stored in the lower word.
3862 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3863 : return __ ChangeInt32ToInt64(__ Word32Shl(value, SmiShiftBitsConstant()));
3864 : }
3865 131245 : return ChangeIntPtrToSmi(ChangeInt32ToIntPtr(value));
3866 : }
3867 :
3868 0 : Node* EffectControlLinearizer::ChangeInt64ToSmi(Node* value) {
3869 : DCHECK(machine()->Is64());
3870 2066 : return ChangeIntPtrToSmi(value);
3871 : }
3872 :
3873 0 : Node* EffectControlLinearizer::ChangeUint32ToUintPtr(Node* value) {
3874 1106 : if (machine()->Is64()) {
3875 1106 : value = __ ChangeUint32ToUint64(value);
3876 : }
3877 0 : return value;
3878 : }
3879 :
3880 556 : Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) {
3881 : // Do shift on 32bit values if Smis are stored in the lower word.
3882 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3883 : return __ ChangeUint32ToUint64(__ Word32Shl(value, SmiShiftBitsConstant()));
3884 : } else {
3885 556 : return __ WordShl(ChangeUint32ToUintPtr(value), SmiShiftBitsConstant());
3886 : }
3887 : }
3888 :
3889 160513 : Node* EffectControlLinearizer::ChangeSmiToIntPtr(Node* value) {
3890 : // Do shift on 32bit values if Smis are stored in the lower word.
3891 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3892 : return __ ChangeInt32ToInt64(
3893 : __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
3894 : }
3895 160514 : return __ WordSar(value, SmiShiftBitsConstant());
3896 : }
3897 :
3898 156095 : Node* EffectControlLinearizer::ChangeSmiToInt32(Node* value) {
3899 : // Do shift on 32bit values if Smis are stored in the lower word.
3900 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3901 : return __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant());
3902 : }
3903 156095 : if (machine()->Is64()) {
3904 156095 : return __ TruncateInt64ToInt32(ChangeSmiToIntPtr(value));
3905 : }
3906 0 : return ChangeSmiToIntPtr(value);
3907 : }
3908 :
3909 1975 : Node* EffectControlLinearizer::ChangeSmiToInt64(Node* value) {
3910 1975 : CHECK(machine()->Is64());
3911 1975 : return ChangeSmiToIntPtr(value);
3912 : }
3913 :
3914 257516 : Node* EffectControlLinearizer::ObjectIsSmi(Node* value) {
3915 : return __ WordEqual(__ WordAnd(value, __ IntPtrConstant(kSmiTagMask)),
3916 257516 : __ IntPtrConstant(kSmiTag));
3917 : }
3918 :
3919 0 : Node* EffectControlLinearizer::SmiMaxValueConstant() {
3920 556 : return __ Int32Constant(Smi::kMaxValue);
3921 : }
3922 :
3923 0 : Node* EffectControlLinearizer::SmiShiftBitsConstant() {
3924 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3925 : return __ Int32Constant(kSmiShiftSize + kSmiTagSize);
3926 : }
3927 296777 : return __ IntPtrConstant(kSmiShiftSize + kSmiTagSize);
3928 : }
3929 :
3930 23 : Node* EffectControlLinearizer::LowerPlainPrimitiveToNumber(Node* node) {
3931 : Node* value = node->InputAt(0);
3932 23 : return __ ToNumber(value);
3933 : }
3934 :
3935 9 : Node* EffectControlLinearizer::LowerPlainPrimitiveToWord32(Node* node) {
3936 : Node* value = node->InputAt(0);
3937 :
3938 9 : auto if_not_smi = __ MakeDeferredLabel();
3939 9 : auto if_to_number_smi = __ MakeLabel();
3940 9 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
3941 :
3942 9 : Node* check0 = ObjectIsSmi(value);
3943 9 : __ GotoIfNot(check0, &if_not_smi);
3944 9 : __ Goto(&done, ChangeSmiToInt32(value));
3945 :
3946 : __ Bind(&if_not_smi);
3947 9 : Node* to_number = __ ToNumber(value);
3948 :
3949 9 : Node* check1 = ObjectIsSmi(to_number);
3950 9 : __ GotoIf(check1, &if_to_number_smi);
3951 9 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
3952 9 : __ Goto(&done, __ TruncateFloat64ToWord32(number));
3953 :
3954 : __ Bind(&if_to_number_smi);
3955 9 : __ Goto(&done, ChangeSmiToInt32(to_number));
3956 :
3957 : __ Bind(&done);
3958 9 : return done.PhiAt(0);
3959 : }
3960 :
3961 0 : Node* EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node) {
3962 : Node* value = node->InputAt(0);
3963 :
3964 0 : auto if_not_smi = __ MakeDeferredLabel();
3965 0 : auto if_to_number_smi = __ MakeLabel();
3966 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
3967 :
3968 0 : Node* check0 = ObjectIsSmi(value);
3969 0 : __ GotoIfNot(check0, &if_not_smi);
3970 0 : Node* from_smi = ChangeSmiToInt32(value);
3971 0 : __ Goto(&done, __ ChangeInt32ToFloat64(from_smi));
3972 :
3973 : __ Bind(&if_not_smi);
3974 0 : Node* to_number = __ ToNumber(value);
3975 0 : Node* check1 = ObjectIsSmi(to_number);
3976 0 : __ GotoIf(check1, &if_to_number_smi);
3977 :
3978 0 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
3979 : __ Goto(&done, number);
3980 :
3981 : __ Bind(&if_to_number_smi);
3982 0 : Node* number_from_smi = ChangeSmiToInt32(to_number);
3983 0 : number_from_smi = __ ChangeInt32ToFloat64(number_from_smi);
3984 : __ Goto(&done, number_from_smi);
3985 :
3986 : __ Bind(&done);
3987 0 : return done.PhiAt(0);
3988 : }
3989 :
3990 1746 : Node* EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node) {
3991 : Node* object = node->InputAt(0);
3992 : Node* elements = node->InputAt(1);
3993 :
3994 1746 : auto if_not_fixed_array = __ MakeDeferredLabel();
3995 1746 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3996 :
3997 : // Load the current map of {elements}.
3998 1746 : Node* elements_map = __ LoadField(AccessBuilder::ForMap(), elements);
3999 :
4000 : // Check if {elements} is not a copy-on-write FixedArray.
4001 1746 : Node* check = __ WordEqual(elements_map, __ FixedArrayMapConstant());
4002 1746 : __ GotoIfNot(check, &if_not_fixed_array);
4003 : // Nothing to do if the {elements} are not copy-on-write.
4004 : __ Goto(&done, elements);
4005 :
4006 : __ Bind(&if_not_fixed_array);
4007 : // We need to take a copy of the {elements} and set them up for {object}.
4008 : Operator::Properties properties = Operator::kEliminatable;
4009 : Callable callable =
4010 1746 : Builtins::CallableFor(isolate(), Builtins::kCopyFastSmiOrObjectElements);
4011 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4012 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4013 : graph()->zone(), callable.descriptor(),
4014 3492 : callable.descriptor().GetStackParameterCount(), flags, properties);
4015 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4016 3492 : object, __ NoContextConstant());
4017 : __ Goto(&done, result);
4018 :
4019 : __ Bind(&done);
4020 1746 : return done.PhiAt(0);
4021 : }
4022 :
4023 3665 : Node* EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node,
4024 : Node* frame_state) {
4025 3665 : GrowFastElementsParameters params = GrowFastElementsParametersOf(node->op());
4026 : Node* object = node->InputAt(0);
4027 : Node* elements = node->InputAt(1);
4028 : Node* index = node->InputAt(2);
4029 : Node* elements_length = node->InputAt(3);
4030 :
4031 3665 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4032 3665 : auto if_grow = __ MakeDeferredLabel();
4033 : auto if_not_grow = __ MakeLabel();
4034 :
4035 : // Check if we need to grow the {elements} backing store.
4036 3665 : Node* check = __ Uint32LessThan(index, elements_length);
4037 3665 : __ GotoIfNot(check, &if_grow);
4038 : __ Goto(&done, elements);
4039 :
4040 : __ Bind(&if_grow);
4041 : // We need to grow the {elements} for {object}.
4042 : Operator::Properties properties = Operator::kEliminatable;
4043 : Callable callable =
4044 3665 : (params.mode() == GrowFastElementsMode::kDoubleElements)
4045 : ? Builtins::CallableFor(isolate(), Builtins::kGrowFastDoubleElements)
4046 : : Builtins::CallableFor(isolate(),
4047 7330 : Builtins::kGrowFastSmiOrObjectElements);
4048 : CallDescriptor::Flags call_flags = CallDescriptor::kNoFlags;
4049 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4050 : graph()->zone(), callable.descriptor(),
4051 7330 : callable.descriptor().GetStackParameterCount(), call_flags, properties);
4052 : Node* new_elements =
4053 : __ Call(call_descriptor, __ HeapConstant(callable.code()), object,
4054 7330 : ChangeInt32ToSmi(index), __ NoContextConstant());
4055 :
4056 : // Ensure that we were able to grow the {elements}.
4057 : __ DeoptimizeIf(DeoptimizeReason::kCouldNotGrowElements, params.feedback(),
4058 3665 : ObjectIsSmi(new_elements), frame_state);
4059 : __ Goto(&done, new_elements);
4060 :
4061 : __ Bind(&done);
4062 3665 : return done.PhiAt(0);
4063 : }
4064 :
4065 592 : void EffectControlLinearizer::LowerTransitionElementsKind(Node* node) {
4066 592 : ElementsTransition const transition = ElementsTransitionOf(node->op());
4067 : Node* object = node->InputAt(0);
4068 :
4069 592 : auto if_map_same = __ MakeDeferredLabel();
4070 592 : auto done = __ MakeLabel();
4071 :
4072 592 : Node* source_map = __ HeapConstant(transition.source());
4073 592 : Node* target_map = __ HeapConstant(transition.target());
4074 :
4075 : // Load the current map of {object}.
4076 592 : Node* object_map = __ LoadField(AccessBuilder::ForMap(), object);
4077 :
4078 : // Check if {object_map} is the same as {source_map}.
4079 592 : Node* check = __ WordEqual(object_map, source_map);
4080 592 : __ GotoIf(check, &if_map_same);
4081 : __ Goto(&done);
4082 :
4083 : __ Bind(&if_map_same);
4084 592 : switch (transition.mode()) {
4085 : case ElementsTransition::kFastTransition:
4086 : // In-place migration of {object}, just store the {target_map}.
4087 214 : __ StoreField(AccessBuilder::ForMap(), object, target_map);
4088 214 : break;
4089 : case ElementsTransition::kSlowTransition: {
4090 : // Instance migration, call out to the runtime for {object}.
4091 378 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4092 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4093 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4094 378 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4095 : __ Call(call_descriptor, __ CEntryStubConstant(1), object, target_map,
4096 : __ ExternalConstant(ExternalReference::Create(id)),
4097 378 : __ Int32Constant(2), __ NoContextConstant());
4098 : break;
4099 : }
4100 : }
4101 : __ Goto(&done);
4102 :
4103 : __ Bind(&done);
4104 592 : }
4105 :
4106 1036 : Node* EffectControlLinearizer::LowerLoadFieldByIndex(Node* node) {
4107 : Node* object = node->InputAt(0);
4108 : Node* index = node->InputAt(1);
4109 1036 : Node* zero = __ IntPtrConstant(0);
4110 1036 : Node* one = __ IntPtrConstant(1);
4111 :
4112 : // Sign-extend the {index} on 64-bit architectures.
4113 1036 : if (machine()->Is64()) {
4114 1036 : index = __ ChangeInt32ToInt64(index);
4115 : }
4116 :
4117 1036 : auto if_double = __ MakeDeferredLabel();
4118 1036 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4119 :
4120 : // Check if field is a mutable double field.
4121 1036 : __ GotoIfNot(__ WordEqual(__ WordAnd(index, one), zero), &if_double);
4122 :
4123 : // The field is a proper Tagged field on {object}. The {index} is shifted
4124 : // to the left by one in the code below.
4125 : {
4126 : // Check if field is in-object or out-of-object.
4127 1036 : auto if_outofobject = __ MakeLabel();
4128 1036 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4129 :
4130 : // The field is located in the {object} itself.
4131 : {
4132 : Node* offset =
4133 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4134 1036 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4135 1036 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4136 : __ Goto(&done, result);
4137 : }
4138 :
4139 : // The field is located in the properties backing store of {object}.
4140 : // The {index} is equal to the negated out of property index plus 1.
4141 : __ Bind(&if_outofobject);
4142 : {
4143 : Node* properties =
4144 1036 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4145 : Node* offset =
4146 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4147 : __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4148 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4149 1036 : kHeapObjectTag));
4150 1036 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4151 : __ Goto(&done, result);
4152 : }
4153 : }
4154 :
4155 : // The field is a Double field, either unboxed in the object on 64-bit
4156 : // architectures, or as MutableHeapNumber.
4157 : __ Bind(&if_double);
4158 : {
4159 1036 : auto done_double = __ MakeLabel(MachineRepresentation::kFloat64);
4160 :
4161 1036 : index = __ WordSar(index, one);
4162 :
4163 : // Check if field is in-object or out-of-object.
4164 1036 : auto if_outofobject = __ MakeLabel();
4165 1036 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4166 :
4167 : // The field is located in the {object} itself.
4168 : {
4169 : Node* offset =
4170 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)),
4171 1036 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4172 : if (FLAG_unbox_double_fields) {
4173 1036 : Node* result = __ Load(MachineType::Float64(), object, offset);
4174 : __ Goto(&done_double, result);
4175 : } else {
4176 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4177 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4178 : __ Goto(&done_double, result);
4179 : }
4180 : }
4181 :
4182 : __ Bind(&if_outofobject);
4183 : {
4184 : Node* properties =
4185 1036 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4186 : Node* offset =
4187 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4188 : __ IntPtrConstant(kTaggedSizeLog2)),
4189 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4190 1036 : kHeapObjectTag));
4191 1036 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4192 1036 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4193 : __ Goto(&done_double, result);
4194 : }
4195 :
4196 : __ Bind(&done_double);
4197 : {
4198 1036 : Node* result = AllocateHeapNumberWithValue(done_double.PhiAt(0));
4199 : __ Goto(&done, result);
4200 : }
4201 : }
4202 :
4203 : __ Bind(&done);
4204 1036 : return done.PhiAt(0);
4205 : }
4206 :
4207 465 : Node* EffectControlLinearizer::BuildReverseBytes(ExternalArrayType type,
4208 : Node* value) {
4209 465 : switch (type) {
4210 : case kExternalInt8Array:
4211 : case kExternalUint8Array:
4212 : case kExternalUint8ClampedArray:
4213 : return value;
4214 :
4215 : case kExternalInt16Array: {
4216 55 : Node* result = __ Word32ReverseBytes(value);
4217 55 : result = __ Word32Sar(result, __ Int32Constant(16));
4218 55 : return result;
4219 : }
4220 :
4221 : case kExternalUint16Array: {
4222 55 : Node* result = __ Word32ReverseBytes(value);
4223 55 : result = __ Word32Shr(result, __ Int32Constant(16));
4224 55 : return result;
4225 : }
4226 :
4227 : case kExternalInt32Array: // Fall through.
4228 : case kExternalUint32Array:
4229 96 : return __ Word32ReverseBytes(value);
4230 :
4231 : case kExternalFloat32Array: {
4232 55 : Node* result = __ BitcastFloat32ToInt32(value);
4233 55 : result = __ Word32ReverseBytes(result);
4234 55 : result = __ BitcastInt32ToFloat32(result);
4235 55 : return result;
4236 : }
4237 :
4238 : case kExternalFloat64Array: {
4239 55 : if (machine()->Is64()) {
4240 55 : Node* result = __ BitcastFloat64ToInt64(value);
4241 55 : result = __ Word64ReverseBytes(result);
4242 55 : result = __ BitcastInt64ToFloat64(result);
4243 55 : return result;
4244 : } else {
4245 0 : Node* lo = __ Word32ReverseBytes(__ Float64ExtractLowWord32(value));
4246 0 : Node* hi = __ Word32ReverseBytes(__ Float64ExtractHighWord32(value));
4247 0 : Node* result = __ Float64Constant(0.0);
4248 0 : result = __ Float64InsertLowWord32(result, hi);
4249 0 : result = __ Float64InsertHighWord32(result, lo);
4250 0 : return result;
4251 : }
4252 : }
4253 :
4254 : case kExternalBigInt64Array:
4255 : case kExternalBigUint64Array:
4256 0 : UNREACHABLE();
4257 : }
4258 0 : }
4259 :
4260 268 : Node* EffectControlLinearizer::LowerLoadDataViewElement(Node* node) {
4261 268 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4262 : Node* buffer = node->InputAt(0);
4263 : Node* storage = node->InputAt(1);
4264 : Node* byte_offset = node->InputAt(2);
4265 : Node* index = node->InputAt(3);
4266 : Node* is_little_endian = node->InputAt(4);
4267 :
4268 : // We need to keep the {buffer} alive so that the GC will not release the
4269 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4270 268 : __ Retain(buffer);
4271 :
4272 : // Compute the effective offset.
4273 268 : Node* offset = __ IntAdd(byte_offset, index);
4274 :
4275 : MachineType const machine_type =
4276 268 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4277 :
4278 268 : Node* value = __ LoadUnaligned(machine_type, storage, offset);
4279 268 : auto big_endian = __ MakeLabel();
4280 268 : auto done = __ MakeLabel(machine_type.representation());
4281 :
4282 268 : __ GotoIfNot(is_little_endian, &big_endian);
4283 : { // Little-endian load.
4284 : #if V8_TARGET_LITTLE_ENDIAN
4285 : __ Goto(&done, value);
4286 : #else
4287 : __ Goto(&done, BuildReverseBytes(element_type, value));
4288 : #endif // V8_TARGET_LITTLE_ENDIAN
4289 : }
4290 :
4291 : __ Bind(&big_endian);
4292 : { // Big-endian load.
4293 : #if V8_TARGET_LITTLE_ENDIAN
4294 268 : __ Goto(&done, BuildReverseBytes(element_type, value));
4295 : #else
4296 : __ Goto(&done, value);
4297 : #endif // V8_TARGET_LITTLE_ENDIAN
4298 : }
4299 :
4300 : // We're done, return {result}.
4301 : __ Bind(&done);
4302 268 : return done.PhiAt(0);
4303 : }
4304 :
4305 197 : void EffectControlLinearizer::LowerStoreDataViewElement(Node* node) {
4306 197 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4307 : Node* buffer = node->InputAt(0);
4308 : Node* storage = node->InputAt(1);
4309 : Node* byte_offset = node->InputAt(2);
4310 : Node* index = node->InputAt(3);
4311 : Node* value = node->InputAt(4);
4312 : Node* is_little_endian = node->InputAt(5);
4313 :
4314 : // We need to keep the {buffer} alive so that the GC will not release the
4315 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4316 197 : __ Retain(buffer);
4317 :
4318 : // Compute the effective offset.
4319 197 : Node* offset = __ IntAdd(byte_offset, index);
4320 :
4321 : MachineType const machine_type =
4322 197 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4323 :
4324 197 : auto big_endian = __ MakeLabel();
4325 197 : auto done = __ MakeLabel(machine_type.representation());
4326 :
4327 197 : __ GotoIfNot(is_little_endian, &big_endian);
4328 : { // Little-endian store.
4329 : #if V8_TARGET_LITTLE_ENDIAN
4330 : __ Goto(&done, value);
4331 : #else
4332 : __ Goto(&done, BuildReverseBytes(element_type, value));
4333 : #endif // V8_TARGET_LITTLE_ENDIAN
4334 : }
4335 :
4336 : __ Bind(&big_endian);
4337 : { // Big-endian store.
4338 : #if V8_TARGET_LITTLE_ENDIAN
4339 197 : __ Goto(&done, BuildReverseBytes(element_type, value));
4340 : #else
4341 : __ Goto(&done, value);
4342 : #endif // V8_TARGET_LITTLE_ENDIAN
4343 : }
4344 :
4345 : __ Bind(&done);
4346 : __ StoreUnaligned(machine_type.representation(), storage, offset,
4347 197 : done.PhiAt(0));
4348 197 : }
4349 :
4350 4594 : Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) {
4351 4594 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4352 : Node* buffer = node->InputAt(0);
4353 : Node* base = node->InputAt(1);
4354 : Node* external = node->InputAt(2);
4355 : Node* index = node->InputAt(3);
4356 :
4357 : // We need to keep the {buffer} alive so that the GC will not release the
4358 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4359 4594 : __ Retain(buffer);
4360 :
4361 : // Compute the effective storage pointer, handling the case where the
4362 : // {external} pointer is the effective storage pointer (i.e. the {base}
4363 : // is Smi zero).
4364 : Node* storage = IntPtrMatcher(base).Is(0)
4365 : ? external
4366 4594 : : __ UnsafePointerAdd(base, external);
4367 :
4368 : // Perform the actual typed element access.
4369 : return __ LoadElement(AccessBuilder::ForTypedArrayElement(
4370 : array_type, true, LoadSensitivity::kCritical),
4371 4594 : storage, index);
4372 : }
4373 :
4374 4162 : void EffectControlLinearizer::LowerStoreTypedElement(Node* node) {
4375 4162 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4376 : Node* buffer = node->InputAt(0);
4377 : Node* base = node->InputAt(1);
4378 : Node* external = node->InputAt(2);
4379 : Node* index = node->InputAt(3);
4380 : Node* value = node->InputAt(4);
4381 :
4382 : // We need to keep the {buffer} alive so that the GC will not release the
4383 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4384 4162 : __ Retain(buffer);
4385 :
4386 : // Compute the effective storage pointer, handling the case where the
4387 : // {external} pointer is the effective storage pointer (i.e. the {base}
4388 : // is Smi zero).
4389 : Node* storage = IntPtrMatcher(base).Is(0)
4390 : ? external
4391 4162 : : __ UnsafePointerAdd(base, external);
4392 :
4393 : // Perform the actual typed element access.
4394 : __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true),
4395 4162 : storage, index, value);
4396 4162 : }
4397 :
4398 614 : void EffectControlLinearizer::TransitionElementsTo(Node* node, Node* array,
4399 : ElementsKind from,
4400 : ElementsKind to) {
4401 : DCHECK(IsMoreGeneralElementsKindTransition(from, to));
4402 : DCHECK(to == HOLEY_ELEMENTS || to == HOLEY_DOUBLE_ELEMENTS);
4403 :
4404 : Handle<Map> target(to == HOLEY_ELEMENTS ? FastMapParameterOf(node->op())
4405 614 : : DoubleMapParameterOf(node->op()));
4406 307 : Node* target_map = __ HeapConstant(target);
4407 :
4408 307 : if (IsSimpleMapChangeTransition(from, to)) {
4409 102 : __ StoreField(AccessBuilder::ForMap(), array, target_map);
4410 : } else {
4411 : // Instance migration, call out to the runtime for {array}.
4412 205 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4413 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4414 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4415 205 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4416 : __ Call(call_descriptor, __ CEntryStubConstant(1), array, target_map,
4417 : __ ExternalConstant(ExternalReference::Create(id)),
4418 205 : __ Int32Constant(2), __ NoContextConstant());
4419 : }
4420 307 : }
4421 :
4422 385 : Node* EffectControlLinearizer::IsElementsKindGreaterThan(
4423 : Node* kind, ElementsKind reference_kind) {
4424 385 : Node* ref_kind = __ Int32Constant(reference_kind);
4425 385 : Node* ret = __ Int32LessThan(ref_kind, kind);
4426 385 : return ret;
4427 : }
4428 :
4429 81 : void EffectControlLinearizer::LowerTransitionAndStoreElement(Node* node) {
4430 : Node* array = node->InputAt(0);
4431 : Node* index = node->InputAt(1);
4432 : Node* value = node->InputAt(2);
4433 :
4434 : // Possibly transition array based on input and store.
4435 : //
4436 : // -- TRANSITION PHASE -----------------
4437 : // kind = ElementsKind(array)
4438 : // if value is not smi {
4439 : // if kind == HOLEY_SMI_ELEMENTS {
4440 : // if value is heap number {
4441 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4442 : // kind = HOLEY_DOUBLE_ELEMENTS
4443 : // } else {
4444 : // Transition array to HOLEY_ELEMENTS
4445 : // kind = HOLEY_ELEMENTS
4446 : // }
4447 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4448 : // if value is not heap number {
4449 : // Transition array to HOLEY_ELEMENTS
4450 : // kind = HOLEY_ELEMENTS
4451 : // }
4452 : // }
4453 : // }
4454 : //
4455 : // -- STORE PHASE ----------------------
4456 : // [make sure {kind} is up-to-date]
4457 : // if kind == HOLEY_DOUBLE_ELEMENTS {
4458 : // if value is smi {
4459 : // float_value = convert smi to float
4460 : // Store array[index] = float_value
4461 : // } else {
4462 : // float_value = value
4463 : // Store array[index] = float_value
4464 : // }
4465 : // } else {
4466 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
4467 : // Store array[index] = value
4468 : // }
4469 : //
4470 81 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4471 : Node* kind;
4472 : {
4473 81 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4474 81 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4475 81 : Node* andit = __ Word32And(bit_field2, mask);
4476 81 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4477 81 : kind = __ Word32Shr(andit, shift);
4478 : }
4479 :
4480 81 : auto do_store = __ MakeLabel(MachineRepresentation::kWord32);
4481 : // We can store a smi anywhere.
4482 81 : __ GotoIf(ObjectIsSmi(value), &do_store, kind);
4483 :
4484 : // {value} is a HeapObject.
4485 81 : auto transition_smi_array = __ MakeDeferredLabel();
4486 81 : auto transition_double_to_fast = __ MakeDeferredLabel();
4487 : {
4488 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4489 81 : &transition_smi_array);
4490 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), &do_store,
4491 81 : kind);
4492 :
4493 : // We have double elements kind. Only a HeapNumber can be stored
4494 : // without effecting a transition.
4495 81 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4496 81 : Node* heap_number_map = __ HeapNumberMapConstant();
4497 81 : Node* check = __ WordEqual(value_map, heap_number_map);
4498 81 : __ GotoIfNot(check, &transition_double_to_fast);
4499 : __ Goto(&do_store, kind);
4500 : }
4501 :
4502 : __ Bind(&transition_smi_array); // deferred code.
4503 : {
4504 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS or
4505 : // to HOLEY_ELEMENTS.
4506 81 : auto if_value_not_heap_number = __ MakeLabel();
4507 81 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4508 81 : Node* heap_number_map = __ HeapNumberMapConstant();
4509 81 : Node* check = __ WordEqual(value_map, heap_number_map);
4510 81 : __ GotoIfNot(check, &if_value_not_heap_number);
4511 : {
4512 : // {value} is a HeapNumber.
4513 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4514 81 : HOLEY_DOUBLE_ELEMENTS);
4515 81 : __ Goto(&do_store, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS));
4516 : }
4517 : __ Bind(&if_value_not_heap_number);
4518 : {
4519 81 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
4520 81 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4521 : }
4522 : }
4523 :
4524 : __ Bind(&transition_double_to_fast); // deferred code.
4525 : {
4526 81 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
4527 81 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4528 : }
4529 :
4530 : // Make sure kind is up-to-date.
4531 : __ Bind(&do_store);
4532 : kind = do_store.PhiAt(0);
4533 :
4534 81 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4535 81 : auto if_kind_is_double = __ MakeLabel();
4536 81 : auto done = __ MakeLabel();
4537 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4538 81 : &if_kind_is_double);
4539 : {
4540 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
4541 : __ StoreElement(AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS),
4542 81 : elements, index, value);
4543 : __ Goto(&done);
4544 : }
4545 : __ Bind(&if_kind_is_double);
4546 : {
4547 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
4548 81 : auto do_double_store = __ MakeLabel();
4549 81 : __ GotoIfNot(ObjectIsSmi(value), &do_double_store);
4550 : {
4551 81 : Node* int_value = ChangeSmiToInt32(value);
4552 81 : Node* float_value = __ ChangeInt32ToFloat64(int_value);
4553 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4554 81 : index, float_value);
4555 : __ Goto(&done);
4556 : }
4557 : __ Bind(&do_double_store);
4558 : {
4559 : Node* float_value =
4560 81 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
4561 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4562 81 : index, float_value);
4563 : __ Goto(&done);
4564 : }
4565 : }
4566 :
4567 : __ Bind(&done);
4568 81 : }
4569 :
4570 22 : void EffectControlLinearizer::LowerTransitionAndStoreNumberElement(Node* node) {
4571 : Node* array = node->InputAt(0);
4572 : Node* index = node->InputAt(1);
4573 : Node* value = node->InputAt(2); // This is a Float64, not tagged.
4574 :
4575 : // Possibly transition array based on input and store.
4576 : //
4577 : // -- TRANSITION PHASE -----------------
4578 : // kind = ElementsKind(array)
4579 : // if kind == HOLEY_SMI_ELEMENTS {
4580 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4581 : // } else if kind != HOLEY_DOUBLE_ELEMENTS {
4582 : // This is UNREACHABLE, execute a debug break.
4583 : // }
4584 : //
4585 : // -- STORE PHASE ----------------------
4586 : // Store array[index] = value (it's a float)
4587 : //
4588 22 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4589 : Node* kind;
4590 : {
4591 22 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4592 22 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4593 22 : Node* andit = __ Word32And(bit_field2, mask);
4594 22 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4595 22 : kind = __ Word32Shr(andit, shift);
4596 : }
4597 :
4598 22 : auto do_store = __ MakeLabel();
4599 :
4600 : // {value} is a float64.
4601 22 : auto transition_smi_array = __ MakeDeferredLabel();
4602 : {
4603 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4604 22 : &transition_smi_array);
4605 : // We expect that our input array started at HOLEY_SMI_ELEMENTS, and
4606 : // climbs the lattice up to HOLEY_DOUBLE_ELEMENTS. Force a debug break
4607 : // if this assumption is broken. It also would be the case that
4608 : // loop peeling can break this assumption.
4609 : __ GotoIf(__ Word32Equal(kind, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)),
4610 22 : &do_store);
4611 : // TODO(turbofan): It would be good to have an "Unreachable()" node type.
4612 22 : __ DebugBreak();
4613 : __ Goto(&do_store);
4614 : }
4615 :
4616 : __ Bind(&transition_smi_array); // deferred code.
4617 : {
4618 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS.
4619 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4620 22 : HOLEY_DOUBLE_ELEMENTS);
4621 : __ Goto(&do_store);
4622 : }
4623 :
4624 : __ Bind(&do_store);
4625 :
4626 22 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4627 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, index,
4628 22 : value);
4629 22 : }
4630 :
4631 21 : void EffectControlLinearizer::LowerTransitionAndStoreNonNumberElement(
4632 21 : Node* node) {
4633 : Node* array = node->InputAt(0);
4634 : Node* index = node->InputAt(1);
4635 : Node* value = node->InputAt(2);
4636 :
4637 : // Possibly transition array based on input and store.
4638 : //
4639 : // -- TRANSITION PHASE -----------------
4640 : // kind = ElementsKind(array)
4641 : // if kind == HOLEY_SMI_ELEMENTS {
4642 : // Transition array to HOLEY_ELEMENTS
4643 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4644 : // Transition array to HOLEY_ELEMENTS
4645 : // }
4646 : //
4647 : // -- STORE PHASE ----------------------
4648 : // // kind is HOLEY_ELEMENTS
4649 : // Store array[index] = value
4650 : //
4651 21 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4652 : Node* kind;
4653 : {
4654 21 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4655 21 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4656 21 : Node* andit = __ Word32And(bit_field2, mask);
4657 21 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4658 21 : kind = __ Word32Shr(andit, shift);
4659 : }
4660 :
4661 21 : auto do_store = __ MakeLabel();
4662 :
4663 21 : auto transition_smi_array = __ MakeDeferredLabel();
4664 21 : auto transition_double_to_fast = __ MakeDeferredLabel();
4665 : {
4666 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4667 21 : &transition_smi_array);
4668 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4669 21 : &transition_double_to_fast);
4670 : __ Goto(&do_store);
4671 : }
4672 :
4673 : __ Bind(&transition_smi_array); // deferred code.
4674 : {
4675 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_ELEMENTS.
4676 21 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
4677 : __ Goto(&do_store);
4678 : }
4679 :
4680 : __ Bind(&transition_double_to_fast); // deferred code.
4681 : {
4682 21 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
4683 : __ Goto(&do_store);
4684 : }
4685 :
4686 : __ Bind(&do_store);
4687 :
4688 21 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4689 : // Our ElementsKind is HOLEY_ELEMENTS.
4690 21 : ElementAccess access = AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS);
4691 21 : Type value_type = ValueTypeParameterOf(node->op());
4692 21 : if (value_type.Is(Type::BooleanOrNullOrUndefined())) {
4693 16 : access.type = value_type;
4694 16 : access.write_barrier_kind = kNoWriteBarrier;
4695 : }
4696 21 : __ StoreElement(access, elements, index, value);
4697 21 : }
4698 :
4699 78 : void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
4700 : Node* array = node->InputAt(0);
4701 : Node* index = node->InputAt(1);
4702 : Node* value = node->InputAt(2); // int32
4703 :
4704 : // Store a signed small in an output array.
4705 : //
4706 : // kind = ElementsKind(array)
4707 : //
4708 : // -- STORE PHASE ----------------------
4709 : // if kind == HOLEY_DOUBLE_ELEMENTS {
4710 : // float_value = convert int32 to float
4711 : // Store array[index] = float_value
4712 : // } else {
4713 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
4714 : // smi_value = convert int32 to smi
4715 : // Store array[index] = smi_value
4716 : // }
4717 : //
4718 78 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4719 : Node* kind;
4720 : {
4721 78 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4722 78 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4723 78 : Node* andit = __ Word32And(bit_field2, mask);
4724 78 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4725 78 : kind = __ Word32Shr(andit, shift);
4726 : }
4727 :
4728 78 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4729 78 : auto if_kind_is_double = __ MakeLabel();
4730 78 : auto done = __ MakeLabel();
4731 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4732 78 : &if_kind_is_double);
4733 : {
4734 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
4735 : // In this case, we know our value is a signed small, and we can optimize
4736 : // the ElementAccess information.
4737 78 : ElementAccess access = AccessBuilder::ForFixedArrayElement();
4738 78 : access.type = Type::SignedSmall();
4739 78 : access.machine_type = MachineType::TaggedSigned();
4740 78 : access.write_barrier_kind = kNoWriteBarrier;
4741 78 : Node* smi_value = ChangeInt32ToSmi(value);
4742 78 : __ StoreElement(access, elements, index, smi_value);
4743 : __ Goto(&done);
4744 : }
4745 : __ Bind(&if_kind_is_double);
4746 : {
4747 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
4748 78 : Node* float_value = __ ChangeInt32ToFloat64(value);
4749 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4750 78 : index, float_value);
4751 : __ Goto(&done);
4752 : }
4753 :
4754 : __ Bind(&done);
4755 78 : }
4756 :
4757 4926 : void EffectControlLinearizer::LowerRuntimeAbort(Node* node) {
4758 2463 : AbortReason reason = AbortReasonOf(node->op());
4759 2463 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4760 : Runtime::FunctionId id = Runtime::kAbort;
4761 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4762 2463 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
4763 : __ Call(call_descriptor, __ CEntryStubConstant(1),
4764 : jsgraph()->SmiConstant(static_cast<int>(reason)),
4765 : __ ExternalConstant(ExternalReference::Create(id)),
4766 4926 : __ Int32Constant(1), __ NoContextConstant());
4767 2463 : }
4768 :
4769 892 : Node* EffectControlLinearizer::LowerConvertReceiver(Node* node) {
4770 892 : ConvertReceiverMode const mode = ConvertReceiverModeOf(node->op());
4771 : Node* value = node->InputAt(0);
4772 : Node* global_proxy = node->InputAt(1);
4773 :
4774 892 : switch (mode) {
4775 : case ConvertReceiverMode::kNullOrUndefined: {
4776 : return global_proxy;
4777 : }
4778 : case ConvertReceiverMode::kNotNullOrUndefined: {
4779 822 : auto convert_to_object = __ MakeDeferredLabel();
4780 822 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
4781 :
4782 : // Check if {value} is already a JSReceiver.
4783 822 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
4784 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
4785 822 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4786 : Node* value_instance_type =
4787 822 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
4788 : Node* check = __ Uint32LessThan(
4789 822 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
4790 822 : __ GotoIf(check, &convert_to_object);
4791 : __ Goto(&done_convert, value);
4792 :
4793 : // Wrap the primitive {value} into a JSValue.
4794 : __ Bind(&convert_to_object);
4795 : Operator::Properties properties = Operator::kEliminatable;
4796 822 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
4797 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4798 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4799 : graph()->zone(), callable.descriptor(),
4800 1644 : callable.descriptor().GetStackParameterCount(), flags, properties);
4801 : Node* native_context = __ LoadField(
4802 822 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
4803 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4804 822 : value, native_context);
4805 : __ Goto(&done_convert, result);
4806 :
4807 : __ Bind(&done_convert);
4808 : return done_convert.PhiAt(0);
4809 : }
4810 : case ConvertReceiverMode::kAny: {
4811 70 : auto convert_to_object = __ MakeDeferredLabel();
4812 70 : auto convert_global_proxy = __ MakeDeferredLabel();
4813 70 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
4814 :
4815 : // Check if {value} is already a JSReceiver, or null/undefined.
4816 70 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
4817 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
4818 70 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4819 : Node* value_instance_type =
4820 70 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
4821 : Node* check = __ Uint32LessThan(
4822 70 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
4823 70 : __ GotoIf(check, &convert_to_object);
4824 : __ Goto(&done_convert, value);
4825 :
4826 : // Wrap the primitive {value} into a JSValue.
4827 : __ Bind(&convert_to_object);
4828 : __ GotoIf(__ WordEqual(value, __ UndefinedConstant()),
4829 70 : &convert_global_proxy);
4830 70 : __ GotoIf(__ WordEqual(value, __ NullConstant()), &convert_global_proxy);
4831 : Operator::Properties properties = Operator::kEliminatable;
4832 70 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
4833 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4834 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4835 : graph()->zone(), callable.descriptor(),
4836 140 : callable.descriptor().GetStackParameterCount(), flags, properties);
4837 : Node* native_context = __ LoadField(
4838 70 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
4839 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4840 70 : value, native_context);
4841 : __ Goto(&done_convert, result);
4842 :
4843 : // Replace the {value} with the {global_proxy}.
4844 : __ Bind(&convert_global_proxy);
4845 : __ Goto(&done_convert, global_proxy);
4846 :
4847 : __ Bind(&done_convert);
4848 : return done_convert.PhiAt(0);
4849 : }
4850 : }
4851 :
4852 0 : UNREACHABLE();
4853 : return nullptr;
4854 : }
4855 :
4856 8223 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundUp(Node* node) {
4857 : // Nothing to be done if a fast hardware instruction is available.
4858 8223 : if (machine()->Float64RoundUp().IsSupported()) {
4859 : return Nothing<Node*>();
4860 : }
4861 :
4862 : Node* const input = node->InputAt(0);
4863 :
4864 : // General case for ceil.
4865 : //
4866 : // if 0.0 < input then
4867 : // if 2^52 <= input then
4868 : // input
4869 : // else
4870 : // let temp1 = (2^52 + input) - 2^52 in
4871 : // if temp1 < input then
4872 : // temp1 + 1
4873 : // else
4874 : // temp1
4875 : // else
4876 : // if input == 0 then
4877 : // input
4878 : // else
4879 : // if input <= -2^52 then
4880 : // input
4881 : // else
4882 : // let temp1 = -0 - input in
4883 : // let temp2 = (2^52 + temp1) - 2^52 in
4884 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
4885 : // -0 - temp3
4886 :
4887 0 : auto if_not_positive = __ MakeDeferredLabel();
4888 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
4889 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
4890 0 : auto if_zero = __ MakeDeferredLabel();
4891 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
4892 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
4893 :
4894 0 : Node* const zero = __ Float64Constant(0.0);
4895 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
4896 0 : Node* const one = __ Float64Constant(1.0);
4897 :
4898 0 : Node* check0 = __ Float64LessThan(zero, input);
4899 0 : __ GotoIfNot(check0, &if_not_positive);
4900 : {
4901 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
4902 0 : __ GotoIf(check1, &if_greater_than_two_52);
4903 : {
4904 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
4905 0 : __ GotoIfNot(__ Float64LessThan(temp1, input), &done, temp1);
4906 0 : __ Goto(&done, __ Float64Add(temp1, one));
4907 : }
4908 :
4909 : __ Bind(&if_greater_than_two_52);
4910 : __ Goto(&done, input);
4911 : }
4912 :
4913 : __ Bind(&if_not_positive);
4914 : {
4915 0 : Node* check1 = __ Float64Equal(input, zero);
4916 0 : __ GotoIf(check1, &if_zero);
4917 :
4918 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
4919 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
4920 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
4921 :
4922 : {
4923 0 : Node* const minus_zero = __ Float64Constant(-0.0);
4924 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
4925 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
4926 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
4927 0 : __ GotoIfNot(check3, &done_temp3, temp2);
4928 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
4929 :
4930 : __ Bind(&done_temp3);
4931 : Node* temp3 = done_temp3.PhiAt(0);
4932 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
4933 : }
4934 : __ Bind(&if_less_than_minus_two_52);
4935 : __ Goto(&done, input);
4936 :
4937 : __ Bind(&if_zero);
4938 : __ Goto(&done, input);
4939 : }
4940 : __ Bind(&done);
4941 : return Just(done.PhiAt(0));
4942 : }
4943 :
4944 0 : Node* EffectControlLinearizer::BuildFloat64RoundDown(Node* value) {
4945 0 : if (machine()->Float64RoundDown().IsSupported()) {
4946 0 : return __ Float64RoundDown(value);
4947 : }
4948 :
4949 : Node* const input = value;
4950 :
4951 : // General case for floor.
4952 : //
4953 : // if 0.0 < input then
4954 : // if 2^52 <= input then
4955 : // input
4956 : // else
4957 : // let temp1 = (2^52 + input) - 2^52 in
4958 : // if input < temp1 then
4959 : // temp1 - 1
4960 : // else
4961 : // temp1
4962 : // else
4963 : // if input == 0 then
4964 : // input
4965 : // else
4966 : // if input <= -2^52 then
4967 : // input
4968 : // else
4969 : // let temp1 = -0 - input in
4970 : // let temp2 = (2^52 + temp1) - 2^52 in
4971 : // if temp2 < temp1 then
4972 : // -1 - temp2
4973 : // else
4974 : // -0 - temp2
4975 :
4976 0 : auto if_not_positive = __ MakeDeferredLabel();
4977 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
4978 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
4979 0 : auto if_temp2_lt_temp1 = __ MakeLabel();
4980 0 : auto if_zero = __ MakeDeferredLabel();
4981 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
4982 :
4983 0 : Node* const zero = __ Float64Constant(0.0);
4984 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
4985 :
4986 0 : Node* check0 = __ Float64LessThan(zero, input);
4987 0 : __ GotoIfNot(check0, &if_not_positive);
4988 : {
4989 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
4990 0 : __ GotoIf(check1, &if_greater_than_two_52);
4991 : {
4992 0 : Node* const one = __ Float64Constant(1.0);
4993 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
4994 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
4995 0 : __ Goto(&done, __ Float64Sub(temp1, one));
4996 : }
4997 :
4998 : __ Bind(&if_greater_than_two_52);
4999 : __ Goto(&done, input);
5000 : }
5001 :
5002 : __ Bind(&if_not_positive);
5003 : {
5004 0 : Node* check1 = __ Float64Equal(input, zero);
5005 0 : __ GotoIf(check1, &if_zero);
5006 :
5007 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5008 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5009 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5010 :
5011 : {
5012 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5013 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5014 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5015 0 : Node* check3 = __ Float64LessThan(temp2, temp1);
5016 0 : __ GotoIf(check3, &if_temp2_lt_temp1);
5017 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp2));
5018 :
5019 : __ Bind(&if_temp2_lt_temp1);
5020 0 : __ Goto(&done, __ Float64Sub(__ Float64Constant(-1.0), temp2));
5021 : }
5022 : __ Bind(&if_less_than_minus_two_52);
5023 : __ Goto(&done, input);
5024 :
5025 : __ Bind(&if_zero);
5026 : __ Goto(&done, input);
5027 : }
5028 : __ Bind(&done);
5029 : return done.PhiAt(0);
5030 : }
5031 :
5032 25908 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundDown(Node* node) {
5033 : // Nothing to be done if a fast hardware instruction is available.
5034 25908 : if (machine()->Float64RoundDown().IsSupported()) {
5035 : return Nothing<Node*>();
5036 : }
5037 :
5038 : Node* const input = node->InputAt(0);
5039 0 : return Just(BuildFloat64RoundDown(input));
5040 : }
5041 :
5042 266 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTiesEven(Node* node) {
5043 : // Nothing to be done if a fast hardware instruction is available.
5044 266 : if (machine()->Float64RoundTiesEven().IsSupported()) {
5045 : return Nothing<Node*>();
5046 : }
5047 :
5048 : Node* const input = node->InputAt(0);
5049 :
5050 : // Generate case for round ties to even:
5051 : //
5052 : // let value = floor(input) in
5053 : // let temp1 = input - value in
5054 : // if temp1 < 0.5 then
5055 : // value
5056 : // else if 0.5 < temp1 then
5057 : // value + 1.0
5058 : // else
5059 : // let temp2 = value % 2.0 in
5060 : // if temp2 == 0.0 then
5061 : // value
5062 : // else
5063 : // value + 1.0
5064 :
5065 0 : auto if_is_half = __ MakeLabel();
5066 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5067 :
5068 0 : Node* value = BuildFloat64RoundDown(input);
5069 0 : Node* temp1 = __ Float64Sub(input, value);
5070 :
5071 0 : Node* const half = __ Float64Constant(0.5);
5072 0 : Node* check0 = __ Float64LessThan(temp1, half);
5073 0 : __ GotoIf(check0, &done, value);
5074 :
5075 0 : Node* const one = __ Float64Constant(1.0);
5076 0 : Node* check1 = __ Float64LessThan(half, temp1);
5077 0 : __ GotoIfNot(check1, &if_is_half);
5078 0 : __ Goto(&done, __ Float64Add(value, one));
5079 :
5080 : __ Bind(&if_is_half);
5081 0 : Node* temp2 = __ Float64Mod(value, __ Float64Constant(2.0));
5082 0 : Node* check2 = __ Float64Equal(temp2, __ Float64Constant(0.0));
5083 0 : __ GotoIf(check2, &done, value);
5084 0 : __ Goto(&done, __ Float64Add(value, one));
5085 :
5086 : __ Bind(&done);
5087 : return Just(done.PhiAt(0));
5088 : }
5089 :
5090 140 : Node* EffectControlLinearizer::BuildFloat64RoundTruncate(Node* input) {
5091 140 : if (machine()->Float64RoundTruncate().IsSupported()) {
5092 140 : return __ Float64RoundTruncate(input);
5093 : }
5094 : // General case for trunc.
5095 : //
5096 : // if 0.0 < input then
5097 : // if 2^52 <= input then
5098 : // input
5099 : // else
5100 : // let temp1 = (2^52 + input) - 2^52 in
5101 : // if input < temp1 then
5102 : // temp1 - 1
5103 : // else
5104 : // temp1
5105 : // else
5106 : // if input == 0 then
5107 : // input
5108 : // else
5109 : // if input <= -2^52 then
5110 : // input
5111 : // else
5112 : // let temp1 = -0 - input in
5113 : // let temp2 = (2^52 + temp1) - 2^52 in
5114 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
5115 : // -0 - temp3
5116 : //
5117 : // Note: We do not use the Diamond helper class here, because it really hurts
5118 : // readability with nested diamonds.
5119 :
5120 0 : auto if_not_positive = __ MakeDeferredLabel();
5121 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5122 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5123 0 : auto if_zero = __ MakeDeferredLabel();
5124 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
5125 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5126 :
5127 0 : Node* const zero = __ Float64Constant(0.0);
5128 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5129 0 : Node* const one = __ Float64Constant(1.0);
5130 :
5131 0 : Node* check0 = __ Float64LessThan(zero, input);
5132 0 : __ GotoIfNot(check0, &if_not_positive);
5133 : {
5134 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5135 0 : __ GotoIf(check1, &if_greater_than_two_52);
5136 : {
5137 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5138 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5139 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5140 : }
5141 :
5142 : __ Bind(&if_greater_than_two_52);
5143 : __ Goto(&done, input);
5144 : }
5145 :
5146 : __ Bind(&if_not_positive);
5147 : {
5148 0 : Node* check1 = __ Float64Equal(input, zero);
5149 0 : __ GotoIf(check1, &if_zero);
5150 :
5151 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5152 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5153 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5154 :
5155 : {
5156 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5157 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5158 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5159 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
5160 0 : __ GotoIfNot(check3, &done_temp3, temp2);
5161 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
5162 :
5163 : __ Bind(&done_temp3);
5164 : Node* temp3 = done_temp3.PhiAt(0);
5165 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
5166 : }
5167 : __ Bind(&if_less_than_minus_two_52);
5168 : __ Goto(&done, input);
5169 :
5170 : __ Bind(&if_zero);
5171 : __ Goto(&done, input);
5172 : }
5173 : __ Bind(&done);
5174 : return done.PhiAt(0);
5175 : }
5176 :
5177 6912 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTruncate(Node* node) {
5178 : // Nothing to be done if a fast hardware instruction is available.
5179 6912 : if (machine()->Float64RoundTruncate().IsSupported()) {
5180 : return Nothing<Node*>();
5181 : }
5182 :
5183 : Node* const input = node->InputAt(0);
5184 0 : return Just(BuildFloat64RoundTruncate(input));
5185 : }
5186 :
5187 380 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntry(Node* node) {
5188 190 : Node* table = NodeProperties::GetValueInput(node, 0);
5189 190 : Node* key = NodeProperties::GetValueInput(node, 1);
5190 :
5191 : {
5192 : Callable const callable =
5193 190 : Builtins::CallableFor(isolate(), Builtins::kFindOrderedHashMapEntry);
5194 190 : Operator::Properties const properties = node->op()->properties();
5195 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
5196 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5197 : graph()->zone(), callable.descriptor(),
5198 380 : callable.descriptor().GetStackParameterCount(), flags, properties);
5199 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), table,
5200 380 : key, __ NoContextConstant());
5201 : }
5202 : }
5203 :
5204 15 : Node* EffectControlLinearizer::ComputeUnseededHash(Node* value) {
5205 : // See v8::internal::ComputeUnseededHash()
5206 : value = __ Int32Add(__ Word32Xor(value, __ Int32Constant(0xFFFFFFFF)),
5207 15 : __ Word32Shl(value, __ Int32Constant(15)));
5208 15 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(12)));
5209 15 : value = __ Int32Add(value, __ Word32Shl(value, __ Int32Constant(2)));
5210 15 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(4)));
5211 15 : value = __ Int32Mul(value, __ Int32Constant(2057));
5212 15 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(16)));
5213 15 : value = __ Word32And(value, __ Int32Constant(0x3FFFFFFF));
5214 15 : return value;
5215 : }
5216 :
5217 15 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntryForInt32Key(
5218 : Node* node) {
5219 15 : Node* table = NodeProperties::GetValueInput(node, 0);
5220 15 : Node* key = NodeProperties::GetValueInput(node, 1);
5221 :
5222 : // Compute the integer hash code.
5223 15 : Node* hash = ChangeUint32ToUintPtr(ComputeUnseededHash(key));
5224 :
5225 : Node* number_of_buckets = ChangeSmiToIntPtr(__ LoadField(
5226 15 : AccessBuilder::ForOrderedHashMapOrSetNumberOfBuckets(), table));
5227 15 : hash = __ WordAnd(hash, __ IntSub(number_of_buckets, __ IntPtrConstant(1)));
5228 : Node* first_entry = ChangeSmiToIntPtr(__ Load(
5229 : MachineType::TaggedSigned(), table,
5230 : __ IntAdd(__ WordShl(hash, __ IntPtrConstant(kTaggedSizeLog2)),
5231 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5232 15 : kHeapObjectTag))));
5233 :
5234 15 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
5235 15 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
5236 : __ Goto(&loop, first_entry);
5237 : __ Bind(&loop);
5238 : {
5239 : Node* entry = loop.PhiAt(0);
5240 : Node* check =
5241 15 : __ WordEqual(entry, __ IntPtrConstant(OrderedHashMap::kNotFound));
5242 15 : __ GotoIf(check, &done, entry);
5243 : entry = __ IntAdd(
5244 : __ IntMul(entry, __ IntPtrConstant(OrderedHashMap::kEntrySize)),
5245 15 : number_of_buckets);
5246 :
5247 : Node* candidate_key = __ Load(
5248 : MachineType::AnyTagged(), table,
5249 : __ IntAdd(__ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5250 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5251 15 : kHeapObjectTag)));
5252 :
5253 15 : auto if_match = __ MakeLabel();
5254 15 : auto if_notmatch = __ MakeLabel();
5255 15 : auto if_notsmi = __ MakeDeferredLabel();
5256 15 : __ GotoIfNot(ObjectIsSmi(candidate_key), &if_notsmi);
5257 : __ Branch(__ Word32Equal(ChangeSmiToInt32(candidate_key), key), &if_match,
5258 15 : &if_notmatch);
5259 :
5260 : __ Bind(&if_notsmi);
5261 : __ GotoIfNot(
5262 : __ WordEqual(__ LoadField(AccessBuilder::ForMap(), candidate_key),
5263 : __ HeapNumberMapConstant()),
5264 15 : &if_notmatch);
5265 : __ Branch(__ Float64Equal(__ LoadField(AccessBuilder::ForHeapNumberValue(),
5266 : candidate_key),
5267 : __ ChangeInt32ToFloat64(key)),
5268 15 : &if_match, &if_notmatch);
5269 :
5270 : __ Bind(&if_match);
5271 : __ Goto(&done, entry);
5272 :
5273 : __ Bind(&if_notmatch);
5274 : {
5275 : Node* next_entry = ChangeSmiToIntPtr(__ Load(
5276 : MachineType::TaggedSigned(), table,
5277 : __ IntAdd(
5278 : __ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5279 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() +
5280 : OrderedHashMap::kChainOffset * kTaggedSize -
5281 15 : kHeapObjectTag))));
5282 : __ Goto(&loop, next_entry);
5283 : }
5284 : }
5285 :
5286 : __ Bind(&done);
5287 15 : return done.PhiAt(0);
5288 : }
5289 :
5290 7 : Node* EffectControlLinearizer::LowerDateNow(Node* node) {
5291 7 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
5292 : Runtime::FunctionId id = Runtime::kDateCurrentTime;
5293 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
5294 7 : graph()->zone(), id, 0, properties, CallDescriptor::kNoFlags);
5295 : return __ Call(call_descriptor, __ CEntryStubConstant(1),
5296 : __ ExternalConstant(ExternalReference::Create(id)),
5297 7 : __ Int32Constant(0), __ NoContextConstant());
5298 : }
5299 :
5300 : #undef __
5301 :
5302 0 : Factory* EffectControlLinearizer::factory() const {
5303 0 : return isolate()->factory();
5304 : }
5305 :
5306 86371 : Isolate* EffectControlLinearizer::isolate() const {
5307 86371 : return jsgraph()->isolate();
5308 : }
5309 :
5310 : } // namespace compiler
5311 : } // namespace internal
5312 183867 : } // namespace v8
|
