Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/effect-control-linearizer.h"
6 :
7 : #include "src/code-factory.h"
8 : #include "src/compiler/access-builder.h"
9 : #include "src/compiler/compiler-source-position-table.h"
10 : #include "src/compiler/js-graph.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-origin-table.h"
14 : #include "src/compiler/node-properties.h"
15 : #include "src/compiler/node.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/heap/factory-inl.h"
18 : #include "src/objects/heap-number.h"
19 : #include "src/objects/oddball.h"
20 :
21 : namespace v8 {
22 : namespace internal {
23 : namespace compiler {
24 :
25 456696 : EffectControlLinearizer::EffectControlLinearizer(
26 : JSGraph* js_graph, Schedule* schedule, Zone* temp_zone,
27 : SourcePositionTable* source_positions, NodeOriginTable* node_origins,
28 : MaskArrayIndexEnable mask_array_index)
29 : : js_graph_(js_graph),
30 : schedule_(schedule),
31 : temp_zone_(temp_zone),
32 : mask_array_index_(mask_array_index),
33 : source_positions_(source_positions),
34 : node_origins_(node_origins),
35 : graph_assembler_(js_graph, nullptr, nullptr, temp_zone),
36 456696 : frame_state_zapper_(nullptr) {}
37 :
38 1721286 : Graph* EffectControlLinearizer::graph() const { return js_graph_->graph(); }
39 0 : CommonOperatorBuilder* EffectControlLinearizer::common() const {
40 1086272 : return js_graph_->common();
41 : }
42 0 : SimplifiedOperatorBuilder* EffectControlLinearizer::simplified() const {
43 0 : return js_graph_->simplified();
44 : }
45 0 : MachineOperatorBuilder* EffectControlLinearizer::machine() const {
46 327502 : return js_graph_->machine();
47 : }
48 :
49 : namespace {
50 :
51 : struct BlockEffectControlData {
52 : Node* current_effect = nullptr; // New effect.
53 : Node* current_control = nullptr; // New control.
54 : Node* current_frame_state = nullptr; // New frame state.
55 : };
56 :
57 : class BlockEffectControlMap {
58 : public:
59 : explicit BlockEffectControlMap(Zone* temp_zone) : map_(temp_zone) {}
60 :
61 19165628 : BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) {
62 19165628 : return map_[std::make_pair(from->rpo_number(), to->rpo_number())];
63 : }
64 :
65 : const BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) const {
66 : return map_.at(std::make_pair(from->rpo_number(), to->rpo_number()));
67 : }
68 :
69 : private:
70 : typedef std::pair<int32_t, int32_t> Key;
71 : typedef ZoneMap<Key, BlockEffectControlData> Map;
72 :
73 : Map map_;
74 : };
75 :
76 : // Effect phis that need to be updated after the first pass.
77 : struct PendingEffectPhi {
78 : Node* effect_phi;
79 : BasicBlock* block;
80 :
81 : PendingEffectPhi(Node* effect_phi, BasicBlock* block)
82 40687 : : effect_phi(effect_phi), block(block) {}
83 : };
84 :
85 1924 : void ConnectUnreachableToEnd(Node* effect, Node* control, JSGraph* jsgraph) {
86 962 : Graph* graph = jsgraph->graph();
87 : CommonOperatorBuilder* common = jsgraph->common();
88 1924 : if (effect->opcode() == IrOpcode::kDead) return;
89 962 : if (effect->opcode() != IrOpcode::kUnreachable) {
90 0 : effect = graph->NewNode(common->Unreachable(), effect, control);
91 : }
92 962 : Node* throw_node = graph->NewNode(common->Throw(), effect, control);
93 962 : NodeProperties::MergeControlToEnd(graph, common, throw_node);
94 : }
95 :
96 3355658 : void UpdateEffectPhi(Node* node, BasicBlock* block,
97 : BlockEffectControlMap* block_effects, JSGraph* jsgraph) {
98 : // Update all inputs to an effect phi with the effects from the given
99 : // block->effect map.
100 : DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
101 : DCHECK_EQ(static_cast<size_t>(node->op()->EffectInputCount()),
102 : block->PredecessorCount());
103 6909390 : for (int i = 0; i < node->op()->EffectInputCount(); i++) {
104 : Node* input = node->InputAt(i);
105 1776858 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
106 : const BlockEffectControlData& block_effect =
107 : block_effects->For(predecessor, block);
108 1776865 : Node* effect = block_effect.current_effect;
109 1776865 : if (input != effect) {
110 133380 : node->ReplaceInput(i, effect);
111 : }
112 : }
113 789404 : }
114 :
115 4328686 : void UpdateBlockControl(BasicBlock* block,
116 : BlockEffectControlMap* block_effects) {
117 12285260 : Node* control = block->NodeAt(0);
118 : DCHECK(NodeProperties::IsControl(control));
119 :
120 : // Do not rewire the end node.
121 4328686 : if (control->opcode() == IrOpcode::kEnd) return;
122 :
123 : // Update all inputs to the given control node with the correct control.
124 : DCHECK(control->opcode() == IrOpcode::kMerge ||
125 : static_cast<size_t>(control->op()->ControlInputCount()) ==
126 : block->PredecessorCount());
127 11618973 : if (static_cast<size_t>(control->op()->ControlInputCount()) !=
128 : block->PredecessorCount()) {
129 : return; // We already re-wired the control inputs of this node.
130 : }
131 20190599 : for (int i = 0; i < control->op()->ControlInputCount(); i++) {
132 4277463 : Node* input = NodeProperties::GetControlInput(control, i);
133 4277454 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
134 : const BlockEffectControlData& block_effect =
135 : block_effects->For(predecessor, block);
136 4277450 : if (input != block_effect.current_control) {
137 : NodeProperties::ReplaceControlInput(control, block_effect.current_control,
138 37290 : i);
139 : }
140 : }
141 : }
142 :
143 6909683 : bool HasIncomingBackEdges(BasicBlock* block) {
144 18845109 : for (BasicBlock* pred : block->predecessors()) {
145 6909683 : if (pred->rpo_number() >= block->rpo_number()) {
146 : return true;
147 : }
148 : }
149 : return false;
150 : }
151 :
152 286158 : void RemoveRenameNode(Node* node) {
153 : DCHECK(IrOpcode::kFinishRegion == node->opcode() ||
154 : IrOpcode::kBeginRegion == node->opcode() ||
155 : IrOpcode::kTypeGuard == node->opcode());
156 : // Update the value/context uses to the value input of the finish node and
157 : // the effect uses to the effect input.
158 3500450 : for (Edge edge : node->use_edges()) {
159 : DCHECK(!edge.from()->IsDead());
160 1464067 : if (NodeProperties::IsEffectEdge(edge)) {
161 287589 : edge.UpdateTo(NodeProperties::GetEffectInput(node));
162 : } else {
163 : DCHECK(!NodeProperties::IsControlEdge(edge));
164 : DCHECK(!NodeProperties::IsFrameStateEdge(edge));
165 1176478 : edge.UpdateTo(node->InputAt(0));
166 : }
167 : }
168 286158 : node->Kill();
169 286157 : }
170 :
171 1172163 : void TryCloneBranch(Node* node, BasicBlock* block, Zone* temp_zone,
172 927690 : Graph* graph, CommonOperatorBuilder* common,
173 : BlockEffectControlMap* block_effects,
174 : SourcePositionTable* source_positions,
175 : NodeOriginTable* node_origins) {
176 : DCHECK_EQ(IrOpcode::kBranch, node->opcode());
177 :
178 : // This optimization is a special case of (super)block cloning. It takes an
179 : // input graph as shown below and clones the Branch node for every predecessor
180 : // to the Merge, essentially removing the Merge completely. This avoids
181 : // materializing the bit for the Phi and may offer potential for further
182 : // branch folding optimizations (i.e. because one or more inputs to the Phi is
183 : // a constant). Note that there may be more Phi nodes hanging off the Merge,
184 : // but we can only a certain subset of them currently (actually only Phi and
185 : // EffectPhi nodes whose uses have either the IfTrue or IfFalse as control
186 : // input).
187 :
188 : // Control1 ... ControlN
189 : // ^ ^
190 : // | | Cond1 ... CondN
191 : // +----+ +----+ ^ ^
192 : // | | | |
193 : // | | +----+ |
194 : // Merge<--+ | +------------+
195 : // ^ \|/
196 : // | Phi
197 : // | |
198 : // Branch----+
199 : // ^
200 : // |
201 : // +-----+-----+
202 : // | |
203 : // IfTrue IfFalse
204 : // ^ ^
205 : // | |
206 :
207 : // The resulting graph (modulo the Phi and EffectPhi nodes) looks like this:
208 :
209 : // Control1 Cond1 ... ControlN CondN
210 : // ^ ^ ^ ^
211 : // \ / \ /
212 : // Branch ... Branch
213 : // ^ ^
214 : // | |
215 : // +---+---+ +---+----+
216 : // | | | |
217 : // IfTrue IfFalse ... IfTrue IfFalse
218 : // ^ ^ ^ ^
219 : // | | | |
220 : // +--+ +-------------+ |
221 : // | | +--------------+ +--+
222 : // | | | |
223 : // Merge Merge
224 : // ^ ^
225 : // | |
226 :
227 : SourcePositionTable::Scope scope(source_positions,
228 1075227 : source_positions->GetSourcePosition(node));
229 : NodeOriginTable::Scope origin_scope(node_origins, "clone branch", node);
230 96936 : Node* branch = node;
231 2126590 : Node* cond = NodeProperties::GetValueInput(branch, 0);
232 2126592 : if (!cond->OwnedBy(branch) || cond->opcode() != IrOpcode::kPhi) return;
233 206458 : Node* merge = NodeProperties::GetControlInput(branch);
234 216673 : if (merge->opcode() != IrOpcode::kMerge ||
235 107151 : NodeProperties::GetControlInput(cond) != merge) {
236 : return;
237 : }
238 : // Grab the IfTrue/IfFalse projections of the Branch.
239 107079 : BranchMatcher matcher(branch);
240 : // Check/collect other Phi/EffectPhi nodes hanging off the Merge.
241 : NodeVector phis(temp_zone);
242 811346 : for (Node* const use : merge->uses()) {
243 303665 : if (use == branch || use == cond) continue;
244 : // We cannot currently deal with non-Phi/EffectPhi nodes hanging off the
245 : // Merge. Ideally, we would just clone the nodes (and everything that
246 : // depends on it to some distant join point), but that requires knowledge
247 : // about dominance/post-dominance.
248 114909 : if (!NodeProperties::IsPhi(use)) return;
249 204493 : for (Edge edge : use->use_edges()) {
250 : // Right now we can only handle Phi/EffectPhi nodes whose uses are
251 : // directly control-dependend on either the IfTrue or the IfFalse
252 : // successor, because we know exactly how to update those uses.
253 11523 : if (edge.from()->op()->ControlInputCount() != 1) return;
254 10410 : Node* control = NodeProperties::GetControlInput(edge.from());
255 10410 : if (NodeProperties::IsPhi(edge.from())) {
256 620 : control = NodeProperties::GetControlInput(control, edge.index());
257 : }
258 10410 : if (control != matcher.IfTrue() && control != matcher.IfFalse()) return;
259 : }
260 94623 : phis.push_back(use);
261 : }
262 96936 : BranchHint const hint = BranchHintOf(branch->op());
263 96936 : int const input_count = merge->op()->ControlInputCount();
264 : DCHECK_LE(1, input_count);
265 96936 : Node** const inputs = graph->zone()->NewArray<Node*>(2 * input_count);
266 : Node** const merge_true_inputs = &inputs[0];
267 96937 : Node** const merge_false_inputs = &inputs[input_count];
268 512313 : for (int index = 0; index < input_count; ++index) {
269 415377 : Node* cond1 = NodeProperties::GetValueInput(cond, index);
270 415377 : Node* control1 = NodeProperties::GetControlInput(merge, index);
271 415376 : Node* branch1 = graph->NewNode(common->Branch(hint), cond1, control1);
272 830753 : merge_true_inputs[index] = graph->NewNode(common->IfTrue(), branch1);
273 830753 : merge_false_inputs[index] = graph->NewNode(common->IfFalse(), branch1);
274 : }
275 96936 : Node* const merge_true = matcher.IfTrue();
276 96936 : Node* const merge_false = matcher.IfFalse();
277 96936 : merge_true->TrimInputCount(0);
278 96936 : merge_false->TrimInputCount(0);
279 512313 : for (int i = 0; i < input_count; ++i) {
280 830754 : merge_true->AppendInput(graph->zone(), merge_true_inputs[i]);
281 830754 : merge_false->AppendInput(graph->zone(), merge_false_inputs[i]);
282 : }
283 : DCHECK_EQ(2u, block->SuccessorCount());
284 96936 : NodeProperties::ChangeOp(matcher.IfTrue(), common->Merge(input_count));
285 96936 : NodeProperties::ChangeOp(matcher.IfFalse(), common->Merge(input_count));
286 : int const true_index =
287 96936 : block->SuccessorAt(0)->NodeAt(0) == matcher.IfTrue() ? 0 : 1;
288 : BlockEffectControlData* true_block_data =
289 96936 : &block_effects->For(block, block->SuccessorAt(true_index));
290 : BlockEffectControlData* false_block_data =
291 96936 : &block_effects->For(block, block->SuccessorAt(true_index ^ 1));
292 572368 : for (Node* const phi : phis) {
293 505056 : for (int index = 0; index < input_count; ++index) {
294 820864 : inputs[index] = phi->InputAt(index);
295 : }
296 94624 : inputs[input_count] = merge_true;
297 189248 : Node* phi_true = graph->NewNode(phi->op(), input_count + 1, inputs);
298 94624 : inputs[input_count] = merge_false;
299 94624 : Node* phi_false = graph->NewNode(phi->op(), input_count + 1, inputs);
300 94624 : if (phi->UseCount() == 0) {
301 : DCHECK_EQ(phi->opcode(), IrOpcode::kEffectPhi);
302 : } else {
303 29242 : for (Edge edge : phi->use_edges()) {
304 7768 : Node* control = NodeProperties::GetControlInput(edge.from());
305 15536 : if (NodeProperties::IsPhi(edge.from())) {
306 615 : control = NodeProperties::GetControlInput(control, edge.index());
307 : }
308 : DCHECK(control == matcher.IfTrue() || control == matcher.IfFalse());
309 7768 : edge.UpdateTo((control == matcher.IfTrue()) ? phi_true : phi_false);
310 : }
311 : }
312 94624 : if (phi->opcode() == IrOpcode::kEffectPhi) {
313 94624 : true_block_data->current_effect = phi_true;
314 94624 : false_block_data->current_effect = phi_false;
315 : }
316 94624 : phi->Kill();
317 : }
318 : // Fix up IfTrue and IfFalse and kill all dead nodes.
319 96936 : if (branch == block->control_input()) {
320 96936 : true_block_data->current_control = merge_true;
321 96936 : false_block_data->current_control = merge_false;
322 : }
323 96936 : branch->Kill();
324 96936 : cond->Kill();
325 96936 : merge->Kill();
326 : }
327 :
328 : } // namespace
329 :
330 9878566 : void EffectControlLinearizer::Run() {
331 : BlockEffectControlMap block_effects(temp_zone());
332 : ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone());
333 : ZoneVector<BasicBlock*> pending_block_controls(temp_zone());
334 : NodeVector inputs_buffer(temp_zone());
335 :
336 5242073 : for (BasicBlock* block : *(schedule()->rpo_order())) {
337 : size_t instr = 0;
338 :
339 : // The control node should be the first.
340 11695014 : Node* control = block->NodeAt(instr);
341 : DCHECK(NodeProperties::IsControl(control));
342 : // Update the control inputs.
343 4328686 : if (HasIncomingBackEdges(block)) {
344 : // If there are back edges, we need to update later because we have not
345 : // computed the control yet. This should only happen for loops.
346 : DCHECK_EQ(IrOpcode::kLoop, control->opcode());
347 40684 : pending_block_controls.push_back(block);
348 : } else {
349 : // If there are no back edges, we can update now.
350 4288002 : UpdateBlockControl(block, &block_effects);
351 : }
352 : instr++;
353 :
354 : // Iterate over the phis and update the effect phis.
355 : Node* effect_phi = nullptr;
356 : Node* terminate = nullptr;
357 10928396 : for (; instr < block->NodeCount(); instr++) {
358 3815120 : Node* node = block->NodeAt(instr);
359 : // Only go through the phis and effect phis.
360 3815120 : if (node->opcode() == IrOpcode::kEffectPhi) {
361 : // There should be at most one effect phi in a block.
362 : DCHECK_NULL(effect_phi);
363 : // IfException blocks should not have effect phis.
364 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
365 : effect_phi = node;
366 3036728 : } else if (node->opcode() == IrOpcode::kPhi) {
367 : // Just skip phis.
368 2720325 : } else if (node->opcode() == IrOpcode::kTerminate) {
369 : DCHECK_NULL(terminate);
370 : terminate = node;
371 : } else {
372 : break;
373 : }
374 : }
375 :
376 4328679 : if (effect_phi) {
377 : // Make sure we update the inputs to the incoming blocks' effects.
378 778420 : if (HasIncomingBackEdges(block)) {
379 : // In case of loops, we do not update the effect phi immediately
380 : // because the back predecessor has not been handled yet. We just
381 : // record the effect phi for later processing.
382 40679 : pending_effect_phis.push_back(PendingEffectPhi(effect_phi, block));
383 : } else {
384 737740 : UpdateEffectPhi(effect_phi, block, &block_effects, jsgraph());
385 : }
386 : }
387 :
388 4328705 : Node* effect = effect_phi;
389 4328705 : if (effect == nullptr) {
390 : // There was no effect phi.
391 3550262 : if (block == schedule()->start()) {
392 : // Start block => effect is start.
393 : DCHECK_EQ(graph()->start(), control);
394 456699 : effect = graph()->start();
395 8824994 : } else if (control->opcode() == IrOpcode::kEnd) {
396 : // End block is just a dummy, no effect needed.
397 : DCHECK_EQ(BasicBlock::kNone, block->control());
398 : DCHECK_EQ(1u, block->size());
399 455697 : effect = nullptr;
400 : } else {
401 : // If all the predecessors have the same effect, we can use it as our
402 : // current effect.
403 8049250 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
404 : const BlockEffectControlData& data =
405 : block_effects.For(block->PredecessorAt(i), block);
406 2716674 : if (!effect) effect = data.current_effect;
407 2716674 : if (data.current_effect != effect) {
408 10982 : effect = nullptr;
409 10982 : break;
410 : }
411 : }
412 2637868 : if (effect == nullptr) {
413 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
414 : // The input blocks do not have the same effect. We have
415 : // to create an effect phi node.
416 : inputs_buffer.clear();
417 21964 : inputs_buffer.resize(block->PredecessorCount(), jsgraph()->Dead());
418 10982 : inputs_buffer.push_back(control);
419 : effect = graph()->NewNode(
420 10982 : common()->EffectPhi(static_cast<int>(block->PredecessorCount())),
421 54910 : static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front()));
422 : // For loops, we update the effect phi node later to break cycles.
423 21964 : if (control->opcode() == IrOpcode::kLoop) {
424 14 : pending_effect_phis.push_back(PendingEffectPhi(effect, block));
425 : } else {
426 10975 : UpdateEffectPhi(effect, block, &block_effects, jsgraph());
427 : }
428 5253772 : } else if (control->opcode() == IrOpcode::kIfException) {
429 : // The IfException is connected into the effect chain, so we need
430 : // to update the effect here.
431 193034 : NodeProperties::ReplaceEffectInput(control, effect);
432 193034 : effect = control;
433 : }
434 : }
435 : }
436 :
437 : // Fixup the Terminate node.
438 4328707 : if (terminate != nullptr) {
439 40685 : NodeProperties::ReplaceEffectInput(terminate, effect);
440 : }
441 :
442 : // The frame state at block entry is determined by the frame states leaving
443 : // all predecessors. In case there is no frame state dominating this block,
444 : // we can rely on a checkpoint being present before the next deoptimization.
445 : // TODO(mstarzinger): Eventually we will need to go hunt for a frame state
446 : // once deoptimizing nodes roam freely through the schedule.
447 4328706 : Node* frame_state = nullptr;
448 4328706 : if (block != schedule()->start()) {
449 : // If all the predecessors have the same effect, we can use it
450 : // as our current effect.
451 : frame_state =
452 3871985 : block_effects.For(block->PredecessorAt(0), block).current_frame_state;
453 9897998 : for (size_t i = 1; i < block->PredecessorCount(); i++) {
454 1170347 : if (block_effects.For(block->PredecessorAt(i), block)
455 1170347 : .current_frame_state != frame_state) {
456 93333 : frame_state = nullptr;
457 93333 : frame_state_zapper_ = graph()->end();
458 93333 : break;
459 : }
460 : }
461 : }
462 :
463 : // Process the ordinary instructions.
464 73029123 : for (; instr < block->NodeCount(); instr++) {
465 : Node* node = block->NodeAt(instr);
466 34350241 : ProcessNode(node, &frame_state, &effect, &control);
467 : }
468 :
469 4328674 : switch (block->control()) {
470 : case BasicBlock::kGoto:
471 : case BasicBlock::kNone:
472 : break;
473 :
474 : case BasicBlock::kCall:
475 : case BasicBlock::kTailCall:
476 : case BasicBlock::kSwitch:
477 : case BasicBlock::kReturn:
478 : case BasicBlock::kDeoptimize:
479 : case BasicBlock::kThrow:
480 887187 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
481 887187 : break;
482 :
483 : case BasicBlock::kBranch:
484 1075231 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
485 : TryCloneBranch(block->control_input(), block, temp_zone(), graph(),
486 : common(), &block_effects, source_positions_,
487 3225708 : node_origins_);
488 1075232 : break;
489 : }
490 :
491 : // Store the effect, control and frame state for later use.
492 18144484 : for (BasicBlock* successor : block->successors()) {
493 5158441 : BlockEffectControlData* data = &block_effects.For(block, successor);
494 5158459 : if (data->current_effect == nullptr) {
495 4969196 : data->current_effect = effect;
496 : }
497 5158459 : if (data->current_control == nullptr) {
498 4964583 : data->current_control = control;
499 : }
500 5158459 : data->current_frame_state = frame_state;
501 : }
502 : }
503 :
504 954081 : for (BasicBlock* pending_block_control : pending_block_controls) {
505 40685 : UpdateBlockControl(pending_block_control, &block_effects);
506 : }
507 : // Update the incoming edges of the effect phis that could not be processed
508 : // during the first pass (because they could have incoming back edges).
509 954085 : for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) {
510 : UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block,
511 40687 : &block_effects, jsgraph());
512 : }
513 456699 : }
514 :
515 186166778 : void EffectControlLinearizer::ProcessNode(Node* node, Node** frame_state,
516 1924 : Node** effect, Node** control) {
517 : SourcePositionTable::Scope scope(source_positions_,
518 36312585 : source_positions_->GetSourcePosition(node));
519 36312590 : NodeOriginTable::Scope origin_scope(node_origins_, "process node", node);
520 :
521 : // If the node needs to be wired into the effect/control chain, do this
522 : // here. Pass current frame state for lowering to eager deoptimization.
523 36312590 : if (TryWireInStateEffect(node, *frame_state, effect, control)) {
524 : return;
525 : }
526 :
527 : // If the node has a visible effect, then there must be a checkpoint in the
528 : // effect chain before we are allowed to place another eager deoptimization
529 : // point. We zap the frame state to ensure this invariant is maintained.
530 68762402 : if (region_observability_ == RegionObservability::kObservable &&
531 : !node->op()->HasProperty(Operator::kNoWrite)) {
532 3852357 : *frame_state = nullptr;
533 3852357 : frame_state_zapper_ = node;
534 : }
535 :
536 : // Remove the end markers of 'atomic' allocation region because the
537 : // region should be wired-in now.
538 35205527 : if (node->opcode() == IrOpcode::kFinishRegion) {
539 : // Reset the current region observability.
540 132404 : region_observability_ = RegionObservability::kObservable;
541 : // Update the value uses to the value input of the finish node and
542 : // the effect uses to the effect input.
543 132404 : return RemoveRenameNode(node);
544 : }
545 35073123 : if (node->opcode() == IrOpcode::kBeginRegion) {
546 : // Determine the observability for this region and use that for all
547 : // nodes inside the region (i.e. ignore the absence of kNoWrite on
548 : // StoreField and other operators).
549 : DCHECK_NE(RegionObservability::kNotObservable, region_observability_);
550 132404 : region_observability_ = RegionObservabilityOf(node->op());
551 : // Update the value uses to the value input of the finish node and
552 : // the effect uses to the effect input.
553 132404 : return RemoveRenameNode(node);
554 : }
555 34940719 : if (node->opcode() == IrOpcode::kTypeGuard) {
556 21350 : return RemoveRenameNode(node);
557 : }
558 :
559 : // Special treatment for checkpoint nodes.
560 34919369 : if (node->opcode() == IrOpcode::kCheckpoint) {
561 : // Unlink the check point; effect uses will be updated to the incoming
562 : // effect that is passed. The frame state is preserved for lowering.
563 : DCHECK_EQ(RegionObservability::kObservable, region_observability_);
564 2982792 : *frame_state = NodeProperties::GetFrameStateInput(node);
565 2982791 : return;
566 : }
567 :
568 : // The IfSuccess nodes should always start a basic block (and basic block
569 : // start nodes are not handled in the ProcessNode method).
570 : DCHECK_NE(IrOpcode::kIfSuccess, node->opcode());
571 :
572 : // If the node takes an effect, replace with the current one.
573 63873154 : if (node->op()->EffectInputCount() > 0) {
574 : DCHECK_EQ(1, node->op()->EffectInputCount());
575 8312168 : Node* input_effect = NodeProperties::GetEffectInput(node);
576 :
577 8312161 : if (input_effect != *effect) {
578 2985563 : NodeProperties::ReplaceEffectInput(node, *effect);
579 : }
580 :
581 : // If the node produces an effect, update our current effect. (However,
582 : // ignore new effect chains started with ValueEffect.)
583 16624276 : if (node->op()->EffectOutputCount() > 0) {
584 : DCHECK_EQ(1, node->op()->EffectOutputCount());
585 7625042 : *effect = node;
586 : }
587 : } else {
588 : // New effect chain is only started with a Start or ValueEffect node.
589 : DCHECK(node->op()->EffectOutputCount() == 0 ||
590 : node->opcode() == IrOpcode::kStart);
591 : }
592 :
593 : // Rewire control inputs.
594 90592795 : for (int i = 0; i < node->op()->ControlInputCount(); i++) {
595 8906575 : NodeProperties::ReplaceControlInput(node, *control, i);
596 : }
597 : // Update the current control.
598 63873078 : if (node->op()->ControlOutputCount() > 0) {
599 4976616 : *control = node;
600 : }
601 :
602 : // Break the effect chain on {Unreachable} and reconnect to the graph end.
603 : // Mark the following code for deletion by connecting to the {Dead} node.
604 31936539 : if (node->opcode() == IrOpcode::kUnreachable) {
605 962 : ConnectUnreachableToEnd(*effect, *control, jsgraph());
606 962 : *effect = *control = jsgraph()->Dead();
607 : }
608 : }
609 :
610 73732306 : bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
611 : Node* frame_state,
612 : Node** effect,
613 : Node** control) {
614 36312594 : gasm()->Reset(*effect, *control);
615 : Node* result = nullptr;
616 36312599 : switch (node->opcode()) {
617 : case IrOpcode::kChangeBitToTagged:
618 9520 : result = LowerChangeBitToTagged(node);
619 9520 : break;
620 : case IrOpcode::kChangeInt31ToTaggedSigned:
621 40470 : result = LowerChangeInt31ToTaggedSigned(node);
622 40470 : break;
623 : case IrOpcode::kChangeInt32ToTagged:
624 36379 : result = LowerChangeInt32ToTagged(node);
625 36379 : break;
626 : case IrOpcode::kChangeInt64ToTagged:
627 252 : result = LowerChangeInt64ToTagged(node);
628 252 : break;
629 : case IrOpcode::kChangeUint32ToTagged:
630 541 : result = LowerChangeUint32ToTagged(node);
631 541 : break;
632 : case IrOpcode::kChangeUint64ToTagged:
633 81 : result = LowerChangeUint64ToTagged(node);
634 81 : break;
635 : case IrOpcode::kChangeFloat64ToTagged:
636 52239 : result = LowerChangeFloat64ToTagged(node);
637 52240 : break;
638 : case IrOpcode::kChangeFloat64ToTaggedPointer:
639 91 : result = LowerChangeFloat64ToTaggedPointer(node);
640 91 : break;
641 : case IrOpcode::kChangeTaggedSignedToInt32:
642 52616 : result = LowerChangeTaggedSignedToInt32(node);
643 52616 : break;
644 : case IrOpcode::kChangeTaggedSignedToInt64:
645 296 : result = LowerChangeTaggedSignedToInt64(node);
646 296 : break;
647 : case IrOpcode::kChangeTaggedToBit:
648 135144 : result = LowerChangeTaggedToBit(node);
649 135144 : break;
650 : case IrOpcode::kChangeTaggedToInt32:
651 2027 : result = LowerChangeTaggedToInt32(node);
652 2027 : break;
653 : case IrOpcode::kChangeTaggedToUint32:
654 452 : result = LowerChangeTaggedToUint32(node);
655 452 : break;
656 : case IrOpcode::kChangeTaggedToInt64:
657 5 : result = LowerChangeTaggedToInt64(node);
658 5 : break;
659 : case IrOpcode::kChangeTaggedToFloat64:
660 : result = LowerChangeTaggedToFloat64(node);
661 11079 : break;
662 : case IrOpcode::kChangeTaggedToTaggedSigned:
663 15 : result = LowerChangeTaggedToTaggedSigned(node);
664 15 : break;
665 : case IrOpcode::kTruncateTaggedToBit:
666 46923 : result = LowerTruncateTaggedToBit(node);
667 46923 : break;
668 : case IrOpcode::kTruncateTaggedPointerToBit:
669 461 : result = LowerTruncateTaggedPointerToBit(node);
670 461 : break;
671 : case IrOpcode::kTruncateTaggedToFloat64:
672 720 : result = LowerTruncateTaggedToFloat64(node);
673 720 : break;
674 : case IrOpcode::kPoisonIndex:
675 1177 : result = LowerPoisonIndex(node);
676 1177 : break;
677 : case IrOpcode::kCheckMaps:
678 56391 : LowerCheckMaps(node, frame_state);
679 56391 : break;
680 : case IrOpcode::kCompareMaps:
681 7486 : result = LowerCompareMaps(node);
682 7486 : break;
683 : case IrOpcode::kCheckNumber:
684 473 : result = LowerCheckNumber(node, frame_state);
685 473 : break;
686 : case IrOpcode::kCheckReceiver:
687 813 : result = LowerCheckReceiver(node, frame_state);
688 813 : break;
689 : case IrOpcode::kCheckReceiverOrNullOrUndefined:
690 72 : result = LowerCheckReceiverOrNullOrUndefined(node, frame_state);
691 72 : break;
692 : case IrOpcode::kCheckSymbol:
693 31 : result = LowerCheckSymbol(node, frame_state);
694 31 : break;
695 : case IrOpcode::kCheckString:
696 6197 : result = LowerCheckString(node, frame_state);
697 6197 : break;
698 : case IrOpcode::kCheckInternalizedString:
699 1415 : result = LowerCheckInternalizedString(node, frame_state);
700 1415 : break;
701 : case IrOpcode::kCheckIf:
702 15132 : LowerCheckIf(node, frame_state);
703 15132 : break;
704 : case IrOpcode::kCheckedInt32Add:
705 55973 : result = LowerCheckedInt32Add(node, frame_state);
706 55970 : break;
707 : case IrOpcode::kCheckedInt32Sub:
708 7449 : result = LowerCheckedInt32Sub(node, frame_state);
709 7449 : break;
710 : case IrOpcode::kCheckedInt32Div:
711 239 : result = LowerCheckedInt32Div(node, frame_state);
712 239 : break;
713 : case IrOpcode::kCheckedInt32Mod:
714 849 : result = LowerCheckedInt32Mod(node, frame_state);
715 849 : break;
716 : case IrOpcode::kCheckedUint32Div:
717 47 : result = LowerCheckedUint32Div(node, frame_state);
718 47 : break;
719 : case IrOpcode::kCheckedUint32Mod:
720 37 : result = LowerCheckedUint32Mod(node, frame_state);
721 37 : break;
722 : case IrOpcode::kCheckedInt32Mul:
723 5679 : result = LowerCheckedInt32Mul(node, frame_state);
724 5679 : break;
725 : case IrOpcode::kCheckedInt32ToTaggedSigned:
726 0 : result = LowerCheckedInt32ToTaggedSigned(node, frame_state);
727 0 : break;
728 : case IrOpcode::kCheckedInt64ToInt32:
729 7 : result = LowerCheckedInt64ToInt32(node, frame_state);
730 7 : break;
731 : case IrOpcode::kCheckedInt64ToTaggedSigned:
732 150 : result = LowerCheckedInt64ToTaggedSigned(node, frame_state);
733 150 : break;
734 : case IrOpcode::kCheckedUint32Bounds:
735 51966 : result = LowerCheckedUint32Bounds(node, frame_state);
736 51966 : break;
737 : case IrOpcode::kCheckedUint32ToInt32:
738 250 : result = LowerCheckedUint32ToInt32(node, frame_state);
739 250 : break;
740 : case IrOpcode::kCheckedUint32ToTaggedSigned:
741 7 : result = LowerCheckedUint32ToTaggedSigned(node, frame_state);
742 7 : break;
743 : case IrOpcode::kCheckedUint64Bounds:
744 176 : result = LowerCheckedUint64Bounds(node, frame_state);
745 176 : break;
746 : case IrOpcode::kCheckedUint64ToInt32:
747 0 : result = LowerCheckedUint64ToInt32(node, frame_state);
748 0 : break;
749 : case IrOpcode::kCheckedUint64ToTaggedSigned:
750 0 : result = LowerCheckedUint64ToTaggedSigned(node, frame_state);
751 0 : break;
752 : case IrOpcode::kCheckedFloat64ToInt32:
753 2331 : result = LowerCheckedFloat64ToInt32(node, frame_state);
754 2331 : break;
755 : case IrOpcode::kCheckedFloat64ToInt64:
756 14 : result = LowerCheckedFloat64ToInt64(node, frame_state);
757 14 : break;
758 : case IrOpcode::kCheckedTaggedSignedToInt32:
759 24599 : if (frame_state == nullptr) {
760 0 : FATAL("No frame state (zapped by #%d: %s)", frame_state_zapper_->id(),
761 0 : frame_state_zapper_->op()->mnemonic());
762 : }
763 24599 : result = LowerCheckedTaggedSignedToInt32(node, frame_state);
764 24599 : break;
765 : case IrOpcode::kCheckedTaggedToInt32:
766 2438 : result = LowerCheckedTaggedToInt32(node, frame_state);
767 2438 : break;
768 : case IrOpcode::kCheckedTaggedToInt64:
769 70 : result = LowerCheckedTaggedToInt64(node, frame_state);
770 70 : break;
771 : case IrOpcode::kCheckedTaggedToFloat64:
772 49750 : result = LowerCheckedTaggedToFloat64(node, frame_state);
773 49750 : break;
774 : case IrOpcode::kCheckedTaggedToTaggedSigned:
775 19187 : result = LowerCheckedTaggedToTaggedSigned(node, frame_state);
776 19187 : break;
777 : case IrOpcode::kCheckedTaggedToTaggedPointer:
778 35410 : result = LowerCheckedTaggedToTaggedPointer(node, frame_state);
779 35410 : break;
780 : case IrOpcode::kTruncateTaggedToWord32:
781 517 : result = LowerTruncateTaggedToWord32(node);
782 517 : break;
783 : case IrOpcode::kCheckedTruncateTaggedToWord32:
784 1169 : result = LowerCheckedTruncateTaggedToWord32(node, frame_state);
785 1169 : break;
786 : case IrOpcode::kNumberToString:
787 1020 : result = LowerNumberToString(node);
788 1020 : break;
789 : case IrOpcode::kObjectIsArrayBufferView:
790 14 : result = LowerObjectIsArrayBufferView(node);
791 14 : break;
792 : case IrOpcode::kObjectIsBigInt:
793 25 : result = LowerObjectIsBigInt(node);
794 25 : break;
795 : case IrOpcode::kObjectIsCallable:
796 93 : result = LowerObjectIsCallable(node);
797 93 : break;
798 : case IrOpcode::kObjectIsConstructor:
799 207 : result = LowerObjectIsConstructor(node);
800 207 : break;
801 : case IrOpcode::kObjectIsDetectableCallable:
802 13787 : result = LowerObjectIsDetectableCallable(node);
803 13787 : break;
804 : case IrOpcode::kObjectIsMinusZero:
805 21 : result = LowerObjectIsMinusZero(node);
806 21 : break;
807 : case IrOpcode::kNumberIsMinusZero:
808 91 : result = LowerNumberIsMinusZero(node);
809 91 : break;
810 : case IrOpcode::kObjectIsNaN:
811 683 : result = LowerObjectIsNaN(node);
812 683 : break;
813 : case IrOpcode::kNumberIsNaN:
814 2608 : result = LowerNumberIsNaN(node);
815 2608 : break;
816 : case IrOpcode::kObjectIsNonCallable:
817 6001 : result = LowerObjectIsNonCallable(node);
818 6001 : break;
819 : case IrOpcode::kObjectIsNumber:
820 7471 : result = LowerObjectIsNumber(node);
821 7471 : break;
822 : case IrOpcode::kObjectIsReceiver:
823 18981 : result = LowerObjectIsReceiver(node);
824 18981 : break;
825 : case IrOpcode::kObjectIsSmi:
826 4134 : result = LowerObjectIsSmi(node);
827 4134 : break;
828 : case IrOpcode::kObjectIsString:
829 2774 : result = LowerObjectIsString(node);
830 2774 : break;
831 : case IrOpcode::kObjectIsSymbol:
832 18 : result = LowerObjectIsSymbol(node);
833 18 : break;
834 : case IrOpcode::kObjectIsUndetectable:
835 1410 : result = LowerObjectIsUndetectable(node);
836 1410 : break;
837 : case IrOpcode::kArgumentsFrame:
838 16586 : result = LowerArgumentsFrame(node);
839 16586 : break;
840 : case IrOpcode::kArgumentsLength:
841 16613 : result = LowerArgumentsLength(node);
842 16613 : break;
843 : case IrOpcode::kToBoolean:
844 201 : result = LowerToBoolean(node);
845 201 : break;
846 : case IrOpcode::kTypeOf:
847 21292 : result = LowerTypeOf(node);
848 21292 : break;
849 : case IrOpcode::kNewDoubleElements:
850 27 : result = LowerNewDoubleElements(node);
851 27 : break;
852 : case IrOpcode::kNewSmiOrObjectElements:
853 484 : result = LowerNewSmiOrObjectElements(node);
854 484 : break;
855 : case IrOpcode::kNewArgumentsElements:
856 16117 : result = LowerNewArgumentsElements(node);
857 16117 : break;
858 : case IrOpcode::kNewConsString:
859 5003 : result = LowerNewConsString(node);
860 5003 : break;
861 : case IrOpcode::kSameValue:
862 10 : result = LowerSameValue(node);
863 10 : break;
864 : case IrOpcode::kDeadValue:
865 1220 : result = LowerDeadValue(node);
866 1220 : break;
867 : case IrOpcode::kStringConcat:
868 20655 : result = LowerStringConcat(node);
869 20655 : break;
870 : case IrOpcode::kStringFromSingleCharCode:
871 938 : result = LowerStringFromSingleCharCode(node);
872 938 : break;
873 : case IrOpcode::kStringFromSingleCodePoint:
874 207 : result = LowerStringFromSingleCodePoint(node);
875 207 : break;
876 : case IrOpcode::kStringIndexOf:
877 285 : result = LowerStringIndexOf(node);
878 285 : break;
879 : case IrOpcode::kStringLength:
880 24359 : result = LowerStringLength(node);
881 24359 : break;
882 : case IrOpcode::kStringToNumber:
883 330 : result = LowerStringToNumber(node);
884 330 : break;
885 : case IrOpcode::kStringCharCodeAt:
886 2379 : result = LowerStringCharCodeAt(node);
887 2379 : break;
888 : case IrOpcode::kStringCodePointAt:
889 215 : result = LowerStringCodePointAt(node, UnicodeEncodingOf(node->op()));
890 215 : break;
891 : case IrOpcode::kStringToLowerCaseIntl:
892 86 : result = LowerStringToLowerCaseIntl(node);
893 86 : break;
894 : case IrOpcode::kStringToUpperCaseIntl:
895 28 : result = LowerStringToUpperCaseIntl(node);
896 28 : break;
897 : case IrOpcode::kStringSubstring:
898 1075 : result = LowerStringSubstring(node);
899 1075 : break;
900 : case IrOpcode::kStringEqual:
901 10252 : result = LowerStringEqual(node);
902 10252 : break;
903 : case IrOpcode::kStringLessThan:
904 121 : result = LowerStringLessThan(node);
905 121 : break;
906 : case IrOpcode::kStringLessThanOrEqual:
907 729 : result = LowerStringLessThanOrEqual(node);
908 729 : break;
909 : case IrOpcode::kNumberIsFloat64Hole:
910 61 : result = LowerNumberIsFloat64Hole(node);
911 61 : break;
912 : case IrOpcode::kNumberIsFinite:
913 126 : result = LowerNumberIsFinite(node);
914 126 : break;
915 : case IrOpcode::kObjectIsFiniteNumber:
916 7 : result = LowerObjectIsFiniteNumber(node);
917 7 : break;
918 : case IrOpcode::kNumberIsInteger:
919 119 : result = LowerNumberIsInteger(node);
920 119 : break;
921 : case IrOpcode::kObjectIsInteger:
922 7 : result = LowerObjectIsInteger(node);
923 7 : break;
924 : case IrOpcode::kNumberIsSafeInteger:
925 7 : result = LowerNumberIsSafeInteger(node);
926 7 : break;
927 : case IrOpcode::kObjectIsSafeInteger:
928 7 : result = LowerObjectIsSafeInteger(node);
929 7 : break;
930 : case IrOpcode::kCheckFloat64Hole:
931 470 : result = LowerCheckFloat64Hole(node, frame_state);
932 470 : break;
933 : case IrOpcode::kCheckNotTaggedHole:
934 358 : result = LowerCheckNotTaggedHole(node, frame_state);
935 358 : break;
936 : case IrOpcode::kConvertTaggedHoleToUndefined:
937 1668 : result = LowerConvertTaggedHoleToUndefined(node);
938 1668 : break;
939 : case IrOpcode::kCheckEqualsInternalizedString:
940 54 : LowerCheckEqualsInternalizedString(node, frame_state);
941 54 : break;
942 : case IrOpcode::kAllocate:
943 112492 : result = LowerAllocate(node);
944 112492 : break;
945 : case IrOpcode::kCheckEqualsSymbol:
946 18 : LowerCheckEqualsSymbol(node, frame_state);
947 18 : break;
948 : case IrOpcode::kPlainPrimitiveToNumber:
949 20 : result = LowerPlainPrimitiveToNumber(node);
950 20 : break;
951 : case IrOpcode::kPlainPrimitiveToWord32:
952 9 : result = LowerPlainPrimitiveToWord32(node);
953 9 : break;
954 : case IrOpcode::kPlainPrimitiveToFloat64:
955 0 : result = LowerPlainPrimitiveToFloat64(node);
956 0 : break;
957 : case IrOpcode::kEnsureWritableFastElements:
958 1690 : result = LowerEnsureWritableFastElements(node);
959 1690 : break;
960 : case IrOpcode::kMaybeGrowFastElements:
961 3538 : result = LowerMaybeGrowFastElements(node, frame_state);
962 3538 : break;
963 : case IrOpcode::kTransitionElementsKind:
964 625 : LowerTransitionElementsKind(node);
965 625 : break;
966 : case IrOpcode::kLoadMessage:
967 7583 : result = LowerLoadMessage(node);
968 7583 : break;
969 : case IrOpcode::kStoreMessage:
970 29409 : LowerStoreMessage(node);
971 29409 : break;
972 : case IrOpcode::kLoadFieldByIndex:
973 578 : result = LowerLoadFieldByIndex(node);
974 578 : break;
975 : case IrOpcode::kLoadTypedElement:
976 4600 : result = LowerLoadTypedElement(node);
977 4600 : break;
978 : case IrOpcode::kLoadDataViewElement:
979 248 : result = LowerLoadDataViewElement(node);
980 248 : break;
981 : case IrOpcode::kStoreTypedElement:
982 4128 : LowerStoreTypedElement(node);
983 4128 : break;
984 : case IrOpcode::kStoreDataViewElement:
985 180 : LowerStoreDataViewElement(node);
986 180 : break;
987 : case IrOpcode::kStoreSignedSmallElement:
988 68 : LowerStoreSignedSmallElement(node);
989 68 : break;
990 : case IrOpcode::kFindOrderedHashMapEntry:
991 174 : result = LowerFindOrderedHashMapEntry(node);
992 174 : break;
993 : case IrOpcode::kFindOrderedHashMapEntryForInt32Key:
994 15 : result = LowerFindOrderedHashMapEntryForInt32Key(node);
995 15 : break;
996 : case IrOpcode::kTransitionAndStoreNumberElement:
997 29 : LowerTransitionAndStoreNumberElement(node);
998 29 : break;
999 : case IrOpcode::kTransitionAndStoreNonNumberElement:
1000 18 : LowerTransitionAndStoreNonNumberElement(node);
1001 18 : break;
1002 : case IrOpcode::kTransitionAndStoreElement:
1003 86 : LowerTransitionAndStoreElement(node);
1004 86 : break;
1005 : case IrOpcode::kRuntimeAbort:
1006 2486 : LowerRuntimeAbort(node);
1007 2486 : break;
1008 : case IrOpcode::kConvertReceiver:
1009 869 : result = LowerConvertReceiver(node);
1010 869 : break;
1011 : case IrOpcode::kFloat64RoundUp:
1012 16430 : if (!LowerFloat64RoundUp(node).To(&result)) {
1013 : return false;
1014 : }
1015 : break;
1016 : case IrOpcode::kFloat64RoundDown:
1017 50610 : if (!LowerFloat64RoundDown(node).To(&result)) {
1018 : return false;
1019 : }
1020 : break;
1021 : case IrOpcode::kFloat64RoundTruncate:
1022 13824 : if (!LowerFloat64RoundTruncate(node).To(&result)) {
1023 : return false;
1024 : }
1025 : break;
1026 : case IrOpcode::kFloat64RoundTiesEven:
1027 530 : if (!LowerFloat64RoundTiesEven(node).To(&result)) {
1028 : return false;
1029 : }
1030 : break;
1031 : case IrOpcode::kDateNow:
1032 7 : result = LowerDateNow(node);
1033 7 : break;
1034 : default:
1035 : return false;
1036 : }
1037 :
1038 3321339 : if ((result ? 1 : 0) != node->op()->ValueOutputCount()) {
1039 0 : FATAL(
1040 : "Effect control linearizer lowering of '%s':"
1041 : " value output count does not agree.",
1042 0 : node->op()->mnemonic());
1043 : }
1044 :
1045 1107113 : *effect = gasm()->ExtractCurrentEffect();
1046 1107114 : *control = gasm()->ExtractCurrentControl();
1047 1107114 : NodeProperties::ReplaceUses(node, result, *effect, *control);
1048 1107115 : return true;
1049 : }
1050 :
1051 : #define __ gasm()->
1052 :
1053 52239 : Node* EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node) {
1054 52239 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1055 : Node* value = node->InputAt(0);
1056 :
1057 52239 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1058 52239 : auto if_heapnumber = __ MakeDeferredLabel();
1059 52239 : auto if_int32 = __ MakeLabel();
1060 :
1061 52239 : Node* value32 = __ RoundFloat64ToInt32(value);
1062 : __ GotoIf(__ Float64Equal(value, __ ChangeInt32ToFloat64(value32)),
1063 52240 : &if_int32);
1064 : __ Goto(&if_heapnumber);
1065 :
1066 : __ Bind(&if_int32);
1067 : {
1068 52240 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1069 39427 : Node* zero = __ Int32Constant(0);
1070 39427 : auto if_zero = __ MakeDeferredLabel();
1071 39427 : auto if_smi = __ MakeLabel();
1072 :
1073 39427 : __ GotoIf(__ Word32Equal(value32, zero), &if_zero);
1074 : __ Goto(&if_smi);
1075 :
1076 : __ Bind(&if_zero);
1077 : {
1078 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
1079 : __ GotoIf(__ Int32LessThan(__ Float64ExtractHighWord32(value), zero),
1080 39427 : &if_heapnumber);
1081 : __ Goto(&if_smi);
1082 : }
1083 :
1084 : __ Bind(&if_smi);
1085 : }
1086 :
1087 : if (SmiValuesAre32Bits()) {
1088 52240 : Node* value_smi = ChangeInt32ToSmi(value32);
1089 : __ Goto(&done, value_smi);
1090 : } else {
1091 : DCHECK(SmiValuesAre31Bits());
1092 : Node* add = __ Int32AddWithOverflow(value32, value32);
1093 : Node* ovf = __ Projection(1, add);
1094 : __ GotoIf(ovf, &if_heapnumber);
1095 : Node* value_smi = __ Projection(0, add);
1096 : value_smi = ChangeInt32ToIntPtr(value_smi);
1097 : __ Goto(&done, value_smi);
1098 : }
1099 : }
1100 :
1101 : __ Bind(&if_heapnumber);
1102 : {
1103 52239 : Node* value_number = AllocateHeapNumberWithValue(value);
1104 : __ Goto(&done, value_number);
1105 : }
1106 :
1107 : __ Bind(&done);
1108 52241 : return done.PhiAt(0);
1109 : }
1110 :
1111 91 : Node* EffectControlLinearizer::LowerChangeFloat64ToTaggedPointer(Node* node) {
1112 : Node* value = node->InputAt(0);
1113 91 : return AllocateHeapNumberWithValue(value);
1114 : }
1115 :
1116 9520 : Node* EffectControlLinearizer::LowerChangeBitToTagged(Node* node) {
1117 : Node* value = node->InputAt(0);
1118 :
1119 9520 : auto if_true = __ MakeLabel();
1120 9520 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1121 :
1122 9520 : __ GotoIf(value, &if_true);
1123 9520 : __ Goto(&done, __ FalseConstant());
1124 :
1125 : __ Bind(&if_true);
1126 9520 : __ Goto(&done, __ TrueConstant());
1127 :
1128 : __ Bind(&done);
1129 9520 : return done.PhiAt(0);
1130 : }
1131 :
1132 40470 : Node* EffectControlLinearizer::LowerChangeInt31ToTaggedSigned(Node* node) {
1133 : Node* value = node->InputAt(0);
1134 40470 : return ChangeInt32ToSmi(value);
1135 : }
1136 :
1137 36379 : Node* EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node) {
1138 : Node* value = node->InputAt(0);
1139 :
1140 : if (SmiValuesAre32Bits()) {
1141 36379 : return ChangeInt32ToSmi(value);
1142 : }
1143 : DCHECK(SmiValuesAre31Bits());
1144 :
1145 : auto if_overflow = __ MakeDeferredLabel();
1146 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1147 :
1148 : Node* add = __ Int32AddWithOverflow(value, value);
1149 : Node* ovf = __ Projection(1, add);
1150 : __ GotoIf(ovf, &if_overflow);
1151 : Node* value_smi = __ Projection(0, add);
1152 : value_smi = ChangeInt32ToIntPtr(value_smi);
1153 : __ Goto(&done, value_smi);
1154 :
1155 : __ Bind(&if_overflow);
1156 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt32ToFloat64(value));
1157 : __ Goto(&done, number);
1158 :
1159 : __ Bind(&done);
1160 : return done.PhiAt(0);
1161 : }
1162 :
1163 252 : Node* EffectControlLinearizer::LowerChangeInt64ToTagged(Node* node) {
1164 : Node* value = node->InputAt(0);
1165 :
1166 252 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1167 252 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1168 :
1169 252 : Node* value32 = __ TruncateInt64ToInt32(value);
1170 : __ GotoIfNot(__ Word64Equal(__ ChangeInt32ToInt64(value32), value),
1171 252 : &if_not_in_smi_range);
1172 :
1173 : if (SmiValuesAre32Bits()) {
1174 : Node* value_smi = ChangeInt64ToSmi(value);
1175 : __ Goto(&done, value_smi);
1176 : } else {
1177 : Node* add = __ Int32AddWithOverflow(value32, value32);
1178 : Node* ovf = __ Projection(1, add);
1179 : __ GotoIf(ovf, &if_not_in_smi_range);
1180 : Node* value_smi = ChangeInt32ToIntPtr(__ Projection(0, add));
1181 : __ Goto(&done, value_smi);
1182 : }
1183 :
1184 : __ Bind(&if_not_in_smi_range);
1185 252 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1186 : __ Goto(&done, number);
1187 :
1188 : __ Bind(&done);
1189 252 : return done.PhiAt(0);
1190 : }
1191 :
1192 541 : Node* EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node) {
1193 : Node* value = node->InputAt(0);
1194 :
1195 541 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1196 541 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1197 :
1198 541 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
1199 541 : __ GotoIfNot(check, &if_not_in_smi_range);
1200 541 : __ Goto(&done, ChangeUint32ToSmi(value));
1201 :
1202 : __ Bind(&if_not_in_smi_range);
1203 541 : Node* number = AllocateHeapNumberWithValue(__ ChangeUint32ToFloat64(value));
1204 :
1205 : __ Goto(&done, number);
1206 : __ Bind(&done);
1207 :
1208 541 : return done.PhiAt(0);
1209 : }
1210 :
1211 81 : Node* EffectControlLinearizer::LowerChangeUint64ToTagged(Node* node) {
1212 : Node* value = node->InputAt(0);
1213 :
1214 81 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1215 81 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1216 :
1217 : Node* check =
1218 81 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
1219 81 : __ GotoIfNot(check, &if_not_in_smi_range);
1220 : __ Goto(&done, ChangeInt64ToSmi(value));
1221 :
1222 : __ Bind(&if_not_in_smi_range);
1223 81 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1224 :
1225 : __ Goto(&done, number);
1226 : __ Bind(&done);
1227 :
1228 81 : return done.PhiAt(0);
1229 : }
1230 :
1231 52616 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt32(Node* node) {
1232 : Node* value = node->InputAt(0);
1233 52616 : return ChangeSmiToInt32(value);
1234 : }
1235 :
1236 296 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt64(Node* node) {
1237 : Node* value = node->InputAt(0);
1238 296 : return ChangeSmiToInt64(value);
1239 : }
1240 :
1241 135144 : Node* EffectControlLinearizer::LowerChangeTaggedToBit(Node* node) {
1242 : Node* value = node->InputAt(0);
1243 135144 : return __ WordEqual(value, __ TrueConstant());
1244 : }
1245 :
1246 47384 : void EffectControlLinearizer::TruncateTaggedPointerToBit(
1247 : Node* node, GraphAssemblerLabel<1>* done) {
1248 : Node* value = node->InputAt(0);
1249 :
1250 47384 : auto if_heapnumber = __ MakeDeferredLabel();
1251 47384 : auto if_bigint = __ MakeDeferredLabel();
1252 :
1253 47384 : Node* zero = __ Int32Constant(0);
1254 47384 : Node* fzero = __ Float64Constant(0.0);
1255 :
1256 : // Check if {value} is false.
1257 47384 : __ GotoIf(__ WordEqual(value, __ FalseConstant()), done, zero);
1258 :
1259 : // Check if {value} is the empty string.
1260 47384 : __ GotoIf(__ WordEqual(value, __ EmptyStringConstant()), done, zero);
1261 :
1262 : // Load the map of {value}.
1263 47384 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1264 :
1265 : // Check if the {value} is undetectable and immediately return false.
1266 : // This includes undefined and null.
1267 : Node* value_map_bitfield =
1268 47384 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1269 : __ GotoIfNot(
1270 : __ Word32Equal(
1271 : __ Word32And(value_map_bitfield,
1272 : __ Int32Constant(Map::IsUndetectableBit::kMask)),
1273 : zero),
1274 47384 : done, zero);
1275 :
1276 : // Check if {value} is a HeapNumber.
1277 : __ GotoIf(__ WordEqual(value_map, __ HeapNumberMapConstant()),
1278 47384 : &if_heapnumber);
1279 :
1280 : // Check if {value} is a BigInt.
1281 : Node* value_instance_type =
1282 47384 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1283 : __ GotoIf(__ Word32Equal(value_instance_type, __ Int32Constant(BIGINT_TYPE)),
1284 47384 : &if_bigint);
1285 :
1286 : // All other values that reach here are true.
1287 47384 : __ Goto(done, __ Int32Constant(1));
1288 :
1289 : __ Bind(&if_heapnumber);
1290 : {
1291 : // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or
1292 : // NaN.
1293 : Node* value_value =
1294 47384 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1295 47384 : __ Goto(done, __ Float64LessThan(fzero, __ Float64Abs(value_value)));
1296 : }
1297 :
1298 : __ Bind(&if_bigint);
1299 : {
1300 47384 : Node* bitfield = __ LoadField(AccessBuilder::ForBigIntBitfield(), value);
1301 : Node* length_is_zero = __ Word32Equal(
1302 : __ Word32And(bitfield, __ Int32Constant(BigInt::LengthBits::kMask)),
1303 47384 : __ Int32Constant(0));
1304 47384 : __ Goto(done, __ Word32Equal(length_is_zero, zero));
1305 : }
1306 47384 : }
1307 :
1308 46923 : Node* EffectControlLinearizer::LowerTruncateTaggedToBit(Node* node) {
1309 46923 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1310 46923 : auto if_smi = __ MakeDeferredLabel();
1311 :
1312 : Node* value = node->InputAt(0);
1313 46923 : __ GotoIf(ObjectIsSmi(value), &if_smi);
1314 :
1315 46923 : TruncateTaggedPointerToBit(node, &done);
1316 :
1317 : __ Bind(&if_smi);
1318 : {
1319 : // If {value} is a Smi, then we only need to check that it's not zero.
1320 : __ Goto(&done, __ Word32Equal(__ WordEqual(value, __ IntPtrConstant(0)),
1321 46923 : __ Int32Constant(0)));
1322 : }
1323 :
1324 : __ Bind(&done);
1325 46923 : return done.PhiAt(0);
1326 : }
1327 :
1328 461 : Node* EffectControlLinearizer::LowerTruncateTaggedPointerToBit(Node* node) {
1329 461 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1330 :
1331 461 : TruncateTaggedPointerToBit(node, &done);
1332 :
1333 : __ Bind(&done);
1334 461 : return done.PhiAt(0);
1335 : }
1336 :
1337 2027 : Node* EffectControlLinearizer::LowerChangeTaggedToInt32(Node* node) {
1338 : Node* value = node->InputAt(0);
1339 :
1340 2027 : auto if_not_smi = __ MakeDeferredLabel();
1341 2027 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1342 :
1343 2027 : Node* check = ObjectIsSmi(value);
1344 2027 : __ GotoIfNot(check, &if_not_smi);
1345 2027 : __ Goto(&done, ChangeSmiToInt32(value));
1346 :
1347 : __ Bind(&if_not_smi);
1348 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1349 2027 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1350 2027 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1351 : __ Goto(&done, vfalse);
1352 :
1353 : __ Bind(&done);
1354 2027 : return done.PhiAt(0);
1355 : }
1356 :
1357 452 : Node* EffectControlLinearizer::LowerChangeTaggedToUint32(Node* node) {
1358 : Node* value = node->InputAt(0);
1359 :
1360 452 : auto if_not_smi = __ MakeDeferredLabel();
1361 452 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1362 :
1363 452 : Node* check = ObjectIsSmi(value);
1364 452 : __ GotoIfNot(check, &if_not_smi);
1365 452 : __ Goto(&done, ChangeSmiToInt32(value));
1366 :
1367 : __ Bind(&if_not_smi);
1368 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1369 452 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1370 452 : vfalse = __ ChangeFloat64ToUint32(vfalse);
1371 : __ Goto(&done, vfalse);
1372 :
1373 : __ Bind(&done);
1374 452 : return done.PhiAt(0);
1375 : }
1376 :
1377 5 : Node* EffectControlLinearizer::LowerChangeTaggedToInt64(Node* node) {
1378 : Node* value = node->InputAt(0);
1379 :
1380 5 : auto if_not_smi = __ MakeDeferredLabel();
1381 5 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
1382 :
1383 5 : Node* check = ObjectIsSmi(value);
1384 5 : __ GotoIfNot(check, &if_not_smi);
1385 5 : __ Goto(&done, ChangeSmiToInt64(value));
1386 :
1387 : __ Bind(&if_not_smi);
1388 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1389 5 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1390 5 : vfalse = __ ChangeFloat64ToInt64(vfalse);
1391 : __ Goto(&done, vfalse);
1392 :
1393 : __ Bind(&done);
1394 5 : return done.PhiAt(0);
1395 : }
1396 :
1397 0 : Node* EffectControlLinearizer::LowerChangeTaggedToFloat64(Node* node) {
1398 11079 : return LowerTruncateTaggedToFloat64(node);
1399 : }
1400 :
1401 15 : Node* EffectControlLinearizer::LowerChangeTaggedToTaggedSigned(Node* node) {
1402 : Node* value = node->InputAt(0);
1403 :
1404 15 : auto if_not_smi = __ MakeDeferredLabel();
1405 15 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1406 :
1407 15 : Node* check = ObjectIsSmi(value);
1408 15 : __ GotoIfNot(check, &if_not_smi);
1409 : __ Goto(&done, value);
1410 :
1411 : __ Bind(&if_not_smi);
1412 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1413 15 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1414 15 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1415 15 : vfalse = ChangeInt32ToSmi(vfalse);
1416 : __ Goto(&done, vfalse);
1417 :
1418 : __ Bind(&done);
1419 15 : return done.PhiAt(0);
1420 : }
1421 :
1422 11799 : Node* EffectControlLinearizer::LowerTruncateTaggedToFloat64(Node* node) {
1423 : Node* value = node->InputAt(0);
1424 :
1425 11799 : auto if_not_smi = __ MakeDeferredLabel();
1426 11799 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
1427 :
1428 11799 : Node* check = ObjectIsSmi(value);
1429 11799 : __ GotoIfNot(check, &if_not_smi);
1430 11799 : Node* vtrue = ChangeSmiToInt32(value);
1431 11799 : vtrue = __ ChangeInt32ToFloat64(vtrue);
1432 : __ Goto(&done, vtrue);
1433 :
1434 : __ Bind(&if_not_smi);
1435 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1436 11799 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1437 : __ Goto(&done, vfalse);
1438 :
1439 : __ Bind(&done);
1440 11799 : return done.PhiAt(0);
1441 : }
1442 :
1443 1177 : Node* EffectControlLinearizer::LowerPoisonIndex(Node* node) {
1444 : Node* index = node->InputAt(0);
1445 1177 : if (mask_array_index_ == kMaskArrayIndex) {
1446 0 : index = __ Word32PoisonOnSpeculation(index);
1447 : }
1448 1177 : return index;
1449 : }
1450 :
1451 56391 : void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
1452 56391 : CheckMapsParameters const& p = CheckMapsParametersOf(node->op());
1453 : Node* value = node->InputAt(0);
1454 :
1455 56391 : ZoneHandleSet<Map> const& maps = p.maps();
1456 : size_t const map_count = maps.size();
1457 :
1458 56391 : if (p.flags() & CheckMapsFlag::kTryMigrateInstance) {
1459 634 : auto done = __ MakeLabel();
1460 634 : auto migrate = __ MakeDeferredLabel();
1461 :
1462 : // Load the current map of the {value}.
1463 634 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1464 :
1465 : // Perform the map checks.
1466 1391 : for (size_t i = 0; i < map_count; ++i) {
1467 757 : Node* map = __ HeapConstant(maps[i]);
1468 757 : Node* check = __ WordEqual(value_map, map);
1469 757 : if (i == map_count - 1) {
1470 634 : __ Branch(check, &done, &migrate, IsSafetyCheck::kCriticalSafetyCheck);
1471 : } else {
1472 123 : auto next_map = __ MakeLabel();
1473 123 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1474 : __ Bind(&next_map);
1475 : }
1476 : }
1477 :
1478 : // Perform the (deferred) instance migration.
1479 : __ Bind(&migrate);
1480 : {
1481 : // If map is not deprecated the migration attempt does not make sense.
1482 : Node* bitfield3 =
1483 634 : __ LoadField(AccessBuilder::ForMapBitField3(), value_map);
1484 : Node* if_not_deprecated = __ WordEqual(
1485 : __ Word32And(bitfield3,
1486 : __ Int32Constant(Map::IsDeprecatedBit::kMask)),
1487 634 : __ Int32Constant(0));
1488 634 : __ DeoptimizeIf(DeoptimizeReason::kWrongMap, p.feedback(),
1489 : if_not_deprecated, frame_state,
1490 634 : IsSafetyCheck::kCriticalSafetyCheck);
1491 :
1492 634 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
1493 : Runtime::FunctionId id = Runtime::kTryMigrateInstance;
1494 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1495 634 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
1496 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1), value,
1497 : __ ExternalConstant(ExternalReference::Create(id)),
1498 634 : __ Int32Constant(1), __ NoContextConstant());
1499 634 : Node* check = ObjectIsSmi(result);
1500 : __ DeoptimizeIf(DeoptimizeReason::kInstanceMigrationFailed, p.feedback(),
1501 634 : check, frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1502 : }
1503 :
1504 : // Reload the current map of the {value}.
1505 634 : value_map = __ LoadField(AccessBuilder::ForMap(), value);
1506 :
1507 : // Perform the map checks again.
1508 1391 : for (size_t i = 0; i < map_count; ++i) {
1509 757 : Node* map = __ HeapConstant(maps[i]);
1510 757 : Node* check = __ WordEqual(value_map, map);
1511 757 : if (i == map_count - 1) {
1512 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1513 634 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1514 : } else {
1515 123 : auto next_map = __ MakeLabel();
1516 123 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1517 : __ Bind(&next_map);
1518 : }
1519 : }
1520 :
1521 : __ Goto(&done);
1522 : __ Bind(&done);
1523 : } else {
1524 55757 : auto done = __ MakeLabel();
1525 :
1526 : // Load the current map of the {value}.
1527 55757 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1528 :
1529 117971 : for (size_t i = 0; i < map_count; ++i) {
1530 62214 : Node* map = __ HeapConstant(maps[i]);
1531 62214 : Node* check = __ WordEqual(value_map, map);
1532 62214 : if (i == map_count - 1) {
1533 55757 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1534 55757 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1535 : } else {
1536 6457 : auto next_map = __ MakeLabel();
1537 6457 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1538 : __ Bind(&next_map);
1539 : }
1540 : }
1541 : __ Goto(&done);
1542 : __ Bind(&done);
1543 : }
1544 56391 : }
1545 :
1546 7486 : Node* EffectControlLinearizer::LowerCompareMaps(Node* node) {
1547 7486 : ZoneHandleSet<Map> const& maps = CompareMapsParametersOf(node->op()).maps();
1548 : size_t const map_count = maps.size();
1549 : Node* value = node->InputAt(0);
1550 :
1551 7486 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1552 :
1553 : // Load the current map of the {value}.
1554 7486 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1555 :
1556 14972 : for (size_t i = 0; i < map_count; ++i) {
1557 7486 : Node* map = __ HeapConstant(maps[i]);
1558 7486 : Node* check = __ WordEqual(value_map, map);
1559 7486 : auto next_map = __ MakeLabel();
1560 7486 : auto passed = __ MakeLabel();
1561 7486 : __ Branch(check, &passed, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1562 :
1563 : __ Bind(&passed);
1564 7486 : __ Goto(&done, __ Int32Constant(1));
1565 :
1566 : __ Bind(&next_map);
1567 : }
1568 7486 : __ Goto(&done, __ Int32Constant(0));
1569 :
1570 : __ Bind(&done);
1571 7486 : return done.PhiAt(0);
1572 : }
1573 :
1574 946 : Node* EffectControlLinearizer::LowerCheckNumber(Node* node, Node* frame_state) {
1575 : Node* value = node->InputAt(0);
1576 473 : const CheckParameters& params = CheckParametersOf(node->op());
1577 :
1578 473 : auto if_not_smi = __ MakeDeferredLabel();
1579 473 : auto done = __ MakeLabel();
1580 :
1581 473 : Node* check0 = ObjectIsSmi(value);
1582 473 : __ GotoIfNot(check0, &if_not_smi);
1583 : __ Goto(&done);
1584 :
1585 : __ Bind(&if_not_smi);
1586 473 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1587 473 : Node* check1 = __ WordEqual(value_map, __ HeapNumberMapConstant());
1588 473 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
1589 473 : check1, frame_state);
1590 : __ Goto(&done);
1591 :
1592 : __ Bind(&done);
1593 473 : return value;
1594 : }
1595 :
1596 813 : Node* EffectControlLinearizer::LowerCheckReceiver(Node* node,
1597 : Node* frame_state) {
1598 : Node* value = node->InputAt(0);
1599 :
1600 813 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1601 : Node* value_instance_type =
1602 813 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1603 :
1604 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1605 : Node* check = __ Uint32LessThanOrEqual(
1606 813 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1607 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObject, VectorSlotPair(),
1608 813 : check, frame_state);
1609 813 : return value;
1610 : }
1611 :
1612 72 : Node* EffectControlLinearizer::LowerCheckReceiverOrNullOrUndefined(
1613 : Node* node, Node* frame_state) {
1614 : Node* value = node->InputAt(0);
1615 :
1616 72 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1617 : Node* value_instance_type =
1618 72 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1619 :
1620 : // Rule out all primitives except oddballs (true, false, undefined, null).
1621 : STATIC_ASSERT(LAST_PRIMITIVE_TYPE == ODDBALL_TYPE);
1622 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1623 : Node* check0 = __ Uint32LessThanOrEqual(__ Uint32Constant(ODDBALL_TYPE),
1624 72 : value_instance_type);
1625 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1626 72 : VectorSlotPair(), check0, frame_state);
1627 :
1628 : // Rule out booleans.
1629 72 : Node* check1 = __ WordEqual(value_map, __ BooleanMapConstant());
1630 : __ DeoptimizeIf(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1631 72 : VectorSlotPair(), check1, frame_state);
1632 72 : return value;
1633 : }
1634 :
1635 31 : Node* EffectControlLinearizer::LowerCheckSymbol(Node* node, Node* frame_state) {
1636 : Node* value = node->InputAt(0);
1637 :
1638 31 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1639 :
1640 : Node* check =
1641 31 : __ WordEqual(value_map, __ HeapConstant(factory()->symbol_map()));
1642 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASymbol, VectorSlotPair(), check,
1643 31 : frame_state);
1644 31 : return value;
1645 : }
1646 :
1647 12394 : Node* EffectControlLinearizer::LowerCheckString(Node* node, Node* frame_state) {
1648 : Node* value = node->InputAt(0);
1649 6197 : const CheckParameters& params = CheckParametersOf(node->op());
1650 :
1651 6197 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1652 : Node* value_instance_type =
1653 6197 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1654 :
1655 : Node* check = __ Uint32LessThan(value_instance_type,
1656 6197 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
1657 6197 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAString, params.feedback(), check,
1658 6197 : frame_state);
1659 6197 : return value;
1660 : }
1661 :
1662 1415 : Node* EffectControlLinearizer::LowerCheckInternalizedString(Node* node,
1663 : Node* frame_state) {
1664 : Node* value = node->InputAt(0);
1665 :
1666 1415 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1667 : Node* value_instance_type =
1668 1415 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1669 :
1670 : Node* check = __ Word32Equal(
1671 : __ Word32And(value_instance_type,
1672 : __ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
1673 1415 : __ Int32Constant(kInternalizedTag));
1674 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1675 1415 : check, frame_state);
1676 :
1677 1415 : return value;
1678 : }
1679 :
1680 30264 : void EffectControlLinearizer::LowerCheckIf(Node* node, Node* frame_state) {
1681 : Node* value = node->InputAt(0);
1682 30264 : const CheckIfParameters& p = CheckIfParametersOf(node->op());
1683 30264 : __ DeoptimizeIfNot(p.reason(), p.feedback(), value, frame_state);
1684 15132 : }
1685 :
1686 41310 : Node* EffectControlLinearizer::LowerStringConcat(Node* node) {
1687 : Node* lhs = node->InputAt(1);
1688 : Node* rhs = node->InputAt(2);
1689 :
1690 : Callable const callable =
1691 20655 : CodeFactory::StringAdd(isolate(), STRING_ADD_CHECK_NONE);
1692 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1693 : graph()->zone(), callable.descriptor(),
1694 : callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags,
1695 41310 : Operator::kNoDeopt | Operator::kNoWrite | Operator::kNoThrow);
1696 :
1697 : Node* value =
1698 : __ Call(call_descriptor, jsgraph()->HeapConstant(callable.code()), lhs,
1699 41310 : rhs, __ NoContextConstant());
1700 :
1701 20655 : return value;
1702 : }
1703 :
1704 55973 : Node* EffectControlLinearizer::LowerCheckedInt32Add(Node* node,
1705 : Node* frame_state) {
1706 : Node* lhs = node->InputAt(0);
1707 : Node* rhs = node->InputAt(1);
1708 :
1709 55973 : Node* value = __ Int32AddWithOverflow(lhs, rhs);
1710 55971 : Node* check = __ Projection(1, value);
1711 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1712 55971 : frame_state);
1713 55973 : return __ Projection(0, value);
1714 : }
1715 :
1716 7449 : Node* EffectControlLinearizer::LowerCheckedInt32Sub(Node* node,
1717 : Node* frame_state) {
1718 : Node* lhs = node->InputAt(0);
1719 : Node* rhs = node->InputAt(1);
1720 :
1721 7449 : Node* value = __ Int32SubWithOverflow(lhs, rhs);
1722 7449 : Node* check = __ Projection(1, value);
1723 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1724 7449 : frame_state);
1725 7449 : return __ Projection(0, value);
1726 : }
1727 :
1728 239 : Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node,
1729 : Node* frame_state) {
1730 : Node* lhs = node->InputAt(0);
1731 : Node* rhs = node->InputAt(1);
1732 239 : Node* zero = __ Int32Constant(0);
1733 :
1734 : // Check if the {rhs} is a known power of two.
1735 : Int32Matcher m(rhs);
1736 239 : if (m.IsPowerOf2()) {
1737 : // Since we know that {rhs} is a power of two, we can perform a fast
1738 : // check to see if the relevant least significant bits of the {lhs}
1739 : // are all zero, and if so we know that we can perform a division
1740 : // safely (and fast by doing an arithmetic - aka sign preserving -
1741 : // right shift on {lhs}).
1742 : int32_t divisor = m.Value();
1743 139 : Node* mask = __ Int32Constant(divisor - 1);
1744 139 : Node* shift = __ Int32Constant(WhichPowerOf2(divisor));
1745 139 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
1746 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1747 139 : check, frame_state);
1748 139 : return __ Word32Sar(lhs, shift);
1749 : } else {
1750 100 : auto if_rhs_positive = __ MakeLabel();
1751 100 : auto if_rhs_negative = __ MakeDeferredLabel();
1752 100 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1753 :
1754 : // Check if {rhs} is positive (and not zero).
1755 100 : Node* check_rhs_positive = __ Int32LessThan(zero, rhs);
1756 100 : __ Branch(check_rhs_positive, &if_rhs_positive, &if_rhs_negative);
1757 :
1758 : __ Bind(&if_rhs_positive);
1759 : {
1760 : // Fast case, no additional checking required.
1761 100 : __ Goto(&done, __ Int32Div(lhs, rhs));
1762 : }
1763 :
1764 : __ Bind(&if_rhs_negative);
1765 : {
1766 100 : auto if_lhs_minint = __ MakeDeferredLabel();
1767 100 : auto if_lhs_notminint = __ MakeLabel();
1768 :
1769 : // Check if {rhs} is zero.
1770 100 : Node* check_rhs_zero = __ Word32Equal(rhs, zero);
1771 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
1772 100 : check_rhs_zero, frame_state);
1773 :
1774 : // Check if {lhs} is zero, as that would produce minus zero.
1775 100 : Node* check_lhs_zero = __ Word32Equal(lhs, zero);
1776 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
1777 100 : check_lhs_zero, frame_state);
1778 :
1779 : // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
1780 : // to return -kMinInt, which is not representable as Word32.
1781 : Node* check_lhs_minint = graph()->NewNode(machine()->Word32Equal(), lhs,
1782 200 : __ Int32Constant(kMinInt));
1783 100 : __ Branch(check_lhs_minint, &if_lhs_minint, &if_lhs_notminint);
1784 :
1785 : __ Bind(&if_lhs_minint);
1786 : {
1787 : // Check that {rhs} is not -1, otherwise result would be -kMinInt.
1788 100 : Node* check_rhs_minusone = __ Word32Equal(rhs, __ Int32Constant(-1));
1789 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(),
1790 100 : check_rhs_minusone, frame_state);
1791 :
1792 : // Perform the actual integer division.
1793 100 : __ Goto(&done, __ Int32Div(lhs, rhs));
1794 : }
1795 :
1796 : __ Bind(&if_lhs_notminint);
1797 : {
1798 : // Perform the actual integer division.
1799 100 : __ Goto(&done, __ Int32Div(lhs, rhs));
1800 : }
1801 : }
1802 :
1803 : __ Bind(&done);
1804 : Node* value = done.PhiAt(0);
1805 :
1806 : // Check if the remainder is non-zero.
1807 100 : Node* check = __ Word32Equal(lhs, __ Int32Mul(value, rhs));
1808 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1809 100 : check, frame_state);
1810 :
1811 : return value;
1812 : }
1813 : }
1814 :
1815 886 : Node* EffectControlLinearizer::BuildUint32Mod(Node* lhs, Node* rhs) {
1816 886 : auto if_rhs_power_of_two = __ MakeLabel();
1817 886 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1818 :
1819 : // Compute the mask for the {rhs}.
1820 886 : Node* one = __ Int32Constant(1);
1821 886 : Node* msk = __ Int32Sub(rhs, one);
1822 :
1823 : // Check if the {rhs} is a power of two.
1824 : __ GotoIf(__ Word32Equal(__ Word32And(rhs, msk), __ Int32Constant(0)),
1825 886 : &if_rhs_power_of_two);
1826 : {
1827 : // The {rhs} is not a power of two, do a generic Uint32Mod.
1828 886 : __ Goto(&done, __ Uint32Mod(lhs, rhs));
1829 : }
1830 :
1831 : __ Bind(&if_rhs_power_of_two);
1832 : {
1833 : // The {rhs} is a power of two, just do a fast bit masking.
1834 886 : __ Goto(&done, __ Word32And(lhs, msk));
1835 : }
1836 :
1837 : __ Bind(&done);
1838 886 : return done.PhiAt(0);
1839 : }
1840 :
1841 849 : Node* EffectControlLinearizer::LowerCheckedInt32Mod(Node* node,
1842 : Node* frame_state) {
1843 : // General case for signed integer modulus, with optimization for (unknown)
1844 : // power of 2 right hand side.
1845 : //
1846 : // if rhs <= 0 then
1847 : // rhs = -rhs
1848 : // deopt if rhs == 0
1849 : // let msk = rhs - 1 in
1850 : // if lhs < 0 then
1851 : // let lhs_abs = -lsh in
1852 : // let res = if rhs & msk == 0 then
1853 : // lhs_abs & msk
1854 : // else
1855 : // lhs_abs % rhs in
1856 : // if lhs < 0 then
1857 : // deopt if res == 0
1858 : // -res
1859 : // else
1860 : // res
1861 : // else
1862 : // if rhs & msk == 0 then
1863 : // lhs & msk
1864 : // else
1865 : // lhs % rhs
1866 : //
1867 : Node* lhs = node->InputAt(0);
1868 : Node* rhs = node->InputAt(1);
1869 :
1870 849 : auto if_rhs_not_positive = __ MakeDeferredLabel();
1871 849 : auto if_lhs_negative = __ MakeDeferredLabel();
1872 : auto if_rhs_power_of_two = __ MakeLabel();
1873 849 : auto rhs_checked = __ MakeLabel(MachineRepresentation::kWord32);
1874 849 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1875 :
1876 849 : Node* zero = __ Int32Constant(0);
1877 :
1878 : // Check if {rhs} is not strictly positive.
1879 849 : Node* check0 = __ Int32LessThanOrEqual(rhs, zero);
1880 849 : __ GotoIf(check0, &if_rhs_not_positive);
1881 : __ Goto(&rhs_checked, rhs);
1882 :
1883 : __ Bind(&if_rhs_not_positive);
1884 : {
1885 : // Negate {rhs}, might still produce a negative result in case of
1886 : // -2^31, but that is handled safely below.
1887 849 : Node* vtrue0 = __ Int32Sub(zero, rhs);
1888 :
1889 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1890 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
1891 849 : __ Word32Equal(vtrue0, zero), frame_state);
1892 : __ Goto(&rhs_checked, vtrue0);
1893 : }
1894 :
1895 : __ Bind(&rhs_checked);
1896 : rhs = rhs_checked.PhiAt(0);
1897 :
1898 849 : __ GotoIf(__ Int32LessThan(lhs, zero), &if_lhs_negative);
1899 : {
1900 : // The {lhs} is a non-negative integer.
1901 849 : __ Goto(&done, BuildUint32Mod(lhs, rhs));
1902 : }
1903 :
1904 : __ Bind(&if_lhs_negative);
1905 : {
1906 : // The {lhs} is a negative integer. This is very unlikely and
1907 : // we intentionally don't use the BuildUint32Mod() here, which
1908 : // would try to figure out whether {rhs} is a power of two,
1909 : // since this is intended to be a slow-path.
1910 849 : Node* res = __ Uint32Mod(__ Int32Sub(zero, lhs), rhs);
1911 :
1912 : // Check if we would have to return -0.
1913 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
1914 849 : __ Word32Equal(res, zero), frame_state);
1915 849 : __ Goto(&done, __ Int32Sub(zero, res));
1916 : }
1917 :
1918 : __ Bind(&done);
1919 849 : return done.PhiAt(0);
1920 : }
1921 :
1922 47 : Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node,
1923 : Node* frame_state) {
1924 : Node* lhs = node->InputAt(0);
1925 : Node* rhs = node->InputAt(1);
1926 47 : Node* zero = __ Int32Constant(0);
1927 :
1928 : // Check if the {rhs} is a known power of two.
1929 : Uint32Matcher m(rhs);
1930 47 : if (m.IsPowerOf2()) {
1931 : // Since we know that {rhs} is a power of two, we can perform a fast
1932 : // check to see if the relevant least significant bits of the {lhs}
1933 : // are all zero, and if so we know that we can perform a division
1934 : // safely (and fast by doing a logical - aka zero extending - right
1935 : // shift on {lhs}).
1936 : uint32_t divisor = m.Value();
1937 36 : Node* mask = __ Uint32Constant(divisor - 1);
1938 36 : Node* shift = __ Uint32Constant(WhichPowerOf2(divisor));
1939 36 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
1940 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1941 36 : check, frame_state);
1942 36 : return __ Word32Shr(lhs, shift);
1943 : } else {
1944 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1945 11 : Node* check = __ Word32Equal(rhs, zero);
1946 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
1947 11 : frame_state);
1948 :
1949 : // Perform the actual unsigned integer division.
1950 11 : Node* value = __ Uint32Div(lhs, rhs);
1951 :
1952 : // Check if the remainder is non-zero.
1953 11 : check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
1954 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1955 11 : check, frame_state);
1956 11 : return value;
1957 : }
1958 : }
1959 :
1960 37 : Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node,
1961 : Node* frame_state) {
1962 : Node* lhs = node->InputAt(0);
1963 : Node* rhs = node->InputAt(1);
1964 :
1965 37 : Node* zero = __ Int32Constant(0);
1966 :
1967 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
1968 37 : Node* check = __ Word32Equal(rhs, zero);
1969 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
1970 37 : frame_state);
1971 :
1972 : // Perform the actual unsigned integer modulus.
1973 37 : return BuildUint32Mod(lhs, rhs);
1974 : }
1975 :
1976 5679 : Node* EffectControlLinearizer::LowerCheckedInt32Mul(Node* node,
1977 : Node* frame_state) {
1978 5679 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1979 : Node* lhs = node->InputAt(0);
1980 : Node* rhs = node->InputAt(1);
1981 :
1982 5679 : Node* projection = __ Int32MulWithOverflow(lhs, rhs);
1983 5679 : Node* check = __ Projection(1, projection);
1984 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1985 5679 : frame_state);
1986 :
1987 5679 : Node* value = __ Projection(0, projection);
1988 :
1989 5679 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1990 3934 : auto if_zero = __ MakeDeferredLabel();
1991 3934 : auto check_done = __ MakeLabel();
1992 3934 : Node* zero = __ Int32Constant(0);
1993 3934 : Node* check_zero = __ Word32Equal(value, zero);
1994 3934 : __ GotoIf(check_zero, &if_zero);
1995 : __ Goto(&check_done);
1996 :
1997 : __ Bind(&if_zero);
1998 : // We may need to return negative zero.
1999 3934 : Node* check_or = __ Int32LessThan(__ Word32Or(lhs, rhs), zero);
2000 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(), check_or,
2001 3934 : frame_state);
2002 : __ Goto(&check_done);
2003 :
2004 : __ Bind(&check_done);
2005 : }
2006 :
2007 5679 : return value;
2008 : }
2009 :
2010 0 : Node* EffectControlLinearizer::LowerCheckedInt32ToTaggedSigned(
2011 0 : Node* node, Node* frame_state) {
2012 : DCHECK(SmiValuesAre31Bits());
2013 : Node* value = node->InputAt(0);
2014 0 : const CheckParameters& params = CheckParametersOf(node->op());
2015 :
2016 0 : Node* add = __ Int32AddWithOverflow(value, value);
2017 0 : Node* check = __ Projection(1, add);
2018 0 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2019 0 : frame_state);
2020 0 : Node* result = __ Projection(0, add);
2021 : result = ChangeInt32ToIntPtr(result);
2022 0 : return result;
2023 : }
2024 :
2025 14 : Node* EffectControlLinearizer::LowerCheckedInt64ToInt32(Node* node,
2026 : Node* frame_state) {
2027 : Node* value = node->InputAt(0);
2028 7 : const CheckParameters& params = CheckParametersOf(node->op());
2029 :
2030 7 : Node* value32 = __ TruncateInt64ToInt32(value);
2031 7 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2032 7 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2033 7 : frame_state);
2034 7 : return value32;
2035 : }
2036 :
2037 150 : Node* EffectControlLinearizer::LowerCheckedInt64ToTaggedSigned(
2038 150 : Node* node, Node* frame_state) {
2039 : Node* value = node->InputAt(0);
2040 150 : const CheckParameters& params = CheckParametersOf(node->op());
2041 :
2042 150 : Node* value32 = __ TruncateInt64ToInt32(value);
2043 150 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2044 150 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2045 150 : frame_state);
2046 :
2047 : if (SmiValuesAre32Bits()) {
2048 : return ChangeInt64ToSmi(value);
2049 : } else {
2050 : Node* add = __ Int32AddWithOverflow(value32, value32);
2051 : Node* check = __ Projection(1, add);
2052 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2053 : frame_state);
2054 : Node* result = __ Projection(0, add);
2055 : result = ChangeInt32ToIntPtr(result);
2056 : return result;
2057 : }
2058 : }
2059 :
2060 103932 : Node* EffectControlLinearizer::LowerCheckedUint32Bounds(Node* node,
2061 : Node* frame_state) {
2062 : Node* index = node->InputAt(0);
2063 : Node* limit = node->InputAt(1);
2064 103932 : const CheckBoundsParameters& params = CheckBoundsParametersOf(node->op());
2065 :
2066 51966 : Node* check = __ Uint32LessThan(index, limit);
2067 51966 : switch (params.mode()) {
2068 : case CheckBoundsParameters::kDeoptOnOutOfBounds:
2069 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds,
2070 47577 : params.check_parameters().feedback(), check,
2071 47577 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2072 47577 : break;
2073 : case CheckBoundsParameters::kAbortOnOutOfBounds: {
2074 4389 : auto if_abort = __ MakeDeferredLabel();
2075 4389 : auto done = __ MakeLabel();
2076 :
2077 4389 : __ Branch(check, &done, &if_abort);
2078 :
2079 : __ Bind(&if_abort);
2080 4389 : __ Unreachable();
2081 : __ Goto(&done);
2082 :
2083 : __ Bind(&done);
2084 : break;
2085 : }
2086 : }
2087 :
2088 51966 : return index;
2089 : }
2090 :
2091 500 : Node* EffectControlLinearizer::LowerCheckedUint32ToInt32(Node* node,
2092 : Node* frame_state) {
2093 : Node* value = node->InputAt(0);
2094 250 : const CheckParameters& params = CheckParametersOf(node->op());
2095 250 : Node* unsafe = __ Int32LessThan(value, __ Int32Constant(0));
2096 250 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), unsafe,
2097 250 : frame_state);
2098 250 : return value;
2099 : }
2100 :
2101 7 : Node* EffectControlLinearizer::LowerCheckedUint32ToTaggedSigned(
2102 7 : Node* node, Node* frame_state) {
2103 : Node* value = node->InputAt(0);
2104 7 : const CheckParameters& params = CheckParametersOf(node->op());
2105 7 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
2106 7 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2107 7 : frame_state);
2108 7 : return ChangeUint32ToSmi(value);
2109 : }
2110 :
2111 176 : Node* EffectControlLinearizer::LowerCheckedUint64Bounds(Node* node,
2112 : Node* frame_state) {
2113 176 : CheckParameters const& params = CheckParametersOf(node->op());
2114 : Node* const index = node->InputAt(0);
2115 : Node* const limit = node->InputAt(1);
2116 :
2117 176 : Node* check = __ Uint64LessThan(index, limit);
2118 176 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, params.feedback(), check,
2119 176 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2120 176 : return index;
2121 : }
2122 :
2123 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToInt32(Node* node,
2124 : Node* frame_state) {
2125 : Node* value = node->InputAt(0);
2126 0 : const CheckParameters& params = CheckParametersOf(node->op());
2127 :
2128 0 : Node* check = __ Uint64LessThanOrEqual(value, __ Int64Constant(kMaxInt));
2129 0 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2130 0 : frame_state);
2131 0 : return __ TruncateInt64ToInt32(value);
2132 : }
2133 :
2134 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToTaggedSigned(
2135 0 : Node* node, Node* frame_state) {
2136 : Node* value = node->InputAt(0);
2137 0 : const CheckParameters& params = CheckParametersOf(node->op());
2138 :
2139 : Node* check =
2140 0 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
2141 0 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2142 0 : frame_state);
2143 0 : return ChangeInt64ToSmi(value);
2144 : }
2145 :
2146 4769 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt32(
2147 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2148 : Node* frame_state) {
2149 4769 : Node* value32 = __ RoundFloat64ToInt32(value);
2150 4769 : Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32));
2151 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2152 4769 : check_same, frame_state);
2153 :
2154 4769 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2155 : // Check if {value} is -0.
2156 2156 : auto if_zero = __ MakeDeferredLabel();
2157 2156 : auto check_done = __ MakeLabel();
2158 :
2159 2156 : Node* check_zero = __ Word32Equal(value32, __ Int32Constant(0));
2160 2156 : __ GotoIf(check_zero, &if_zero);
2161 : __ Goto(&check_done);
2162 :
2163 : __ Bind(&if_zero);
2164 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2165 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2166 2156 : __ Int32Constant(0));
2167 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2168 2156 : frame_state);
2169 : __ Goto(&check_done);
2170 :
2171 : __ Bind(&check_done);
2172 : }
2173 4769 : return value32;
2174 : }
2175 :
2176 2331 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt32(Node* node,
2177 : Node* frame_state) {
2178 2331 : const CheckMinusZeroParameters& params =
2179 2331 : CheckMinusZeroParametersOf(node->op());
2180 : Node* value = node->InputAt(0);
2181 2331 : return BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), value,
2182 2331 : frame_state);
2183 : }
2184 :
2185 84 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt64(
2186 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2187 : Node* frame_state) {
2188 84 : Node* value64 = __ TruncateFloat64ToInt64(value);
2189 84 : Node* check_same = __ Float64Equal(value, __ ChangeInt64ToFloat64(value64));
2190 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2191 84 : check_same, frame_state);
2192 :
2193 84 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2194 : // Check if {value} is -0.
2195 0 : auto if_zero = __ MakeDeferredLabel();
2196 0 : auto check_done = __ MakeLabel();
2197 :
2198 0 : Node* check_zero = __ Word64Equal(value64, __ Int64Constant(0));
2199 0 : __ GotoIf(check_zero, &if_zero);
2200 : __ Goto(&check_done);
2201 :
2202 : __ Bind(&if_zero);
2203 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2204 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2205 0 : __ Int32Constant(0));
2206 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2207 0 : frame_state);
2208 : __ Goto(&check_done);
2209 :
2210 : __ Bind(&check_done);
2211 : }
2212 84 : return value64;
2213 : }
2214 :
2215 14 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt64(Node* node,
2216 : Node* frame_state) {
2217 14 : const CheckMinusZeroParameters& params =
2218 14 : CheckMinusZeroParametersOf(node->op());
2219 : Node* value = node->InputAt(0);
2220 14 : return BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), value,
2221 14 : frame_state);
2222 : }
2223 :
2224 24599 : Node* EffectControlLinearizer::LowerCheckedTaggedSignedToInt32(
2225 24599 : Node* node, Node* frame_state) {
2226 : Node* value = node->InputAt(0);
2227 24599 : const CheckParameters& params = CheckParametersOf(node->op());
2228 24599 : Node* check = ObjectIsSmi(value);
2229 24599 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2230 24599 : frame_state);
2231 24599 : return ChangeSmiToInt32(value);
2232 : }
2233 :
2234 2438 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt32(Node* node,
2235 : Node* frame_state) {
2236 2438 : const CheckMinusZeroParameters& params =
2237 2438 : CheckMinusZeroParametersOf(node->op());
2238 : Node* value = node->InputAt(0);
2239 :
2240 2438 : auto if_not_smi = __ MakeDeferredLabel();
2241 2438 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2242 :
2243 2438 : Node* check = ObjectIsSmi(value);
2244 2438 : __ GotoIfNot(check, &if_not_smi);
2245 : // In the Smi case, just convert to int32.
2246 2438 : __ Goto(&done, ChangeSmiToInt32(value));
2247 :
2248 : // In the non-Smi case, check the heap numberness, load the number and convert
2249 : // to int32.
2250 : __ Bind(&if_not_smi);
2251 2438 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2252 2438 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2253 2438 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2254 2438 : check_map, frame_state);
2255 2438 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2256 : vfalse = BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), vfalse,
2257 2438 : frame_state);
2258 : __ Goto(&done, vfalse);
2259 :
2260 : __ Bind(&done);
2261 2438 : return done.PhiAt(0);
2262 : }
2263 :
2264 70 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt64(Node* node,
2265 : Node* frame_state) {
2266 70 : const CheckMinusZeroParameters& params =
2267 70 : CheckMinusZeroParametersOf(node->op());
2268 : Node* value = node->InputAt(0);
2269 :
2270 70 : auto if_not_smi = __ MakeDeferredLabel();
2271 70 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
2272 :
2273 70 : Node* check = ObjectIsSmi(value);
2274 70 : __ GotoIfNot(check, &if_not_smi);
2275 : // In the Smi case, just convert to int64.
2276 70 : __ Goto(&done, ChangeSmiToInt64(value));
2277 :
2278 : // In the non-Smi case, check the heap numberness, load the number and convert
2279 : // to int64.
2280 : __ Bind(&if_not_smi);
2281 70 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2282 70 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2283 70 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2284 70 : check_map, frame_state);
2285 70 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2286 : vfalse = BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), vfalse,
2287 70 : frame_state);
2288 : __ Goto(&done, vfalse);
2289 :
2290 : __ Bind(&done);
2291 70 : return done.PhiAt(0);
2292 : }
2293 :
2294 50919 : Node* EffectControlLinearizer::BuildCheckedHeapNumberOrOddballToFloat64(
2295 : CheckTaggedInputMode mode, const VectorSlotPair& feedback, Node* value,
2296 : Node* frame_state) {
2297 50919 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2298 50919 : Node* check_number = __ WordEqual(value_map, __ HeapNumberMapConstant());
2299 50919 : switch (mode) {
2300 : case CheckTaggedInputMode::kNumber: {
2301 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, feedback,
2302 8294 : check_number, frame_state);
2303 8294 : break;
2304 : }
2305 : case CheckTaggedInputMode::kNumberOrOddball: {
2306 42625 : auto check_done = __ MakeLabel();
2307 :
2308 42625 : __ GotoIf(check_number, &check_done);
2309 : // For oddballs also contain the numeric value, let us just check that
2310 : // we have an oddball here.
2311 : Node* instance_type =
2312 42625 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2313 : Node* check_oddball =
2314 42625 : __ Word32Equal(instance_type, __ Int32Constant(ODDBALL_TYPE));
2315 : __ DeoptimizeIfNot(DeoptimizeReason::kNotANumberOrOddball, feedback,
2316 42625 : check_oddball, frame_state);
2317 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
2318 : __ Goto(&check_done);
2319 :
2320 : __ Bind(&check_done);
2321 : break;
2322 : }
2323 : }
2324 50919 : return __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2325 : }
2326 :
2327 49750 : Node* EffectControlLinearizer::LowerCheckedTaggedToFloat64(Node* node,
2328 : Node* frame_state) {
2329 49750 : CheckTaggedInputParameters const& p =
2330 49750 : CheckTaggedInputParametersOf(node->op());
2331 : Node* value = node->InputAt(0);
2332 :
2333 49750 : auto if_smi = __ MakeLabel();
2334 49750 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
2335 :
2336 49750 : Node* check = ObjectIsSmi(value);
2337 49750 : __ GotoIf(check, &if_smi);
2338 :
2339 : // In the Smi case, just convert to int32 and then float64.
2340 : // Otherwise, check heap numberness and load the number.
2341 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2342 99500 : p.mode(), p.feedback(), value, frame_state);
2343 : __ Goto(&done, number);
2344 :
2345 : __ Bind(&if_smi);
2346 49750 : Node* from_smi = ChangeSmiToInt32(value);
2347 49750 : from_smi = __ ChangeInt32ToFloat64(from_smi);
2348 : __ Goto(&done, from_smi);
2349 :
2350 : __ Bind(&done);
2351 49750 : return done.PhiAt(0);
2352 : }
2353 :
2354 19187 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedSigned(
2355 19187 : Node* node, Node* frame_state) {
2356 : Node* value = node->InputAt(0);
2357 19187 : const CheckParameters& params = CheckParametersOf(node->op());
2358 :
2359 19187 : Node* check = ObjectIsSmi(value);
2360 19187 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2361 19187 : frame_state);
2362 :
2363 19187 : return value;
2364 : }
2365 :
2366 35410 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedPointer(
2367 35410 : Node* node, Node* frame_state) {
2368 : Node* value = node->InputAt(0);
2369 35410 : const CheckParameters& params = CheckParametersOf(node->op());
2370 :
2371 35410 : Node* check = ObjectIsSmi(value);
2372 35410 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2373 35410 : frame_state);
2374 35410 : return value;
2375 : }
2376 :
2377 517 : Node* EffectControlLinearizer::LowerTruncateTaggedToWord32(Node* node) {
2378 : Node* value = node->InputAt(0);
2379 :
2380 517 : auto if_not_smi = __ MakeDeferredLabel();
2381 517 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2382 :
2383 517 : Node* check = ObjectIsSmi(value);
2384 517 : __ GotoIfNot(check, &if_not_smi);
2385 517 : __ Goto(&done, ChangeSmiToInt32(value));
2386 :
2387 : __ Bind(&if_not_smi);
2388 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
2389 517 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2390 517 : vfalse = __ TruncateFloat64ToWord32(vfalse);
2391 : __ Goto(&done, vfalse);
2392 :
2393 : __ Bind(&done);
2394 517 : return done.PhiAt(0);
2395 : }
2396 :
2397 1169 : Node* EffectControlLinearizer::LowerCheckedTruncateTaggedToWord32(
2398 1169 : Node* node, Node* frame_state) {
2399 1169 : const CheckTaggedInputParameters& params =
2400 1169 : CheckTaggedInputParametersOf(node->op());
2401 : Node* value = node->InputAt(0);
2402 :
2403 1169 : auto if_not_smi = __ MakeLabel();
2404 1169 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2405 :
2406 1169 : Node* check = ObjectIsSmi(value);
2407 1169 : __ GotoIfNot(check, &if_not_smi);
2408 : // In the Smi case, just convert to int32.
2409 1169 : __ Goto(&done, ChangeSmiToInt32(value));
2410 :
2411 : // Otherwise, check that it's a heap number or oddball and truncate the value
2412 : // to int32.
2413 : __ Bind(&if_not_smi);
2414 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2415 2338 : params.mode(), params.feedback(), value, frame_state);
2416 1169 : number = __ TruncateFloat64ToWord32(number);
2417 : __ Goto(&done, number);
2418 :
2419 : __ Bind(&done);
2420 1169 : return done.PhiAt(0);
2421 : }
2422 :
2423 224984 : Node* EffectControlLinearizer::LowerAllocate(Node* node) {
2424 : Node* size = node->InputAt(0);
2425 112492 : PretenureFlag pretenure = PretenureFlagOf(node->op());
2426 112492 : Node* new_node = __ Allocate(pretenure, size);
2427 112492 : return new_node;
2428 : }
2429 :
2430 1020 : Node* EffectControlLinearizer::LowerNumberToString(Node* node) {
2431 : Node* argument = node->InputAt(0);
2432 :
2433 : Callable const callable =
2434 1020 : Builtins::CallableFor(isolate(), Builtins::kNumberToString);
2435 : Operator::Properties properties = Operator::kEliminatable;
2436 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2437 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2438 : graph()->zone(), callable.descriptor(),
2439 2040 : callable.descriptor().GetStackParameterCount(), flags, properties);
2440 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), argument,
2441 2040 : __ NoContextConstant());
2442 : }
2443 :
2444 14 : Node* EffectControlLinearizer::LowerObjectIsArrayBufferView(Node* node) {
2445 : Node* value = node->InputAt(0);
2446 :
2447 14 : auto if_smi = __ MakeDeferredLabel();
2448 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2449 :
2450 14 : Node* check = ObjectIsSmi(value);
2451 14 : __ GotoIf(check, &if_smi);
2452 :
2453 14 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2454 : Node* value_instance_type =
2455 14 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2456 : STATIC_ASSERT(JS_TYPED_ARRAY_TYPE + 1 == JS_DATA_VIEW_TYPE);
2457 : Node* vfalse = __ Uint32LessThan(
2458 : __ Int32Sub(value_instance_type, __ Int32Constant(JS_TYPED_ARRAY_TYPE)),
2459 14 : __ Int32Constant(2));
2460 : __ Goto(&done, vfalse);
2461 :
2462 : __ Bind(&if_smi);
2463 14 : __ Goto(&done, __ Int32Constant(0));
2464 :
2465 : __ Bind(&done);
2466 14 : return done.PhiAt(0);
2467 : }
2468 :
2469 25 : Node* EffectControlLinearizer::LowerObjectIsBigInt(Node* node) {
2470 : Node* value = node->InputAt(0);
2471 :
2472 25 : auto if_smi = __ MakeDeferredLabel();
2473 25 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2474 :
2475 25 : Node* check = ObjectIsSmi(value);
2476 25 : __ GotoIf(check, &if_smi);
2477 25 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2478 : Node* value_instance_type =
2479 25 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2480 : Node* vfalse =
2481 25 : __ Word32Equal(value_instance_type, __ Uint32Constant(BIGINT_TYPE));
2482 : __ Goto(&done, vfalse);
2483 :
2484 : __ Bind(&if_smi);
2485 25 : __ Goto(&done, __ Int32Constant(0));
2486 :
2487 : __ Bind(&done);
2488 25 : return done.PhiAt(0);
2489 : }
2490 :
2491 93 : Node* EffectControlLinearizer::LowerObjectIsCallable(Node* node) {
2492 : Node* value = node->InputAt(0);
2493 :
2494 93 : auto if_smi = __ MakeDeferredLabel();
2495 93 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2496 :
2497 93 : Node* check = ObjectIsSmi(value);
2498 93 : __ GotoIf(check, &if_smi);
2499 :
2500 93 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2501 : Node* value_bit_field =
2502 93 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2503 : Node* vfalse =
2504 : __ Word32Equal(__ Int32Constant(Map::IsCallableBit::kMask),
2505 : __ Word32And(value_bit_field,
2506 93 : __ Int32Constant(Map::IsCallableBit::kMask)));
2507 : __ Goto(&done, vfalse);
2508 :
2509 : __ Bind(&if_smi);
2510 93 : __ Goto(&done, __ Int32Constant(0));
2511 :
2512 : __ Bind(&done);
2513 93 : return done.PhiAt(0);
2514 : }
2515 :
2516 207 : Node* EffectControlLinearizer::LowerObjectIsConstructor(Node* node) {
2517 : Node* value = node->InputAt(0);
2518 :
2519 207 : auto if_smi = __ MakeDeferredLabel();
2520 207 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2521 :
2522 207 : Node* check = ObjectIsSmi(value);
2523 207 : __ GotoIf(check, &if_smi);
2524 :
2525 207 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2526 : Node* value_bit_field =
2527 207 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2528 : Node* vfalse = __ Word32Equal(
2529 : __ Int32Constant(Map::IsConstructorBit::kMask),
2530 : __ Word32And(value_bit_field,
2531 207 : __ Int32Constant(Map::IsConstructorBit::kMask)));
2532 : __ Goto(&done, vfalse);
2533 :
2534 : __ Bind(&if_smi);
2535 207 : __ Goto(&done, __ Int32Constant(0));
2536 :
2537 : __ Bind(&done);
2538 207 : return done.PhiAt(0);
2539 : }
2540 :
2541 13787 : Node* EffectControlLinearizer::LowerObjectIsDetectableCallable(Node* node) {
2542 : Node* value = node->InputAt(0);
2543 :
2544 13787 : auto if_smi = __ MakeDeferredLabel();
2545 13787 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2546 :
2547 13787 : Node* check = ObjectIsSmi(value);
2548 13787 : __ GotoIf(check, &if_smi);
2549 :
2550 13787 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2551 : Node* value_bit_field =
2552 13787 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2553 : Node* vfalse = __ Word32Equal(
2554 : __ Int32Constant(Map::IsCallableBit::kMask),
2555 : __ Word32And(value_bit_field,
2556 : __ Int32Constant((Map::IsCallableBit::kMask) |
2557 13787 : (Map::IsUndetectableBit::kMask))));
2558 : __ Goto(&done, vfalse);
2559 :
2560 : __ Bind(&if_smi);
2561 13787 : __ Goto(&done, __ Int32Constant(0));
2562 :
2563 : __ Bind(&done);
2564 13787 : return done.PhiAt(0);
2565 : }
2566 :
2567 61 : Node* EffectControlLinearizer::LowerNumberIsFloat64Hole(Node* node) {
2568 : Node* value = node->InputAt(0);
2569 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
2570 61 : __ Int32Constant(kHoleNanUpper32));
2571 61 : return check;
2572 : }
2573 :
2574 126 : Node* EffectControlLinearizer::LowerNumberIsFinite(Node* node) {
2575 : Node* number = node->InputAt(0);
2576 126 : Node* diff = __ Float64Sub(number, number);
2577 126 : Node* check = __ Float64Equal(diff, diff);
2578 126 : return check;
2579 : }
2580 :
2581 7 : Node* EffectControlLinearizer::LowerObjectIsFiniteNumber(Node* node) {
2582 : Node* object = node->InputAt(0);
2583 7 : Node* zero = __ Int32Constant(0);
2584 7 : Node* one = __ Int32Constant(1);
2585 :
2586 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2587 :
2588 : // Check if {object} is a Smi.
2589 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2590 :
2591 : // Check if {object} is a HeapNumber.
2592 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2593 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2594 7 : zero);
2595 :
2596 : // {object} is a HeapNumber.
2597 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2598 7 : Node* diff = __ Float64Sub(value, value);
2599 7 : Node* check = __ Float64Equal(diff, diff);
2600 : __ Goto(&done, check);
2601 :
2602 : __ Bind(&done);
2603 7 : return done.PhiAt(0);
2604 : }
2605 :
2606 119 : Node* EffectControlLinearizer::LowerNumberIsInteger(Node* node) {
2607 : Node* number = node->InputAt(0);
2608 119 : Node* trunc = BuildFloat64RoundTruncate(number);
2609 119 : Node* diff = __ Float64Sub(number, trunc);
2610 119 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2611 119 : return check;
2612 : }
2613 :
2614 7 : Node* EffectControlLinearizer::LowerObjectIsInteger(Node* node) {
2615 : Node* object = node->InputAt(0);
2616 7 : Node* zero = __ Int32Constant(0);
2617 7 : Node* one = __ Int32Constant(1);
2618 :
2619 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2620 :
2621 : // Check if {object} is a Smi.
2622 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2623 :
2624 : // Check if {object} is a HeapNumber.
2625 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2626 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2627 7 : zero);
2628 :
2629 : // {object} is a HeapNumber.
2630 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2631 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2632 7 : Node* diff = __ Float64Sub(value, trunc);
2633 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2634 : __ Goto(&done, check);
2635 :
2636 : __ Bind(&done);
2637 7 : return done.PhiAt(0);
2638 : }
2639 :
2640 7 : Node* EffectControlLinearizer::LowerNumberIsSafeInteger(Node* node) {
2641 : Node* number = node->InputAt(0);
2642 7 : Node* zero = __ Int32Constant(0);
2643 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2644 :
2645 7 : Node* trunc = BuildFloat64RoundTruncate(number);
2646 7 : Node* diff = __ Float64Sub(number, trunc);
2647 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2648 7 : __ GotoIfNot(check, &done, zero);
2649 : Node* in_range = __ Float64LessThanOrEqual(
2650 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2651 : __ Goto(&done, in_range);
2652 :
2653 : __ Bind(&done);
2654 7 : return done.PhiAt(0);
2655 : }
2656 :
2657 7 : Node* EffectControlLinearizer::LowerObjectIsSafeInteger(Node* node) {
2658 : Node* object = node->InputAt(0);
2659 7 : Node* zero = __ Int32Constant(0);
2660 7 : Node* one = __ Int32Constant(1);
2661 :
2662 7 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2663 :
2664 : // Check if {object} is a Smi.
2665 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2666 :
2667 : // Check if {object} is a HeapNumber.
2668 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2669 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2670 7 : zero);
2671 :
2672 : // {object} is a HeapNumber.
2673 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2674 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2675 7 : Node* diff = __ Float64Sub(value, trunc);
2676 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2677 7 : __ GotoIfNot(check, &done, zero);
2678 : Node* in_range = __ Float64LessThanOrEqual(
2679 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2680 : __ Goto(&done, in_range);
2681 :
2682 : __ Bind(&done);
2683 7 : return done.PhiAt(0);
2684 : }
2685 :
2686 : namespace {
2687 :
2688 59593 : const int64_t kMinusZeroBits = bit_cast<int64_t>(-0.0);
2689 59593 : const int32_t kMinusZeroLoBits = static_cast<int32_t>(kMinusZeroBits);
2690 59593 : const int32_t kMinusZeroHiBits = static_cast<int32_t>(kMinusZeroBits >> 32);
2691 :
2692 : } // namespace
2693 :
2694 21 : Node* EffectControlLinearizer::LowerObjectIsMinusZero(Node* node) {
2695 : Node* value = node->InputAt(0);
2696 21 : Node* zero = __ Int32Constant(0);
2697 :
2698 21 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2699 :
2700 : // Check if {value} is a Smi.
2701 21 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2702 :
2703 : // Check if {value} is a HeapNumber.
2704 21 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2705 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2706 21 : zero);
2707 :
2708 : // Check if {value} contains -0.
2709 21 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2710 21 : if (machine()->Is64()) {
2711 21 : Node* value64 = __ BitcastFloat64ToInt64(value_value);
2712 21 : __ Goto(&done, __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits)));
2713 : } else {
2714 0 : Node* value_lo = __ Float64ExtractLowWord32(value_value);
2715 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2716 0 : &done, zero);
2717 0 : Node* value_hi = __ Float64ExtractHighWord32(value_value);
2718 : __ Goto(&done,
2719 0 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2720 : }
2721 :
2722 : __ Bind(&done);
2723 21 : return done.PhiAt(0);
2724 : }
2725 :
2726 91 : Node* EffectControlLinearizer::LowerNumberIsMinusZero(Node* node) {
2727 : Node* value = node->InputAt(0);
2728 :
2729 91 : if (machine()->Is64()) {
2730 91 : Node* value64 = __ BitcastFloat64ToInt64(value);
2731 91 : return __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits));
2732 : } else {
2733 0 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2734 :
2735 0 : Node* value_lo = __ Float64ExtractLowWord32(value);
2736 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2737 0 : &done, __ Int32Constant(0));
2738 0 : Node* value_hi = __ Float64ExtractHighWord32(value);
2739 : __ Goto(&done,
2740 0 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2741 :
2742 : __ Bind(&done);
2743 : return done.PhiAt(0);
2744 : }
2745 : }
2746 :
2747 683 : Node* EffectControlLinearizer::LowerObjectIsNaN(Node* node) {
2748 : Node* value = node->InputAt(0);
2749 683 : Node* zero = __ Int32Constant(0);
2750 :
2751 683 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2752 :
2753 : // Check if {value} is a Smi.
2754 683 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2755 :
2756 : // Check if {value} is a HeapNumber.
2757 683 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2758 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2759 683 : zero);
2760 :
2761 : // Check if {value} contains a NaN.
2762 683 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2763 : __ Goto(&done,
2764 683 : __ Word32Equal(__ Float64Equal(value_value, value_value), zero));
2765 :
2766 : __ Bind(&done);
2767 683 : return done.PhiAt(0);
2768 : }
2769 :
2770 2608 : Node* EffectControlLinearizer::LowerNumberIsNaN(Node* node) {
2771 : Node* number = node->InputAt(0);
2772 2608 : Node* diff = __ Float64Equal(number, number);
2773 2608 : Node* check = __ Word32Equal(diff, __ Int32Constant(0));
2774 2608 : return check;
2775 : }
2776 :
2777 6001 : Node* EffectControlLinearizer::LowerObjectIsNonCallable(Node* node) {
2778 : Node* value = node->InputAt(0);
2779 :
2780 6001 : auto if_primitive = __ MakeDeferredLabel();
2781 6001 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2782 :
2783 6001 : Node* check0 = ObjectIsSmi(value);
2784 6001 : __ GotoIf(check0, &if_primitive);
2785 :
2786 6001 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2787 : Node* value_instance_type =
2788 6001 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2789 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
2790 : Node* check1 = __ Uint32LessThanOrEqual(
2791 6001 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
2792 6001 : __ GotoIfNot(check1, &if_primitive);
2793 :
2794 : Node* value_bit_field =
2795 6001 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2796 : Node* check2 =
2797 : __ Word32Equal(__ Int32Constant(0),
2798 : __ Word32And(value_bit_field,
2799 6001 : __ Int32Constant(Map::IsCallableBit::kMask)));
2800 : __ Goto(&done, check2);
2801 :
2802 : __ Bind(&if_primitive);
2803 6001 : __ Goto(&done, __ Int32Constant(0));
2804 :
2805 : __ Bind(&done);
2806 6001 : return done.PhiAt(0);
2807 : }
2808 :
2809 7471 : Node* EffectControlLinearizer::LowerObjectIsNumber(Node* node) {
2810 : Node* value = node->InputAt(0);
2811 :
2812 7471 : auto if_smi = __ MakeLabel();
2813 7471 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2814 :
2815 7471 : __ GotoIf(ObjectIsSmi(value), &if_smi);
2816 7471 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2817 7471 : __ Goto(&done, __ WordEqual(value_map, __ HeapNumberMapConstant()));
2818 :
2819 : __ Bind(&if_smi);
2820 7471 : __ Goto(&done, __ Int32Constant(1));
2821 :
2822 : __ Bind(&done);
2823 7471 : return done.PhiAt(0);
2824 : }
2825 :
2826 18981 : Node* EffectControlLinearizer::LowerObjectIsReceiver(Node* node) {
2827 : Node* value = node->InputAt(0);
2828 :
2829 18981 : auto if_smi = __ MakeDeferredLabel();
2830 18981 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2831 :
2832 18981 : __ GotoIf(ObjectIsSmi(value), &if_smi);
2833 :
2834 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
2835 18981 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2836 : Node* value_instance_type =
2837 18981 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2838 : Node* result = __ Uint32LessThanOrEqual(
2839 18981 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
2840 : __ Goto(&done, result);
2841 :
2842 : __ Bind(&if_smi);
2843 18981 : __ Goto(&done, __ Int32Constant(0));
2844 :
2845 : __ Bind(&done);
2846 18981 : return done.PhiAt(0);
2847 : }
2848 :
2849 4134 : Node* EffectControlLinearizer::LowerObjectIsSmi(Node* node) {
2850 : Node* value = node->InputAt(0);
2851 4134 : return ObjectIsSmi(value);
2852 : }
2853 :
2854 2774 : Node* EffectControlLinearizer::LowerObjectIsString(Node* node) {
2855 : Node* value = node->InputAt(0);
2856 :
2857 2774 : auto if_smi = __ MakeDeferredLabel();
2858 2774 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2859 :
2860 2774 : Node* check = ObjectIsSmi(value);
2861 2774 : __ GotoIf(check, &if_smi);
2862 2774 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2863 : Node* value_instance_type =
2864 2774 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2865 : Node* vfalse = __ Uint32LessThan(value_instance_type,
2866 2774 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
2867 : __ Goto(&done, vfalse);
2868 :
2869 : __ Bind(&if_smi);
2870 2774 : __ Goto(&done, __ Int32Constant(0));
2871 :
2872 : __ Bind(&done);
2873 2774 : return done.PhiAt(0);
2874 : }
2875 :
2876 18 : Node* EffectControlLinearizer::LowerObjectIsSymbol(Node* node) {
2877 : Node* value = node->InputAt(0);
2878 :
2879 18 : auto if_smi = __ MakeDeferredLabel();
2880 18 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2881 :
2882 18 : Node* check = ObjectIsSmi(value);
2883 18 : __ GotoIf(check, &if_smi);
2884 18 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2885 : Node* value_instance_type =
2886 18 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2887 : Node* vfalse =
2888 18 : __ Word32Equal(value_instance_type, __ Uint32Constant(SYMBOL_TYPE));
2889 : __ Goto(&done, vfalse);
2890 :
2891 : __ Bind(&if_smi);
2892 18 : __ Goto(&done, __ Int32Constant(0));
2893 :
2894 : __ Bind(&done);
2895 18 : return done.PhiAt(0);
2896 : }
2897 :
2898 1410 : Node* EffectControlLinearizer::LowerObjectIsUndetectable(Node* node) {
2899 : Node* value = node->InputAt(0);
2900 :
2901 1410 : auto if_smi = __ MakeDeferredLabel();
2902 1410 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2903 :
2904 1410 : Node* check = ObjectIsSmi(value);
2905 1410 : __ GotoIf(check, &if_smi);
2906 :
2907 1410 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2908 : Node* value_bit_field =
2909 1410 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2910 : Node* vfalse = __ Word32Equal(
2911 : __ Word32Equal(
2912 : __ Int32Constant(0),
2913 : __ Word32And(value_bit_field,
2914 : __ Int32Constant(Map::IsUndetectableBit::kMask))),
2915 1410 : __ Int32Constant(0));
2916 : __ Goto(&done, vfalse);
2917 :
2918 : __ Bind(&if_smi);
2919 1410 : __ Goto(&done, __ Int32Constant(0));
2920 :
2921 : __ Bind(&done);
2922 1410 : return done.PhiAt(0);
2923 : }
2924 :
2925 21292 : Node* EffectControlLinearizer::LowerTypeOf(Node* node) {
2926 : Node* obj = node->InputAt(0);
2927 21292 : Callable const callable = Builtins::CallableFor(isolate(), Builtins::kTypeof);
2928 : Operator::Properties const properties = Operator::kEliminatable;
2929 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
2930 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2931 : graph()->zone(), callable.descriptor(),
2932 42584 : callable.descriptor().GetStackParameterCount(), flags, properties);
2933 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
2934 42584 : __ NoContextConstant());
2935 : }
2936 :
2937 201 : Node* EffectControlLinearizer::LowerToBoolean(Node* node) {
2938 : Node* obj = node->InputAt(0);
2939 : Callable const callable =
2940 201 : Builtins::CallableFor(isolate(), Builtins::kToBoolean);
2941 : Operator::Properties const properties = Operator::kEliminatable;
2942 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
2943 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2944 : graph()->zone(), callable.descriptor(),
2945 402 : callable.descriptor().GetStackParameterCount(), flags, properties);
2946 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
2947 402 : __ NoContextConstant());
2948 : }
2949 :
2950 49839 : Node* EffectControlLinearizer::LowerArgumentsLength(Node* node) {
2951 16613 : Node* arguments_frame = NodeProperties::GetValueInput(node, 0);
2952 16613 : int formal_parameter_count = FormalParameterCountOf(node->op());
2953 16613 : bool is_rest_length = IsRestLengthOf(node->op());
2954 : DCHECK_LE(0, formal_parameter_count);
2955 :
2956 16613 : if (is_rest_length) {
2957 : // The ArgumentsLength node is computing the number of rest parameters,
2958 : // which is max(0, actual_parameter_count - formal_parameter_count).
2959 : // We have to distinguish the case, when there is an arguments adaptor frame
2960 : // (i.e., arguments_frame != LoadFramePointer()).
2961 230 : auto if_adaptor_frame = __ MakeLabel();
2962 230 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
2963 :
2964 230 : Node* frame = __ LoadFramePointer();
2965 230 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done, __ SmiConstant(0));
2966 : __ Goto(&if_adaptor_frame);
2967 :
2968 : __ Bind(&if_adaptor_frame);
2969 : Node* arguments_length = __ Load(
2970 : MachineType::TaggedSigned(), arguments_frame,
2971 230 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
2972 :
2973 : Node* rest_length =
2974 230 : __ IntSub(arguments_length, __ SmiConstant(formal_parameter_count));
2975 : __ GotoIf(__ IntLessThan(rest_length, __ SmiConstant(0)), &done,
2976 230 : __ SmiConstant(0));
2977 : __ Goto(&done, rest_length);
2978 :
2979 : __ Bind(&done);
2980 : return done.PhiAt(0);
2981 : } else {
2982 : // The ArgumentsLength node is computing the actual number of arguments.
2983 : // We have to distinguish the case when there is an arguments adaptor frame
2984 : // (i.e., arguments_frame != LoadFramePointer()).
2985 16383 : auto if_adaptor_frame = __ MakeLabel();
2986 16383 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
2987 :
2988 16383 : Node* frame = __ LoadFramePointer();
2989 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done,
2990 16383 : __ SmiConstant(formal_parameter_count));
2991 : __ Goto(&if_adaptor_frame);
2992 :
2993 : __ Bind(&if_adaptor_frame);
2994 : Node* arguments_length = __ Load(
2995 : MachineType::TaggedSigned(), arguments_frame,
2996 16383 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
2997 : __ Goto(&done, arguments_length);
2998 :
2999 : __ Bind(&done);
3000 : return done.PhiAt(0);
3001 : }
3002 : }
3003 :
3004 16586 : Node* EffectControlLinearizer::LowerArgumentsFrame(Node* node) {
3005 16586 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
3006 :
3007 16586 : Node* frame = __ LoadFramePointer();
3008 : Node* parent_frame =
3009 : __ Load(MachineType::Pointer(), frame,
3010 16586 : __ IntPtrConstant(StandardFrameConstants::kCallerFPOffset));
3011 : Node* parent_frame_type = __ Load(
3012 : MachineType::AnyTagged(), parent_frame,
3013 16586 : __ IntPtrConstant(CommonFrameConstants::kContextOrFrameTypeOffset));
3014 : __ GotoIf(__ WordEqual(parent_frame_type,
3015 : __ IntPtrConstant(StackFrame::TypeToMarker(
3016 : StackFrame::ARGUMENTS_ADAPTOR))),
3017 16586 : &done, parent_frame);
3018 : __ Goto(&done, frame);
3019 :
3020 : __ Bind(&done);
3021 16586 : return done.PhiAt(0);
3022 : }
3023 :
3024 27 : Node* EffectControlLinearizer::LowerNewDoubleElements(Node* node) {
3025 27 : PretenureFlag const pretenure = PretenureFlagOf(node->op());
3026 : Node* length = node->InputAt(0);
3027 :
3028 27 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3029 27 : Node* zero_length = __ Word32Equal(length, __ Int32Constant(0));
3030 : __ GotoIf(zero_length, &done,
3031 27 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3032 :
3033 : // Compute the effective size of the backing store.
3034 : Node* size =
3035 : __ Int32Add(__ Word32Shl(length, __ Int32Constant(kDoubleSizeLog2)),
3036 27 : __ Int32Constant(FixedDoubleArray::kHeaderSize));
3037 :
3038 : // Allocate the result and initialize the header.
3039 27 : Node* result = __ Allocate(pretenure, size);
3040 : __ StoreField(AccessBuilder::ForMap(), result,
3041 27 : __ FixedDoubleArrayMapConstant());
3042 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3043 27 : ChangeInt32ToSmi(length));
3044 :
3045 : // Initialize the backing store with holes.
3046 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
3047 : Node* limit = ChangeUint32ToUintPtr(length);
3048 : Node* the_hole =
3049 27 : __ LoadField(AccessBuilder::ForHeapNumberValue(), __ TheHoleConstant());
3050 27 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3051 27 : __ Goto(&loop, __ IntPtrConstant(0));
3052 : __ Bind(&loop);
3053 : {
3054 : // Check if we've initialized everything.
3055 : Node* index = loop.PhiAt(0);
3056 27 : Node* check = __ UintLessThan(index, limit);
3057 27 : __ GotoIfNot(check, &done, result);
3058 :
3059 : // Storing "the_hole" doesn't need a write barrier.
3060 : StoreRepresentation rep(MachineRepresentation::kFloat64, kNoWriteBarrier);
3061 : Node* offset = __ IntAdd(
3062 : __ WordShl(index, __ IntPtrConstant(kDoubleSizeLog2)),
3063 27 : __ IntPtrConstant(FixedDoubleArray::kHeaderSize - kHeapObjectTag));
3064 27 : __ Store(rep, result, offset, the_hole);
3065 :
3066 : // Advance the {index}.
3067 27 : index = __ IntAdd(index, __ IntPtrConstant(1));
3068 : __ Goto(&loop, index);
3069 : }
3070 :
3071 : __ Bind(&done);
3072 27 : return done.PhiAt(0);
3073 : }
3074 :
3075 484 : Node* EffectControlLinearizer::LowerNewSmiOrObjectElements(Node* node) {
3076 484 : PretenureFlag const pretenure = PretenureFlagOf(node->op());
3077 : Node* length = node->InputAt(0);
3078 :
3079 484 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3080 484 : Node* zero_length = __ Word32Equal(length, __ Int32Constant(0));
3081 : __ GotoIf(zero_length, &done,
3082 484 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3083 :
3084 : // Compute the effective size of the backing store.
3085 : Node* size =
3086 : __ Int32Add(__ Word32Shl(length, __ Int32Constant(kTaggedSizeLog2)),
3087 484 : __ Int32Constant(FixedArray::kHeaderSize));
3088 :
3089 : // Allocate the result and initialize the header.
3090 484 : Node* result = __ Allocate(pretenure, size);
3091 484 : __ StoreField(AccessBuilder::ForMap(), result, __ FixedArrayMapConstant());
3092 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3093 484 : ChangeInt32ToSmi(length));
3094 :
3095 : // Initialize the backing store with holes.
3096 : Node* limit = ChangeUint32ToUintPtr(length);
3097 484 : Node* the_hole = __ TheHoleConstant();
3098 484 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3099 484 : __ Goto(&loop, __ IntPtrConstant(0));
3100 : __ Bind(&loop);
3101 : {
3102 : // Check if we've initialized everything.
3103 : Node* index = loop.PhiAt(0);
3104 484 : Node* check = __ UintLessThan(index, limit);
3105 484 : __ GotoIfNot(check, &done, result);
3106 :
3107 : // Storing "the_hole" doesn't need a write barrier.
3108 : StoreRepresentation rep(MachineRepresentation::kTagged, kNoWriteBarrier);
3109 : Node* offset =
3110 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)),
3111 484 : __ IntPtrConstant(FixedArray::kHeaderSize - kHeapObjectTag));
3112 484 : __ Store(rep, result, offset, the_hole);
3113 :
3114 : // Advance the {index}.
3115 484 : index = __ IntAdd(index, __ IntPtrConstant(1));
3116 : __ Goto(&loop, index);
3117 : }
3118 :
3119 : __ Bind(&done);
3120 484 : return done.PhiAt(0);
3121 : }
3122 :
3123 48351 : Node* EffectControlLinearizer::LowerNewArgumentsElements(Node* node) {
3124 16117 : Node* frame = NodeProperties::GetValueInput(node, 0);
3125 16117 : Node* length = NodeProperties::GetValueInput(node, 1);
3126 16117 : int mapped_count = NewArgumentsElementsMappedCountOf(node->op());
3127 :
3128 : Callable const callable =
3129 16117 : Builtins::CallableFor(isolate(), Builtins::kNewArgumentsElements);
3130 16117 : Operator::Properties const properties = node->op()->properties();
3131 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
3132 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3133 : graph()->zone(), callable.descriptor(),
3134 32234 : callable.descriptor().GetStackParameterCount(), flags, properties);
3135 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), frame,
3136 32234 : length, __ SmiConstant(mapped_count), __ NoContextConstant());
3137 : }
3138 :
3139 5003 : Node* EffectControlLinearizer::LowerNewConsString(Node* node) {
3140 : Node* length = node->InputAt(0);
3141 : Node* first = node->InputAt(1);
3142 : Node* second = node->InputAt(2);
3143 :
3144 : // Determine the instance types of {first} and {second}.
3145 5003 : Node* first_map = __ LoadField(AccessBuilder::ForMap(), first);
3146 : Node* first_instance_type =
3147 5003 : __ LoadField(AccessBuilder::ForMapInstanceType(), first_map);
3148 5003 : Node* second_map = __ LoadField(AccessBuilder::ForMap(), second);
3149 : Node* second_instance_type =
3150 5003 : __ LoadField(AccessBuilder::ForMapInstanceType(), second_map);
3151 :
3152 : // Determine the proper map for the resulting ConsString.
3153 : // If both {first} and {second} are one-byte strings, we
3154 : // create a new ConsOneByteString, otherwise we create a
3155 : // new ConsString instead.
3156 5003 : auto if_onebyte = __ MakeLabel();
3157 5003 : auto if_twobyte = __ MakeLabel();
3158 5003 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3159 : STATIC_ASSERT(kOneByteStringTag != 0);
3160 : STATIC_ASSERT(kTwoByteStringTag == 0);
3161 5003 : Node* instance_type = __ Word32And(first_instance_type, second_instance_type);
3162 : Node* encoding =
3163 5003 : __ Word32And(instance_type, __ Int32Constant(kStringEncodingMask));
3164 : __ Branch(__ Word32Equal(encoding, __ Int32Constant(kTwoByteStringTag)),
3165 5003 : &if_twobyte, &if_onebyte);
3166 : __ Bind(&if_onebyte);
3167 : __ Goto(&done,
3168 10006 : jsgraph()->HeapConstant(factory()->cons_one_byte_string_map()));
3169 : __ Bind(&if_twobyte);
3170 10006 : __ Goto(&done, jsgraph()->HeapConstant(factory()->cons_string_map()));
3171 : __ Bind(&done);
3172 : Node* result_map = done.PhiAt(0);
3173 :
3174 : // Allocate the resulting ConsString.
3175 5003 : Node* result = __ Allocate(NOT_TENURED, __ Int32Constant(ConsString::kSize));
3176 5003 : __ StoreField(AccessBuilder::ForMap(), result, result_map);
3177 : __ StoreField(AccessBuilder::ForNameHashField(), result,
3178 5003 : __ Int32Constant(Name::kEmptyHashField));
3179 5003 : __ StoreField(AccessBuilder::ForStringLength(), result, length);
3180 5003 : __ StoreField(AccessBuilder::ForConsStringFirst(), result, first);
3181 5003 : __ StoreField(AccessBuilder::ForConsStringSecond(), result, second);
3182 5003 : return result;
3183 : }
3184 :
3185 10 : Node* EffectControlLinearizer::LowerSameValue(Node* node) {
3186 : Node* lhs = node->InputAt(0);
3187 : Node* rhs = node->InputAt(1);
3188 :
3189 : Callable const callable =
3190 10 : Builtins::CallableFor(isolate(), Builtins::kSameValue);
3191 : Operator::Properties properties = Operator::kEliminatable;
3192 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3193 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3194 : graph()->zone(), callable.descriptor(),
3195 20 : callable.descriptor().GetStackParameterCount(), flags, properties);
3196 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3197 20 : __ NoContextConstant());
3198 : }
3199 :
3200 1220 : Node* EffectControlLinearizer::LowerDeadValue(Node* node) {
3201 1220 : Node* input = NodeProperties::GetValueInput(node, 0);
3202 1220 : if (input->opcode() != IrOpcode::kUnreachable) {
3203 937 : Node* unreachable = __ Unreachable();
3204 937 : NodeProperties::ReplaceValueInput(node, unreachable, 0);
3205 : }
3206 1220 : return node;
3207 : }
3208 :
3209 330 : Node* EffectControlLinearizer::LowerStringToNumber(Node* node) {
3210 : Node* string = node->InputAt(0);
3211 :
3212 : Callable const callable =
3213 330 : Builtins::CallableFor(isolate(), Builtins::kStringToNumber);
3214 : Operator::Properties properties = Operator::kEliminatable;
3215 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3216 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3217 : graph()->zone(), callable.descriptor(),
3218 660 : callable.descriptor().GetStackParameterCount(), flags, properties);
3219 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), string,
3220 660 : __ NoContextConstant());
3221 : }
3222 :
3223 2379 : Node* EffectControlLinearizer::LowerStringCharCodeAt(Node* node) {
3224 : Node* receiver = node->InputAt(0);
3225 : Node* position = node->InputAt(1);
3226 :
3227 : // We need a loop here to properly deal with indirect strings
3228 : // (SlicedString, ConsString and ThinString).
3229 : auto loop = __ MakeLoopLabel(MachineRepresentation::kTagged,
3230 2379 : MachineType::PointerRepresentation());
3231 : auto loop_next = __ MakeLabel(MachineRepresentation::kTagged,
3232 2379 : MachineType::PointerRepresentation());
3233 2379 : auto loop_done = __ MakeLabel(MachineRepresentation::kWord32);
3234 2379 : __ Goto(&loop, receiver, position);
3235 : __ Bind(&loop);
3236 : {
3237 : Node* receiver = loop.PhiAt(0);
3238 : Node* position = loop.PhiAt(1);
3239 2379 : Node* receiver_map = __ LoadField(AccessBuilder::ForMap(), receiver);
3240 : Node* receiver_instance_type =
3241 2379 : __ LoadField(AccessBuilder::ForMapInstanceType(), receiver_map);
3242 : Node* receiver_representation = __ Word32And(
3243 2379 : receiver_instance_type, __ Int32Constant(kStringRepresentationMask));
3244 :
3245 : // Dispatch on the current {receiver}s string representation.
3246 2379 : auto if_seqstring = __ MakeLabel();
3247 2379 : auto if_consstring = __ MakeLabel();
3248 2379 : auto if_thinstring = __ MakeLabel();
3249 2379 : auto if_externalstring = __ MakeLabel();
3250 2379 : auto if_slicedstring = __ MakeLabel();
3251 2379 : auto if_runtime = __ MakeDeferredLabel();
3252 : __ GotoIf(__ Word32Equal(receiver_representation,
3253 : __ Int32Constant(kSeqStringTag)),
3254 2379 : &if_seqstring);
3255 : __ GotoIf(__ Word32Equal(receiver_representation,
3256 : __ Int32Constant(kConsStringTag)),
3257 2379 : &if_consstring);
3258 : __ GotoIf(__ Word32Equal(receiver_representation,
3259 : __ Int32Constant(kThinStringTag)),
3260 2379 : &if_thinstring);
3261 : __ GotoIf(__ Word32Equal(receiver_representation,
3262 : __ Int32Constant(kExternalStringTag)),
3263 2379 : &if_externalstring);
3264 : __ Branch(__ Word32Equal(receiver_representation,
3265 : __ Int32Constant(kSlicedStringTag)),
3266 2379 : &if_slicedstring, &if_runtime);
3267 :
3268 : __ Bind(&if_seqstring);
3269 : {
3270 : Node* receiver_is_onebyte = __ Word32Equal(
3271 : __ Word32Equal(__ Word32And(receiver_instance_type,
3272 : __ Int32Constant(kStringEncodingMask)),
3273 : __ Int32Constant(kTwoByteStringTag)),
3274 2379 : __ Int32Constant(0));
3275 2379 : Node* result = LoadFromSeqString(receiver, position, receiver_is_onebyte);
3276 : __ Goto(&loop_done, result);
3277 : }
3278 :
3279 : __ Bind(&if_thinstring);
3280 : {
3281 : Node* receiver_actual =
3282 2379 : __ LoadField(AccessBuilder::ForThinStringActual(), receiver);
3283 : __ Goto(&loop_next, receiver_actual, position);
3284 : }
3285 :
3286 : __ Bind(&if_consstring);
3287 : {
3288 : Node* receiver_second =
3289 2379 : __ LoadField(AccessBuilder::ForConsStringSecond(), receiver);
3290 : __ GotoIfNot(__ WordEqual(receiver_second, __ EmptyStringConstant()),
3291 2379 : &if_runtime);
3292 : Node* receiver_first =
3293 2379 : __ LoadField(AccessBuilder::ForConsStringFirst(), receiver);
3294 : __ Goto(&loop_next, receiver_first, position);
3295 : }
3296 :
3297 : __ Bind(&if_externalstring);
3298 : {
3299 : // We need to bailout to the runtime for uncached external strings.
3300 : __ GotoIf(__ Word32Equal(
3301 : __ Word32And(receiver_instance_type,
3302 : __ Int32Constant(kUncachedExternalStringMask)),
3303 : __ Int32Constant(kUncachedExternalStringTag)),
3304 2379 : &if_runtime);
3305 :
3306 : Node* receiver_data = __ LoadField(
3307 2379 : AccessBuilder::ForExternalStringResourceData(), receiver);
3308 :
3309 2379 : auto if_onebyte = __ MakeLabel();
3310 2379 : auto if_twobyte = __ MakeLabel();
3311 : __ Branch(
3312 : __ Word32Equal(__ Word32And(receiver_instance_type,
3313 : __ Int32Constant(kStringEncodingMask)),
3314 : __ Int32Constant(kTwoByteStringTag)),
3315 2379 : &if_twobyte, &if_onebyte);
3316 :
3317 : __ Bind(&if_onebyte);
3318 : {
3319 2379 : Node* result = __ Load(MachineType::Uint8(), receiver_data, position);
3320 : __ Goto(&loop_done, result);
3321 : }
3322 :
3323 : __ Bind(&if_twobyte);
3324 : {
3325 : Node* result = __ Load(MachineType::Uint16(), receiver_data,
3326 2379 : __ WordShl(position, __ IntPtrConstant(1)));
3327 : __ Goto(&loop_done, result);
3328 : }
3329 : }
3330 :
3331 : __ Bind(&if_slicedstring);
3332 : {
3333 : Node* receiver_offset =
3334 2379 : __ LoadField(AccessBuilder::ForSlicedStringOffset(), receiver);
3335 : Node* receiver_parent =
3336 2379 : __ LoadField(AccessBuilder::ForSlicedStringParent(), receiver);
3337 : __ Goto(&loop_next, receiver_parent,
3338 2379 : __ IntAdd(position, ChangeSmiToIntPtr(receiver_offset)));
3339 : }
3340 :
3341 : __ Bind(&if_runtime);
3342 : {
3343 2379 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3344 : Runtime::FunctionId id = Runtime::kStringCharCodeAt;
3345 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3346 2379 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
3347 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1),
3348 : receiver, ChangeIntPtrToSmi(position),
3349 : __ ExternalConstant(ExternalReference::Create(id)),
3350 2379 : __ Int32Constant(2), __ NoContextConstant());
3351 2379 : __ Goto(&loop_done, ChangeSmiToInt32(result));
3352 : }
3353 :
3354 : __ Bind(&loop_next);
3355 : __ Goto(&loop, loop_next.PhiAt(0), loop_next.PhiAt(1));
3356 : }
3357 : __ Bind(&loop_done);
3358 2379 : return loop_done.PhiAt(0);
3359 : }
3360 :
3361 215 : Node* EffectControlLinearizer::LowerStringCodePointAt(
3362 : Node* node, UnicodeEncoding encoding) {
3363 : Node* receiver = node->InputAt(0);
3364 : Node* position = node->InputAt(1);
3365 :
3366 : Builtins::Name builtin = encoding == UnicodeEncoding::UTF16
3367 : ? Builtins::kStringCodePointAtUTF16
3368 215 : : Builtins::kStringCodePointAtUTF32;
3369 :
3370 215 : Callable const callable = Builtins::CallableFor(isolate(), builtin);
3371 215 : Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite;
3372 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3373 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3374 : graph()->zone(), callable.descriptor(),
3375 430 : callable.descriptor().GetStackParameterCount(), flags, properties);
3376 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3377 430 : position, __ NoContextConstant());
3378 : }
3379 :
3380 2379 : Node* EffectControlLinearizer::LoadFromSeqString(Node* receiver, Node* position,
3381 : Node* is_one_byte) {
3382 2379 : auto one_byte_load = __ MakeLabel();
3383 2379 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
3384 2379 : __ GotoIf(is_one_byte, &one_byte_load);
3385 : Node* two_byte_result = __ LoadElement(
3386 2379 : AccessBuilder::ForSeqTwoByteStringCharacter(), receiver, position);
3387 : __ Goto(&done, two_byte_result);
3388 :
3389 : __ Bind(&one_byte_load);
3390 : Node* one_byte_element = __ LoadElement(
3391 2379 : AccessBuilder::ForSeqOneByteStringCharacter(), receiver, position);
3392 : __ Goto(&done, one_byte_element);
3393 :
3394 : __ Bind(&done);
3395 2379 : return done.PhiAt(0);
3396 : }
3397 :
3398 938 : Node* EffectControlLinearizer::LowerStringFromSingleCharCode(Node* node) {
3399 : Node* value = node->InputAt(0);
3400 938 : Node* code = __ Word32And(value, __ Uint32Constant(0xFFFF));
3401 :
3402 938 : auto if_not_one_byte = __ MakeDeferredLabel();
3403 938 : auto cache_miss = __ MakeDeferredLabel();
3404 938 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3405 :
3406 : // Check if the {code} is a one byte character
3407 : Node* check1 = __ Uint32LessThanOrEqual(
3408 938 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3409 938 : __ GotoIfNot(check1, &if_not_one_byte);
3410 : {
3411 : // Load the isolate wide single character string cache.
3412 938 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3413 :
3414 : // Compute the {cache} index for {code}.
3415 938 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3416 :
3417 : // Check if we have an entry for the {code} in the single character string
3418 : // cache already.
3419 : Node* entry =
3420 938 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3421 :
3422 938 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3423 938 : __ GotoIf(check2, &cache_miss);
3424 :
3425 : // Use the {entry} from the {cache}.
3426 : __ Goto(&done, entry);
3427 :
3428 : __ Bind(&cache_miss);
3429 : {
3430 : // Allocate a new SeqOneByteString for {code}.
3431 : Node* vtrue2 = __ Allocate(
3432 938 : NOT_TENURED, __ Int32Constant(SeqOneByteString::SizeFor(1)));
3433 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3434 938 : __ HeapConstant(factory()->one_byte_string_map()));
3435 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3436 938 : __ Int32Constant(Name::kEmptyHashField));
3437 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3438 938 : __ Int32Constant(1));
3439 : __ Store(
3440 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3441 : vtrue2,
3442 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3443 1876 : code);
3444 :
3445 : // Remember it in the {cache}.
3446 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3447 938 : vtrue2);
3448 : __ Goto(&done, vtrue2);
3449 : }
3450 : }
3451 :
3452 : __ Bind(&if_not_one_byte);
3453 : {
3454 : // Allocate a new SeqTwoByteString for {code}.
3455 : Node* vfalse1 = __ Allocate(NOT_TENURED,
3456 938 : __ Int32Constant(SeqTwoByteString::SizeFor(1)));
3457 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3458 938 : __ HeapConstant(factory()->string_map()));
3459 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3460 938 : __ Int32Constant(Name::kEmptyHashField));
3461 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3462 938 : __ Int32Constant(1));
3463 : __ Store(
3464 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3465 : vfalse1,
3466 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3467 1876 : code);
3468 : __ Goto(&done, vfalse1);
3469 : }
3470 :
3471 : __ Bind(&done);
3472 938 : return done.PhiAt(0);
3473 : }
3474 :
3475 : #ifdef V8_INTL_SUPPORT
3476 :
3477 86 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3478 : Node* receiver = node->InputAt(0);
3479 :
3480 : Callable callable =
3481 86 : Builtins::CallableFor(isolate(), Builtins::kStringToLowerCaseIntl);
3482 86 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3483 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3484 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3485 : graph()->zone(), callable.descriptor(),
3486 172 : callable.descriptor().GetStackParameterCount(), flags, properties);
3487 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3488 172 : __ NoContextConstant());
3489 : }
3490 :
3491 28 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3492 : Node* receiver = node->InputAt(0);
3493 28 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3494 : Runtime::FunctionId id = Runtime::kStringToUpperCaseIntl;
3495 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3496 28 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
3497 : return __ Call(call_descriptor, __ CEntryStubConstant(1), receiver,
3498 : __ ExternalConstant(ExternalReference::Create(id)),
3499 28 : __ Int32Constant(1), __ NoContextConstant());
3500 : }
3501 :
3502 : #else
3503 :
3504 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3505 : UNREACHABLE();
3506 : return nullptr;
3507 : }
3508 :
3509 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3510 : UNREACHABLE();
3511 : return nullptr;
3512 : }
3513 :
3514 : #endif // V8_INTL_SUPPORT
3515 :
3516 414 : Node* EffectControlLinearizer::LowerStringFromSingleCodePoint(Node* node) {
3517 : Node* value = node->InputAt(0);
3518 : Node* code = value;
3519 :
3520 207 : auto if_not_single_code = __ MakeDeferredLabel();
3521 207 : auto if_not_one_byte = __ MakeDeferredLabel();
3522 207 : auto cache_miss = __ MakeDeferredLabel();
3523 207 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3524 :
3525 : // Check if the {code} is a single code unit
3526 207 : Node* check0 = __ Uint32LessThanOrEqual(code, __ Uint32Constant(0xFFFF));
3527 207 : __ GotoIfNot(check0, &if_not_single_code);
3528 :
3529 : {
3530 : // Check if the {code} is a one byte character
3531 : Node* check1 = __ Uint32LessThanOrEqual(
3532 207 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3533 207 : __ GotoIfNot(check1, &if_not_one_byte);
3534 : {
3535 : // Load the isolate wide single character string cache.
3536 207 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3537 :
3538 : // Compute the {cache} index for {code}.
3539 207 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3540 :
3541 : // Check if we have an entry for the {code} in the single character string
3542 : // cache already.
3543 : Node* entry =
3544 207 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3545 :
3546 207 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3547 207 : __ GotoIf(check2, &cache_miss);
3548 :
3549 : // Use the {entry} from the {cache}.
3550 : __ Goto(&done, entry);
3551 :
3552 : __ Bind(&cache_miss);
3553 : {
3554 : // Allocate a new SeqOneByteString for {code}.
3555 : Node* vtrue2 = __ Allocate(
3556 207 : NOT_TENURED, __ Int32Constant(SeqOneByteString::SizeFor(1)));
3557 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3558 207 : __ HeapConstant(factory()->one_byte_string_map()));
3559 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3560 207 : __ Int32Constant(Name::kEmptyHashField));
3561 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3562 207 : __ Int32Constant(1));
3563 : __ Store(
3564 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3565 : vtrue2,
3566 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3567 414 : code);
3568 :
3569 : // Remember it in the {cache}.
3570 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3571 207 : vtrue2);
3572 : __ Goto(&done, vtrue2);
3573 : }
3574 : }
3575 :
3576 : __ Bind(&if_not_one_byte);
3577 : {
3578 : // Allocate a new SeqTwoByteString for {code}.
3579 : Node* vfalse1 = __ Allocate(
3580 207 : NOT_TENURED, __ Int32Constant(SeqTwoByteString::SizeFor(1)));
3581 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3582 207 : __ HeapConstant(factory()->string_map()));
3583 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3584 207 : __ IntPtrConstant(Name::kEmptyHashField));
3585 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3586 207 : __ Int32Constant(1));
3587 : __ Store(
3588 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3589 : vfalse1,
3590 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3591 414 : code);
3592 : __ Goto(&done, vfalse1);
3593 : }
3594 : }
3595 :
3596 : __ Bind(&if_not_single_code);
3597 : // Generate surrogate pair string
3598 : {
3599 207 : switch (UnicodeEncodingOf(node->op())) {
3600 : case UnicodeEncoding::UTF16:
3601 : break;
3602 :
3603 : case UnicodeEncoding::UTF32: {
3604 : // Convert UTF32 to UTF16 code units, and store as a 32 bit word.
3605 92 : Node* lead_offset = __ Int32Constant(0xD800 - (0x10000 >> 10));
3606 :
3607 : // lead = (codepoint >> 10) + LEAD_OFFSET
3608 : Node* lead =
3609 92 : __ Int32Add(__ Word32Shr(code, __ Int32Constant(10)), lead_offset);
3610 :
3611 : // trail = (codepoint & 0x3FF) + 0xDC00;
3612 : Node* trail = __ Int32Add(__ Word32And(code, __ Int32Constant(0x3FF)),
3613 92 : __ Int32Constant(0xDC00));
3614 :
3615 : // codpoint = (trail << 16) | lead;
3616 : #if V8_TARGET_BIG_ENDIAN
3617 : code = __ Word32Or(__ Word32Shl(lead, __ Int32Constant(16)), trail);
3618 : #else
3619 92 : code = __ Word32Or(__ Word32Shl(trail, __ Int32Constant(16)), lead);
3620 : #endif
3621 92 : break;
3622 : }
3623 : }
3624 :
3625 : // Allocate a new SeqTwoByteString for {code}.
3626 : Node* vfalse0 = __ Allocate(NOT_TENURED,
3627 207 : __ Int32Constant(SeqTwoByteString::SizeFor(2)));
3628 : __ StoreField(AccessBuilder::ForMap(), vfalse0,
3629 207 : __ HeapConstant(factory()->string_map()));
3630 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse0,
3631 207 : __ Int32Constant(Name::kEmptyHashField));
3632 : __ StoreField(AccessBuilder::ForStringLength(), vfalse0,
3633 207 : __ Int32Constant(2));
3634 : __ Store(
3635 : StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier),
3636 : vfalse0,
3637 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3638 414 : code);
3639 : __ Goto(&done, vfalse0);
3640 : }
3641 :
3642 : __ Bind(&done);
3643 207 : return done.PhiAt(0);
3644 : }
3645 :
3646 285 : Node* EffectControlLinearizer::LowerStringIndexOf(Node* node) {
3647 : Node* subject = node->InputAt(0);
3648 : Node* search_string = node->InputAt(1);
3649 : Node* position = node->InputAt(2);
3650 :
3651 : Callable callable =
3652 285 : Builtins::CallableFor(isolate(), Builtins::kStringIndexOf);
3653 : Operator::Properties properties = Operator::kEliminatable;
3654 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3655 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3656 : graph()->zone(), callable.descriptor(),
3657 570 : callable.descriptor().GetStackParameterCount(), flags, properties);
3658 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), subject,
3659 570 : search_string, position, __ NoContextConstant());
3660 : }
3661 :
3662 24359 : Node* EffectControlLinearizer::LowerStringLength(Node* node) {
3663 : Node* subject = node->InputAt(0);
3664 :
3665 24359 : return __ LoadField(AccessBuilder::ForStringLength(), subject);
3666 : }
3667 :
3668 11102 : Node* EffectControlLinearizer::LowerStringComparison(Callable const& callable,
3669 : Node* node) {
3670 : Node* lhs = node->InputAt(0);
3671 : Node* rhs = node->InputAt(1);
3672 :
3673 : Operator::Properties properties = Operator::kEliminatable;
3674 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3675 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3676 : graph()->zone(), callable.descriptor(),
3677 22204 : callable.descriptor().GetStackParameterCount(), flags, properties);
3678 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3679 22204 : __ NoContextConstant());
3680 : }
3681 :
3682 1075 : Node* EffectControlLinearizer::LowerStringSubstring(Node* node) {
3683 : Node* receiver = node->InputAt(0);
3684 : Node* start = ChangeInt32ToIntPtr(node->InputAt(1));
3685 : Node* end = ChangeInt32ToIntPtr(node->InputAt(2));
3686 :
3687 : Callable callable =
3688 1075 : Builtins::CallableFor(isolate(), Builtins::kStringSubstring);
3689 : Operator::Properties properties = Operator::kEliminatable;
3690 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3691 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3692 : graph()->zone(), callable.descriptor(),
3693 2150 : callable.descriptor().GetStackParameterCount(), flags, properties);
3694 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3695 2150 : start, end, __ NoContextConstant());
3696 : }
3697 :
3698 10252 : Node* EffectControlLinearizer::LowerStringEqual(Node* node) {
3699 : return LowerStringComparison(
3700 10252 : Builtins::CallableFor(isolate(), Builtins::kStringEqual), node);
3701 : }
3702 :
3703 121 : Node* EffectControlLinearizer::LowerStringLessThan(Node* node) {
3704 : return LowerStringComparison(
3705 121 : Builtins::CallableFor(isolate(), Builtins::kStringLessThan), node);
3706 : }
3707 :
3708 729 : Node* EffectControlLinearizer::LowerStringLessThanOrEqual(Node* node) {
3709 : return LowerStringComparison(
3710 729 : Builtins::CallableFor(isolate(), Builtins::kStringLessThanOrEqual), node);
3711 : }
3712 :
3713 470 : Node* EffectControlLinearizer::LowerCheckFloat64Hole(Node* node,
3714 : Node* frame_state) {
3715 : // If we reach this point w/o eliminating the {node} that's marked
3716 : // with allow-return-hole, we cannot do anything, so just deoptimize
3717 : // in case of the hole NaN.
3718 : CheckFloat64HoleParameters const& params =
3719 470 : CheckFloat64HoleParametersOf(node->op());
3720 : Node* value = node->InputAt(0);
3721 :
3722 470 : auto if_nan = __ MakeDeferredLabel();
3723 470 : auto done = __ MakeLabel();
3724 :
3725 : // First check whether {value} is a NaN at all...
3726 470 : __ Branch(__ Float64Equal(value, value), &done, &if_nan);
3727 :
3728 : __ Bind(&if_nan);
3729 : {
3730 : // ...and only if {value} is a NaN, perform the expensive bit
3731 : // check. See http://crbug.com/v8/8264 for details.
3732 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
3733 470 : __ Int32Constant(kHoleNanUpper32));
3734 470 : __ DeoptimizeIf(DeoptimizeReason::kHole, params.feedback(), check,
3735 470 : frame_state);
3736 : __ Goto(&done);
3737 : }
3738 :
3739 : __ Bind(&done);
3740 470 : return value;
3741 : }
3742 :
3743 358 : Node* EffectControlLinearizer::LowerCheckNotTaggedHole(Node* node,
3744 : Node* frame_state) {
3745 : Node* value = node->InputAt(0);
3746 358 : Node* check = __ WordEqual(value, __ TheHoleConstant());
3747 : __ DeoptimizeIf(DeoptimizeReason::kHole, VectorSlotPair(), check,
3748 358 : frame_state);
3749 358 : return value;
3750 : }
3751 :
3752 1668 : Node* EffectControlLinearizer::LowerConvertTaggedHoleToUndefined(Node* node) {
3753 : Node* value = node->InputAt(0);
3754 :
3755 1668 : auto if_is_hole = __ MakeDeferredLabel();
3756 1668 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3757 :
3758 1668 : Node* check = __ WordEqual(value, __ TheHoleConstant());
3759 1668 : __ GotoIf(check, &if_is_hole);
3760 : __ Goto(&done, value);
3761 :
3762 : __ Bind(&if_is_hole);
3763 1668 : __ Goto(&done, __ UndefinedConstant());
3764 :
3765 : __ Bind(&done);
3766 1668 : return done.PhiAt(0);
3767 : }
3768 :
3769 54 : void EffectControlLinearizer::LowerCheckEqualsInternalizedString(
3770 : Node* node, Node* frame_state) {
3771 : Node* exp = node->InputAt(0);
3772 : Node* val = node->InputAt(1);
3773 :
3774 54 : auto if_same = __ MakeLabel();
3775 54 : auto if_notsame = __ MakeDeferredLabel();
3776 54 : auto if_thinstring = __ MakeLabel();
3777 54 : auto if_notthinstring = __ MakeLabel();
3778 :
3779 : // Check if {exp} and {val} are the same, which is the likely case.
3780 54 : __ Branch(__ WordEqual(exp, val), &if_same, &if_notsame);
3781 :
3782 : __ Bind(&if_notsame);
3783 : {
3784 : // Now {val} could still be a non-internalized String that matches {exp}.
3785 : __ DeoptimizeIf(DeoptimizeReason::kWrongName, VectorSlotPair(),
3786 54 : ObjectIsSmi(val), frame_state);
3787 54 : Node* val_map = __ LoadField(AccessBuilder::ForMap(), val);
3788 : Node* val_instance_type =
3789 54 : __ LoadField(AccessBuilder::ForMapInstanceType(), val_map);
3790 :
3791 : // Check for the common case of ThinString first.
3792 : __ GotoIf(__ Word32Equal(val_instance_type,
3793 : __ Int32Constant(THIN_ONE_BYTE_STRING_TYPE)),
3794 54 : &if_thinstring);
3795 : __ Branch(
3796 : __ Word32Equal(val_instance_type, __ Int32Constant(THIN_STRING_TYPE)),
3797 54 : &if_thinstring, &if_notthinstring);
3798 :
3799 : __ Bind(&if_notthinstring);
3800 : {
3801 : // Check that the {val} is a non-internalized String, if it's anything
3802 : // else it cannot match the recorded feedback {exp} anyways.
3803 : __ DeoptimizeIfNot(
3804 : DeoptimizeReason::kWrongName, VectorSlotPair(),
3805 : __ Word32Equal(__ Word32And(val_instance_type,
3806 : __ Int32Constant(kIsNotStringMask |
3807 : kIsNotInternalizedMask)),
3808 : __ Int32Constant(kStringTag | kNotInternalizedTag)),
3809 54 : frame_state);
3810 :
3811 : // Try to find the {val} in the string table.
3812 54 : MachineSignature::Builder builder(graph()->zone(), 1, 2);
3813 : builder.AddReturn(MachineType::AnyTagged());
3814 : builder.AddParam(MachineType::Pointer());
3815 : builder.AddParam(MachineType::AnyTagged());
3816 : Node* try_internalize_string_function = __ ExternalConstant(
3817 54 : ExternalReference::try_internalize_string_function());
3818 : Node* const isolate_ptr =
3819 54 : __ ExternalConstant(ExternalReference::isolate_address(isolate()));
3820 : auto call_descriptor =
3821 108 : Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build());
3822 : Node* val_internalized =
3823 : __ Call(common()->Call(call_descriptor),
3824 54 : try_internalize_string_function, isolate_ptr, val);
3825 :
3826 : // Now see if the results match.
3827 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
3828 54 : __ WordEqual(exp, val_internalized), frame_state);
3829 : __ Goto(&if_same);
3830 : }
3831 :
3832 : __ Bind(&if_thinstring);
3833 : {
3834 : // The {val} is a ThinString, let's check the actual value.
3835 : Node* val_actual =
3836 54 : __ LoadField(AccessBuilder::ForThinStringActual(), val);
3837 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
3838 54 : __ WordEqual(exp, val_actual), frame_state);
3839 : __ Goto(&if_same);
3840 : }
3841 : }
3842 :
3843 : __ Bind(&if_same);
3844 54 : }
3845 :
3846 18 : void EffectControlLinearizer::LowerCheckEqualsSymbol(Node* node,
3847 : Node* frame_state) {
3848 : Node* exp = node->InputAt(0);
3849 : Node* val = node->InputAt(1);
3850 18 : Node* check = __ WordEqual(exp, val);
3851 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(), check,
3852 18 : frame_state);
3853 18 : }
3854 :
3855 53781 : Node* EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value) {
3856 53781 : Node* result = __ Allocate(NOT_TENURED, __ Int32Constant(HeapNumber::kSize));
3857 53784 : __ StoreField(AccessBuilder::ForMap(), result, __ HeapNumberMapConstant());
3858 53784 : __ StoreField(AccessBuilder::ForHeapNumberValue(), result, value);
3859 53784 : return result;
3860 : }
3861 :
3862 136084 : Node* EffectControlLinearizer::ChangeIntPtrToSmi(Node* value) {
3863 : // Do shift on 32bit values if Smis are stored in the lower word.
3864 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3865 : return __ ChangeInt32ToInt64(
3866 : __ Word32Shl(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
3867 : }
3868 136084 : return __ WordShl(value, SmiShiftBitsConstant());
3869 : }
3870 :
3871 0 : Node* EffectControlLinearizer::ChangeInt32ToIntPtr(Node* value) {
3872 135370 : if (machine()->Is64()) {
3873 135370 : value = __ ChangeInt32ToInt64(value);
3874 : }
3875 0 : return value;
3876 : }
3877 :
3878 0 : Node* EffectControlLinearizer::ChangeIntPtrToInt32(Node* value) {
3879 0 : if (machine()->Is64()) {
3880 0 : value = __ TruncateInt64ToInt32(value);
3881 : }
3882 0 : return value;
3883 : }
3884 :
3885 133220 : Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) {
3886 : // Do shift on 32bit values if Smis are stored in the lower word.
3887 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3888 : return __ ChangeInt32ToInt64(__ Word32Shl(value, SmiShiftBitsConstant()));
3889 : }
3890 133222 : return ChangeIntPtrToSmi(ChangeInt32ToIntPtr(value));
3891 : }
3892 :
3893 0 : Node* EffectControlLinearizer::ChangeInt64ToSmi(Node* value) {
3894 : DCHECK(machine()->Is64());
3895 483 : return ChangeIntPtrToSmi(value);
3896 : }
3897 :
3898 0 : Node* EffectControlLinearizer::ChangeUint32ToUintPtr(Node* value) {
3899 1074 : if (machine()->Is64()) {
3900 1074 : value = __ ChangeUint32ToUint64(value);
3901 : }
3902 0 : return value;
3903 : }
3904 :
3905 548 : Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) {
3906 : // Do shift on 32bit values if Smis are stored in the lower word.
3907 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3908 : return __ ChangeUint32ToUint64(__ Word32Shl(value, SmiShiftBitsConstant()));
3909 : } else {
3910 548 : return __ WordShl(ChangeUint32ToUintPtr(value), SmiShiftBitsConstant());
3911 : }
3912 : }
3913 :
3914 150659 : Node* EffectControlLinearizer::ChangeSmiToIntPtr(Node* value) {
3915 : // Do shift on 32bit values if Smis are stored in the lower word.
3916 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3917 : return __ ChangeInt32ToInt64(
3918 : __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
3919 : }
3920 150660 : return __ WordSar(value, SmiShiftBitsConstant());
3921 : }
3922 :
3923 147864 : Node* EffectControlLinearizer::ChangeSmiToInt32(Node* value) {
3924 : // Do shift on 32bit values if Smis are stored in the lower word.
3925 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3926 : return __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant());
3927 : }
3928 147864 : if (machine()->Is64()) {
3929 147864 : return __ TruncateInt64ToInt32(ChangeSmiToIntPtr(value));
3930 : }
3931 0 : return ChangeSmiToIntPtr(value);
3932 : }
3933 :
3934 371 : Node* EffectControlLinearizer::ChangeSmiToInt64(Node* value) {
3935 371 : CHECK(machine()->Is64());
3936 371 : return ChangeSmiToIntPtr(value);
3937 : }
3938 :
3939 255774 : Node* EffectControlLinearizer::ObjectIsSmi(Node* value) {
3940 : return __ WordEqual(__ WordAnd(value, __ IntPtrConstant(kSmiTagMask)),
3941 255774 : __ IntPtrConstant(kSmiTag));
3942 : }
3943 :
3944 0 : Node* EffectControlLinearizer::SmiMaxValueConstant() {
3945 548 : return __ Int32Constant(Smi::kMaxValue);
3946 : }
3947 :
3948 0 : Node* EffectControlLinearizer::SmiShiftBitsConstant() {
3949 : if (machine()->Is64() && SmiValuesAre31Bits()) {
3950 : return __ Int32Constant(kSmiShiftSize + kSmiTagSize);
3951 : }
3952 287291 : return __ IntPtrConstant(kSmiShiftSize + kSmiTagSize);
3953 : }
3954 :
3955 20 : Node* EffectControlLinearizer::LowerPlainPrimitiveToNumber(Node* node) {
3956 : Node* value = node->InputAt(0);
3957 20 : return __ ToNumber(value);
3958 : }
3959 :
3960 9 : Node* EffectControlLinearizer::LowerPlainPrimitiveToWord32(Node* node) {
3961 : Node* value = node->InputAt(0);
3962 :
3963 9 : auto if_not_smi = __ MakeDeferredLabel();
3964 9 : auto if_to_number_smi = __ MakeLabel();
3965 9 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
3966 :
3967 9 : Node* check0 = ObjectIsSmi(value);
3968 9 : __ GotoIfNot(check0, &if_not_smi);
3969 9 : __ Goto(&done, ChangeSmiToInt32(value));
3970 :
3971 : __ Bind(&if_not_smi);
3972 9 : Node* to_number = __ ToNumber(value);
3973 :
3974 9 : Node* check1 = ObjectIsSmi(to_number);
3975 9 : __ GotoIf(check1, &if_to_number_smi);
3976 9 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
3977 9 : __ Goto(&done, __ TruncateFloat64ToWord32(number));
3978 :
3979 : __ Bind(&if_to_number_smi);
3980 9 : __ Goto(&done, ChangeSmiToInt32(to_number));
3981 :
3982 : __ Bind(&done);
3983 9 : return done.PhiAt(0);
3984 : }
3985 :
3986 0 : Node* EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node) {
3987 : Node* value = node->InputAt(0);
3988 :
3989 0 : auto if_not_smi = __ MakeDeferredLabel();
3990 0 : auto if_to_number_smi = __ MakeLabel();
3991 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
3992 :
3993 0 : Node* check0 = ObjectIsSmi(value);
3994 0 : __ GotoIfNot(check0, &if_not_smi);
3995 0 : Node* from_smi = ChangeSmiToInt32(value);
3996 0 : __ Goto(&done, __ ChangeInt32ToFloat64(from_smi));
3997 :
3998 : __ Bind(&if_not_smi);
3999 0 : Node* to_number = __ ToNumber(value);
4000 0 : Node* check1 = ObjectIsSmi(to_number);
4001 0 : __ GotoIf(check1, &if_to_number_smi);
4002 :
4003 0 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
4004 : __ Goto(&done, number);
4005 :
4006 : __ Bind(&if_to_number_smi);
4007 0 : Node* number_from_smi = ChangeSmiToInt32(to_number);
4008 0 : number_from_smi = __ ChangeInt32ToFloat64(number_from_smi);
4009 : __ Goto(&done, number_from_smi);
4010 :
4011 : __ Bind(&done);
4012 0 : return done.PhiAt(0);
4013 : }
4014 :
4015 1690 : Node* EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node) {
4016 : Node* object = node->InputAt(0);
4017 : Node* elements = node->InputAt(1);
4018 :
4019 1690 : auto if_not_fixed_array = __ MakeDeferredLabel();
4020 1690 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4021 :
4022 : // Load the current map of {elements}.
4023 1690 : Node* elements_map = __ LoadField(AccessBuilder::ForMap(), elements);
4024 :
4025 : // Check if {elements} is not a copy-on-write FixedArray.
4026 1690 : Node* check = __ WordEqual(elements_map, __ FixedArrayMapConstant());
4027 1690 : __ GotoIfNot(check, &if_not_fixed_array);
4028 : // Nothing to do if the {elements} are not copy-on-write.
4029 : __ Goto(&done, elements);
4030 :
4031 : __ Bind(&if_not_fixed_array);
4032 : // We need to take a copy of the {elements} and set them up for {object}.
4033 : Operator::Properties properties = Operator::kEliminatable;
4034 : Callable callable =
4035 1690 : Builtins::CallableFor(isolate(), Builtins::kCopyFastSmiOrObjectElements);
4036 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4037 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4038 : graph()->zone(), callable.descriptor(),
4039 3380 : callable.descriptor().GetStackParameterCount(), flags, properties);
4040 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4041 3380 : object, __ NoContextConstant());
4042 : __ Goto(&done, result);
4043 :
4044 : __ Bind(&done);
4045 1690 : return done.PhiAt(0);
4046 : }
4047 :
4048 3538 : Node* EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node,
4049 : Node* frame_state) {
4050 3538 : GrowFastElementsParameters params = GrowFastElementsParametersOf(node->op());
4051 : Node* object = node->InputAt(0);
4052 : Node* elements = node->InputAt(1);
4053 : Node* index = node->InputAt(2);
4054 : Node* elements_length = node->InputAt(3);
4055 :
4056 3538 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4057 3538 : auto if_grow = __ MakeDeferredLabel();
4058 : auto if_not_grow = __ MakeLabel();
4059 :
4060 : // Check if we need to grow the {elements} backing store.
4061 3538 : Node* check = __ Uint32LessThan(index, elements_length);
4062 3538 : __ GotoIfNot(check, &if_grow);
4063 : __ Goto(&done, elements);
4064 :
4065 : __ Bind(&if_grow);
4066 : // We need to grow the {elements} for {object}.
4067 : Operator::Properties properties = Operator::kEliminatable;
4068 : Callable callable =
4069 3538 : (params.mode() == GrowFastElementsMode::kDoubleElements)
4070 : ? Builtins::CallableFor(isolate(), Builtins::kGrowFastDoubleElements)
4071 : : Builtins::CallableFor(isolate(),
4072 7076 : Builtins::kGrowFastSmiOrObjectElements);
4073 : CallDescriptor::Flags call_flags = CallDescriptor::kNoFlags;
4074 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4075 : graph()->zone(), callable.descriptor(),
4076 7076 : callable.descriptor().GetStackParameterCount(), call_flags, properties);
4077 : Node* new_elements =
4078 : __ Call(call_descriptor, __ HeapConstant(callable.code()), object,
4079 7076 : ChangeInt32ToSmi(index), __ NoContextConstant());
4080 :
4081 : // Ensure that we were able to grow the {elements}.
4082 : __ DeoptimizeIf(DeoptimizeReason::kCouldNotGrowElements, params.feedback(),
4083 3538 : ObjectIsSmi(new_elements), frame_state);
4084 : __ Goto(&done, new_elements);
4085 :
4086 : __ Bind(&done);
4087 3538 : return done.PhiAt(0);
4088 : }
4089 :
4090 625 : void EffectControlLinearizer::LowerTransitionElementsKind(Node* node) {
4091 625 : ElementsTransition const transition = ElementsTransitionOf(node->op());
4092 : Node* object = node->InputAt(0);
4093 :
4094 625 : auto if_map_same = __ MakeDeferredLabel();
4095 625 : auto done = __ MakeLabel();
4096 :
4097 625 : Node* source_map = __ HeapConstant(transition.source());
4098 625 : Node* target_map = __ HeapConstant(transition.target());
4099 :
4100 : // Load the current map of {object}.
4101 625 : Node* object_map = __ LoadField(AccessBuilder::ForMap(), object);
4102 :
4103 : // Check if {object_map} is the same as {source_map}.
4104 625 : Node* check = __ WordEqual(object_map, source_map);
4105 625 : __ GotoIf(check, &if_map_same);
4106 : __ Goto(&done);
4107 :
4108 : __ Bind(&if_map_same);
4109 625 : switch (transition.mode()) {
4110 : case ElementsTransition::kFastTransition:
4111 : // In-place migration of {object}, just store the {target_map}.
4112 209 : __ StoreField(AccessBuilder::ForMap(), object, target_map);
4113 209 : break;
4114 : case ElementsTransition::kSlowTransition: {
4115 : // Instance migration, call out to the runtime for {object}.
4116 416 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4117 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4118 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4119 416 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4120 : __ Call(call_descriptor, __ CEntryStubConstant(1), object, target_map,
4121 : __ ExternalConstant(ExternalReference::Create(id)),
4122 416 : __ Int32Constant(2), __ NoContextConstant());
4123 : break;
4124 : }
4125 : }
4126 : __ Goto(&done);
4127 :
4128 : __ Bind(&done);
4129 625 : }
4130 :
4131 7583 : Node* EffectControlLinearizer::LowerLoadMessage(Node* node) {
4132 : Node* offset = node->InputAt(0);
4133 : Node* object_pattern =
4134 7583 : __ LoadField(AccessBuilder::ForExternalIntPtr(), offset);
4135 7583 : return __ BitcastWordToTagged(object_pattern);
4136 : }
4137 :
4138 29409 : void EffectControlLinearizer::LowerStoreMessage(Node* node) {
4139 : Node* offset = node->InputAt(0);
4140 : Node* object = node->InputAt(1);
4141 29409 : Node* object_pattern = __ BitcastTaggedToWord(object);
4142 29409 : __ StoreField(AccessBuilder::ForExternalIntPtr(), offset, object_pattern);
4143 29409 : }
4144 :
4145 578 : Node* EffectControlLinearizer::LowerLoadFieldByIndex(Node* node) {
4146 : Node* object = node->InputAt(0);
4147 : Node* index = node->InputAt(1);
4148 578 : Node* zero = __ IntPtrConstant(0);
4149 578 : Node* one = __ IntPtrConstant(1);
4150 :
4151 : // Sign-extend the {index} on 64-bit architectures.
4152 578 : if (machine()->Is64()) {
4153 578 : index = __ ChangeInt32ToInt64(index);
4154 : }
4155 :
4156 578 : auto if_double = __ MakeDeferredLabel();
4157 578 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4158 :
4159 : // Check if field is a mutable double field.
4160 578 : __ GotoIfNot(__ WordEqual(__ WordAnd(index, one), zero), &if_double);
4161 :
4162 : // The field is a proper Tagged field on {object}. The {index} is shifted
4163 : // to the left by one in the code below.
4164 : {
4165 : // Check if field is in-object or out-of-object.
4166 578 : auto if_outofobject = __ MakeLabel();
4167 578 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4168 :
4169 : // The field is located in the {object} itself.
4170 : {
4171 : Node* offset =
4172 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4173 578 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4174 578 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4175 : __ Goto(&done, result);
4176 : }
4177 :
4178 : // The field is located in the properties backing store of {object}.
4179 : // The {index} is equal to the negated out of property index plus 1.
4180 : __ Bind(&if_outofobject);
4181 : {
4182 : Node* properties =
4183 578 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4184 : Node* offset =
4185 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4186 : __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4187 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4188 578 : kHeapObjectTag));
4189 578 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4190 : __ Goto(&done, result);
4191 : }
4192 : }
4193 :
4194 : // The field is a Double field, either unboxed in the object on 64-bit
4195 : // architectures, or as MutableHeapNumber.
4196 : __ Bind(&if_double);
4197 : {
4198 578 : auto done_double = __ MakeLabel(MachineRepresentation::kFloat64);
4199 :
4200 578 : index = __ WordSar(index, one);
4201 :
4202 : // Check if field is in-object or out-of-object.
4203 578 : auto if_outofobject = __ MakeLabel();
4204 578 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4205 :
4206 : // The field is located in the {object} itself.
4207 : {
4208 : Node* offset =
4209 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)),
4210 578 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4211 : if (FLAG_unbox_double_fields) {
4212 578 : Node* result = __ Load(MachineType::Float64(), object, offset);
4213 : __ Goto(&done_double, result);
4214 : } else {
4215 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4216 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4217 : __ Goto(&done_double, result);
4218 : }
4219 : }
4220 :
4221 : __ Bind(&if_outofobject);
4222 : {
4223 : Node* properties =
4224 578 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4225 : Node* offset =
4226 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4227 : __ IntPtrConstant(kTaggedSizeLog2)),
4228 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4229 578 : kHeapObjectTag));
4230 578 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4231 578 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4232 : __ Goto(&done_double, result);
4233 : }
4234 :
4235 : __ Bind(&done_double);
4236 : {
4237 578 : Node* result = AllocateHeapNumberWithValue(done_double.PhiAt(0));
4238 : __ Goto(&done, result);
4239 : }
4240 : }
4241 :
4242 : __ Bind(&done);
4243 578 : return done.PhiAt(0);
4244 : }
4245 :
4246 428 : Node* EffectControlLinearizer::BuildReverseBytes(ExternalArrayType type,
4247 : Node* value) {
4248 428 : switch (type) {
4249 : case kExternalInt8Array:
4250 : case kExternalUint8Array:
4251 : case kExternalUint8ClampedArray:
4252 : return value;
4253 :
4254 : case kExternalInt16Array: {
4255 51 : Node* result = __ Word32ReverseBytes(value);
4256 51 : result = __ Word32Sar(result, __ Int32Constant(16));
4257 51 : return result;
4258 : }
4259 :
4260 : case kExternalUint16Array: {
4261 51 : Node* result = __ Word32ReverseBytes(value);
4262 51 : result = __ Word32Shr(result, __ Int32Constant(16));
4263 51 : return result;
4264 : }
4265 :
4266 : case kExternalInt32Array: // Fall through.
4267 : case kExternalUint32Array:
4268 88 : return __ Word32ReverseBytes(value);
4269 :
4270 : case kExternalFloat32Array: {
4271 51 : Node* result = __ BitcastFloat32ToInt32(value);
4272 51 : result = __ Word32ReverseBytes(result);
4273 51 : result = __ BitcastInt32ToFloat32(result);
4274 51 : return result;
4275 : }
4276 :
4277 : case kExternalFloat64Array: {
4278 51 : if (machine()->Is64()) {
4279 51 : Node* result = __ BitcastFloat64ToInt64(value);
4280 51 : result = __ Word64ReverseBytes(result);
4281 51 : result = __ BitcastInt64ToFloat64(result);
4282 51 : return result;
4283 : } else {
4284 0 : Node* lo = __ Word32ReverseBytes(__ Float64ExtractLowWord32(value));
4285 0 : Node* hi = __ Word32ReverseBytes(__ Float64ExtractHighWord32(value));
4286 0 : Node* result = __ Float64Constant(0.0);
4287 0 : result = __ Float64InsertLowWord32(result, hi);
4288 0 : result = __ Float64InsertHighWord32(result, lo);
4289 0 : return result;
4290 : }
4291 : }
4292 :
4293 : case kExternalBigInt64Array:
4294 : case kExternalBigUint64Array:
4295 0 : UNREACHABLE();
4296 : }
4297 0 : }
4298 :
4299 248 : Node* EffectControlLinearizer::LowerLoadDataViewElement(Node* node) {
4300 248 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4301 : Node* buffer = node->InputAt(0);
4302 : Node* storage = node->InputAt(1);
4303 : Node* byte_offset = node->InputAt(2);
4304 : Node* index = node->InputAt(3);
4305 : Node* is_little_endian = node->InputAt(4);
4306 :
4307 : // We need to keep the {buffer} alive so that the GC will not release the
4308 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4309 248 : __ Retain(buffer);
4310 :
4311 : // Compute the effective offset.
4312 248 : Node* offset = __ IntAdd(byte_offset, index);
4313 :
4314 : MachineType const machine_type =
4315 248 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4316 :
4317 248 : Node* value = __ LoadUnaligned(machine_type, storage, offset);
4318 248 : auto big_endian = __ MakeLabel();
4319 248 : auto done = __ MakeLabel(machine_type.representation());
4320 :
4321 248 : __ GotoIfNot(is_little_endian, &big_endian);
4322 : { // Little-endian load.
4323 : #if V8_TARGET_LITTLE_ENDIAN
4324 : __ Goto(&done, value);
4325 : #else
4326 : __ Goto(&done, BuildReverseBytes(element_type, value));
4327 : #endif // V8_TARGET_LITTLE_ENDIAN
4328 : }
4329 :
4330 : __ Bind(&big_endian);
4331 : { // Big-endian load.
4332 : #if V8_TARGET_LITTLE_ENDIAN
4333 248 : __ Goto(&done, BuildReverseBytes(element_type, value));
4334 : #else
4335 : __ Goto(&done, value);
4336 : #endif // V8_TARGET_LITTLE_ENDIAN
4337 : }
4338 :
4339 : // We're done, return {result}.
4340 : __ Bind(&done);
4341 248 : return done.PhiAt(0);
4342 : }
4343 :
4344 180 : void EffectControlLinearizer::LowerStoreDataViewElement(Node* node) {
4345 180 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4346 : Node* buffer = node->InputAt(0);
4347 : Node* storage = node->InputAt(1);
4348 : Node* byte_offset = node->InputAt(2);
4349 : Node* index = node->InputAt(3);
4350 : Node* value = node->InputAt(4);
4351 : Node* is_little_endian = node->InputAt(5);
4352 :
4353 : // We need to keep the {buffer} alive so that the GC will not release the
4354 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4355 180 : __ Retain(buffer);
4356 :
4357 : // Compute the effective offset.
4358 180 : Node* offset = __ IntAdd(byte_offset, index);
4359 :
4360 : MachineType const machine_type =
4361 180 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4362 :
4363 180 : auto big_endian = __ MakeLabel();
4364 180 : auto done = __ MakeLabel(machine_type.representation());
4365 :
4366 180 : __ GotoIfNot(is_little_endian, &big_endian);
4367 : { // Little-endian store.
4368 : #if V8_TARGET_LITTLE_ENDIAN
4369 : __ Goto(&done, value);
4370 : #else
4371 : __ Goto(&done, BuildReverseBytes(element_type, value));
4372 : #endif // V8_TARGET_LITTLE_ENDIAN
4373 : }
4374 :
4375 : __ Bind(&big_endian);
4376 : { // Big-endian store.
4377 : #if V8_TARGET_LITTLE_ENDIAN
4378 180 : __ Goto(&done, BuildReverseBytes(element_type, value));
4379 : #else
4380 : __ Goto(&done, value);
4381 : #endif // V8_TARGET_LITTLE_ENDIAN
4382 : }
4383 :
4384 : __ Bind(&done);
4385 : __ StoreUnaligned(machine_type.representation(), storage, offset,
4386 180 : done.PhiAt(0));
4387 180 : }
4388 :
4389 4600 : Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) {
4390 4600 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4391 : Node* buffer = node->InputAt(0);
4392 : Node* base = node->InputAt(1);
4393 : Node* external = node->InputAt(2);
4394 : Node* index = node->InputAt(3);
4395 :
4396 : // We need to keep the {buffer} alive so that the GC will not release the
4397 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4398 4600 : __ Retain(buffer);
4399 :
4400 : // Compute the effective storage pointer, handling the case where the
4401 : // {external} pointer is the effective storage pointer (i.e. the {base}
4402 : // is Smi zero).
4403 : Node* storage = IntPtrMatcher(base).Is(0)
4404 : ? external
4405 4600 : : __ UnsafePointerAdd(base, external);
4406 :
4407 : // Perform the actual typed element access.
4408 : return __ LoadElement(AccessBuilder::ForTypedArrayElement(
4409 : array_type, true, LoadSensitivity::kCritical),
4410 4600 : storage, index);
4411 : }
4412 :
4413 4128 : void EffectControlLinearizer::LowerStoreTypedElement(Node* node) {
4414 4128 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4415 : Node* buffer = node->InputAt(0);
4416 : Node* base = node->InputAt(1);
4417 : Node* external = node->InputAt(2);
4418 : Node* index = node->InputAt(3);
4419 : Node* value = node->InputAt(4);
4420 :
4421 : // We need to keep the {buffer} alive so that the GC will not release the
4422 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4423 4128 : __ Retain(buffer);
4424 :
4425 : // Compute the effective storage pointer, handling the case where the
4426 : // {external} pointer is the effective storage pointer (i.e. the {base}
4427 : // is Smi zero).
4428 : Node* storage = IntPtrMatcher(base).Is(0)
4429 : ? external
4430 4128 : : __ UnsafePointerAdd(base, external);
4431 :
4432 : // Perform the actual typed element access.
4433 : __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true),
4434 4128 : storage, index, value);
4435 4128 : }
4436 :
4437 646 : void EffectControlLinearizer::TransitionElementsTo(Node* node, Node* array,
4438 : ElementsKind from,
4439 : ElementsKind to) {
4440 : DCHECK(IsMoreGeneralElementsKindTransition(from, to));
4441 : DCHECK(to == HOLEY_ELEMENTS || to == HOLEY_DOUBLE_ELEMENTS);
4442 :
4443 : Handle<Map> target(to == HOLEY_ELEMENTS ? FastMapParameterOf(node->op())
4444 646 : : DoubleMapParameterOf(node->op()));
4445 323 : Node* target_map = __ HeapConstant(target);
4446 :
4447 323 : if (IsSimpleMapChangeTransition(from, to)) {
4448 104 : __ StoreField(AccessBuilder::ForMap(), array, target_map);
4449 : } else {
4450 : // Instance migration, call out to the runtime for {array}.
4451 219 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4452 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4453 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4454 219 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4455 : __ Call(call_descriptor, __ CEntryStubConstant(1), array, target_map,
4456 : __ ExternalConstant(ExternalReference::Create(id)),
4457 219 : __ Int32Constant(2), __ NoContextConstant());
4458 : }
4459 323 : }
4460 :
4461 391 : Node* EffectControlLinearizer::IsElementsKindGreaterThan(
4462 : Node* kind, ElementsKind reference_kind) {
4463 391 : Node* ref_kind = __ Int32Constant(reference_kind);
4464 391 : Node* ret = __ Int32LessThan(ref_kind, kind);
4465 391 : return ret;
4466 : }
4467 :
4468 86 : void EffectControlLinearizer::LowerTransitionAndStoreElement(Node* node) {
4469 : Node* array = node->InputAt(0);
4470 : Node* index = node->InputAt(1);
4471 : Node* value = node->InputAt(2);
4472 :
4473 : // Possibly transition array based on input and store.
4474 : //
4475 : // -- TRANSITION PHASE -----------------
4476 : // kind = ElementsKind(array)
4477 : // if value is not smi {
4478 : // if kind == HOLEY_SMI_ELEMENTS {
4479 : // if value is heap number {
4480 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4481 : // kind = HOLEY_DOUBLE_ELEMENTS
4482 : // } else {
4483 : // Transition array to HOLEY_ELEMENTS
4484 : // kind = HOLEY_ELEMENTS
4485 : // }
4486 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4487 : // if value is not heap number {
4488 : // Transition array to HOLEY_ELEMENTS
4489 : // kind = HOLEY_ELEMENTS
4490 : // }
4491 : // }
4492 : // }
4493 : //
4494 : // -- STORE PHASE ----------------------
4495 : // [make sure {kind} is up-to-date]
4496 : // if kind == HOLEY_DOUBLE_ELEMENTS {
4497 : // if value is smi {
4498 : // float_value = convert smi to float
4499 : // Store array[index] = float_value
4500 : // } else {
4501 : // float_value = value
4502 : // Store array[index] = float_value
4503 : // }
4504 : // } else {
4505 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
4506 : // Store array[index] = value
4507 : // }
4508 : //
4509 86 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4510 : Node* kind;
4511 : {
4512 86 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4513 86 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4514 86 : Node* andit = __ Word32And(bit_field2, mask);
4515 86 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4516 86 : kind = __ Word32Shr(andit, shift);
4517 : }
4518 :
4519 86 : auto do_store = __ MakeLabel(MachineRepresentation::kWord32);
4520 : // We can store a smi anywhere.
4521 86 : __ GotoIf(ObjectIsSmi(value), &do_store, kind);
4522 :
4523 : // {value} is a HeapObject.
4524 86 : auto transition_smi_array = __ MakeDeferredLabel();
4525 86 : auto transition_double_to_fast = __ MakeDeferredLabel();
4526 : {
4527 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4528 86 : &transition_smi_array);
4529 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), &do_store,
4530 86 : kind);
4531 :
4532 : // We have double elements kind. Only a HeapNumber can be stored
4533 : // without effecting a transition.
4534 86 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4535 86 : Node* heap_number_map = __ HeapNumberMapConstant();
4536 86 : Node* check = __ WordEqual(value_map, heap_number_map);
4537 86 : __ GotoIfNot(check, &transition_double_to_fast);
4538 : __ Goto(&do_store, kind);
4539 : }
4540 :
4541 : __ Bind(&transition_smi_array); // deferred code.
4542 : {
4543 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS or
4544 : // to HOLEY_ELEMENTS.
4545 86 : auto if_value_not_heap_number = __ MakeLabel();
4546 86 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4547 86 : Node* heap_number_map = __ HeapNumberMapConstant();
4548 86 : Node* check = __ WordEqual(value_map, heap_number_map);
4549 86 : __ GotoIfNot(check, &if_value_not_heap_number);
4550 : {
4551 : // {value} is a HeapNumber.
4552 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4553 86 : HOLEY_DOUBLE_ELEMENTS);
4554 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS));
4555 : }
4556 : __ Bind(&if_value_not_heap_number);
4557 : {
4558 86 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
4559 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4560 : }
4561 : }
4562 :
4563 : __ Bind(&transition_double_to_fast); // deferred code.
4564 : {
4565 86 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
4566 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4567 : }
4568 :
4569 : // Make sure kind is up-to-date.
4570 : __ Bind(&do_store);
4571 : kind = do_store.PhiAt(0);
4572 :
4573 86 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4574 86 : auto if_kind_is_double = __ MakeLabel();
4575 86 : auto done = __ MakeLabel();
4576 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4577 86 : &if_kind_is_double);
4578 : {
4579 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
4580 : __ StoreElement(AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS),
4581 86 : elements, index, value);
4582 : __ Goto(&done);
4583 : }
4584 : __ Bind(&if_kind_is_double);
4585 : {
4586 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
4587 86 : auto do_double_store = __ MakeLabel();
4588 86 : __ GotoIfNot(ObjectIsSmi(value), &do_double_store);
4589 : {
4590 86 : Node* int_value = ChangeSmiToInt32(value);
4591 86 : Node* float_value = __ ChangeInt32ToFloat64(int_value);
4592 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4593 86 : index, float_value);
4594 : __ Goto(&done);
4595 : }
4596 : __ Bind(&do_double_store);
4597 : {
4598 : Node* float_value =
4599 86 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
4600 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4601 86 : index, __ Float64SilenceNaN(float_value));
4602 : __ Goto(&done);
4603 : }
4604 : }
4605 :
4606 : __ Bind(&done);
4607 86 : }
4608 :
4609 29 : void EffectControlLinearizer::LowerTransitionAndStoreNumberElement(Node* node) {
4610 : Node* array = node->InputAt(0);
4611 : Node* index = node->InputAt(1);
4612 : Node* value = node->InputAt(2); // This is a Float64, not tagged.
4613 :
4614 : // Possibly transition array based on input and store.
4615 : //
4616 : // -- TRANSITION PHASE -----------------
4617 : // kind = ElementsKind(array)
4618 : // if kind == HOLEY_SMI_ELEMENTS {
4619 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4620 : // } else if kind != HOLEY_DOUBLE_ELEMENTS {
4621 : // This is UNREACHABLE, execute a debug break.
4622 : // }
4623 : //
4624 : // -- STORE PHASE ----------------------
4625 : // Store array[index] = value (it's a float)
4626 : //
4627 29 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4628 : Node* kind;
4629 : {
4630 29 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4631 29 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4632 29 : Node* andit = __ Word32And(bit_field2, mask);
4633 29 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4634 29 : kind = __ Word32Shr(andit, shift);
4635 : }
4636 :
4637 29 : auto do_store = __ MakeLabel();
4638 :
4639 : // {value} is a float64.
4640 29 : auto transition_smi_array = __ MakeDeferredLabel();
4641 : {
4642 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4643 29 : &transition_smi_array);
4644 : // We expect that our input array started at HOLEY_SMI_ELEMENTS, and
4645 : // climbs the lattice up to HOLEY_DOUBLE_ELEMENTS. Force a debug break
4646 : // if this assumption is broken. It also would be the case that
4647 : // loop peeling can break this assumption.
4648 : __ GotoIf(__ Word32Equal(kind, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)),
4649 29 : &do_store);
4650 : // TODO(turbofan): It would be good to have an "Unreachable()" node type.
4651 29 : __ DebugBreak();
4652 : __ Goto(&do_store);
4653 : }
4654 :
4655 : __ Bind(&transition_smi_array); // deferred code.
4656 : {
4657 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS.
4658 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4659 29 : HOLEY_DOUBLE_ELEMENTS);
4660 : __ Goto(&do_store);
4661 : }
4662 :
4663 : __ Bind(&do_store);
4664 :
4665 29 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4666 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, index,
4667 29 : __ Float64SilenceNaN(value));
4668 29 : }
4669 :
4670 18 : void EffectControlLinearizer::LowerTransitionAndStoreNonNumberElement(
4671 18 : Node* node) {
4672 : Node* array = node->InputAt(0);
4673 : Node* index = node->InputAt(1);
4674 : Node* value = node->InputAt(2);
4675 :
4676 : // Possibly transition array based on input and store.
4677 : //
4678 : // -- TRANSITION PHASE -----------------
4679 : // kind = ElementsKind(array)
4680 : // if kind == HOLEY_SMI_ELEMENTS {
4681 : // Transition array to HOLEY_ELEMENTS
4682 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4683 : // Transition array to HOLEY_ELEMENTS
4684 : // }
4685 : //
4686 : // -- STORE PHASE ----------------------
4687 : // // kind is HOLEY_ELEMENTS
4688 : // Store array[index] = value
4689 : //
4690 18 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4691 : Node* kind;
4692 : {
4693 18 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4694 18 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4695 18 : Node* andit = __ Word32And(bit_field2, mask);
4696 18 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4697 18 : kind = __ Word32Shr(andit, shift);
4698 : }
4699 :
4700 18 : auto do_store = __ MakeLabel();
4701 :
4702 18 : auto transition_smi_array = __ MakeDeferredLabel();
4703 18 : auto transition_double_to_fast = __ MakeDeferredLabel();
4704 : {
4705 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4706 18 : &transition_smi_array);
4707 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4708 18 : &transition_double_to_fast);
4709 : __ Goto(&do_store);
4710 : }
4711 :
4712 : __ Bind(&transition_smi_array); // deferred code.
4713 : {
4714 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_ELEMENTS.
4715 18 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
4716 : __ Goto(&do_store);
4717 : }
4718 :
4719 : __ Bind(&transition_double_to_fast); // deferred code.
4720 : {
4721 18 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
4722 : __ Goto(&do_store);
4723 : }
4724 :
4725 : __ Bind(&do_store);
4726 :
4727 18 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4728 : // Our ElementsKind is HOLEY_ELEMENTS.
4729 18 : ElementAccess access = AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS);
4730 18 : Type value_type = ValueTypeParameterOf(node->op());
4731 18 : if (value_type.Is(Type::BooleanOrNullOrUndefined())) {
4732 14 : access.type = value_type;
4733 14 : access.write_barrier_kind = kNoWriteBarrier;
4734 : }
4735 18 : __ StoreElement(access, elements, index, value);
4736 18 : }
4737 :
4738 68 : void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
4739 : Node* array = node->InputAt(0);
4740 : Node* index = node->InputAt(1);
4741 : Node* value = node->InputAt(2); // int32
4742 :
4743 : // Store a signed small in an output array.
4744 : //
4745 : // kind = ElementsKind(array)
4746 : //
4747 : // -- STORE PHASE ----------------------
4748 : // if kind == HOLEY_DOUBLE_ELEMENTS {
4749 : // float_value = convert int32 to float
4750 : // Store array[index] = float_value
4751 : // } else {
4752 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
4753 : // smi_value = convert int32 to smi
4754 : // Store array[index] = smi_value
4755 : // }
4756 : //
4757 68 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4758 : Node* kind;
4759 : {
4760 68 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4761 68 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4762 68 : Node* andit = __ Word32And(bit_field2, mask);
4763 68 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4764 68 : kind = __ Word32Shr(andit, shift);
4765 : }
4766 :
4767 68 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4768 68 : auto if_kind_is_double = __ MakeLabel();
4769 68 : auto done = __ MakeLabel();
4770 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4771 68 : &if_kind_is_double);
4772 : {
4773 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
4774 : // In this case, we know our value is a signed small, and we can optimize
4775 : // the ElementAccess information.
4776 68 : ElementAccess access = AccessBuilder::ForFixedArrayElement();
4777 68 : access.type = Type::SignedSmall();
4778 68 : access.machine_type = MachineType::TaggedSigned();
4779 68 : access.write_barrier_kind = kNoWriteBarrier;
4780 68 : Node* smi_value = ChangeInt32ToSmi(value);
4781 68 : __ StoreElement(access, elements, index, smi_value);
4782 : __ Goto(&done);
4783 : }
4784 : __ Bind(&if_kind_is_double);
4785 : {
4786 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
4787 68 : Node* float_value = __ ChangeInt32ToFloat64(value);
4788 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4789 68 : index, float_value);
4790 : __ Goto(&done);
4791 : }
4792 :
4793 : __ Bind(&done);
4794 68 : }
4795 :
4796 4972 : void EffectControlLinearizer::LowerRuntimeAbort(Node* node) {
4797 2486 : AbortReason reason = AbortReasonOf(node->op());
4798 2486 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4799 : Runtime::FunctionId id = Runtime::kAbort;
4800 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4801 2486 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
4802 : __ Call(call_descriptor, __ CEntryStubConstant(1),
4803 : jsgraph()->SmiConstant(static_cast<int>(reason)),
4804 : __ ExternalConstant(ExternalReference::Create(id)),
4805 4972 : __ Int32Constant(1), __ NoContextConstant());
4806 2486 : }
4807 :
4808 869 : Node* EffectControlLinearizer::LowerConvertReceiver(Node* node) {
4809 869 : ConvertReceiverMode const mode = ConvertReceiverModeOf(node->op());
4810 : Node* value = node->InputAt(0);
4811 : Node* global_proxy = node->InputAt(1);
4812 :
4813 869 : switch (mode) {
4814 : case ConvertReceiverMode::kNullOrUndefined: {
4815 : return global_proxy;
4816 : }
4817 : case ConvertReceiverMode::kNotNullOrUndefined: {
4818 804 : auto convert_to_object = __ MakeDeferredLabel();
4819 804 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
4820 :
4821 : // Check if {value} is already a JSReceiver.
4822 804 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
4823 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
4824 804 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4825 : Node* value_instance_type =
4826 804 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
4827 : Node* check = __ Uint32LessThan(
4828 804 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
4829 804 : __ GotoIf(check, &convert_to_object);
4830 : __ Goto(&done_convert, value);
4831 :
4832 : // Wrap the primitive {value} into a JSValue.
4833 : __ Bind(&convert_to_object);
4834 : Operator::Properties properties = Operator::kEliminatable;
4835 804 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
4836 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4837 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4838 : graph()->zone(), callable.descriptor(),
4839 1608 : callable.descriptor().GetStackParameterCount(), flags, properties);
4840 : Node* native_context = __ LoadField(
4841 804 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
4842 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4843 804 : value, native_context);
4844 : __ Goto(&done_convert, result);
4845 :
4846 : __ Bind(&done_convert);
4847 : return done_convert.PhiAt(0);
4848 : }
4849 : case ConvertReceiverMode::kAny: {
4850 65 : auto convert_to_object = __ MakeDeferredLabel();
4851 65 : auto convert_global_proxy = __ MakeDeferredLabel();
4852 65 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
4853 :
4854 : // Check if {value} is already a JSReceiver, or null/undefined.
4855 65 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
4856 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
4857 65 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4858 : Node* value_instance_type =
4859 65 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
4860 : Node* check = __ Uint32LessThan(
4861 65 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
4862 65 : __ GotoIf(check, &convert_to_object);
4863 : __ Goto(&done_convert, value);
4864 :
4865 : // Wrap the primitive {value} into a JSValue.
4866 : __ Bind(&convert_to_object);
4867 : __ GotoIf(__ WordEqual(value, __ UndefinedConstant()),
4868 65 : &convert_global_proxy);
4869 65 : __ GotoIf(__ WordEqual(value, __ NullConstant()), &convert_global_proxy);
4870 : Operator::Properties properties = Operator::kEliminatable;
4871 65 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
4872 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4873 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4874 : graph()->zone(), callable.descriptor(),
4875 130 : callable.descriptor().GetStackParameterCount(), flags, properties);
4876 : Node* native_context = __ LoadField(
4877 65 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
4878 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4879 65 : value, native_context);
4880 : __ Goto(&done_convert, result);
4881 :
4882 : // Replace the {value} with the {global_proxy}.
4883 : __ Bind(&convert_global_proxy);
4884 : __ Goto(&done_convert, global_proxy);
4885 :
4886 : __ Bind(&done_convert);
4887 : return done_convert.PhiAt(0);
4888 : }
4889 : }
4890 :
4891 0 : UNREACHABLE();
4892 : return nullptr;
4893 : }
4894 :
4895 8215 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundUp(Node* node) {
4896 : // Nothing to be done if a fast hardware instruction is available.
4897 8215 : if (machine()->Float64RoundUp().IsSupported()) {
4898 : return Nothing<Node*>();
4899 : }
4900 :
4901 : Node* const input = node->InputAt(0);
4902 :
4903 : // General case for ceil.
4904 : //
4905 : // if 0.0 < input then
4906 : // if 2^52 <= input then
4907 : // input
4908 : // else
4909 : // let temp1 = (2^52 + input) - 2^52 in
4910 : // if temp1 < input then
4911 : // temp1 + 1
4912 : // else
4913 : // temp1
4914 : // else
4915 : // if input == 0 then
4916 : // input
4917 : // else
4918 : // if input <= -2^52 then
4919 : // input
4920 : // else
4921 : // let temp1 = -0 - input in
4922 : // let temp2 = (2^52 + temp1) - 2^52 in
4923 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
4924 : // -0 - temp3
4925 :
4926 0 : auto if_not_positive = __ MakeDeferredLabel();
4927 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
4928 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
4929 0 : auto if_zero = __ MakeDeferredLabel();
4930 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
4931 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
4932 :
4933 0 : Node* const zero = __ Float64Constant(0.0);
4934 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
4935 0 : Node* const one = __ Float64Constant(1.0);
4936 :
4937 0 : Node* check0 = __ Float64LessThan(zero, input);
4938 0 : __ GotoIfNot(check0, &if_not_positive);
4939 : {
4940 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
4941 0 : __ GotoIf(check1, &if_greater_than_two_52);
4942 : {
4943 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
4944 0 : __ GotoIfNot(__ Float64LessThan(temp1, input), &done, temp1);
4945 0 : __ Goto(&done, __ Float64Add(temp1, one));
4946 : }
4947 :
4948 : __ Bind(&if_greater_than_two_52);
4949 : __ Goto(&done, input);
4950 : }
4951 :
4952 : __ Bind(&if_not_positive);
4953 : {
4954 0 : Node* check1 = __ Float64Equal(input, zero);
4955 0 : __ GotoIf(check1, &if_zero);
4956 :
4957 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
4958 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
4959 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
4960 :
4961 : {
4962 0 : Node* const minus_zero = __ Float64Constant(-0.0);
4963 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
4964 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
4965 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
4966 0 : __ GotoIfNot(check3, &done_temp3, temp2);
4967 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
4968 :
4969 : __ Bind(&done_temp3);
4970 : Node* temp3 = done_temp3.PhiAt(0);
4971 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
4972 : }
4973 : __ Bind(&if_less_than_minus_two_52);
4974 : __ Goto(&done, input);
4975 :
4976 : __ Bind(&if_zero);
4977 : __ Goto(&done, input);
4978 : }
4979 : __ Bind(&done);
4980 : return Just(done.PhiAt(0));
4981 : }
4982 :
4983 0 : Node* EffectControlLinearizer::BuildFloat64RoundDown(Node* value) {
4984 0 : if (machine()->Float64RoundDown().IsSupported()) {
4985 0 : return __ Float64RoundDown(value);
4986 : }
4987 :
4988 : Node* const input = value;
4989 :
4990 : // General case for floor.
4991 : //
4992 : // if 0.0 < input then
4993 : // if 2^52 <= input then
4994 : // input
4995 : // else
4996 : // let temp1 = (2^52 + input) - 2^52 in
4997 : // if input < temp1 then
4998 : // temp1 - 1
4999 : // else
5000 : // temp1
5001 : // else
5002 : // if input == 0 then
5003 : // input
5004 : // else
5005 : // if input <= -2^52 then
5006 : // input
5007 : // else
5008 : // let temp1 = -0 - input in
5009 : // let temp2 = (2^52 + temp1) - 2^52 in
5010 : // if temp2 < temp1 then
5011 : // -1 - temp2
5012 : // else
5013 : // -0 - temp2
5014 :
5015 0 : auto if_not_positive = __ MakeDeferredLabel();
5016 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5017 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5018 0 : auto if_temp2_lt_temp1 = __ MakeLabel();
5019 0 : auto if_zero = __ MakeDeferredLabel();
5020 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5021 :
5022 0 : Node* const zero = __ Float64Constant(0.0);
5023 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5024 :
5025 0 : Node* check0 = __ Float64LessThan(zero, input);
5026 0 : __ GotoIfNot(check0, &if_not_positive);
5027 : {
5028 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5029 0 : __ GotoIf(check1, &if_greater_than_two_52);
5030 : {
5031 0 : Node* const one = __ Float64Constant(1.0);
5032 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5033 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5034 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5035 : }
5036 :
5037 : __ Bind(&if_greater_than_two_52);
5038 : __ Goto(&done, input);
5039 : }
5040 :
5041 : __ Bind(&if_not_positive);
5042 : {
5043 0 : Node* check1 = __ Float64Equal(input, zero);
5044 0 : __ GotoIf(check1, &if_zero);
5045 :
5046 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5047 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5048 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5049 :
5050 : {
5051 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5052 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5053 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5054 0 : Node* check3 = __ Float64LessThan(temp2, temp1);
5055 0 : __ GotoIf(check3, &if_temp2_lt_temp1);
5056 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp2));
5057 :
5058 : __ Bind(&if_temp2_lt_temp1);
5059 0 : __ Goto(&done, __ Float64Sub(__ Float64Constant(-1.0), temp2));
5060 : }
5061 : __ Bind(&if_less_than_minus_two_52);
5062 : __ Goto(&done, input);
5063 :
5064 : __ Bind(&if_zero);
5065 : __ Goto(&done, input);
5066 : }
5067 : __ Bind(&done);
5068 : return done.PhiAt(0);
5069 : }
5070 :
5071 25305 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundDown(Node* node) {
5072 : // Nothing to be done if a fast hardware instruction is available.
5073 25305 : if (machine()->Float64RoundDown().IsSupported()) {
5074 : return Nothing<Node*>();
5075 : }
5076 :
5077 : Node* const input = node->InputAt(0);
5078 0 : return Just(BuildFloat64RoundDown(input));
5079 : }
5080 :
5081 265 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTiesEven(Node* node) {
5082 : // Nothing to be done if a fast hardware instruction is available.
5083 265 : if (machine()->Float64RoundTiesEven().IsSupported()) {
5084 : return Nothing<Node*>();
5085 : }
5086 :
5087 : Node* const input = node->InputAt(0);
5088 :
5089 : // Generate case for round ties to even:
5090 : //
5091 : // let value = floor(input) in
5092 : // let temp1 = input - value in
5093 : // if temp1 < 0.5 then
5094 : // value
5095 : // else if 0.5 < temp1 then
5096 : // value + 1.0
5097 : // else
5098 : // let temp2 = value % 2.0 in
5099 : // if temp2 == 0.0 then
5100 : // value
5101 : // else
5102 : // value + 1.0
5103 :
5104 0 : auto if_is_half = __ MakeLabel();
5105 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5106 :
5107 0 : Node* value = BuildFloat64RoundDown(input);
5108 0 : Node* temp1 = __ Float64Sub(input, value);
5109 :
5110 0 : Node* const half = __ Float64Constant(0.5);
5111 0 : Node* check0 = __ Float64LessThan(temp1, half);
5112 0 : __ GotoIf(check0, &done, value);
5113 :
5114 0 : Node* const one = __ Float64Constant(1.0);
5115 0 : Node* check1 = __ Float64LessThan(half, temp1);
5116 0 : __ GotoIfNot(check1, &if_is_half);
5117 0 : __ Goto(&done, __ Float64Add(value, one));
5118 :
5119 : __ Bind(&if_is_half);
5120 0 : Node* temp2 = __ Float64Mod(value, __ Float64Constant(2.0));
5121 0 : Node* check2 = __ Float64Equal(temp2, __ Float64Constant(0.0));
5122 0 : __ GotoIf(check2, &done, value);
5123 0 : __ Goto(&done, __ Float64Add(value, one));
5124 :
5125 : __ Bind(&done);
5126 : return Just(done.PhiAt(0));
5127 : }
5128 :
5129 140 : Node* EffectControlLinearizer::BuildFloat64RoundTruncate(Node* input) {
5130 140 : if (machine()->Float64RoundTruncate().IsSupported()) {
5131 140 : return __ Float64RoundTruncate(input);
5132 : }
5133 : // General case for trunc.
5134 : //
5135 : // if 0.0 < input then
5136 : // if 2^52 <= input then
5137 : // input
5138 : // else
5139 : // let temp1 = (2^52 + input) - 2^52 in
5140 : // if input < temp1 then
5141 : // temp1 - 1
5142 : // else
5143 : // temp1
5144 : // else
5145 : // if input == 0 then
5146 : // input
5147 : // else
5148 : // if input <= -2^52 then
5149 : // input
5150 : // else
5151 : // let temp1 = -0 - input in
5152 : // let temp2 = (2^52 + temp1) - 2^52 in
5153 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
5154 : // -0 - temp3
5155 : //
5156 : // Note: We do not use the Diamond helper class here, because it really hurts
5157 : // readability with nested diamonds.
5158 :
5159 0 : auto if_not_positive = __ MakeDeferredLabel();
5160 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5161 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5162 0 : auto if_zero = __ MakeDeferredLabel();
5163 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
5164 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5165 :
5166 0 : Node* const zero = __ Float64Constant(0.0);
5167 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5168 0 : Node* const one = __ Float64Constant(1.0);
5169 :
5170 0 : Node* check0 = __ Float64LessThan(zero, input);
5171 0 : __ GotoIfNot(check0, &if_not_positive);
5172 : {
5173 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5174 0 : __ GotoIf(check1, &if_greater_than_two_52);
5175 : {
5176 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5177 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5178 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5179 : }
5180 :
5181 : __ Bind(&if_greater_than_two_52);
5182 : __ Goto(&done, input);
5183 : }
5184 :
5185 : __ Bind(&if_not_positive);
5186 : {
5187 0 : Node* check1 = __ Float64Equal(input, zero);
5188 0 : __ GotoIf(check1, &if_zero);
5189 :
5190 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5191 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5192 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5193 :
5194 : {
5195 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5196 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5197 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5198 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
5199 0 : __ GotoIfNot(check3, &done_temp3, temp2);
5200 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
5201 :
5202 : __ Bind(&done_temp3);
5203 : Node* temp3 = done_temp3.PhiAt(0);
5204 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
5205 : }
5206 : __ Bind(&if_less_than_minus_two_52);
5207 : __ Goto(&done, input);
5208 :
5209 : __ Bind(&if_zero);
5210 : __ Goto(&done, input);
5211 : }
5212 : __ Bind(&done);
5213 : return done.PhiAt(0);
5214 : }
5215 :
5216 6912 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTruncate(Node* node) {
5217 : // Nothing to be done if a fast hardware instruction is available.
5218 6912 : if (machine()->Float64RoundTruncate().IsSupported()) {
5219 : return Nothing<Node*>();
5220 : }
5221 :
5222 : Node* const input = node->InputAt(0);
5223 0 : return Just(BuildFloat64RoundTruncate(input));
5224 : }
5225 :
5226 348 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntry(Node* node) {
5227 174 : Node* table = NodeProperties::GetValueInput(node, 0);
5228 174 : Node* key = NodeProperties::GetValueInput(node, 1);
5229 :
5230 : {
5231 : Callable const callable =
5232 174 : Builtins::CallableFor(isolate(), Builtins::kFindOrderedHashMapEntry);
5233 174 : Operator::Properties const properties = node->op()->properties();
5234 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
5235 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5236 : graph()->zone(), callable.descriptor(),
5237 348 : callable.descriptor().GetStackParameterCount(), flags, properties);
5238 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), table,
5239 348 : key, __ NoContextConstant());
5240 : }
5241 : }
5242 :
5243 15 : Node* EffectControlLinearizer::ComputeUnseededHash(Node* value) {
5244 : // See v8::internal::ComputeUnseededHash()
5245 : value = __ Int32Add(__ Word32Xor(value, __ Int32Constant(0xFFFFFFFF)),
5246 15 : __ Word32Shl(value, __ Int32Constant(15)));
5247 15 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(12)));
5248 15 : value = __ Int32Add(value, __ Word32Shl(value, __ Int32Constant(2)));
5249 15 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(4)));
5250 15 : value = __ Int32Mul(value, __ Int32Constant(2057));
5251 15 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(16)));
5252 15 : value = __ Word32And(value, __ Int32Constant(0x3FFFFFFF));
5253 15 : return value;
5254 : }
5255 :
5256 15 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntryForInt32Key(
5257 : Node* node) {
5258 15 : Node* table = NodeProperties::GetValueInput(node, 0);
5259 15 : Node* key = NodeProperties::GetValueInput(node, 1);
5260 :
5261 : // Compute the integer hash code.
5262 15 : Node* hash = ChangeUint32ToUintPtr(ComputeUnseededHash(key));
5263 :
5264 : Node* number_of_buckets = ChangeSmiToIntPtr(__ LoadField(
5265 15 : AccessBuilder::ForOrderedHashMapOrSetNumberOfBuckets(), table));
5266 15 : hash = __ WordAnd(hash, __ IntSub(number_of_buckets, __ IntPtrConstant(1)));
5267 : Node* first_entry = ChangeSmiToIntPtr(__ Load(
5268 : MachineType::TaggedSigned(), table,
5269 : __ IntAdd(__ WordShl(hash, __ IntPtrConstant(kTaggedSizeLog2)),
5270 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5271 15 : kHeapObjectTag))));
5272 :
5273 15 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
5274 15 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
5275 : __ Goto(&loop, first_entry);
5276 : __ Bind(&loop);
5277 : {
5278 : Node* entry = loop.PhiAt(0);
5279 : Node* check =
5280 15 : __ WordEqual(entry, __ IntPtrConstant(OrderedHashMap::kNotFound));
5281 15 : __ GotoIf(check, &done, entry);
5282 : entry = __ IntAdd(
5283 : __ IntMul(entry, __ IntPtrConstant(OrderedHashMap::kEntrySize)),
5284 15 : number_of_buckets);
5285 :
5286 : Node* candidate_key = __ Load(
5287 : MachineType::AnyTagged(), table,
5288 : __ IntAdd(__ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5289 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5290 15 : kHeapObjectTag)));
5291 :
5292 15 : auto if_match = __ MakeLabel();
5293 15 : auto if_notmatch = __ MakeLabel();
5294 15 : auto if_notsmi = __ MakeDeferredLabel();
5295 15 : __ GotoIfNot(ObjectIsSmi(candidate_key), &if_notsmi);
5296 : __ Branch(__ Word32Equal(ChangeSmiToInt32(candidate_key), key), &if_match,
5297 15 : &if_notmatch);
5298 :
5299 : __ Bind(&if_notsmi);
5300 : __ GotoIfNot(
5301 : __ WordEqual(__ LoadField(AccessBuilder::ForMap(), candidate_key),
5302 : __ HeapNumberMapConstant()),
5303 15 : &if_notmatch);
5304 : __ Branch(__ Float64Equal(__ LoadField(AccessBuilder::ForHeapNumberValue(),
5305 : candidate_key),
5306 : __ ChangeInt32ToFloat64(key)),
5307 15 : &if_match, &if_notmatch);
5308 :
5309 : __ Bind(&if_match);
5310 : __ Goto(&done, entry);
5311 :
5312 : __ Bind(&if_notmatch);
5313 : {
5314 : Node* next_entry = ChangeSmiToIntPtr(__ Load(
5315 : MachineType::TaggedSigned(), table,
5316 : __ IntAdd(
5317 : __ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5318 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() +
5319 : OrderedHashMap::kChainOffset * kTaggedSize -
5320 15 : kHeapObjectTag))));
5321 : __ Goto(&loop, next_entry);
5322 : }
5323 : }
5324 :
5325 : __ Bind(&done);
5326 15 : return done.PhiAt(0);
5327 : }
5328 :
5329 7 : Node* EffectControlLinearizer::LowerDateNow(Node* node) {
5330 7 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
5331 : Runtime::FunctionId id = Runtime::kDateCurrentTime;
5332 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
5333 7 : graph()->zone(), id, 0, properties, CallDescriptor::kNoFlags);
5334 : return __ Call(call_descriptor, __ CEntryStubConstant(1),
5335 : __ ExternalConstant(ExternalReference::Create(id)),
5336 7 : __ Int32Constant(0), __ NoContextConstant());
5337 : }
5338 :
5339 : #undef __
5340 :
5341 0 : Factory* EffectControlLinearizer::factory() const {
5342 0 : return isolate()->factory();
5343 : }
5344 :
5345 92903 : Isolate* EffectControlLinearizer::isolate() const {
5346 92903 : return jsgraph()->isolate();
5347 : }
5348 :
5349 : } // namespace compiler
5350 : } // namespace internal
5351 178779 : } // namespace v8
|
