Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/effect-control-linearizer.h"
6 :
7 : #include "src/code-factory.h"
8 : #include "src/compiler/access-builder.h"
9 : #include "src/compiler/compiler-source-position-table.h"
10 : #include "src/compiler/js-graph.h"
11 : #include "src/compiler/linkage.h"
12 : #include "src/compiler/node-matchers.h"
13 : #include "src/compiler/node-origin-table.h"
14 : #include "src/compiler/node-properties.h"
15 : #include "src/compiler/node.h"
16 : #include "src/compiler/schedule.h"
17 : #include "src/heap/factory-inl.h"
18 : #include "src/machine-type.h"
19 : #include "src/objects/heap-number.h"
20 : #include "src/objects/oddball.h"
21 : #include "src/objects/ordered-hash-table.h"
22 :
23 : #include "src/ptr-compr-inl.h"
24 :
25 : namespace v8 {
26 : namespace internal {
27 : namespace compiler {
28 :
29 : namespace {
30 : bool UsingCompressedPointers() { return false; }
31 :
32 : } // namespace
33 :
34 464081 : EffectControlLinearizer::EffectControlLinearizer(
35 : JSGraph* js_graph, Schedule* schedule, Zone* temp_zone,
36 : SourcePositionTable* source_positions, NodeOriginTable* node_origins,
37 : MaskArrayIndexEnable mask_array_index,
38 : std::vector<Handle<Map>>* embedded_maps)
39 : : js_graph_(js_graph),
40 : schedule_(schedule),
41 : temp_zone_(temp_zone),
42 : mask_array_index_(mask_array_index),
43 : source_positions_(source_positions),
44 : node_origins_(node_origins),
45 : graph_assembler_(js_graph, nullptr, nullptr, temp_zone),
46 : frame_state_zapper_(nullptr),
47 464081 : embedded_maps_(embedded_maps) {}
48 :
49 651845 : Graph* EffectControlLinearizer::graph() const { return js_graph_->graph(); }
50 0 : CommonOperatorBuilder* EffectControlLinearizer::common() const {
51 1098207 : return js_graph_->common();
52 : }
53 0 : SimplifiedOperatorBuilder* EffectControlLinearizer::simplified() const {
54 0 : return js_graph_->simplified();
55 : }
56 0 : MachineOperatorBuilder* EffectControlLinearizer::machine() const {
57 326070 : return js_graph_->machine();
58 : }
59 :
60 : namespace {
61 :
62 : struct BlockEffectControlData {
63 : Node* current_effect = nullptr; // New effect.
64 : Node* current_control = nullptr; // New control.
65 : Node* current_frame_state = nullptr; // New frame state.
66 : };
67 :
68 464083 : class BlockEffectControlMap {
69 : public:
70 : explicit BlockEffectControlMap(Zone* temp_zone) : map_(temp_zone) {}
71 :
72 : BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) {
73 19389097 : return map_[std::make_pair(from->rpo_number(), to->rpo_number())];
74 : }
75 :
76 : const BlockEffectControlData& For(BasicBlock* from, BasicBlock* to) const {
77 : return map_.at(std::make_pair(from->rpo_number(), to->rpo_number()));
78 : }
79 :
80 : private:
81 : using Key = std::pair<int32_t, int32_t>;
82 : using Map = ZoneMap<Key, BlockEffectControlData>;
83 :
84 : Map map_;
85 : };
86 :
87 : // Effect phis that need to be updated after the first pass.
88 : struct PendingEffectPhi {
89 : Node* effect_phi;
90 : BasicBlock* block;
91 :
92 : PendingEffectPhi(Node* effect_phi, BasicBlock* block)
93 39805 : : effect_phi(effect_phi), block(block) {}
94 : };
95 :
96 1126 : void ConnectUnreachableToEnd(Node* effect, Node* control, JSGraph* jsgraph) {
97 : Graph* graph = jsgraph->graph();
98 : CommonOperatorBuilder* common = jsgraph->common();
99 1126 : if (effect->opcode() == IrOpcode::kDead) return;
100 1126 : if (effect->opcode() != IrOpcode::kUnreachable) {
101 0 : effect = graph->NewNode(common->Unreachable(), effect, control);
102 : }
103 1126 : Node* throw_node = graph->NewNode(common->Throw(), effect, control);
104 1126 : NodeProperties::MergeControlToEnd(graph, common, throw_node);
105 : }
106 :
107 796596 : void UpdateEffectPhi(Node* node, BasicBlock* block,
108 : BlockEffectControlMap* block_effects, JSGraph* jsgraph) {
109 : // Update all inputs to an effect phi with the effects from the given
110 : // block->effect map.
111 : DCHECK_EQ(IrOpcode::kEffectPhi, node->opcode());
112 : DCHECK_EQ(static_cast<size_t>(node->op()->EffectInputCount()),
113 : block->PredecessorCount());
114 4395572 : for (int i = 0; i < node->op()->EffectInputCount(); i++) {
115 : Node* input = node->InputAt(i);
116 1799483 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
117 : const BlockEffectControlData& block_effect =
118 : block_effects->For(predecessor, block);
119 1799491 : Node* effect = block_effect.current_effect;
120 1799491 : if (input != effect) {
121 133069 : node->ReplaceInput(i, effect);
122 : }
123 : }
124 796601 : }
125 :
126 4382016 : void UpdateBlockControl(BasicBlock* block,
127 : BlockEffectControlMap* block_effects) {
128 : Node* control = block->NodeAt(0);
129 : DCHECK(NodeProperties::IsControl(control));
130 :
131 : // Do not rewire the end node.
132 4382016 : if (control->opcode() == IrOpcode::kEnd) return;
133 :
134 : // Update all inputs to the given control node with the correct control.
135 : DCHECK(control->opcode() == IrOpcode::kMerge ||
136 : static_cast<size_t>(control->op()->ControlInputCount()) ==
137 : block->PredecessorCount());
138 7837868 : if (static_cast<size_t>(control->op()->ControlInputCount()) !=
139 : block->PredecessorCount()) {
140 : return; // We already re-wired the control inputs of this node.
141 : }
142 12378678 : for (int i = 0; i < control->op()->ControlInputCount(); i++) {
143 4328156 : Node* input = NodeProperties::GetControlInput(control, i);
144 4328025 : BasicBlock* predecessor = block->PredecessorAt(static_cast<size_t>(i));
145 : const BlockEffectControlData& block_effect =
146 : block_effects->For(predecessor, block);
147 4328037 : if (input != block_effect.current_control) {
148 : NodeProperties::ReplaceControlInput(control, block_effect.current_control,
149 36764 : i);
150 : }
151 : }
152 : }
153 :
154 : bool HasIncomingBackEdges(BasicBlock* block) {
155 12080503 : for (BasicBlock* pred : block->predecessors()) {
156 6992586 : if (pred->rpo_number() >= block->rpo_number()) {
157 : return true;
158 : }
159 : }
160 : return false;
161 : }
162 :
163 288286 : void RemoveRenameNode(Node* node) {
164 : DCHECK(IrOpcode::kFinishRegion == node->opcode() ||
165 : IrOpcode::kBeginRegion == node->opcode() ||
166 : IrOpcode::kTypeGuard == node->opcode());
167 : // Update the value/context uses to the value input of the finish node and
168 : // the effect uses to the effect input.
169 3341300 : for (Edge edge : node->use_edges()) {
170 : DCHECK(!edge.from()->IsDead());
171 1526507 : if (NodeProperties::IsEffectEdge(edge)) {
172 289736 : edge.UpdateTo(NodeProperties::GetEffectInput(node));
173 : } else {
174 : DCHECK(!NodeProperties::IsControlEdge(edge));
175 : DCHECK(!NodeProperties::IsFrameStateEdge(edge));
176 1236771 : edge.UpdateTo(node->InputAt(0));
177 : }
178 : }
179 288286 : node->Kill();
180 288284 : }
181 :
182 1086991 : void TryCloneBranch(Node* node, BasicBlock* block, Zone* temp_zone,
183 : Graph* graph, CommonOperatorBuilder* common,
184 : BlockEffectControlMap* block_effects,
185 : SourcePositionTable* source_positions,
186 : NodeOriginTable* node_origins) {
187 : DCHECK_EQ(IrOpcode::kBranch, node->opcode());
188 :
189 : // This optimization is a special case of (super)block cloning. It takes an
190 : // input graph as shown below and clones the Branch node for every predecessor
191 : // to the Merge, essentially removing the Merge completely. This avoids
192 : // materializing the bit for the Phi and may offer potential for further
193 : // branch folding optimizations (i.e. because one or more inputs to the Phi is
194 : // a constant). Note that there may be more Phi nodes hanging off the Merge,
195 : // but we can only a certain subset of them currently (actually only Phi and
196 : // EffectPhi nodes whose uses have either the IfTrue or IfFalse as control
197 : // input).
198 :
199 : // Control1 ... ControlN
200 : // ^ ^
201 : // | | Cond1 ... CondN
202 : // +----+ +----+ ^ ^
203 : // | | | |
204 : // | | +----+ |
205 : // Merge<--+ | +------------+
206 : // ^ \|/
207 : // | Phi
208 : // | |
209 : // Branch----+
210 : // ^
211 : // |
212 : // +-----+-----+
213 : // | |
214 : // IfTrue IfFalse
215 : // ^ ^
216 : // | |
217 :
218 : // The resulting graph (modulo the Phi and EffectPhi nodes) looks like this:
219 :
220 : // Control1 Cond1 ... ControlN CondN
221 : // ^ ^ ^ ^
222 : // \ / \ /
223 : // Branch ... Branch
224 : // ^ ^
225 : // | |
226 : // +---+---+ +---+----+
227 : // | | | |
228 : // IfTrue IfFalse ... IfTrue IfFalse
229 : // ^ ^ ^ ^
230 : // | | | |
231 : // +--+ +-------------+ |
232 : // | | +--------------+ +--+
233 : // | | | |
234 : // Merge Merge
235 : // ^ ^
236 : // | |
237 :
238 : SourcePositionTable::Scope scope(source_positions,
239 1086991 : source_positions->GetSourcePosition(node));
240 : NodeOriginTable::Scope origin_scope(node_origins, "clone branch", node);
241 : Node* branch = node;
242 1086994 : Node* cond = NodeProperties::GetValueInput(branch, 0);
243 2149536 : if (!cond->OwnedBy(branch) || cond->opcode() != IrOpcode::kPhi) return;
244 110502 : Node* merge = NodeProperties::GetControlInput(branch);
245 218604 : if (merge->opcode() != IrOpcode::kMerge ||
246 108102 : NodeProperties::GetControlInput(cond) != merge) {
247 : return;
248 : }
249 : // Grab the IfTrue/IfFalse projections of the Branch.
250 108029 : BranchMatcher matcher(branch);
251 : // Check/collect other Phi/EffectPhi nodes hanging off the Merge.
252 : NodeVector phis(temp_zone);
253 712285 : for (Node* const use : merge->uses()) {
254 307056 : if (use == branch || use == cond) continue;
255 : // We cannot currently deal with non-Phi/EffectPhi nodes hanging off the
256 : // Merge. Ideally, we would just clone the nodes (and everything that
257 : // depends on it to some distant join point), but that requires knowledge
258 : // about dominance/post-dominance.
259 115590 : if (!NodeProperties::IsPhi(use)) return;
260 107358 : for (Edge edge : use->use_edges()) {
261 : // Right now we can only handle Phi/EffectPhi nodes whose uses are
262 : // directly control-dependend on either the IfTrue or the IfFalse
263 : // successor, because we know exactly how to update those uses.
264 11480 : if (edge.from()->op()->ControlInputCount() != 1) return;
265 10392 : Node* control = NodeProperties::GetControlInput(edge.from());
266 10392 : if (NodeProperties::IsPhi(edge.from())) {
267 735 : control = NodeProperties::GetControlInput(control, edge.index());
268 : }
269 10392 : if (control != matcher.IfTrue() && control != matcher.IfFalse()) return;
270 : }
271 95878 : phis.push_back(use);
272 : }
273 98173 : BranchHint const hint = BranchHintOf(branch->op());
274 : int const input_count = merge->op()->ControlInputCount();
275 : DCHECK_LE(1, input_count);
276 98173 : Node** const inputs = graph->zone()->NewArray<Node*>(2 * input_count);
277 : Node** const merge_true_inputs = &inputs[0];
278 98173 : Node** const merge_false_inputs = &inputs[input_count];
279 939491 : for (int index = 0; index < input_count; ++index) {
280 420659 : Node* cond1 = NodeProperties::GetValueInput(cond, index);
281 420659 : Node* control1 = NodeProperties::GetControlInput(merge, index);
282 420659 : Node* branch1 = graph->NewNode(common->Branch(hint), cond1, control1);
283 841318 : merge_true_inputs[index] = graph->NewNode(common->IfTrue(), branch1);
284 841318 : merge_false_inputs[index] = graph->NewNode(common->IfFalse(), branch1);
285 : }
286 : Node* const merge_true = matcher.IfTrue();
287 : Node* const merge_false = matcher.IfFalse();
288 98173 : merge_true->TrimInputCount(0);
289 98173 : merge_false->TrimInputCount(0);
290 939491 : for (int i = 0; i < input_count; ++i) {
291 420659 : merge_true->AppendInput(graph->zone(), merge_true_inputs[i]);
292 420659 : merge_false->AppendInput(graph->zone(), merge_false_inputs[i]);
293 : }
294 : DCHECK_EQ(2u, block->SuccessorCount());
295 98173 : NodeProperties::ChangeOp(matcher.IfTrue(), common->Merge(input_count));
296 98173 : NodeProperties::ChangeOp(matcher.IfFalse(), common->Merge(input_count));
297 : int const true_index =
298 98173 : block->SuccessorAt(0)->NodeAt(0) == matcher.IfTrue() ? 0 : 1;
299 : BlockEffectControlData* true_block_data =
300 98173 : &block_effects->For(block, block->SuccessorAt(true_index));
301 : BlockEffectControlData* false_block_data =
302 98173 : &block_effects->For(block, block->SuccessorAt(true_index ^ 1));
303 194051 : for (Node* const phi : phis) {
304 927378 : for (int index = 0; index < input_count; ++index) {
305 831500 : inputs[index] = phi->InputAt(index);
306 : }
307 95878 : inputs[input_count] = merge_true;
308 95878 : Node* phi_true = graph->NewNode(phi->op(), input_count + 1, inputs);
309 95878 : inputs[input_count] = merge_false;
310 95878 : Node* phi_false = graph->NewNode(phi->op(), input_count + 1, inputs);
311 95878 : if (phi->UseCount() == 0) {
312 : DCHECK_EQ(phi->opcode(), IrOpcode::kEffectPhi);
313 : } else {
314 22431 : for (Edge edge : phi->use_edges()) {
315 7768 : Node* control = NodeProperties::GetControlInput(edge.from());
316 7768 : if (NodeProperties::IsPhi(edge.from())) {
317 730 : control = NodeProperties::GetControlInput(control, edge.index());
318 : }
319 : DCHECK(control == matcher.IfTrue() || control == matcher.IfFalse());
320 7768 : edge.UpdateTo((control == matcher.IfTrue()) ? phi_true : phi_false);
321 : }
322 : }
323 95878 : if (phi->opcode() == IrOpcode::kEffectPhi) {
324 95878 : true_block_data->current_effect = phi_true;
325 95878 : false_block_data->current_effect = phi_false;
326 : }
327 95878 : phi->Kill();
328 : }
329 : // Fix up IfTrue and IfFalse and kill all dead nodes.
330 98173 : if (branch == block->control_input()) {
331 98173 : true_block_data->current_control = merge_true;
332 98173 : false_block_data->current_control = merge_false;
333 : }
334 98173 : branch->Kill();
335 98173 : cond->Kill();
336 98173 : merge->Kill();
337 : }
338 :
339 : } // namespace
340 :
341 464087 : void EffectControlLinearizer::Run() {
342 : BlockEffectControlMap block_effects(temp_zone());
343 : ZoneVector<PendingEffectPhi> pending_effect_phis(temp_zone());
344 : ZoneVector<BasicBlock*> pending_block_controls(temp_zone());
345 : NodeVector inputs_buffer(temp_zone());
346 :
347 4846108 : for (BasicBlock* block : *(schedule()->rpo_order())) {
348 : size_t instr = 0;
349 :
350 : // The control node should be the first.
351 4382025 : Node* control = block->NodeAt(instr);
352 : DCHECK(NodeProperties::IsControl(control));
353 : // Update the control inputs.
354 4382025 : if (HasIncomingBackEdges(block)) {
355 : // If there are back edges, we need to update later because we have not
356 : // computed the control yet. This should only happen for loops.
357 : DCHECK_EQ(IrOpcode::kLoop, control->opcode());
358 39805 : pending_block_controls.push_back(block);
359 : } else {
360 : // If there are no back edges, we can update now.
361 4342220 : UpdateBlockControl(block, &block_effects);
362 : }
363 : instr++;
364 :
365 : // Iterate over the phis and update the effect phis.
366 : Node* effect_phi = nullptr;
367 : Node* terminate = nullptr;
368 12178224 : for (; instr < block->NodeCount(); instr++) {
369 : Node* node = block->NodeAt(instr);
370 : // Only go through the phis and effect phis.
371 3848231 : if (node->opcode() == IrOpcode::kEffectPhi) {
372 : // There should be at most one effect phi in a block.
373 : DCHECK_NULL(effect_phi);
374 : // IfException blocks should not have effect phis.
375 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
376 : effect_phi = node;
377 3062749 : } else if (node->opcode() == IrOpcode::kPhi) {
378 : // Just skip phis.
379 2749926 : } else if (node->opcode() == IrOpcode::kTerminate) {
380 : DCHECK_NULL(terminate);
381 : terminate = node;
382 : } else {
383 : break;
384 : }
385 : }
386 :
387 4381923 : if (effect_phi) {
388 : // Make sure we update the inputs to the incoming blocks' effects.
389 785495 : if (HasIncomingBackEdges(block)) {
390 : // In case of loops, we do not update the effect phi immediately
391 : // because the back predecessor has not been handled yet. We just
392 : // record the effect phi for later processing.
393 39796 : pending_effect_phis.push_back(PendingEffectPhi(effect_phi, block));
394 : } else {
395 745697 : UpdateEffectPhi(effect_phi, block, &block_effects, jsgraph());
396 : }
397 : }
398 :
399 4382040 : Node* effect = effect_phi;
400 4382040 : if (effect == nullptr) {
401 : // There was no effect phi.
402 3596492 : if (block == schedule()->start()) {
403 : // Start block => effect is start.
404 : DCHECK_EQ(graph()->start(), control);
405 464080 : effect = graph()->start();
406 6264824 : } else if (control->opcode() == IrOpcode::kEnd) {
407 : // End block is just a dummy, no effect needed.
408 : DCHECK_EQ(BasicBlock::kNone, block->control());
409 : DCHECK_EQ(1u, block->size());
410 463079 : effect = nullptr;
411 : } else {
412 : // If all the predecessors have the same effect, we can use it as our
413 : // current effect.
414 13547788 : for (size_t i = 0; i < block->PredecessorCount(); ++i) {
415 : const BlockEffectControlData& data =
416 : block_effects.For(block->PredecessorAt(i), block);
417 2747479 : if (!effect) effect = data.current_effect;
418 2747479 : if (data.current_effect != effect) {
419 11105 : effect = nullptr;
420 11105 : break;
421 : }
422 : }
423 2669349 : if (effect == nullptr) {
424 : DCHECK_NE(IrOpcode::kIfException, control->opcode());
425 : // The input blocks do not have the same effect. We have
426 : // to create an effect phi node.
427 : inputs_buffer.clear();
428 22210 : inputs_buffer.resize(block->PredecessorCount(), jsgraph()->Dead());
429 11105 : inputs_buffer.push_back(control);
430 44420 : effect = graph()->NewNode(
431 11105 : common()->EffectPhi(static_cast<int>(block->PredecessorCount())),
432 11105 : static_cast<int>(inputs_buffer.size()), &(inputs_buffer.front()));
433 : // For loops, we update the effect phi node later to break cycles.
434 22210 : if (control->opcode() == IrOpcode::kLoop) {
435 14 : pending_effect_phis.push_back(PendingEffectPhi(effect, block));
436 : } else {
437 11098 : UpdateEffectPhi(effect, block, &block_effects, jsgraph());
438 : }
439 5316488 : } else if (control->opcode() == IrOpcode::kIfException) {
440 : // The IfException is connected into the effect chain, so we need
441 : // to update the effect here.
442 200270 : NodeProperties::ReplaceEffectInput(control, effect);
443 200270 : effect = control;
444 : }
445 : }
446 : }
447 :
448 : // Fixup the Terminate node.
449 4382056 : if (terminate != nullptr) {
450 39802 : NodeProperties::ReplaceEffectInput(terminate, effect);
451 : }
452 :
453 : // The frame state at block entry is determined by the frame states leaving
454 : // all predecessors. In case there is no frame state dominating this block,
455 : // we can rely on a checkpoint being present before the next deoptimization.
456 : // TODO(mstarzinger): Eventually we will need to go hunt for a frame state
457 : // once deoptimizing nodes roam freely through the schedule.
458 4382057 : Node* frame_state = nullptr;
459 4382057 : if (block != schedule()->start()) {
460 : // If all the predecessors have the same effect, we can use it
461 : // as our current effect.
462 : frame_state =
463 3917914 : block_effects.For(block->PredecessorAt(0), block).current_frame_state;
464 11104427 : for (size_t i = 1; i < block->PredecessorCount(); i++) {
465 1180919 : if (block_effects.For(block->PredecessorAt(i), block)
466 1180919 : .current_frame_state != frame_state) {
467 91386 : frame_state = nullptr;
468 91386 : frame_state_zapper_ = graph()->end();
469 91386 : break;
470 : }
471 : }
472 : }
473 :
474 : // Process the ordinary instructions.
475 109813238 : for (; instr < block->NodeCount(); instr++) {
476 : Node* node = block->NodeAt(instr);
477 33683136 : ProcessNode(node, &frame_state, &effect, &control);
478 : }
479 :
480 4381959 : switch (block->control()) {
481 : case BasicBlock::kGoto:
482 : case BasicBlock::kNone:
483 : break;
484 :
485 : case BasicBlock::kCall:
486 : case BasicBlock::kTailCall:
487 : case BasicBlock::kSwitch:
488 : case BasicBlock::kReturn:
489 : case BasicBlock::kDeoptimize:
490 : case BasicBlock::kThrow:
491 900294 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
492 900295 : break;
493 :
494 : case BasicBlock::kBranch:
495 1086995 : ProcessNode(block->control_input(), &frame_state, &effect, &control);
496 2173986 : TryCloneBranch(block->control_input(), block, temp_zone(), graph(),
497 : common(), &block_effects, source_positions_,
498 1086993 : node_origins_);
499 1087005 : break;
500 : }
501 :
502 : // Store the effect, control and frame state for later use.
503 13982928 : for (BasicBlock* successor : block->successors()) {
504 5218937 : BlockEffectControlData* data = &block_effects.For(block, successor);
505 5218988 : if (data->current_effect == nullptr) {
506 5027206 : data->current_effect = effect;
507 : }
508 5218988 : if (data->current_control == nullptr) {
509 5022631 : data->current_control = control;
510 : }
511 5218988 : data->current_frame_state = frame_state;
512 : }
513 : }
514 :
515 503887 : for (BasicBlock* pending_block_control : pending_block_controls) {
516 39804 : UpdateBlockControl(pending_block_control, &block_effects);
517 : }
518 : // Update the incoming edges of the effect phis that could not be processed
519 : // during the first pass (because they could have incoming back edges).
520 503887 : for (const PendingEffectPhi& pending_effect_phi : pending_effect_phis) {
521 39804 : UpdateEffectPhi(pending_effect_phi.effect_phi, pending_effect_phi.block,
522 39804 : &block_effects, jsgraph());
523 : }
524 464083 : }
525 :
526 35670278 : void EffectControlLinearizer::ProcessNode(Node* node, Node** frame_state,
527 : Node** effect, Node** control) {
528 : SourcePositionTable::Scope scope(source_positions_,
529 35670278 : source_positions_->GetSourcePosition(node));
530 35670246 : NodeOriginTable::Scope origin_scope(node_origins_, "process node", node);
531 :
532 : // If the node needs to be wired into the effect/control chain, do this
533 : // here. Pass current frame state for lowering to eager deoptimization.
534 35670246 : if (TryWireInStateEffect(node, *frame_state, effect, control)) {
535 : return;
536 : }
537 :
538 : // If the node has a visible effect, then there must be a checkpoint in the
539 : // effect chain before we are allowed to place another eager deoptimization
540 : // point. We zap the frame state to ensure this invariant is maintained.
541 67469395 : if (region_observability_ == RegionObservability::kObservable &&
542 : !node->op()->HasProperty(Operator::kNoWrite)) {
543 3648997 : *frame_state = nullptr;
544 3648997 : frame_state_zapper_ = node;
545 : }
546 :
547 : // Remove the end markers of 'atomic' allocation region because the
548 : // region should be wired-in now.
549 34561797 : if (node->opcode() == IrOpcode::kFinishRegion) {
550 : // Reset the current region observability.
551 132941 : region_observability_ = RegionObservability::kObservable;
552 : // Update the value uses to the value input of the finish node and
553 : // the effect uses to the effect input.
554 132941 : return RemoveRenameNode(node);
555 : }
556 34428856 : if (node->opcode() == IrOpcode::kBeginRegion) {
557 : // Determine the observability for this region and use that for all
558 : // nodes inside the region (i.e. ignore the absence of kNoWrite on
559 : // StoreField and other operators).
560 : DCHECK_NE(RegionObservability::kNotObservable, region_observability_);
561 132941 : region_observability_ = RegionObservabilityOf(node->op());
562 : // Update the value uses to the value input of the finish node and
563 : // the effect uses to the effect input.
564 132941 : return RemoveRenameNode(node);
565 : }
566 34295915 : if (node->opcode() == IrOpcode::kTypeGuard) {
567 22404 : return RemoveRenameNode(node);
568 : }
569 :
570 : // Special treatment for checkpoint nodes.
571 34273511 : if (node->opcode() == IrOpcode::kCheckpoint) {
572 : // Unlink the check point; effect uses will be updated to the incoming
573 : // effect that is passed. The frame state is preserved for lowering.
574 : DCHECK_EQ(RegionObservability::kObservable, region_observability_);
575 2749067 : *frame_state = NodeProperties::GetFrameStateInput(node);
576 2749062 : return;
577 : }
578 :
579 : // The IfSuccess nodes should always start a basic block (and basic block
580 : // start nodes are not handled in the ProcessNode method).
581 : DCHECK_NE(IrOpcode::kIfSuccess, node->opcode());
582 :
583 : // If the node takes an effect, replace with the current one.
584 31524444 : if (node->op()->EffectInputCount() > 0) {
585 : DCHECK_EQ(1, node->op()->EffectInputCount());
586 8134849 : Node* input_effect = NodeProperties::GetEffectInput(node);
587 :
588 8134844 : if (input_effect != *effect) {
589 2756308 : NodeProperties::ReplaceEffectInput(node, *effect);
590 : }
591 :
592 : // If the node produces an effect, update our current effect. (However,
593 : // ignore new effect chains started with ValueEffect.)
594 8134757 : if (node->op()->EffectOutputCount() > 0) {
595 : DCHECK_EQ(1, node->op()->EffectOutputCount());
596 7440384 : *effect = node;
597 : }
598 : } else {
599 : // New effect chain is only started with a Start or ValueEffect node.
600 : DCHECK(node->op()->EffectOutputCount() == 0 ||
601 : node->opcode() == IrOpcode::kStart);
602 : }
603 :
604 : // Rewire control inputs.
605 48987372 : for (int i = 0; i < node->op()->ControlInputCount(); i++) {
606 8731525 : NodeProperties::ReplaceControlInput(node, *control, i);
607 : }
608 : // Update the current control.
609 31524337 : if (node->op()->ControlOutputCount() > 0) {
610 4759136 : *control = node;
611 : }
612 :
613 : // Break the effect chain on {Unreachable} and reconnect to the graph end.
614 : // Mark the following code for deletion by connecting to the {Dead} node.
615 31524337 : if (node->opcode() == IrOpcode::kUnreachable) {
616 1126 : ConnectUnreachableToEnd(*effect, *control, jsgraph());
617 1126 : *effect = *control = jsgraph()->Dead();
618 : }
619 : }
620 :
621 35670276 : bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
622 : Node* frame_state,
623 : Node** effect,
624 : Node** control) {
625 71340552 : gasm()->Reset(*effect, *control);
626 : Node* result = nullptr;
627 35670264 : switch (node->opcode()) {
628 : case IrOpcode::kChangeBitToTagged:
629 9647 : result = LowerChangeBitToTagged(node);
630 9647 : break;
631 : case IrOpcode::kChangeInt31ToTaggedSigned:
632 39324 : result = LowerChangeInt31ToTaggedSigned(node);
633 39324 : break;
634 : case IrOpcode::kChangeInt32ToTagged:
635 35972 : result = LowerChangeInt32ToTagged(node);
636 35972 : break;
637 : case IrOpcode::kChangeInt64ToTagged:
638 45 : result = LowerChangeInt64ToTagged(node);
639 45 : break;
640 : case IrOpcode::kChangeUint32ToTagged:
641 653 : result = LowerChangeUint32ToTagged(node);
642 653 : break;
643 : case IrOpcode::kChangeUint64ToTagged:
644 53 : result = LowerChangeUint64ToTagged(node);
645 53 : break;
646 : case IrOpcode::kChangeFloat64ToTagged:
647 53387 : result = LowerChangeFloat64ToTagged(node);
648 53386 : break;
649 : case IrOpcode::kChangeFloat64ToTaggedPointer:
650 91 : result = LowerChangeFloat64ToTaggedPointer(node);
651 91 : break;
652 : case IrOpcode::kChangeTaggedSignedToInt32:
653 49908 : result = LowerChangeTaggedSignedToInt32(node);
654 49908 : break;
655 : case IrOpcode::kChangeTaggedSignedToInt64:
656 297 : result = LowerChangeTaggedSignedToInt64(node);
657 297 : break;
658 : case IrOpcode::kChangeTaggedToBit:
659 139067 : result = LowerChangeTaggedToBit(node);
660 139067 : break;
661 : case IrOpcode::kChangeTaggedToInt32:
662 1754 : result = LowerChangeTaggedToInt32(node);
663 1754 : break;
664 : case IrOpcode::kChangeTaggedToUint32:
665 394 : result = LowerChangeTaggedToUint32(node);
666 394 : break;
667 : case IrOpcode::kChangeTaggedToInt64:
668 2 : result = LowerChangeTaggedToInt64(node);
669 2 : break;
670 : case IrOpcode::kChangeTaggedToFloat64:
671 : result = LowerChangeTaggedToFloat64(node);
672 11274 : break;
673 : case IrOpcode::kChangeTaggedToTaggedSigned:
674 14 : result = LowerChangeTaggedToTaggedSigned(node);
675 14 : break;
676 : case IrOpcode::kChangeCompressedToTaggedSigned:
677 0 : result = LowerChangeCompressedToTaggedSigned(node);
678 0 : break;
679 : case IrOpcode::kChangeTaggedToCompressedSigned:
680 0 : result = LowerChangeTaggedToCompressedSigned(node);
681 0 : break;
682 : case IrOpcode::kTruncateTaggedToBit:
683 47628 : result = LowerTruncateTaggedToBit(node);
684 47628 : break;
685 : case IrOpcode::kTruncateTaggedPointerToBit:
686 391 : result = LowerTruncateTaggedPointerToBit(node);
687 391 : break;
688 : case IrOpcode::kTruncateTaggedToFloat64:
689 716 : result = LowerTruncateTaggedToFloat64(node);
690 716 : break;
691 : case IrOpcode::kPoisonIndex:
692 1033 : result = LowerPoisonIndex(node);
693 1033 : break;
694 : case IrOpcode::kCheckMaps:
695 51503 : LowerCheckMaps(node, frame_state);
696 51503 : break;
697 : case IrOpcode::kCompareMaps:
698 7530 : result = LowerCompareMaps(node);
699 7530 : break;
700 : case IrOpcode::kCheckNumber:
701 475 : result = LowerCheckNumber(node, frame_state);
702 475 : break;
703 : case IrOpcode::kCheckReceiver:
704 817 : result = LowerCheckReceiver(node, frame_state);
705 817 : break;
706 : case IrOpcode::kCheckReceiverOrNullOrUndefined:
707 72 : result = LowerCheckReceiverOrNullOrUndefined(node, frame_state);
708 72 : break;
709 : case IrOpcode::kCheckSymbol:
710 32 : result = LowerCheckSymbol(node, frame_state);
711 32 : break;
712 : case IrOpcode::kCheckString:
713 5348 : result = LowerCheckString(node, frame_state);
714 5348 : break;
715 : case IrOpcode::kCheckInternalizedString:
716 1393 : result = LowerCheckInternalizedString(node, frame_state);
717 1393 : break;
718 : case IrOpcode::kCheckNonEmptyOneByteString:
719 1424 : result = LowerCheckNonEmptyOneByteString(node, frame_state);
720 1424 : break;
721 : case IrOpcode::kCheckNonEmptyTwoByteString:
722 209 : result = LowerCheckNonEmptyTwoByteString(node, frame_state);
723 209 : break;
724 : case IrOpcode::kCheckNonEmptyString:
725 18 : result = LowerCheckNonEmptyString(node, frame_state);
726 18 : break;
727 : case IrOpcode::kCheckIf:
728 16409 : LowerCheckIf(node, frame_state);
729 16409 : break;
730 : case IrOpcode::kCheckedInt32Add:
731 55672 : result = LowerCheckedInt32Add(node, frame_state);
732 55673 : break;
733 : case IrOpcode::kCheckedInt32Sub:
734 7631 : result = LowerCheckedInt32Sub(node, frame_state);
735 7631 : break;
736 : case IrOpcode::kCheckedInt32Div:
737 236 : result = LowerCheckedInt32Div(node, frame_state);
738 236 : break;
739 : case IrOpcode::kCheckedInt32Mod:
740 980 : result = LowerCheckedInt32Mod(node, frame_state);
741 980 : break;
742 : case IrOpcode::kCheckedUint32Div:
743 52 : result = LowerCheckedUint32Div(node, frame_state);
744 52 : break;
745 : case IrOpcode::kCheckedUint32Mod:
746 37 : result = LowerCheckedUint32Mod(node, frame_state);
747 37 : break;
748 : case IrOpcode::kCheckedInt32Mul:
749 5906 : result = LowerCheckedInt32Mul(node, frame_state);
750 5906 : break;
751 : case IrOpcode::kCheckedInt32ToTaggedSigned:
752 0 : result = LowerCheckedInt32ToTaggedSigned(node, frame_state);
753 0 : break;
754 : case IrOpcode::kCheckedInt64ToInt32:
755 8 : result = LowerCheckedInt64ToInt32(node, frame_state);
756 8 : break;
757 : case IrOpcode::kCheckedInt64ToTaggedSigned:
758 151 : result = LowerCheckedInt64ToTaggedSigned(node, frame_state);
759 151 : break;
760 : case IrOpcode::kCheckedUint32Bounds:
761 52339 : result = LowerCheckedUint32Bounds(node, frame_state);
762 52339 : break;
763 : case IrOpcode::kCheckedUint32ToInt32:
764 396 : result = LowerCheckedUint32ToInt32(node, frame_state);
765 396 : break;
766 : case IrOpcode::kCheckedUint32ToTaggedSigned:
767 7 : result = LowerCheckedUint32ToTaggedSigned(node, frame_state);
768 7 : break;
769 : case IrOpcode::kCheckedUint64Bounds:
770 195 : result = LowerCheckedUint64Bounds(node, frame_state);
771 195 : break;
772 : case IrOpcode::kCheckedUint64ToInt32:
773 0 : result = LowerCheckedUint64ToInt32(node, frame_state);
774 0 : break;
775 : case IrOpcode::kCheckedUint64ToTaggedSigned:
776 0 : result = LowerCheckedUint64ToTaggedSigned(node, frame_state);
777 0 : break;
778 : case IrOpcode::kCheckedFloat64ToInt32:
779 2315 : result = LowerCheckedFloat64ToInt32(node, frame_state);
780 2315 : break;
781 : case IrOpcode::kCheckedFloat64ToInt64:
782 16 : result = LowerCheckedFloat64ToInt64(node, frame_state);
783 16 : break;
784 : case IrOpcode::kCheckedTaggedSignedToInt32:
785 24775 : if (frame_state == nullptr) {
786 0 : FATAL("No frame state (zapped by #%d: %s)", frame_state_zapper_->id(),
787 0 : frame_state_zapper_->op()->mnemonic());
788 : }
789 24775 : result = LowerCheckedTaggedSignedToInt32(node, frame_state);
790 24778 : break;
791 : case IrOpcode::kCheckedTaggedToInt32:
792 2274 : result = LowerCheckedTaggedToInt32(node, frame_state);
793 2274 : break;
794 : case IrOpcode::kCheckedTaggedToInt64:
795 87 : result = LowerCheckedTaggedToInt64(node, frame_state);
796 87 : break;
797 : case IrOpcode::kCheckedTaggedToFloat64:
798 51080 : result = LowerCheckedTaggedToFloat64(node, frame_state);
799 51080 : break;
800 : case IrOpcode::kCheckedTaggedToTaggedSigned:
801 18916 : result = LowerCheckedTaggedToTaggedSigned(node, frame_state);
802 18916 : break;
803 : case IrOpcode::kCheckedTaggedToTaggedPointer:
804 34653 : result = LowerCheckedTaggedToTaggedPointer(node, frame_state);
805 34653 : break;
806 : case IrOpcode::kCheckedCompressedToTaggedSigned:
807 0 : result = LowerCheckedCompressedToTaggedSigned(node, frame_state);
808 0 : break;
809 : case IrOpcode::kCheckedCompressedToTaggedPointer:
810 0 : result = LowerCheckedCompressedToTaggedPointer(node, frame_state);
811 0 : break;
812 : case IrOpcode::kCheckedTaggedToCompressedSigned:
813 0 : result = LowerCheckedTaggedToCompressedSigned(node, frame_state);
814 0 : break;
815 : case IrOpcode::kCheckedTaggedToCompressedPointer:
816 0 : result = LowerCheckedTaggedToCompressedPointer(node, frame_state);
817 0 : break;
818 : case IrOpcode::kTruncateTaggedToWord32:
819 507 : result = LowerTruncateTaggedToWord32(node);
820 507 : break;
821 : case IrOpcode::kCheckedTruncateTaggedToWord32:
822 1177 : result = LowerCheckedTruncateTaggedToWord32(node, frame_state);
823 1177 : break;
824 : case IrOpcode::kNumberToString:
825 968 : result = LowerNumberToString(node);
826 968 : break;
827 : case IrOpcode::kObjectIsArrayBufferView:
828 16 : result = LowerObjectIsArrayBufferView(node);
829 16 : break;
830 : case IrOpcode::kObjectIsBigInt:
831 25 : result = LowerObjectIsBigInt(node);
832 25 : break;
833 : case IrOpcode::kObjectIsCallable:
834 103 : result = LowerObjectIsCallable(node);
835 103 : break;
836 : case IrOpcode::kObjectIsConstructor:
837 378 : result = LowerObjectIsConstructor(node);
838 378 : break;
839 : case IrOpcode::kObjectIsDetectableCallable:
840 14095 : result = LowerObjectIsDetectableCallable(node);
841 14095 : break;
842 : case IrOpcode::kObjectIsMinusZero:
843 23 : result = LowerObjectIsMinusZero(node);
844 23 : break;
845 : case IrOpcode::kNumberIsMinusZero:
846 98 : result = LowerNumberIsMinusZero(node);
847 98 : break;
848 : case IrOpcode::kObjectIsNaN:
849 716 : result = LowerObjectIsNaN(node);
850 716 : break;
851 : case IrOpcode::kNumberIsNaN:
852 2527 : result = LowerNumberIsNaN(node);
853 2527 : break;
854 : case IrOpcode::kObjectIsNonCallable:
855 6067 : result = LowerObjectIsNonCallable(node);
856 6067 : break;
857 : case IrOpcode::kObjectIsNumber:
858 7594 : result = LowerObjectIsNumber(node);
859 7594 : break;
860 : case IrOpcode::kObjectIsReceiver:
861 19519 : result = LowerObjectIsReceiver(node);
862 19519 : break;
863 : case IrOpcode::kObjectIsSmi:
864 4131 : result = LowerObjectIsSmi(node);
865 4131 : break;
866 : case IrOpcode::kObjectIsString:
867 2221 : result = LowerObjectIsString(node);
868 2221 : break;
869 : case IrOpcode::kObjectIsSymbol:
870 23 : result = LowerObjectIsSymbol(node);
871 23 : break;
872 : case IrOpcode::kObjectIsUndetectable:
873 1426 : result = LowerObjectIsUndetectable(node);
874 1426 : break;
875 : case IrOpcode::kArgumentsFrame:
876 16894 : result = LowerArgumentsFrame(node);
877 16894 : break;
878 : case IrOpcode::kArgumentsLength:
879 16921 : result = LowerArgumentsLength(node);
880 16921 : break;
881 : case IrOpcode::kToBoolean:
882 196 : result = LowerToBoolean(node);
883 196 : break;
884 : case IrOpcode::kTypeOf:
885 21328 : result = LowerTypeOf(node);
886 21328 : break;
887 : case IrOpcode::kNewDoubleElements:
888 28 : result = LowerNewDoubleElements(node);
889 28 : break;
890 : case IrOpcode::kNewSmiOrObjectElements:
891 464 : result = LowerNewSmiOrObjectElements(node);
892 464 : break;
893 : case IrOpcode::kNewArgumentsElements:
894 16418 : result = LowerNewArgumentsElements(node);
895 16418 : break;
896 : case IrOpcode::kNewConsOneByteString:
897 3813 : result = LowerNewConsOneByteString(node);
898 3813 : break;
899 : case IrOpcode::kNewConsTwoByteString:
900 35 : result = LowerNewConsTwoByteString(node);
901 35 : break;
902 : case IrOpcode::kNewConsString:
903 1080 : result = LowerNewConsString(node);
904 1080 : break;
905 : case IrOpcode::kSameValue:
906 1 : result = LowerSameValue(node);
907 1 : break;
908 : case IrOpcode::kNumberSameValue:
909 77 : result = LowerNumberSameValue(node);
910 77 : break;
911 : case IrOpcode::kDeadValue:
912 1356 : result = LowerDeadValue(node);
913 1356 : break;
914 : case IrOpcode::kStringConcat:
915 21110 : result = LowerStringConcat(node);
916 21110 : break;
917 : case IrOpcode::kStringFromSingleCharCode:
918 970 : result = LowerStringFromSingleCharCode(node);
919 970 : break;
920 : case IrOpcode::kStringFromSingleCodePoint:
921 207 : result = LowerStringFromSingleCodePoint(node);
922 207 : break;
923 : case IrOpcode::kStringIndexOf:
924 258 : result = LowerStringIndexOf(node);
925 258 : break;
926 : case IrOpcode::kStringLength:
927 23424 : result = LowerStringLength(node);
928 23424 : break;
929 : case IrOpcode::kStringToNumber:
930 331 : result = LowerStringToNumber(node);
931 331 : break;
932 : case IrOpcode::kStringCharCodeAt:
933 2247 : result = LowerStringCharCodeAt(node);
934 2247 : break;
935 : case IrOpcode::kStringCodePointAt:
936 223 : result = LowerStringCodePointAt(node, UnicodeEncodingOf(node->op()));
937 223 : break;
938 : case IrOpcode::kStringToLowerCaseIntl:
939 103 : result = LowerStringToLowerCaseIntl(node);
940 103 : break;
941 : case IrOpcode::kStringToUpperCaseIntl:
942 43 : result = LowerStringToUpperCaseIntl(node);
943 43 : break;
944 : case IrOpcode::kStringSubstring:
945 1080 : result = LowerStringSubstring(node);
946 1080 : break;
947 : case IrOpcode::kStringEqual:
948 10379 : result = LowerStringEqual(node);
949 10379 : break;
950 : case IrOpcode::kStringLessThan:
951 121 : result = LowerStringLessThan(node);
952 121 : break;
953 : case IrOpcode::kStringLessThanOrEqual:
954 707 : result = LowerStringLessThanOrEqual(node);
955 707 : break;
956 : case IrOpcode::kNumberIsFloat64Hole:
957 61 : result = LowerNumberIsFloat64Hole(node);
958 61 : break;
959 : case IrOpcode::kNumberIsFinite:
960 126 : result = LowerNumberIsFinite(node);
961 126 : break;
962 : case IrOpcode::kObjectIsFiniteNumber:
963 7 : result = LowerObjectIsFiniteNumber(node);
964 7 : break;
965 : case IrOpcode::kNumberIsInteger:
966 119 : result = LowerNumberIsInteger(node);
967 119 : break;
968 : case IrOpcode::kObjectIsInteger:
969 7 : result = LowerObjectIsInteger(node);
970 7 : break;
971 : case IrOpcode::kNumberIsSafeInteger:
972 7 : result = LowerNumberIsSafeInteger(node);
973 7 : break;
974 : case IrOpcode::kObjectIsSafeInteger:
975 7 : result = LowerObjectIsSafeInteger(node);
976 7 : break;
977 : case IrOpcode::kCheckFloat64Hole:
978 470 : result = LowerCheckFloat64Hole(node, frame_state);
979 470 : break;
980 : case IrOpcode::kCheckNotTaggedHole:
981 104 : result = LowerCheckNotTaggedHole(node, frame_state);
982 104 : break;
983 : case IrOpcode::kConvertTaggedHoleToUndefined:
984 1667 : result = LowerConvertTaggedHoleToUndefined(node);
985 1667 : break;
986 : case IrOpcode::kCheckEqualsInternalizedString:
987 109 : LowerCheckEqualsInternalizedString(node, frame_state);
988 109 : break;
989 : case IrOpcode::kAllocate:
990 112826 : result = LowerAllocate(node);
991 112826 : break;
992 : case IrOpcode::kCheckEqualsSymbol:
993 18 : LowerCheckEqualsSymbol(node, frame_state);
994 18 : break;
995 : case IrOpcode::kPlainPrimitiveToNumber:
996 20 : result = LowerPlainPrimitiveToNumber(node);
997 20 : break;
998 : case IrOpcode::kPlainPrimitiveToWord32:
999 9 : result = LowerPlainPrimitiveToWord32(node);
1000 9 : break;
1001 : case IrOpcode::kPlainPrimitiveToFloat64:
1002 0 : result = LowerPlainPrimitiveToFloat64(node);
1003 0 : break;
1004 : case IrOpcode::kEnsureWritableFastElements:
1005 1669 : result = LowerEnsureWritableFastElements(node);
1006 1669 : break;
1007 : case IrOpcode::kMaybeGrowFastElements:
1008 3485 : result = LowerMaybeGrowFastElements(node, frame_state);
1009 3485 : break;
1010 : case IrOpcode::kTransitionElementsKind:
1011 618 : LowerTransitionElementsKind(node);
1012 618 : break;
1013 : case IrOpcode::kLoadMessage:
1014 7217 : result = LowerLoadMessage(node);
1015 7217 : break;
1016 : case IrOpcode::kStoreMessage:
1017 29702 : LowerStoreMessage(node);
1018 29702 : break;
1019 : case IrOpcode::kLoadFieldByIndex:
1020 582 : result = LowerLoadFieldByIndex(node);
1021 582 : break;
1022 : case IrOpcode::kLoadTypedElement:
1023 4510 : result = LowerLoadTypedElement(node);
1024 4510 : break;
1025 : case IrOpcode::kLoadDataViewElement:
1026 268 : result = LowerLoadDataViewElement(node);
1027 268 : break;
1028 : case IrOpcode::kLoadStackArgument:
1029 571 : result = LowerLoadStackArgument(node);
1030 571 : break;
1031 : case IrOpcode::kStoreTypedElement:
1032 4342 : LowerStoreTypedElement(node);
1033 4342 : break;
1034 : case IrOpcode::kStoreDataViewElement:
1035 196 : LowerStoreDataViewElement(node);
1036 196 : break;
1037 : case IrOpcode::kStoreSignedSmallElement:
1038 70 : LowerStoreSignedSmallElement(node);
1039 70 : break;
1040 : case IrOpcode::kFindOrderedHashMapEntry:
1041 176 : result = LowerFindOrderedHashMapEntry(node);
1042 176 : break;
1043 : case IrOpcode::kFindOrderedHashMapEntryForInt32Key:
1044 16 : result = LowerFindOrderedHashMapEntryForInt32Key(node);
1045 16 : break;
1046 : case IrOpcode::kTransitionAndStoreNumberElement:
1047 28 : LowerTransitionAndStoreNumberElement(node);
1048 28 : break;
1049 : case IrOpcode::kTransitionAndStoreNonNumberElement:
1050 20 : LowerTransitionAndStoreNonNumberElement(node);
1051 20 : break;
1052 : case IrOpcode::kTransitionAndStoreElement:
1053 86 : LowerTransitionAndStoreElement(node);
1054 86 : break;
1055 : case IrOpcode::kRuntimeAbort:
1056 2557 : LowerRuntimeAbort(node);
1057 2557 : break;
1058 : case IrOpcode::kConvertReceiver:
1059 876 : result = LowerConvertReceiver(node);
1060 876 : break;
1061 : case IrOpcode::kFloat64RoundUp:
1062 16444 : if (!LowerFloat64RoundUp(node).To(&result)) {
1063 : return false;
1064 : }
1065 : break;
1066 : case IrOpcode::kFloat64RoundDown:
1067 53586 : if (!LowerFloat64RoundDown(node).To(&result)) {
1068 : return false;
1069 : }
1070 : break;
1071 : case IrOpcode::kFloat64RoundTruncate:
1072 13828 : if (!LowerFloat64RoundTruncate(node).To(&result)) {
1073 : return false;
1074 : }
1075 : break;
1076 : case IrOpcode::kFloat64RoundTiesEven:
1077 532 : if (!LowerFloat64RoundTiesEven(node).To(&result)) {
1078 : return false;
1079 : }
1080 : break;
1081 : case IrOpcode::kDateNow:
1082 13 : result = LowerDateNow(node);
1083 13 : break;
1084 : default:
1085 : return false;
1086 : }
1087 :
1088 2217184 : if ((result ? 1 : 0) != node->op()->ValueOutputCount()) {
1089 : FATAL(
1090 : "Effect control linearizer lowering of '%s':"
1091 : " value output count does not agree.",
1092 0 : node->op()->mnemonic());
1093 : }
1094 :
1095 1108592 : *effect = gasm()->ExtractCurrentEffect();
1096 1108591 : *control = gasm()->ExtractCurrentControl();
1097 1108592 : NodeProperties::ReplaceUses(node, result, *effect, *control);
1098 1108590 : return true;
1099 : }
1100 :
1101 : #define __ gasm()->
1102 :
1103 53387 : Node* EffectControlLinearizer::LowerChangeFloat64ToTagged(Node* node) {
1104 53387 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
1105 : Node* value = node->InputAt(0);
1106 :
1107 106774 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1108 53387 : auto if_heapnumber = __ MakeDeferredLabel();
1109 53387 : auto if_int32 = __ MakeLabel();
1110 :
1111 53387 : Node* value32 = __ RoundFloat64ToInt32(value);
1112 53388 : __ GotoIf(__ Float64Equal(value, __ ChangeInt32ToFloat64(value32)),
1113 53387 : &if_int32);
1114 : __ Goto(&if_heapnumber);
1115 :
1116 : __ Bind(&if_int32);
1117 : {
1118 53388 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
1119 40494 : Node* zero = __ Int32Constant(0);
1120 40494 : auto if_zero = __ MakeDeferredLabel();
1121 40494 : auto if_smi = __ MakeLabel();
1122 :
1123 40494 : __ GotoIf(__ Word32Equal(value32, zero), &if_zero);
1124 : __ Goto(&if_smi);
1125 :
1126 : __ Bind(&if_zero);
1127 : {
1128 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
1129 40494 : __ GotoIf(__ Int32LessThan(__ Float64ExtractHighWord32(value), zero),
1130 40494 : &if_heapnumber);
1131 : __ Goto(&if_smi);
1132 : }
1133 :
1134 : __ Bind(&if_smi);
1135 : }
1136 :
1137 : if (SmiValuesAre32Bits()) {
1138 53388 : Node* value_smi = ChangeInt32ToSmi(value32);
1139 : __ Goto(&done, value_smi);
1140 : } else {
1141 : DCHECK(SmiValuesAre31Bits());
1142 : Node* add = __ Int32AddWithOverflow(value32, value32);
1143 : Node* ovf = __ Projection(1, add);
1144 : __ GotoIf(ovf, &if_heapnumber);
1145 : Node* value_smi = __ Projection(0, add);
1146 : value_smi = ChangeInt32ToIntPtr(value_smi);
1147 : __ Goto(&done, value_smi);
1148 : }
1149 : }
1150 :
1151 : __ Bind(&if_heapnumber);
1152 : {
1153 53385 : Node* value_number = AllocateHeapNumberWithValue(value);
1154 : __ Goto(&done, value_number);
1155 : }
1156 :
1157 : __ Bind(&done);
1158 53388 : return done.PhiAt(0);
1159 : }
1160 :
1161 91 : Node* EffectControlLinearizer::LowerChangeFloat64ToTaggedPointer(Node* node) {
1162 : Node* value = node->InputAt(0);
1163 91 : return AllocateHeapNumberWithValue(value);
1164 : }
1165 :
1166 9647 : Node* EffectControlLinearizer::LowerChangeBitToTagged(Node* node) {
1167 : Node* value = node->InputAt(0);
1168 :
1169 9647 : auto if_true = __ MakeLabel();
1170 19294 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1171 :
1172 9647 : __ GotoIf(value, &if_true);
1173 9647 : __ Goto(&done, __ FalseConstant());
1174 :
1175 : __ Bind(&if_true);
1176 9647 : __ Goto(&done, __ TrueConstant());
1177 :
1178 : __ Bind(&done);
1179 9647 : return done.PhiAt(0);
1180 : }
1181 :
1182 39324 : Node* EffectControlLinearizer::LowerChangeInt31ToTaggedSigned(Node* node) {
1183 : Node* value = node->InputAt(0);
1184 39324 : return ChangeInt32ToSmi(value);
1185 : }
1186 :
1187 35972 : Node* EffectControlLinearizer::LowerChangeInt32ToTagged(Node* node) {
1188 : Node* value = node->InputAt(0);
1189 :
1190 : if (SmiValuesAre32Bits()) {
1191 35972 : return ChangeInt32ToSmi(value);
1192 : }
1193 : DCHECK(SmiValuesAre31Bits());
1194 :
1195 : auto if_overflow = __ MakeDeferredLabel();
1196 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1197 :
1198 : Node* add = __ Int32AddWithOverflow(value, value);
1199 : Node* ovf = __ Projection(1, add);
1200 : __ GotoIf(ovf, &if_overflow);
1201 : Node* value_smi = __ Projection(0, add);
1202 : value_smi = ChangeInt32ToIntPtr(value_smi);
1203 : __ Goto(&done, value_smi);
1204 :
1205 : __ Bind(&if_overflow);
1206 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt32ToFloat64(value));
1207 : __ Goto(&done, number);
1208 :
1209 : __ Bind(&done);
1210 : return done.PhiAt(0);
1211 : }
1212 :
1213 45 : Node* EffectControlLinearizer::LowerChangeInt64ToTagged(Node* node) {
1214 : Node* value = node->InputAt(0);
1215 :
1216 45 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1217 90 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1218 :
1219 45 : Node* value32 = __ TruncateInt64ToInt32(value);
1220 45 : __ GotoIfNot(__ Word64Equal(__ ChangeInt32ToInt64(value32), value),
1221 45 : &if_not_in_smi_range);
1222 :
1223 : if (SmiValuesAre32Bits()) {
1224 : Node* value_smi = ChangeInt64ToSmi(value);
1225 : __ Goto(&done, value_smi);
1226 : } else {
1227 : Node* add = __ Int32AddWithOverflow(value32, value32);
1228 : Node* ovf = __ Projection(1, add);
1229 : __ GotoIf(ovf, &if_not_in_smi_range);
1230 : Node* value_smi = ChangeInt32ToIntPtr(__ Projection(0, add));
1231 : __ Goto(&done, value_smi);
1232 : }
1233 :
1234 : __ Bind(&if_not_in_smi_range);
1235 45 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1236 : __ Goto(&done, number);
1237 :
1238 : __ Bind(&done);
1239 45 : return done.PhiAt(0);
1240 : }
1241 :
1242 653 : Node* EffectControlLinearizer::LowerChangeUint32ToTagged(Node* node) {
1243 : Node* value = node->InputAt(0);
1244 :
1245 653 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1246 1306 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1247 :
1248 653 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
1249 653 : __ GotoIfNot(check, &if_not_in_smi_range);
1250 653 : __ Goto(&done, ChangeUint32ToSmi(value));
1251 :
1252 : __ Bind(&if_not_in_smi_range);
1253 653 : Node* number = AllocateHeapNumberWithValue(__ ChangeUint32ToFloat64(value));
1254 :
1255 : __ Goto(&done, number);
1256 : __ Bind(&done);
1257 :
1258 653 : return done.PhiAt(0);
1259 : }
1260 :
1261 53 : Node* EffectControlLinearizer::LowerChangeUint64ToTagged(Node* node) {
1262 : Node* value = node->InputAt(0);
1263 :
1264 53 : auto if_not_in_smi_range = __ MakeDeferredLabel();
1265 106 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
1266 :
1267 : Node* check =
1268 53 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
1269 53 : __ GotoIfNot(check, &if_not_in_smi_range);
1270 : __ Goto(&done, ChangeInt64ToSmi(value));
1271 :
1272 : __ Bind(&if_not_in_smi_range);
1273 53 : Node* number = AllocateHeapNumberWithValue(__ ChangeInt64ToFloat64(value));
1274 :
1275 : __ Goto(&done, number);
1276 : __ Bind(&done);
1277 :
1278 53 : return done.PhiAt(0);
1279 : }
1280 :
1281 49908 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt32(Node* node) {
1282 : Node* value = node->InputAt(0);
1283 49908 : return ChangeSmiToInt32(value);
1284 : }
1285 :
1286 297 : Node* EffectControlLinearizer::LowerChangeTaggedSignedToInt64(Node* node) {
1287 : Node* value = node->InputAt(0);
1288 297 : return ChangeSmiToInt64(value);
1289 : }
1290 :
1291 139067 : Node* EffectControlLinearizer::LowerChangeTaggedToBit(Node* node) {
1292 : Node* value = node->InputAt(0);
1293 139067 : return __ WordEqual(value, __ TrueConstant());
1294 : }
1295 :
1296 48019 : void EffectControlLinearizer::TruncateTaggedPointerToBit(
1297 : Node* node, GraphAssemblerLabel<1>* done) {
1298 : Node* value = node->InputAt(0);
1299 :
1300 48019 : auto if_heapnumber = __ MakeDeferredLabel();
1301 48019 : auto if_bigint = __ MakeDeferredLabel();
1302 :
1303 48019 : Node* zero = __ Int32Constant(0);
1304 48019 : Node* fzero = __ Float64Constant(0.0);
1305 :
1306 : // Check if {value} is false.
1307 48019 : __ GotoIf(__ WordEqual(value, __ FalseConstant()), done, zero);
1308 :
1309 : // Check if {value} is the empty string.
1310 48019 : __ GotoIf(__ WordEqual(value, __ EmptyStringConstant()), done, zero);
1311 :
1312 : // Load the map of {value}.
1313 48019 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1314 :
1315 : // Check if the {value} is undetectable and immediately return false.
1316 : // This includes undefined and null.
1317 : Node* value_map_bitfield =
1318 48019 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
1319 48019 : __ GotoIfNot(
1320 : __ Word32Equal(
1321 : __ Word32And(value_map_bitfield,
1322 : __ Int32Constant(Map::IsUndetectableBit::kMask)),
1323 : zero),
1324 48019 : done, zero);
1325 :
1326 : // Check if {value} is a HeapNumber.
1327 48019 : __ GotoIf(__ WordEqual(value_map, __ HeapNumberMapConstant()),
1328 48019 : &if_heapnumber);
1329 :
1330 : // Check if {value} is a BigInt.
1331 48019 : __ GotoIf(__ WordEqual(value_map, __ BigIntMapConstant()), &if_bigint);
1332 :
1333 : // All other values that reach here are true.
1334 48019 : __ Goto(done, __ Int32Constant(1));
1335 :
1336 : __ Bind(&if_heapnumber);
1337 : {
1338 : // For HeapNumber {value}, just check that its value is not 0.0, -0.0 or
1339 : // NaN.
1340 : Node* value_value =
1341 48019 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1342 48019 : __ Goto(done, __ Float64LessThan(fzero, __ Float64Abs(value_value)));
1343 : }
1344 :
1345 : __ Bind(&if_bigint);
1346 : {
1347 48019 : Node* bitfield = __ LoadField(AccessBuilder::ForBigIntBitfield(), value);
1348 48019 : Node* length_is_zero = __ Word32Equal(
1349 : __ Word32And(bitfield, __ Int32Constant(BigInt::LengthBits::kMask)),
1350 48019 : __ Int32Constant(0));
1351 48019 : __ Goto(done, __ Word32Equal(length_is_zero, zero));
1352 : }
1353 48019 : }
1354 :
1355 47628 : Node* EffectControlLinearizer::LowerTruncateTaggedToBit(Node* node) {
1356 95256 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1357 47628 : auto if_smi = __ MakeDeferredLabel();
1358 :
1359 : Node* value = node->InputAt(0);
1360 95256 : __ GotoIf(ObjectIsSmi(value), &if_smi);
1361 :
1362 47628 : TruncateTaggedPointerToBit(node, &done);
1363 :
1364 : __ Bind(&if_smi);
1365 : {
1366 : // If {value} is a Smi, then we only need to check that it's not zero.
1367 47628 : __ Goto(&done, __ Word32Equal(__ WordEqual(value, __ IntPtrConstant(0)),
1368 : __ Int32Constant(0)));
1369 : }
1370 :
1371 : __ Bind(&done);
1372 47628 : return done.PhiAt(0);
1373 : }
1374 :
1375 391 : Node* EffectControlLinearizer::LowerTruncateTaggedPointerToBit(Node* node) {
1376 782 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1377 :
1378 391 : TruncateTaggedPointerToBit(node, &done);
1379 :
1380 : __ Bind(&done);
1381 391 : return done.PhiAt(0);
1382 : }
1383 :
1384 1754 : Node* EffectControlLinearizer::LowerChangeTaggedToInt32(Node* node) {
1385 : Node* value = node->InputAt(0);
1386 :
1387 1754 : auto if_not_smi = __ MakeDeferredLabel();
1388 3508 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1389 :
1390 1754 : Node* check = ObjectIsSmi(value);
1391 1754 : __ GotoIfNot(check, &if_not_smi);
1392 1754 : __ Goto(&done, ChangeSmiToInt32(value));
1393 :
1394 : __ Bind(&if_not_smi);
1395 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1396 : Oddball::kToNumberRawOffset);
1397 1754 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1398 1754 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1399 : __ Goto(&done, vfalse);
1400 :
1401 : __ Bind(&done);
1402 1754 : return done.PhiAt(0);
1403 : }
1404 :
1405 394 : Node* EffectControlLinearizer::LowerChangeTaggedToUint32(Node* node) {
1406 : Node* value = node->InputAt(0);
1407 :
1408 394 : auto if_not_smi = __ MakeDeferredLabel();
1409 788 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1410 :
1411 394 : Node* check = ObjectIsSmi(value);
1412 394 : __ GotoIfNot(check, &if_not_smi);
1413 394 : __ Goto(&done, ChangeSmiToInt32(value));
1414 :
1415 : __ Bind(&if_not_smi);
1416 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1417 : Oddball::kToNumberRawOffset);
1418 394 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1419 394 : vfalse = __ ChangeFloat64ToUint32(vfalse);
1420 : __ Goto(&done, vfalse);
1421 :
1422 : __ Bind(&done);
1423 394 : return done.PhiAt(0);
1424 : }
1425 :
1426 2 : Node* EffectControlLinearizer::LowerChangeTaggedToInt64(Node* node) {
1427 : Node* value = node->InputAt(0);
1428 :
1429 2 : auto if_not_smi = __ MakeDeferredLabel();
1430 4 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
1431 :
1432 2 : Node* check = ObjectIsSmi(value);
1433 2 : __ GotoIfNot(check, &if_not_smi);
1434 2 : __ Goto(&done, ChangeSmiToInt64(value));
1435 :
1436 : __ Bind(&if_not_smi);
1437 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1438 : Oddball::kToNumberRawOffset);
1439 2 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1440 2 : vfalse = __ ChangeFloat64ToInt64(vfalse);
1441 : __ Goto(&done, vfalse);
1442 :
1443 : __ Bind(&done);
1444 2 : return done.PhiAt(0);
1445 : }
1446 :
1447 0 : Node* EffectControlLinearizer::LowerChangeTaggedToFloat64(Node* node) {
1448 11274 : return LowerTruncateTaggedToFloat64(node);
1449 : }
1450 :
1451 14 : Node* EffectControlLinearizer::LowerChangeTaggedToTaggedSigned(Node* node) {
1452 : Node* value = node->InputAt(0);
1453 :
1454 14 : auto if_not_smi = __ MakeDeferredLabel();
1455 28 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1456 :
1457 14 : Node* check = ObjectIsSmi(value);
1458 14 : __ GotoIfNot(check, &if_not_smi);
1459 : __ Goto(&done, value);
1460 :
1461 : __ Bind(&if_not_smi);
1462 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1463 : Oddball::kToNumberRawOffset);
1464 14 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1465 14 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1466 14 : vfalse = ChangeInt32ToSmi(vfalse);
1467 : __ Goto(&done, vfalse);
1468 :
1469 : __ Bind(&done);
1470 14 : return done.PhiAt(0);
1471 : }
1472 :
1473 0 : Node* EffectControlLinearizer::LowerChangeCompressedToTaggedSigned(Node* node) {
1474 : Node* value = node->InputAt(0);
1475 :
1476 0 : auto if_not_smi = __ MakeDeferredLabel();
1477 0 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1478 :
1479 0 : Node* check = ObjectIsSmi(value);
1480 0 : __ GotoIfNot(check, &if_not_smi);
1481 0 : __ Goto(&done, __ ChangeCompressedSignedToTaggedSigned(value));
1482 :
1483 : __ Bind(&if_not_smi);
1484 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1485 0 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(),
1486 0 : __ ChangeCompressedToTagged(value));
1487 0 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1488 0 : vfalse = ChangeInt32ToSmi(vfalse);
1489 : __ Goto(&done, vfalse);
1490 :
1491 : __ Bind(&done);
1492 0 : return done.PhiAt(0);
1493 : }
1494 :
1495 0 : Node* EffectControlLinearizer::LowerChangeTaggedToCompressedSigned(Node* node) {
1496 : Node* value = node->InputAt(0);
1497 :
1498 0 : auto if_not_smi = __ MakeDeferredLabel();
1499 0 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1500 :
1501 0 : Node* check = ObjectIsSmi(value);
1502 0 : __ GotoIfNot(check, &if_not_smi);
1503 0 : __ Goto(&done, __ ChangeTaggedSignedToCompressedSigned(value));
1504 :
1505 : __ Bind(&if_not_smi);
1506 : STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
1507 0 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1508 0 : vfalse = __ ChangeFloat64ToInt32(vfalse);
1509 0 : vfalse = ChangeInt32ToSmi(vfalse);
1510 0 : vfalse = __ ChangeTaggedSignedToCompressedSigned(vfalse);
1511 : __ Goto(&done, vfalse);
1512 :
1513 : __ Bind(&done);
1514 0 : return done.PhiAt(0);
1515 : }
1516 :
1517 11990 : Node* EffectControlLinearizer::LowerTruncateTaggedToFloat64(Node* node) {
1518 : Node* value = node->InputAt(0);
1519 :
1520 11990 : auto if_not_smi = __ MakeDeferredLabel();
1521 23980 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
1522 :
1523 11990 : Node* check = ObjectIsSmi(value);
1524 11990 : __ GotoIfNot(check, &if_not_smi);
1525 11990 : Node* vtrue = ChangeSmiToInt32(value);
1526 11990 : vtrue = __ ChangeInt32ToFloat64(vtrue);
1527 : __ Goto(&done, vtrue);
1528 :
1529 : __ Bind(&if_not_smi);
1530 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
1531 : Oddball::kToNumberRawOffset);
1532 11990 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
1533 : __ Goto(&done, vfalse);
1534 :
1535 : __ Bind(&done);
1536 11990 : return done.PhiAt(0);
1537 : }
1538 :
1539 1033 : Node* EffectControlLinearizer::LowerPoisonIndex(Node* node) {
1540 : Node* index = node->InputAt(0);
1541 1033 : if (mask_array_index_ == kMaskArrayIndex) {
1542 0 : index = __ Word32PoisonOnSpeculation(index);
1543 : }
1544 1033 : return index;
1545 : }
1546 :
1547 51503 : void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
1548 51503 : CheckMapsParameters const& p = CheckMapsParametersOf(node->op());
1549 : Node* value = node->InputAt(0);
1550 :
1551 : ZoneHandleSet<Map> const& maps = p.maps();
1552 : size_t const map_count = maps.size();
1553 :
1554 51503 : if (p.flags() & CheckMapsFlag::kTryMigrateInstance) {
1555 56 : auto done = __ MakeLabel();
1556 56 : auto migrate = __ MakeDeferredLabel();
1557 :
1558 : // Load the current map of the {value}.
1559 112 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1560 :
1561 : // Perform the map checks.
1562 168 : for (size_t i = 0; i < map_count; ++i) {
1563 56 : Node* map = __ HeapConstant(maps[i]);
1564 56 : Node* check = __ WordEqual(value_map, map);
1565 56 : if (i == map_count - 1) {
1566 56 : __ Branch(check, &done, &migrate, IsSafetyCheck::kCriticalSafetyCheck);
1567 : } else {
1568 0 : auto next_map = __ MakeLabel();
1569 0 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1570 : __ Bind(&next_map);
1571 : }
1572 : }
1573 :
1574 : // Perform the (deferred) instance migration.
1575 : __ Bind(&migrate);
1576 : {
1577 : // If map is not deprecated the migration attempt does not make sense.
1578 : Node* bitfield3 =
1579 56 : __ LoadField(AccessBuilder::ForMapBitField3(), value_map);
1580 56 : Node* if_not_deprecated = __ WordEqual(
1581 : __ Word32And(bitfield3,
1582 : __ Int32Constant(Map::IsDeprecatedBit::kMask)),
1583 56 : __ Int32Constant(0));
1584 : __ DeoptimizeIf(DeoptimizeReason::kWrongMap, p.feedback(),
1585 : if_not_deprecated, frame_state,
1586 56 : IsSafetyCheck::kCriticalSafetyCheck);
1587 :
1588 56 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
1589 : Runtime::FunctionId id = Runtime::kTryMigrateInstance;
1590 56 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1591 56 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
1592 56 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1), value,
1593 : __ ExternalConstant(ExternalReference::Create(id)),
1594 56 : __ Int32Constant(1), __ NoContextConstant());
1595 56 : Node* check = ObjectIsSmi(result);
1596 : __ DeoptimizeIf(DeoptimizeReason::kInstanceMigrationFailed, p.feedback(),
1597 56 : check, frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1598 : }
1599 :
1600 : // Reload the current map of the {value}.
1601 56 : value_map = __ LoadField(AccessBuilder::ForMap(), value);
1602 :
1603 : // Perform the map checks again.
1604 168 : for (size_t i = 0; i < map_count; ++i) {
1605 56 : Node* map = __ HeapConstant(maps[i]);
1606 56 : Node* check = __ WordEqual(value_map, map);
1607 56 : if (i == map_count - 1) {
1608 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1609 56 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1610 : } else {
1611 0 : auto next_map = __ MakeLabel();
1612 0 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1613 : __ Bind(&next_map);
1614 : }
1615 : }
1616 :
1617 : __ Goto(&done);
1618 : __ Bind(&done);
1619 : } else {
1620 51447 : auto done = __ MakeLabel();
1621 :
1622 : // Load the current map of the {value}.
1623 : Node* value_map =
1624 : UsingCompressedPointers()
1625 : ? __ LoadField(AccessBuilder::ForCompressedMap(), value)
1626 102894 : : __ LoadField(AccessBuilder::ForMap(), value);
1627 :
1628 166027 : for (size_t i = 0; i < map_count; ++i) {
1629 : Node* check;
1630 :
1631 : if (UsingCompressedPointers()) {
1632 : // We need the dereference scope to embed the map pointer value as an
1633 : // int32. We don't visit the pointer.
1634 : AllowHandleDereference allow_map_dereference;
1635 : int32_t int32Map = static_cast<int32_t>(CompressTagged(maps[i]->ptr()));
1636 : Node* map = __ Int32Constant(int32Map);
1637 : check = __ Word32Equal(value_map, map);
1638 : this->embedded_maps()->push_back(maps[i]);
1639 : } else {
1640 57290 : Node* map = __ HeapConstant(maps[i]);
1641 57290 : check = __ WordEqual(value_map, map);
1642 : }
1643 :
1644 57290 : if (i == map_count - 1) {
1645 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongMap, p.feedback(), check,
1646 51447 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
1647 : } else {
1648 5843 : auto next_map = __ MakeLabel();
1649 5843 : __ Branch(check, &done, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1650 : __ Bind(&next_map);
1651 : }
1652 : }
1653 : __ Goto(&done);
1654 : __ Bind(&done);
1655 : }
1656 51503 : }
1657 :
1658 7530 : Node* EffectControlLinearizer::LowerCompareMaps(Node* node) {
1659 7530 : ZoneHandleSet<Map> const& maps = CompareMapsParametersOf(node->op());
1660 : size_t const map_count = maps.size();
1661 : Node* value = node->InputAt(0);
1662 :
1663 15060 : auto done = __ MakeLabel(MachineRepresentation::kBit);
1664 :
1665 : // Load the current map of the {value}.
1666 : Node* value_map = UsingCompressedPointers()
1667 : ? __ LoadField(AccessBuilder::ForCompressedMap(), value)
1668 15060 : : __ LoadField(AccessBuilder::ForMap(), value);
1669 :
1670 22606 : for (size_t i = 0; i < map_count; ++i) {
1671 : Node* check;
1672 :
1673 : if (UsingCompressedPointers()) {
1674 : // We need the dereference scope to embed the map pointer value as an
1675 : // int32. We don't visit the pointer.
1676 : AllowHandleDereference allow_map_dereference;
1677 : int32_t int32Map = static_cast<int32_t>(CompressTagged(maps[i]->ptr()));
1678 : Node* map = __ Int32Constant(int32Map);
1679 : check = __ Word32Equal(value_map, map);
1680 : this->embedded_maps()->push_back(maps[i]);
1681 : } else {
1682 7538 : Node* map = __ HeapConstant(maps[i]);
1683 7538 : check = __ WordEqual(value_map, map);
1684 : }
1685 :
1686 7538 : auto next_map = __ MakeLabel();
1687 7538 : auto passed = __ MakeLabel();
1688 7538 : __ Branch(check, &passed, &next_map, IsSafetyCheck::kCriticalSafetyCheck);
1689 :
1690 : __ Bind(&passed);
1691 7538 : __ Goto(&done, __ Int32Constant(1));
1692 :
1693 : __ Bind(&next_map);
1694 : }
1695 7530 : __ Goto(&done, __ Int32Constant(0));
1696 :
1697 : __ Bind(&done);
1698 7530 : return done.PhiAt(0);
1699 : }
1700 :
1701 475 : Node* EffectControlLinearizer::LowerCheckNumber(Node* node, Node* frame_state) {
1702 : Node* value = node->InputAt(0);
1703 475 : const CheckParameters& params = CheckParametersOf(node->op());
1704 :
1705 475 : auto if_not_smi = __ MakeDeferredLabel();
1706 475 : auto done = __ MakeLabel();
1707 :
1708 475 : Node* check0 = ObjectIsSmi(value);
1709 475 : __ GotoIfNot(check0, &if_not_smi);
1710 : __ Goto(&done);
1711 :
1712 : __ Bind(&if_not_smi);
1713 475 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1714 475 : Node* check1 = __ WordEqual(value_map, __ HeapNumberMapConstant());
1715 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
1716 475 : check1, frame_state);
1717 : __ Goto(&done);
1718 :
1719 : __ Bind(&done);
1720 475 : return value;
1721 : }
1722 :
1723 817 : Node* EffectControlLinearizer::LowerCheckReceiver(Node* node,
1724 : Node* frame_state) {
1725 : Node* value = node->InputAt(0);
1726 :
1727 1634 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1728 : Node* value_instance_type =
1729 817 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1730 :
1731 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1732 817 : Node* check = __ Uint32LessThanOrEqual(
1733 817 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
1734 1634 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObject, VectorSlotPair(),
1735 817 : check, frame_state);
1736 817 : return value;
1737 : }
1738 :
1739 72 : Node* EffectControlLinearizer::LowerCheckReceiverOrNullOrUndefined(
1740 : Node* node, Node* frame_state) {
1741 : Node* value = node->InputAt(0);
1742 :
1743 144 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1744 : Node* value_instance_type =
1745 72 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1746 :
1747 : // Rule out all primitives except oddballs (true, false, undefined, null).
1748 : STATIC_ASSERT(LAST_PRIMITIVE_TYPE == ODDBALL_TYPE);
1749 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
1750 72 : Node* check0 = __ Uint32LessThanOrEqual(__ Uint32Constant(ODDBALL_TYPE),
1751 72 : value_instance_type);
1752 144 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1753 72 : VectorSlotPair(), check0, frame_state);
1754 :
1755 : // Rule out booleans.
1756 72 : Node* check1 = __ WordEqual(value_map, __ BooleanMapConstant());
1757 144 : __ DeoptimizeIf(DeoptimizeReason::kNotAJavaScriptObjectOrNullOrUndefined,
1758 72 : VectorSlotPair(), check1, frame_state);
1759 72 : return value;
1760 : }
1761 :
1762 32 : Node* EffectControlLinearizer::LowerCheckSymbol(Node* node, Node* frame_state) {
1763 : Node* value = node->InputAt(0);
1764 :
1765 64 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1766 :
1767 : Node* check =
1768 32 : __ WordEqual(value_map, __ HeapConstant(factory()->symbol_map()));
1769 64 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASymbol, VectorSlotPair(), check,
1770 32 : frame_state);
1771 32 : return value;
1772 : }
1773 :
1774 5348 : Node* EffectControlLinearizer::LowerCheckString(Node* node, Node* frame_state) {
1775 : Node* value = node->InputAt(0);
1776 5348 : const CheckParameters& params = CheckParametersOf(node->op());
1777 :
1778 10696 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1779 : Node* value_instance_type =
1780 5348 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1781 :
1782 5348 : Node* check = __ Uint32LessThan(value_instance_type,
1783 5348 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
1784 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAString, params.feedback(), check,
1785 5348 : frame_state);
1786 5348 : return value;
1787 : }
1788 :
1789 1393 : Node* EffectControlLinearizer::LowerCheckInternalizedString(Node* node,
1790 : Node* frame_state) {
1791 : Node* value = node->InputAt(0);
1792 :
1793 2786 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1794 : Node* value_instance_type =
1795 1393 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1796 :
1797 1393 : Node* check = __ Word32Equal(
1798 : __ Word32And(value_instance_type,
1799 : __ Int32Constant(kIsNotStringMask | kIsNotInternalizedMask)),
1800 1393 : __ Int32Constant(kInternalizedTag));
1801 2786 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1802 1393 : check, frame_state);
1803 :
1804 1393 : return value;
1805 : }
1806 :
1807 1424 : Node* EffectControlLinearizer::LowerCheckNonEmptyOneByteString(
1808 : Node* node, Node* frame_state) {
1809 : Node* value = node->InputAt(0);
1810 :
1811 2848 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1812 : Node* value_instance_type =
1813 1424 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1814 :
1815 1424 : Node* check = __ Word32Equal(
1816 : __ Word32And(value_instance_type,
1817 : __ Int32Constant(kIsNotStringMask | kStringEncodingMask |
1818 : kIsEmptyStringMask)),
1819 1424 : __ Int32Constant(kStringTag | kOneByteStringTag | kIsNotEmptyStringTag));
1820 2848 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1821 1424 : check, frame_state);
1822 :
1823 1424 : return value;
1824 : }
1825 :
1826 209 : Node* EffectControlLinearizer::LowerCheckNonEmptyTwoByteString(
1827 : Node* node, Node* frame_state) {
1828 : Node* value = node->InputAt(0);
1829 :
1830 418 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1831 : Node* value_instance_type =
1832 209 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1833 :
1834 209 : Node* check = __ Word32Equal(
1835 : __ Word32And(value_instance_type,
1836 : __ Int32Constant(kIsNotStringMask | kStringEncodingMask |
1837 : kIsEmptyStringMask)),
1838 209 : __ Int32Constant(kStringTag | kTwoByteStringTag | kIsNotEmptyStringTag));
1839 418 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1840 209 : check, frame_state);
1841 :
1842 209 : return value;
1843 : }
1844 :
1845 18 : Node* EffectControlLinearizer::LowerCheckNonEmptyString(Node* node,
1846 : Node* frame_state) {
1847 : Node* value = node->InputAt(0);
1848 :
1849 36 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
1850 : Node* value_instance_type =
1851 18 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
1852 :
1853 18 : Node* check = __ Word32Equal(
1854 : __ Word32And(value_instance_type,
1855 : __ Int32Constant(kIsNotStringMask | kIsEmptyStringMask)),
1856 18 : __ Int32Constant(kStringTag | kIsNotEmptyStringTag));
1857 36 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongInstanceType, VectorSlotPair(),
1858 18 : check, frame_state);
1859 :
1860 18 : return value;
1861 : }
1862 :
1863 16409 : void EffectControlLinearizer::LowerCheckIf(Node* node, Node* frame_state) {
1864 : Node* value = node->InputAt(0);
1865 16409 : const CheckIfParameters& p = CheckIfParametersOf(node->op());
1866 16409 : __ DeoptimizeIfNot(p.reason(), p.feedback(), value, frame_state);
1867 16409 : }
1868 :
1869 21110 : Node* EffectControlLinearizer::LowerStringConcat(Node* node) {
1870 : Node* lhs = node->InputAt(1);
1871 : Node* rhs = node->InputAt(2);
1872 :
1873 : Callable const callable =
1874 21110 : CodeFactory::StringAdd(isolate(), STRING_ADD_CHECK_NONE);
1875 21110 : auto call_descriptor = Linkage::GetStubCallDescriptor(
1876 21110 : graph()->zone(), callable.descriptor(),
1877 : callable.descriptor().GetStackParameterCount(), CallDescriptor::kNoFlags,
1878 21110 : Operator::kNoDeopt | Operator::kNoWrite | Operator::kNoThrow);
1879 :
1880 : Node* value =
1881 63330 : __ Call(call_descriptor, jsgraph()->HeapConstant(callable.code()), lhs,
1882 21110 : rhs, __ NoContextConstant());
1883 :
1884 21110 : return value;
1885 : }
1886 :
1887 55669 : Node* EffectControlLinearizer::LowerCheckedInt32Add(Node* node,
1888 : Node* frame_state) {
1889 : Node* lhs = node->InputAt(0);
1890 : Node* rhs = node->InputAt(1);
1891 :
1892 55669 : Node* value = __ Int32AddWithOverflow(lhs, rhs);
1893 55669 : Node* check = __ Projection(1, value);
1894 111345 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1895 55673 : frame_state);
1896 55673 : return __ Projection(0, value);
1897 : }
1898 :
1899 7631 : Node* EffectControlLinearizer::LowerCheckedInt32Sub(Node* node,
1900 : Node* frame_state) {
1901 : Node* lhs = node->InputAt(0);
1902 : Node* rhs = node->InputAt(1);
1903 :
1904 7631 : Node* value = __ Int32SubWithOverflow(lhs, rhs);
1905 7631 : Node* check = __ Projection(1, value);
1906 15262 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
1907 7631 : frame_state);
1908 7631 : return __ Projection(0, value);
1909 : }
1910 :
1911 236 : Node* EffectControlLinearizer::LowerCheckedInt32Div(Node* node,
1912 : Node* frame_state) {
1913 : Node* lhs = node->InputAt(0);
1914 : Node* rhs = node->InputAt(1);
1915 236 : Node* zero = __ Int32Constant(0);
1916 :
1917 : // Check if the {rhs} is a known power of two.
1918 : Int32Matcher m(rhs);
1919 236 : if (m.IsPowerOf2()) {
1920 : // Since we know that {rhs} is a power of two, we can perform a fast
1921 : // check to see if the relevant least significant bits of the {lhs}
1922 : // are all zero, and if so we know that we can perform a division
1923 : // safely (and fast by doing an arithmetic - aka sign preserving -
1924 : // right shift on {lhs}).
1925 : int32_t divisor = m.Value();
1926 135 : Node* mask = __ Int32Constant(divisor - 1);
1927 135 : Node* shift = __ Int32Constant(WhichPowerOf2(divisor));
1928 135 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
1929 270 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1930 135 : check, frame_state);
1931 135 : return __ Word32Sar(lhs, shift);
1932 : } else {
1933 101 : auto if_rhs_positive = __ MakeLabel();
1934 101 : auto if_rhs_negative = __ MakeDeferredLabel();
1935 202 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
1936 :
1937 : // Check if {rhs} is positive (and not zero).
1938 101 : Node* check_rhs_positive = __ Int32LessThan(zero, rhs);
1939 101 : __ Branch(check_rhs_positive, &if_rhs_positive, &if_rhs_negative);
1940 :
1941 : __ Bind(&if_rhs_positive);
1942 : {
1943 : // Fast case, no additional checking required.
1944 101 : __ Goto(&done, __ Int32Div(lhs, rhs));
1945 : }
1946 :
1947 : __ Bind(&if_rhs_negative);
1948 : {
1949 101 : auto if_lhs_minint = __ MakeDeferredLabel();
1950 101 : auto if_lhs_notminint = __ MakeLabel();
1951 :
1952 : // Check if {rhs} is zero.
1953 101 : Node* check_rhs_zero = __ Word32Equal(rhs, zero);
1954 202 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
1955 101 : check_rhs_zero, frame_state);
1956 :
1957 : // Check if {lhs} is zero, as that would produce minus zero.
1958 101 : Node* check_lhs_zero = __ Word32Equal(lhs, zero);
1959 202 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
1960 101 : check_lhs_zero, frame_state);
1961 :
1962 : // Check if {lhs} is kMinInt and {rhs} is -1, in which case we'd have
1963 : // to return -kMinInt, which is not representable as Word32.
1964 202 : Node* check_lhs_minint = graph()->NewNode(machine()->Word32Equal(), lhs,
1965 : __ Int32Constant(kMinInt));
1966 101 : __ Branch(check_lhs_minint, &if_lhs_minint, &if_lhs_notminint);
1967 :
1968 : __ Bind(&if_lhs_minint);
1969 : {
1970 : // Check that {rhs} is not -1, otherwise result would be -kMinInt.
1971 101 : Node* check_rhs_minusone = __ Word32Equal(rhs, __ Int32Constant(-1));
1972 202 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(),
1973 101 : check_rhs_minusone, frame_state);
1974 :
1975 : // Perform the actual integer division.
1976 101 : __ Goto(&done, __ Int32Div(lhs, rhs));
1977 : }
1978 :
1979 : __ Bind(&if_lhs_notminint);
1980 : {
1981 : // Perform the actual integer division.
1982 101 : __ Goto(&done, __ Int32Div(lhs, rhs));
1983 : }
1984 : }
1985 :
1986 : __ Bind(&done);
1987 : Node* value = done.PhiAt(0);
1988 :
1989 : // Check if the remainder is non-zero.
1990 101 : Node* check = __ Word32Equal(lhs, __ Int32Mul(value, rhs));
1991 202 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
1992 101 : check, frame_state);
1993 :
1994 : return value;
1995 : }
1996 : }
1997 :
1998 1017 : Node* EffectControlLinearizer::BuildUint32Mod(Node* lhs, Node* rhs) {
1999 1017 : auto if_rhs_power_of_two = __ MakeLabel();
2000 2034 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2001 :
2002 : // Compute the mask for the {rhs}.
2003 1017 : Node* one = __ Int32Constant(1);
2004 1017 : Node* msk = __ Int32Sub(rhs, one);
2005 :
2006 : // Check if the {rhs} is a power of two.
2007 1017 : __ GotoIf(__ Word32Equal(__ Word32And(rhs, msk), __ Int32Constant(0)),
2008 1017 : &if_rhs_power_of_two);
2009 : {
2010 : // The {rhs} is not a power of two, do a generic Uint32Mod.
2011 1017 : __ Goto(&done, __ Uint32Mod(lhs, rhs));
2012 : }
2013 :
2014 : __ Bind(&if_rhs_power_of_two);
2015 : {
2016 : // The {rhs} is a power of two, just do a fast bit masking.
2017 1017 : __ Goto(&done, __ Word32And(lhs, msk));
2018 : }
2019 :
2020 : __ Bind(&done);
2021 1017 : return done.PhiAt(0);
2022 : }
2023 :
2024 980 : Node* EffectControlLinearizer::LowerCheckedInt32Mod(Node* node,
2025 : Node* frame_state) {
2026 : // General case for signed integer modulus, with optimization for (unknown)
2027 : // power of 2 right hand side.
2028 : //
2029 : // if rhs <= 0 then
2030 : // rhs = -rhs
2031 : // deopt if rhs == 0
2032 : // let msk = rhs - 1 in
2033 : // if lhs < 0 then
2034 : // let lhs_abs = -lsh in
2035 : // let res = if rhs & msk == 0 then
2036 : // lhs_abs & msk
2037 : // else
2038 : // lhs_abs % rhs in
2039 : // if lhs < 0 then
2040 : // deopt if res == 0
2041 : // -res
2042 : // else
2043 : // res
2044 : // else
2045 : // if rhs & msk == 0 then
2046 : // lhs & msk
2047 : // else
2048 : // lhs % rhs
2049 : //
2050 : Node* lhs = node->InputAt(0);
2051 : Node* rhs = node->InputAt(1);
2052 :
2053 980 : auto if_rhs_not_positive = __ MakeDeferredLabel();
2054 980 : auto if_lhs_negative = __ MakeDeferredLabel();
2055 : auto if_rhs_power_of_two = __ MakeLabel();
2056 1960 : auto rhs_checked = __ MakeLabel(MachineRepresentation::kWord32);
2057 1960 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2058 :
2059 980 : Node* zero = __ Int32Constant(0);
2060 :
2061 : // Check if {rhs} is not strictly positive.
2062 980 : Node* check0 = __ Int32LessThanOrEqual(rhs, zero);
2063 980 : __ GotoIf(check0, &if_rhs_not_positive);
2064 : __ Goto(&rhs_checked, rhs);
2065 :
2066 : __ Bind(&if_rhs_not_positive);
2067 : {
2068 : // Negate {rhs}, might still produce a negative result in case of
2069 : // -2^31, but that is handled safely below.
2070 980 : Node* vtrue0 = __ Int32Sub(zero, rhs);
2071 :
2072 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
2073 1960 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(),
2074 980 : __ Word32Equal(vtrue0, zero), frame_state);
2075 : __ Goto(&rhs_checked, vtrue0);
2076 : }
2077 :
2078 : __ Bind(&rhs_checked);
2079 : rhs = rhs_checked.PhiAt(0);
2080 :
2081 980 : __ GotoIf(__ Int32LessThan(lhs, zero), &if_lhs_negative);
2082 : {
2083 : // The {lhs} is a non-negative integer.
2084 980 : __ Goto(&done, BuildUint32Mod(lhs, rhs));
2085 : }
2086 :
2087 : __ Bind(&if_lhs_negative);
2088 : {
2089 : // The {lhs} is a negative integer. This is very unlikely and
2090 : // we intentionally don't use the BuildUint32Mod() here, which
2091 : // would try to figure out whether {rhs} is a power of two,
2092 : // since this is intended to be a slow-path.
2093 980 : Node* res = __ Uint32Mod(__ Int32Sub(zero, lhs), rhs);
2094 :
2095 : // Check if we would have to return -0.
2096 1960 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(),
2097 980 : __ Word32Equal(res, zero), frame_state);
2098 980 : __ Goto(&done, __ Int32Sub(zero, res));
2099 : }
2100 :
2101 : __ Bind(&done);
2102 980 : return done.PhiAt(0);
2103 : }
2104 :
2105 52 : Node* EffectControlLinearizer::LowerCheckedUint32Div(Node* node,
2106 : Node* frame_state) {
2107 : Node* lhs = node->InputAt(0);
2108 : Node* rhs = node->InputAt(1);
2109 52 : Node* zero = __ Int32Constant(0);
2110 :
2111 : // Check if the {rhs} is a known power of two.
2112 : Uint32Matcher m(rhs);
2113 52 : if (m.IsPowerOf2()) {
2114 : // Since we know that {rhs} is a power of two, we can perform a fast
2115 : // check to see if the relevant least significant bits of the {lhs}
2116 : // are all zero, and if so we know that we can perform a division
2117 : // safely (and fast by doing a logical - aka zero extending - right
2118 : // shift on {lhs}).
2119 : uint32_t divisor = m.Value();
2120 41 : Node* mask = __ Uint32Constant(divisor - 1);
2121 41 : Node* shift = __ Uint32Constant(WhichPowerOf2(divisor));
2122 41 : Node* check = __ Word32Equal(__ Word32And(lhs, mask), zero);
2123 82 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
2124 41 : check, frame_state);
2125 41 : return __ Word32Shr(lhs, shift);
2126 : } else {
2127 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
2128 11 : Node* check = __ Word32Equal(rhs, zero);
2129 22 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
2130 11 : frame_state);
2131 :
2132 : // Perform the actual unsigned integer division.
2133 11 : Node* value = __ Uint32Div(lhs, rhs);
2134 :
2135 : // Check if the remainder is non-zero.
2136 11 : check = __ Word32Equal(lhs, __ Int32Mul(rhs, value));
2137 22 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, VectorSlotPair(),
2138 11 : check, frame_state);
2139 11 : return value;
2140 : }
2141 : }
2142 :
2143 37 : Node* EffectControlLinearizer::LowerCheckedUint32Mod(Node* node,
2144 : Node* frame_state) {
2145 : Node* lhs = node->InputAt(0);
2146 : Node* rhs = node->InputAt(1);
2147 :
2148 37 : Node* zero = __ Int32Constant(0);
2149 :
2150 : // Ensure that {rhs} is not zero, otherwise we'd have to return NaN.
2151 37 : Node* check = __ Word32Equal(rhs, zero);
2152 74 : __ DeoptimizeIf(DeoptimizeReason::kDivisionByZero, VectorSlotPair(), check,
2153 37 : frame_state);
2154 :
2155 : // Perform the actual unsigned integer modulus.
2156 37 : return BuildUint32Mod(lhs, rhs);
2157 : }
2158 :
2159 5906 : Node* EffectControlLinearizer::LowerCheckedInt32Mul(Node* node,
2160 : Node* frame_state) {
2161 5906 : CheckForMinusZeroMode mode = CheckMinusZeroModeOf(node->op());
2162 : Node* lhs = node->InputAt(0);
2163 : Node* rhs = node->InputAt(1);
2164 :
2165 5906 : Node* projection = __ Int32MulWithOverflow(lhs, rhs);
2166 5906 : Node* check = __ Projection(1, projection);
2167 11812 : __ DeoptimizeIf(DeoptimizeReason::kOverflow, VectorSlotPair(), check,
2168 5906 : frame_state);
2169 :
2170 5906 : Node* value = __ Projection(0, projection);
2171 :
2172 5906 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2173 3936 : auto if_zero = __ MakeDeferredLabel();
2174 3936 : auto check_done = __ MakeLabel();
2175 3936 : Node* zero = __ Int32Constant(0);
2176 3936 : Node* check_zero = __ Word32Equal(value, zero);
2177 3936 : __ GotoIf(check_zero, &if_zero);
2178 : __ Goto(&check_done);
2179 :
2180 : __ Bind(&if_zero);
2181 : // We may need to return negative zero.
2182 3936 : Node* check_or = __ Int32LessThan(__ Word32Or(lhs, rhs), zero);
2183 7872 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, VectorSlotPair(), check_or,
2184 3936 : frame_state);
2185 : __ Goto(&check_done);
2186 :
2187 : __ Bind(&check_done);
2188 : }
2189 :
2190 5906 : return value;
2191 : }
2192 :
2193 0 : Node* EffectControlLinearizer::LowerCheckedInt32ToTaggedSigned(
2194 : Node* node, Node* frame_state) {
2195 : DCHECK(SmiValuesAre31Bits());
2196 : Node* value = node->InputAt(0);
2197 0 : const CheckParameters& params = CheckParametersOf(node->op());
2198 :
2199 0 : Node* add = __ Int32AddWithOverflow(value, value);
2200 0 : Node* check = __ Projection(1, add);
2201 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2202 0 : frame_state);
2203 0 : Node* result = __ Projection(0, add);
2204 : result = ChangeInt32ToIntPtr(result);
2205 0 : return result;
2206 : }
2207 :
2208 8 : Node* EffectControlLinearizer::LowerCheckedInt64ToInt32(Node* node,
2209 : Node* frame_state) {
2210 : Node* value = node->InputAt(0);
2211 8 : const CheckParameters& params = CheckParametersOf(node->op());
2212 :
2213 8 : Node* value32 = __ TruncateInt64ToInt32(value);
2214 8 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2215 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2216 8 : frame_state);
2217 8 : return value32;
2218 : }
2219 :
2220 151 : Node* EffectControlLinearizer::LowerCheckedInt64ToTaggedSigned(
2221 : Node* node, Node* frame_state) {
2222 : Node* value = node->InputAt(0);
2223 151 : const CheckParameters& params = CheckParametersOf(node->op());
2224 :
2225 151 : Node* value32 = __ TruncateInt64ToInt32(value);
2226 151 : Node* check = __ Word64Equal(__ ChangeInt32ToInt64(value32), value);
2227 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2228 151 : frame_state);
2229 :
2230 : if (SmiValuesAre32Bits()) {
2231 : return ChangeInt64ToSmi(value);
2232 : } else {
2233 : Node* add = __ Int32AddWithOverflow(value32, value32);
2234 : Node* check = __ Projection(1, add);
2235 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2236 : frame_state);
2237 : Node* result = __ Projection(0, add);
2238 : result = ChangeInt32ToIntPtr(result);
2239 : return result;
2240 : }
2241 : }
2242 :
2243 52339 : Node* EffectControlLinearizer::LowerCheckedUint32Bounds(Node* node,
2244 : Node* frame_state) {
2245 : Node* index = node->InputAt(0);
2246 : Node* limit = node->InputAt(1);
2247 52339 : const CheckBoundsParameters& params = CheckBoundsParametersOf(node->op());
2248 :
2249 52339 : Node* check = __ Uint32LessThan(index, limit);
2250 52339 : switch (params.mode()) {
2251 : case CheckBoundsParameters::kDeoptOnOutOfBounds:
2252 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds,
2253 : params.check_parameters().feedback(), check,
2254 47742 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2255 47742 : break;
2256 : case CheckBoundsParameters::kAbortOnOutOfBounds: {
2257 4597 : auto if_abort = __ MakeDeferredLabel();
2258 4597 : auto done = __ MakeLabel();
2259 :
2260 4597 : __ Branch(check, &done, &if_abort);
2261 :
2262 : __ Bind(&if_abort);
2263 4597 : __ Unreachable();
2264 : __ Goto(&done);
2265 :
2266 : __ Bind(&done);
2267 : break;
2268 : }
2269 : }
2270 :
2271 52339 : return index;
2272 : }
2273 :
2274 396 : Node* EffectControlLinearizer::LowerCheckedUint32ToInt32(Node* node,
2275 : Node* frame_state) {
2276 : Node* value = node->InputAt(0);
2277 396 : const CheckParameters& params = CheckParametersOf(node->op());
2278 396 : Node* unsafe = __ Int32LessThan(value, __ Int32Constant(0));
2279 : __ DeoptimizeIf(DeoptimizeReason::kLostPrecision, params.feedback(), unsafe,
2280 396 : frame_state);
2281 396 : return value;
2282 : }
2283 :
2284 7 : Node* EffectControlLinearizer::LowerCheckedUint32ToTaggedSigned(
2285 : Node* node, Node* frame_state) {
2286 : Node* value = node->InputAt(0);
2287 7 : const CheckParameters& params = CheckParametersOf(node->op());
2288 7 : Node* check = __ Uint32LessThanOrEqual(value, SmiMaxValueConstant());
2289 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2290 7 : frame_state);
2291 7 : return ChangeUint32ToSmi(value);
2292 : }
2293 :
2294 195 : Node* EffectControlLinearizer::LowerCheckedUint64Bounds(Node* node,
2295 : Node* frame_state) {
2296 195 : CheckParameters const& params = CheckParametersOf(node->op());
2297 : Node* const index = node->InputAt(0);
2298 : Node* const limit = node->InputAt(1);
2299 :
2300 195 : Node* check = __ Uint64LessThan(index, limit);
2301 : __ DeoptimizeIfNot(DeoptimizeReason::kOutOfBounds, params.feedback(), check,
2302 195 : frame_state, IsSafetyCheck::kCriticalSafetyCheck);
2303 195 : return index;
2304 : }
2305 :
2306 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToInt32(Node* node,
2307 : Node* frame_state) {
2308 : Node* value = node->InputAt(0);
2309 0 : const CheckParameters& params = CheckParametersOf(node->op());
2310 :
2311 0 : Node* check = __ Uint64LessThanOrEqual(value, __ Int64Constant(kMaxInt));
2312 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2313 0 : frame_state);
2314 0 : return __ TruncateInt64ToInt32(value);
2315 : }
2316 :
2317 0 : Node* EffectControlLinearizer::LowerCheckedUint64ToTaggedSigned(
2318 : Node* node, Node* frame_state) {
2319 : Node* value = node->InputAt(0);
2320 0 : const CheckParameters& params = CheckParametersOf(node->op());
2321 :
2322 : Node* check =
2323 0 : __ Uint64LessThanOrEqual(value, __ Int64Constant(Smi::kMaxValue));
2324 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecision, params.feedback(), check,
2325 0 : frame_state);
2326 0 : return ChangeInt64ToSmi(value);
2327 : }
2328 :
2329 4589 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt32(
2330 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2331 : Node* frame_state) {
2332 4589 : Node* value32 = __ RoundFloat64ToInt32(value);
2333 4589 : Node* check_same = __ Float64Equal(value, __ ChangeInt32ToFloat64(value32));
2334 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2335 4589 : check_same, frame_state);
2336 :
2337 4589 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2338 : // Check if {value} is -0.
2339 2156 : auto if_zero = __ MakeDeferredLabel();
2340 2156 : auto check_done = __ MakeLabel();
2341 :
2342 2156 : Node* check_zero = __ Word32Equal(value32, __ Int32Constant(0));
2343 2156 : __ GotoIf(check_zero, &if_zero);
2344 : __ Goto(&check_done);
2345 :
2346 : __ Bind(&if_zero);
2347 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2348 2156 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2349 2156 : __ Int32Constant(0));
2350 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2351 2156 : frame_state);
2352 : __ Goto(&check_done);
2353 :
2354 : __ Bind(&check_done);
2355 : }
2356 4589 : return value32;
2357 : }
2358 :
2359 2315 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt32(Node* node,
2360 : Node* frame_state) {
2361 : const CheckMinusZeroParameters& params =
2362 2315 : CheckMinusZeroParametersOf(node->op());
2363 : Node* value = node->InputAt(0);
2364 : return BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), value,
2365 2315 : frame_state);
2366 : }
2367 :
2368 103 : Node* EffectControlLinearizer::BuildCheckedFloat64ToInt64(
2369 : CheckForMinusZeroMode mode, const VectorSlotPair& feedback, Node* value,
2370 : Node* frame_state) {
2371 103 : Node* value64 = __ TruncateFloat64ToInt64(value);
2372 103 : Node* check_same = __ Float64Equal(value, __ ChangeInt64ToFloat64(value64));
2373 : __ DeoptimizeIfNot(DeoptimizeReason::kLostPrecisionOrNaN, feedback,
2374 103 : check_same, frame_state);
2375 :
2376 103 : if (mode == CheckForMinusZeroMode::kCheckForMinusZero) {
2377 : // Check if {value} is -0.
2378 0 : auto if_zero = __ MakeDeferredLabel();
2379 0 : auto check_done = __ MakeLabel();
2380 :
2381 0 : Node* check_zero = __ Word64Equal(value64, __ Int64Constant(0));
2382 0 : __ GotoIf(check_zero, &if_zero);
2383 : __ Goto(&check_done);
2384 :
2385 : __ Bind(&if_zero);
2386 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
2387 0 : Node* check_negative = __ Int32LessThan(__ Float64ExtractHighWord32(value),
2388 0 : __ Int32Constant(0));
2389 : __ DeoptimizeIf(DeoptimizeReason::kMinusZero, feedback, check_negative,
2390 0 : frame_state);
2391 : __ Goto(&check_done);
2392 :
2393 : __ Bind(&check_done);
2394 : }
2395 103 : return value64;
2396 : }
2397 :
2398 16 : Node* EffectControlLinearizer::LowerCheckedFloat64ToInt64(Node* node,
2399 : Node* frame_state) {
2400 : const CheckMinusZeroParameters& params =
2401 16 : CheckMinusZeroParametersOf(node->op());
2402 : Node* value = node->InputAt(0);
2403 : return BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), value,
2404 16 : frame_state);
2405 : }
2406 :
2407 24775 : Node* EffectControlLinearizer::LowerCheckedTaggedSignedToInt32(
2408 : Node* node, Node* frame_state) {
2409 : Node* value = node->InputAt(0);
2410 24775 : const CheckParameters& params = CheckParametersOf(node->op());
2411 24778 : Node* check = ObjectIsSmi(value);
2412 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2413 24778 : frame_state);
2414 24778 : return ChangeSmiToInt32(value);
2415 : }
2416 :
2417 2274 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt32(Node* node,
2418 : Node* frame_state) {
2419 : const CheckMinusZeroParameters& params =
2420 2274 : CheckMinusZeroParametersOf(node->op());
2421 : Node* value = node->InputAt(0);
2422 :
2423 2274 : auto if_not_smi = __ MakeDeferredLabel();
2424 4548 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2425 :
2426 2274 : Node* check = ObjectIsSmi(value);
2427 2274 : __ GotoIfNot(check, &if_not_smi);
2428 : // In the Smi case, just convert to int32.
2429 2274 : __ Goto(&done, ChangeSmiToInt32(value));
2430 :
2431 : // In the non-Smi case, check the heap numberness, load the number and convert
2432 : // to int32.
2433 : __ Bind(&if_not_smi);
2434 2274 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2435 2274 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2436 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2437 2274 : check_map, frame_state);
2438 2274 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2439 : vfalse = BuildCheckedFloat64ToInt32(params.mode(), params.feedback(), vfalse,
2440 2274 : frame_state);
2441 : __ Goto(&done, vfalse);
2442 :
2443 : __ Bind(&done);
2444 2274 : return done.PhiAt(0);
2445 : }
2446 :
2447 87 : Node* EffectControlLinearizer::LowerCheckedTaggedToInt64(Node* node,
2448 : Node* frame_state) {
2449 : const CheckMinusZeroParameters& params =
2450 87 : CheckMinusZeroParametersOf(node->op());
2451 : Node* value = node->InputAt(0);
2452 :
2453 87 : auto if_not_smi = __ MakeDeferredLabel();
2454 174 : auto done = __ MakeLabel(MachineRepresentation::kWord64);
2455 :
2456 87 : Node* check = ObjectIsSmi(value);
2457 87 : __ GotoIfNot(check, &if_not_smi);
2458 : // In the Smi case, just convert to int64.
2459 87 : __ Goto(&done, ChangeSmiToInt64(value));
2460 :
2461 : // In the non-Smi case, check the heap numberness, load the number and convert
2462 : // to int64.
2463 : __ Bind(&if_not_smi);
2464 87 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2465 87 : Node* check_map = __ WordEqual(value_map, __ HeapNumberMapConstant());
2466 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, params.feedback(),
2467 87 : check_map, frame_state);
2468 87 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2469 : vfalse = BuildCheckedFloat64ToInt64(params.mode(), params.feedback(), vfalse,
2470 87 : frame_state);
2471 : __ Goto(&done, vfalse);
2472 :
2473 : __ Bind(&done);
2474 87 : return done.PhiAt(0);
2475 : }
2476 :
2477 52257 : Node* EffectControlLinearizer::BuildCheckedHeapNumberOrOddballToFloat64(
2478 : CheckTaggedInputMode mode, const VectorSlotPair& feedback, Node* value,
2479 : Node* frame_state) {
2480 104513 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2481 52257 : Node* check_number = __ WordEqual(value_map, __ HeapNumberMapConstant());
2482 52257 : switch (mode) {
2483 : case CheckTaggedInputMode::kNumber: {
2484 : __ DeoptimizeIfNot(DeoptimizeReason::kNotAHeapNumber, feedback,
2485 8293 : check_number, frame_state);
2486 8293 : break;
2487 : }
2488 : case CheckTaggedInputMode::kNumberOrOddball: {
2489 43964 : auto check_done = __ MakeLabel();
2490 :
2491 43964 : __ GotoIf(check_number, &check_done);
2492 : // For oddballs also contain the numeric value, let us just check that
2493 : // we have an oddball here.
2494 : Node* instance_type =
2495 43964 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2496 : Node* check_oddball =
2497 43963 : __ Word32Equal(instance_type, __ Int32Constant(ODDBALL_TYPE));
2498 : __ DeoptimizeIfNot(DeoptimizeReason::kNotANumberOrOddball, feedback,
2499 43964 : check_oddball, frame_state);
2500 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
2501 : Oddball::kToNumberRawOffset);
2502 : __ Goto(&check_done);
2503 :
2504 : __ Bind(&check_done);
2505 : break;
2506 : }
2507 : }
2508 52257 : return __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2509 : }
2510 :
2511 51080 : Node* EffectControlLinearizer::LowerCheckedTaggedToFloat64(Node* node,
2512 : Node* frame_state) {
2513 : CheckTaggedInputParameters const& p =
2514 51080 : CheckTaggedInputParametersOf(node->op());
2515 : Node* value = node->InputAt(0);
2516 :
2517 51080 : auto if_smi = __ MakeLabel();
2518 102160 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
2519 :
2520 51080 : Node* check = ObjectIsSmi(value);
2521 51080 : __ GotoIf(check, &if_smi);
2522 :
2523 : // In the Smi case, just convert to int32 and then float64.
2524 : // Otherwise, check heap numberness and load the number.
2525 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2526 51080 : p.mode(), p.feedback(), value, frame_state);
2527 : __ Goto(&done, number);
2528 :
2529 : __ Bind(&if_smi);
2530 51080 : Node* from_smi = ChangeSmiToInt32(value);
2531 51080 : from_smi = __ ChangeInt32ToFloat64(from_smi);
2532 : __ Goto(&done, from_smi);
2533 :
2534 : __ Bind(&done);
2535 51080 : return done.PhiAt(0);
2536 : }
2537 :
2538 18916 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedSigned(
2539 : Node* node, Node* frame_state) {
2540 : Node* value = node->InputAt(0);
2541 18916 : const CheckParameters& params = CheckParametersOf(node->op());
2542 :
2543 18916 : Node* check = ObjectIsSmi(value);
2544 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2545 18916 : frame_state);
2546 :
2547 18916 : return value;
2548 : }
2549 :
2550 34653 : Node* EffectControlLinearizer::LowerCheckedTaggedToTaggedPointer(
2551 : Node* node, Node* frame_state) {
2552 : Node* value = node->InputAt(0);
2553 34653 : const CheckParameters& params = CheckParametersOf(node->op());
2554 :
2555 34653 : Node* check = ObjectIsSmi(value);
2556 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2557 34653 : frame_state);
2558 34653 : return value;
2559 : }
2560 :
2561 0 : Node* EffectControlLinearizer::LowerCheckedCompressedToTaggedSigned(
2562 : Node* node, Node* frame_state) {
2563 : Node* value = node->InputAt(0);
2564 0 : const CheckParameters& params = CheckParametersOf(node->op());
2565 :
2566 0 : Node* check = ObjectIsSmi(value);
2567 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2568 0 : frame_state);
2569 :
2570 0 : return __ ChangeCompressedSignedToTaggedSigned(value);
2571 : }
2572 :
2573 0 : Node* EffectControlLinearizer::LowerCheckedCompressedToTaggedPointer(
2574 : Node* node, Node* frame_state) {
2575 : Node* value = node->InputAt(0);
2576 0 : const CheckParameters& params = CheckParametersOf(node->op());
2577 :
2578 0 : Node* check = ObjectIsSmi(value);
2579 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2580 0 : frame_state);
2581 0 : return __ ChangeCompressedPointerToTaggedPointer(value);
2582 : }
2583 :
2584 0 : Node* EffectControlLinearizer::LowerCheckedTaggedToCompressedSigned(
2585 : Node* node, Node* frame_state) {
2586 : Node* value = node->InputAt(0);
2587 0 : const CheckParameters& params = CheckParametersOf(node->op());
2588 :
2589 0 : Node* check = ObjectIsSmi(value);
2590 : __ DeoptimizeIfNot(DeoptimizeReason::kNotASmi, params.feedback(), check,
2591 0 : frame_state);
2592 :
2593 0 : return __ ChangeTaggedSignedToCompressedSigned(value);
2594 : }
2595 :
2596 0 : Node* EffectControlLinearizer::LowerCheckedTaggedToCompressedPointer(
2597 : Node* node, Node* frame_state) {
2598 : Node* value = node->InputAt(0);
2599 0 : const CheckParameters& params = CheckParametersOf(node->op());
2600 :
2601 0 : Node* check = ObjectIsSmi(value);
2602 : __ DeoptimizeIf(DeoptimizeReason::kSmi, params.feedback(), check,
2603 0 : frame_state);
2604 0 : return __ ChangeTaggedPointerToCompressedPointer(value);
2605 : }
2606 :
2607 507 : Node* EffectControlLinearizer::LowerTruncateTaggedToWord32(Node* node) {
2608 : Node* value = node->InputAt(0);
2609 :
2610 507 : auto if_not_smi = __ MakeDeferredLabel();
2611 1014 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2612 :
2613 507 : Node* check = ObjectIsSmi(value);
2614 507 : __ GotoIfNot(check, &if_not_smi);
2615 507 : __ Goto(&done, ChangeSmiToInt32(value));
2616 :
2617 : __ Bind(&if_not_smi);
2618 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
2619 : Oddball::kToNumberRawOffset);
2620 507 : Node* vfalse = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2621 507 : vfalse = __ TruncateFloat64ToWord32(vfalse);
2622 : __ Goto(&done, vfalse);
2623 :
2624 : __ Bind(&done);
2625 507 : return done.PhiAt(0);
2626 : }
2627 :
2628 1177 : Node* EffectControlLinearizer::LowerCheckedTruncateTaggedToWord32(
2629 : Node* node, Node* frame_state) {
2630 : const CheckTaggedInputParameters& params =
2631 1177 : CheckTaggedInputParametersOf(node->op());
2632 : Node* value = node->InputAt(0);
2633 :
2634 1177 : auto if_not_smi = __ MakeLabel();
2635 2354 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
2636 :
2637 1177 : Node* check = ObjectIsSmi(value);
2638 1177 : __ GotoIfNot(check, &if_not_smi);
2639 : // In the Smi case, just convert to int32.
2640 1176 : __ Goto(&done, ChangeSmiToInt32(value));
2641 :
2642 : // Otherwise, check that it's a heap number or oddball and truncate the value
2643 : // to int32.
2644 : __ Bind(&if_not_smi);
2645 : Node* number = BuildCheckedHeapNumberOrOddballToFloat64(
2646 1177 : params.mode(), params.feedback(), value, frame_state);
2647 1177 : number = __ TruncateFloat64ToWord32(number);
2648 : __ Goto(&done, number);
2649 :
2650 : __ Bind(&done);
2651 1177 : return done.PhiAt(0);
2652 : }
2653 :
2654 112826 : Node* EffectControlLinearizer::LowerAllocate(Node* node) {
2655 : Node* size = node->InputAt(0);
2656 112826 : AllocationType allocation = AllocationTypeOf(node->op());
2657 112826 : Node* new_node = __ Allocate(allocation, size);
2658 112826 : return new_node;
2659 : }
2660 :
2661 968 : Node* EffectControlLinearizer::LowerNumberToString(Node* node) {
2662 : Node* argument = node->InputAt(0);
2663 :
2664 : Callable const callable =
2665 968 : Builtins::CallableFor(isolate(), Builtins::kNumberToString);
2666 : Operator::Properties properties = Operator::kEliminatable;
2667 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
2668 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2669 968 : graph()->zone(), callable.descriptor(),
2670 968 : callable.descriptor().GetStackParameterCount(), flags, properties);
2671 2904 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), argument,
2672 1936 : __ NoContextConstant());
2673 : }
2674 :
2675 16 : Node* EffectControlLinearizer::LowerObjectIsArrayBufferView(Node* node) {
2676 : Node* value = node->InputAt(0);
2677 :
2678 16 : auto if_smi = __ MakeDeferredLabel();
2679 32 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2680 :
2681 16 : Node* check = ObjectIsSmi(value);
2682 16 : __ GotoIf(check, &if_smi);
2683 :
2684 16 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2685 : Node* value_instance_type =
2686 16 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
2687 : STATIC_ASSERT(JS_TYPED_ARRAY_TYPE + 1 == JS_DATA_VIEW_TYPE);
2688 16 : Node* vfalse = __ Uint32LessThan(
2689 : __ Int32Sub(value_instance_type, __ Int32Constant(JS_TYPED_ARRAY_TYPE)),
2690 16 : __ Int32Constant(2));
2691 : __ Goto(&done, vfalse);
2692 :
2693 : __ Bind(&if_smi);
2694 16 : __ Goto(&done, __ Int32Constant(0));
2695 :
2696 : __ Bind(&done);
2697 16 : return done.PhiAt(0);
2698 : }
2699 :
2700 25 : Node* EffectControlLinearizer::LowerObjectIsBigInt(Node* node) {
2701 : Node* value = node->InputAt(0);
2702 :
2703 25 : auto if_smi = __ MakeDeferredLabel();
2704 50 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2705 :
2706 25 : Node* check = ObjectIsSmi(value);
2707 25 : __ GotoIf(check, &if_smi);
2708 25 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2709 25 : Node* vfalse = __ WordEqual(value_map, __ BigIntMapConstant());
2710 : __ Goto(&done, vfalse);
2711 :
2712 : __ Bind(&if_smi);
2713 25 : __ Goto(&done, __ Int32Constant(0));
2714 :
2715 : __ Bind(&done);
2716 25 : return done.PhiAt(0);
2717 : }
2718 :
2719 103 : Node* EffectControlLinearizer::LowerObjectIsCallable(Node* node) {
2720 : Node* value = node->InputAt(0);
2721 :
2722 103 : auto if_smi = __ MakeDeferredLabel();
2723 206 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2724 :
2725 103 : Node* check = ObjectIsSmi(value);
2726 103 : __ GotoIf(check, &if_smi);
2727 :
2728 103 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2729 : Node* value_bit_field =
2730 103 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2731 : Node* vfalse =
2732 103 : __ Word32Equal(__ Int32Constant(Map::IsCallableBit::kMask),
2733 : __ Word32And(value_bit_field,
2734 103 : __ Int32Constant(Map::IsCallableBit::kMask)));
2735 : __ Goto(&done, vfalse);
2736 :
2737 : __ Bind(&if_smi);
2738 103 : __ Goto(&done, __ Int32Constant(0));
2739 :
2740 : __ Bind(&done);
2741 103 : return done.PhiAt(0);
2742 : }
2743 :
2744 378 : Node* EffectControlLinearizer::LowerObjectIsConstructor(Node* node) {
2745 : Node* value = node->InputAt(0);
2746 :
2747 378 : auto if_smi = __ MakeDeferredLabel();
2748 756 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2749 :
2750 378 : Node* check = ObjectIsSmi(value);
2751 378 : __ GotoIf(check, &if_smi);
2752 :
2753 378 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2754 : Node* value_bit_field =
2755 378 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2756 378 : Node* vfalse = __ Word32Equal(
2757 : __ Int32Constant(Map::IsConstructorBit::kMask),
2758 : __ Word32And(value_bit_field,
2759 378 : __ Int32Constant(Map::IsConstructorBit::kMask)));
2760 : __ Goto(&done, vfalse);
2761 :
2762 : __ Bind(&if_smi);
2763 378 : __ Goto(&done, __ Int32Constant(0));
2764 :
2765 : __ Bind(&done);
2766 378 : return done.PhiAt(0);
2767 : }
2768 :
2769 14095 : Node* EffectControlLinearizer::LowerObjectIsDetectableCallable(Node* node) {
2770 : Node* value = node->InputAt(0);
2771 :
2772 14095 : auto if_smi = __ MakeDeferredLabel();
2773 28190 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2774 :
2775 14095 : Node* check = ObjectIsSmi(value);
2776 14095 : __ GotoIf(check, &if_smi);
2777 :
2778 14095 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2779 : Node* value_bit_field =
2780 14095 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
2781 14095 : Node* vfalse = __ Word32Equal(
2782 : __ Int32Constant(Map::IsCallableBit::kMask),
2783 : __ Word32And(value_bit_field,
2784 : __ Int32Constant((Map::IsCallableBit::kMask) |
2785 14095 : (Map::IsUndetectableBit::kMask))));
2786 : __ Goto(&done, vfalse);
2787 :
2788 : __ Bind(&if_smi);
2789 14095 : __ Goto(&done, __ Int32Constant(0));
2790 :
2791 : __ Bind(&done);
2792 14095 : return done.PhiAt(0);
2793 : }
2794 :
2795 61 : Node* EffectControlLinearizer::LowerNumberIsFloat64Hole(Node* node) {
2796 : Node* value = node->InputAt(0);
2797 61 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
2798 61 : __ Int32Constant(kHoleNanUpper32));
2799 61 : return check;
2800 : }
2801 :
2802 126 : Node* EffectControlLinearizer::LowerNumberIsFinite(Node* node) {
2803 : Node* number = node->InputAt(0);
2804 126 : Node* diff = __ Float64Sub(number, number);
2805 126 : Node* check = __ Float64Equal(diff, diff);
2806 126 : return check;
2807 : }
2808 :
2809 7 : Node* EffectControlLinearizer::LowerObjectIsFiniteNumber(Node* node) {
2810 : Node* object = node->InputAt(0);
2811 7 : Node* zero = __ Int32Constant(0);
2812 7 : Node* one = __ Int32Constant(1);
2813 :
2814 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2815 :
2816 : // Check if {object} is a Smi.
2817 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2818 :
2819 : // Check if {object} is a HeapNumber.
2820 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2821 7 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2822 7 : zero);
2823 :
2824 : // {object} is a HeapNumber.
2825 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2826 7 : Node* diff = __ Float64Sub(value, value);
2827 7 : Node* check = __ Float64Equal(diff, diff);
2828 : __ Goto(&done, check);
2829 :
2830 : __ Bind(&done);
2831 7 : return done.PhiAt(0);
2832 : }
2833 :
2834 119 : Node* EffectControlLinearizer::LowerNumberIsInteger(Node* node) {
2835 : Node* number = node->InputAt(0);
2836 119 : Node* trunc = BuildFloat64RoundTruncate(number);
2837 119 : Node* diff = __ Float64Sub(number, trunc);
2838 119 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2839 119 : return check;
2840 : }
2841 :
2842 7 : Node* EffectControlLinearizer::LowerObjectIsInteger(Node* node) {
2843 : Node* object = node->InputAt(0);
2844 7 : Node* zero = __ Int32Constant(0);
2845 7 : Node* one = __ Int32Constant(1);
2846 :
2847 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2848 :
2849 : // Check if {object} is a Smi.
2850 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2851 :
2852 : // Check if {object} is a HeapNumber.
2853 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2854 7 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2855 7 : zero);
2856 :
2857 : // {object} is a HeapNumber.
2858 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2859 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2860 7 : Node* diff = __ Float64Sub(value, trunc);
2861 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2862 : __ Goto(&done, check);
2863 :
2864 : __ Bind(&done);
2865 7 : return done.PhiAt(0);
2866 : }
2867 :
2868 7 : Node* EffectControlLinearizer::LowerNumberIsSafeInteger(Node* node) {
2869 : Node* number = node->InputAt(0);
2870 7 : Node* zero = __ Int32Constant(0);
2871 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2872 :
2873 7 : Node* trunc = BuildFloat64RoundTruncate(number);
2874 7 : Node* diff = __ Float64Sub(number, trunc);
2875 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2876 7 : __ GotoIfNot(check, &done, zero);
2877 7 : Node* in_range = __ Float64LessThanOrEqual(
2878 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2879 : __ Goto(&done, in_range);
2880 :
2881 : __ Bind(&done);
2882 7 : return done.PhiAt(0);
2883 : }
2884 :
2885 7 : Node* EffectControlLinearizer::LowerObjectIsSafeInteger(Node* node) {
2886 : Node* object = node->InputAt(0);
2887 7 : Node* zero = __ Int32Constant(0);
2888 7 : Node* one = __ Int32Constant(1);
2889 :
2890 14 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2891 :
2892 : // Check if {object} is a Smi.
2893 7 : __ GotoIf(ObjectIsSmi(object), &done, one);
2894 :
2895 : // Check if {object} is a HeapNumber.
2896 7 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), object);
2897 7 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2898 7 : zero);
2899 :
2900 : // {object} is a HeapNumber.
2901 7 : Node* value = __ LoadField(AccessBuilder::ForHeapNumberValue(), object);
2902 7 : Node* trunc = BuildFloat64RoundTruncate(value);
2903 7 : Node* diff = __ Float64Sub(value, trunc);
2904 7 : Node* check = __ Float64Equal(diff, __ Float64Constant(0));
2905 7 : __ GotoIfNot(check, &done, zero);
2906 7 : Node* in_range = __ Float64LessThanOrEqual(
2907 7 : __ Float64Abs(trunc), __ Float64Constant(kMaxSafeInteger));
2908 : __ Goto(&done, in_range);
2909 :
2910 : __ Bind(&done);
2911 7 : return done.PhiAt(0);
2912 : }
2913 :
2914 : namespace {
2915 :
2916 60998 : const int64_t kMinusZeroBits = bit_cast<int64_t>(-0.0);
2917 60998 : const int32_t kMinusZeroLoBits = static_cast<int32_t>(kMinusZeroBits);
2918 60998 : const int32_t kMinusZeroHiBits = static_cast<int32_t>(kMinusZeroBits >> 32);
2919 :
2920 : } // namespace
2921 :
2922 23 : Node* EffectControlLinearizer::LowerObjectIsMinusZero(Node* node) {
2923 : Node* value = node->InputAt(0);
2924 23 : Node* zero = __ Int32Constant(0);
2925 :
2926 46 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2927 :
2928 : // Check if {value} is a Smi.
2929 23 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2930 :
2931 : // Check if {value} is a HeapNumber.
2932 23 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2933 23 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2934 23 : zero);
2935 :
2936 : // Check if {value} contains -0.
2937 23 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2938 23 : if (machine()->Is64()) {
2939 23 : Node* value64 = __ BitcastFloat64ToInt64(value_value);
2940 23 : __ Goto(&done, __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits)));
2941 : } else {
2942 0 : Node* value_lo = __ Float64ExtractLowWord32(value_value);
2943 0 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2944 0 : &done, zero);
2945 0 : Node* value_hi = __ Float64ExtractHighWord32(value_value);
2946 0 : __ Goto(&done,
2947 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2948 : }
2949 :
2950 : __ Bind(&done);
2951 23 : return done.PhiAt(0);
2952 : }
2953 :
2954 98 : Node* EffectControlLinearizer::LowerNumberIsMinusZero(Node* node) {
2955 : Node* value = node->InputAt(0);
2956 :
2957 98 : if (machine()->Is64()) {
2958 98 : Node* value64 = __ BitcastFloat64ToInt64(value);
2959 98 : return __ Word64Equal(value64, __ Int64Constant(kMinusZeroBits));
2960 : } else {
2961 0 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2962 :
2963 0 : Node* value_lo = __ Float64ExtractLowWord32(value);
2964 0 : __ GotoIfNot(__ Word32Equal(value_lo, __ Int32Constant(kMinusZeroLoBits)),
2965 0 : &done, __ Int32Constant(0));
2966 0 : Node* value_hi = __ Float64ExtractHighWord32(value);
2967 0 : __ Goto(&done,
2968 : __ Word32Equal(value_hi, __ Int32Constant(kMinusZeroHiBits)));
2969 :
2970 : __ Bind(&done);
2971 : return done.PhiAt(0);
2972 : }
2973 : }
2974 :
2975 716 : Node* EffectControlLinearizer::LowerObjectIsNaN(Node* node) {
2976 : Node* value = node->InputAt(0);
2977 716 : Node* zero = __ Int32Constant(0);
2978 :
2979 1432 : auto done = __ MakeLabel(MachineRepresentation::kBit);
2980 :
2981 : // Check if {value} is a Smi.
2982 716 : __ GotoIf(ObjectIsSmi(value), &done, zero);
2983 :
2984 : // Check if {value} is a HeapNumber.
2985 716 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
2986 716 : __ GotoIfNot(__ WordEqual(value_map, __ HeapNumberMapConstant()), &done,
2987 716 : zero);
2988 :
2989 : // Check if {value} contains a NaN.
2990 716 : Node* value_value = __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
2991 716 : __ Goto(&done,
2992 : __ Word32Equal(__ Float64Equal(value_value, value_value), zero));
2993 :
2994 : __ Bind(&done);
2995 716 : return done.PhiAt(0);
2996 : }
2997 :
2998 2527 : Node* EffectControlLinearizer::LowerNumberIsNaN(Node* node) {
2999 : Node* number = node->InputAt(0);
3000 2527 : Node* diff = __ Float64Equal(number, number);
3001 2527 : Node* check = __ Word32Equal(diff, __ Int32Constant(0));
3002 2527 : return check;
3003 : }
3004 :
3005 6067 : Node* EffectControlLinearizer::LowerObjectIsNonCallable(Node* node) {
3006 : Node* value = node->InputAt(0);
3007 :
3008 6067 : auto if_primitive = __ MakeDeferredLabel();
3009 12134 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3010 :
3011 6067 : Node* check0 = ObjectIsSmi(value);
3012 6067 : __ GotoIf(check0, &if_primitive);
3013 :
3014 6067 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3015 : Node* value_instance_type =
3016 6067 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3017 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
3018 6067 : Node* check1 = __ Uint32LessThanOrEqual(
3019 6067 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
3020 6067 : __ GotoIfNot(check1, &if_primitive);
3021 :
3022 : Node* value_bit_field =
3023 6067 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
3024 : Node* check2 =
3025 6067 : __ Word32Equal(__ Int32Constant(0),
3026 : __ Word32And(value_bit_field,
3027 6067 : __ Int32Constant(Map::IsCallableBit::kMask)));
3028 : __ Goto(&done, check2);
3029 :
3030 : __ Bind(&if_primitive);
3031 6067 : __ Goto(&done, __ Int32Constant(0));
3032 :
3033 : __ Bind(&done);
3034 6067 : return done.PhiAt(0);
3035 : }
3036 :
3037 7594 : Node* EffectControlLinearizer::LowerObjectIsNumber(Node* node) {
3038 : Node* value = node->InputAt(0);
3039 :
3040 7594 : auto if_smi = __ MakeLabel();
3041 15188 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3042 :
3043 15188 : __ GotoIf(ObjectIsSmi(value), &if_smi);
3044 7594 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3045 7594 : __ Goto(&done, __ WordEqual(value_map, __ HeapNumberMapConstant()));
3046 :
3047 : __ Bind(&if_smi);
3048 7594 : __ Goto(&done, __ Int32Constant(1));
3049 :
3050 : __ Bind(&done);
3051 7594 : return done.PhiAt(0);
3052 : }
3053 :
3054 19519 : Node* EffectControlLinearizer::LowerObjectIsReceiver(Node* node) {
3055 : Node* value = node->InputAt(0);
3056 :
3057 19519 : auto if_smi = __ MakeDeferredLabel();
3058 39038 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3059 :
3060 39038 : __ GotoIf(ObjectIsSmi(value), &if_smi);
3061 :
3062 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
3063 19519 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3064 : Node* value_instance_type =
3065 19519 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3066 19519 : Node* result = __ Uint32LessThanOrEqual(
3067 19519 : __ Uint32Constant(FIRST_JS_RECEIVER_TYPE), value_instance_type);
3068 : __ Goto(&done, result);
3069 :
3070 : __ Bind(&if_smi);
3071 19519 : __ Goto(&done, __ Int32Constant(0));
3072 :
3073 : __ Bind(&done);
3074 19519 : return done.PhiAt(0);
3075 : }
3076 :
3077 4131 : Node* EffectControlLinearizer::LowerObjectIsSmi(Node* node) {
3078 : Node* value = node->InputAt(0);
3079 4131 : return ObjectIsSmi(value);
3080 : }
3081 :
3082 2221 : Node* EffectControlLinearizer::LowerObjectIsString(Node* node) {
3083 : Node* value = node->InputAt(0);
3084 :
3085 2221 : auto if_smi = __ MakeDeferredLabel();
3086 4442 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3087 :
3088 2221 : Node* check = ObjectIsSmi(value);
3089 2221 : __ GotoIf(check, &if_smi);
3090 2221 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3091 : Node* value_instance_type =
3092 2221 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3093 2221 : Node* vfalse = __ Uint32LessThan(value_instance_type,
3094 2221 : __ Uint32Constant(FIRST_NONSTRING_TYPE));
3095 : __ Goto(&done, vfalse);
3096 :
3097 : __ Bind(&if_smi);
3098 2221 : __ Goto(&done, __ Int32Constant(0));
3099 :
3100 : __ Bind(&done);
3101 2221 : return done.PhiAt(0);
3102 : }
3103 :
3104 23 : Node* EffectControlLinearizer::LowerObjectIsSymbol(Node* node) {
3105 : Node* value = node->InputAt(0);
3106 :
3107 23 : auto if_smi = __ MakeDeferredLabel();
3108 46 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3109 :
3110 23 : Node* check = ObjectIsSmi(value);
3111 23 : __ GotoIf(check, &if_smi);
3112 23 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3113 : Node* value_instance_type =
3114 23 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
3115 : Node* vfalse =
3116 23 : __ Word32Equal(value_instance_type, __ Uint32Constant(SYMBOL_TYPE));
3117 : __ Goto(&done, vfalse);
3118 :
3119 : __ Bind(&if_smi);
3120 23 : __ Goto(&done, __ Int32Constant(0));
3121 :
3122 : __ Bind(&done);
3123 23 : return done.PhiAt(0);
3124 : }
3125 :
3126 1426 : Node* EffectControlLinearizer::LowerObjectIsUndetectable(Node* node) {
3127 : Node* value = node->InputAt(0);
3128 :
3129 1426 : auto if_smi = __ MakeDeferredLabel();
3130 2852 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3131 :
3132 1426 : Node* check = ObjectIsSmi(value);
3133 1426 : __ GotoIf(check, &if_smi);
3134 :
3135 1426 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
3136 : Node* value_bit_field =
3137 1426 : __ LoadField(AccessBuilder::ForMapBitField(), value_map);
3138 1426 : Node* vfalse = __ Word32Equal(
3139 : __ Word32Equal(
3140 : __ Int32Constant(0),
3141 : __ Word32And(value_bit_field,
3142 : __ Int32Constant(Map::IsUndetectableBit::kMask))),
3143 1426 : __ Int32Constant(0));
3144 : __ Goto(&done, vfalse);
3145 :
3146 : __ Bind(&if_smi);
3147 1426 : __ Goto(&done, __ Int32Constant(0));
3148 :
3149 : __ Bind(&done);
3150 1426 : return done.PhiAt(0);
3151 : }
3152 :
3153 21328 : Node* EffectControlLinearizer::LowerTypeOf(Node* node) {
3154 : Node* obj = node->InputAt(0);
3155 21328 : Callable const callable = Builtins::CallableFor(isolate(), Builtins::kTypeof);
3156 : Operator::Properties const properties = Operator::kEliminatable;
3157 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
3158 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3159 21328 : graph()->zone(), callable.descriptor(),
3160 21328 : callable.descriptor().GetStackParameterCount(), flags, properties);
3161 63984 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
3162 42656 : __ NoContextConstant());
3163 : }
3164 :
3165 196 : Node* EffectControlLinearizer::LowerToBoolean(Node* node) {
3166 : Node* obj = node->InputAt(0);
3167 : Callable const callable =
3168 196 : Builtins::CallableFor(isolate(), Builtins::kToBoolean);
3169 : Operator::Properties const properties = Operator::kEliminatable;
3170 : CallDescriptor::Flags const flags = CallDescriptor::kNoAllocate;
3171 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3172 196 : graph()->zone(), callable.descriptor(),
3173 196 : callable.descriptor().GetStackParameterCount(), flags, properties);
3174 588 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), obj,
3175 392 : __ NoContextConstant());
3176 : }
3177 :
3178 16921 : Node* EffectControlLinearizer::LowerArgumentsLength(Node* node) {
3179 16921 : Node* arguments_frame = NodeProperties::GetValueInput(node, 0);
3180 16921 : int formal_parameter_count = FormalParameterCountOf(node->op());
3181 16921 : bool is_rest_length = IsRestLengthOf(node->op());
3182 : DCHECK_LE(0, formal_parameter_count);
3183 :
3184 16921 : if (is_rest_length) {
3185 : // The ArgumentsLength node is computing the number of rest parameters,
3186 : // which is max(0, actual_parameter_count - formal_parameter_count).
3187 : // We have to distinguish the case, when there is an arguments adaptor frame
3188 : // (i.e., arguments_frame != LoadFramePointer()).
3189 229 : auto if_adaptor_frame = __ MakeLabel();
3190 458 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
3191 :
3192 229 : Node* frame = __ LoadFramePointer();
3193 229 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done, __ SmiConstant(0));
3194 : __ Goto(&if_adaptor_frame);
3195 :
3196 : __ Bind(&if_adaptor_frame);
3197 229 : Node* arguments_length = __ Load(
3198 : MachineType::TaggedSigned(), arguments_frame,
3199 229 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
3200 :
3201 : Node* rest_length =
3202 229 : __ IntSub(arguments_length, __ SmiConstant(formal_parameter_count));
3203 229 : __ GotoIf(__ IntLessThan(rest_length, __ SmiConstant(0)), &done,
3204 229 : __ SmiConstant(0));
3205 : __ Goto(&done, rest_length);
3206 :
3207 : __ Bind(&done);
3208 : return done.PhiAt(0);
3209 : } else {
3210 : // The ArgumentsLength node is computing the actual number of arguments.
3211 : // We have to distinguish the case when there is an arguments adaptor frame
3212 : // (i.e., arguments_frame != LoadFramePointer()).
3213 16692 : auto if_adaptor_frame = __ MakeLabel();
3214 33384 : auto done = __ MakeLabel(MachineRepresentation::kTaggedSigned);
3215 :
3216 16692 : Node* frame = __ LoadFramePointer();
3217 16692 : __ GotoIf(__ WordEqual(arguments_frame, frame), &done,
3218 16692 : __ SmiConstant(formal_parameter_count));
3219 : __ Goto(&if_adaptor_frame);
3220 :
3221 : __ Bind(&if_adaptor_frame);
3222 16692 : Node* arguments_length = __ Load(
3223 : MachineType::TaggedSigned(), arguments_frame,
3224 16692 : __ IntPtrConstant(ArgumentsAdaptorFrameConstants::kLengthOffset));
3225 : __ Goto(&done, arguments_length);
3226 :
3227 : __ Bind(&done);
3228 : return done.PhiAt(0);
3229 : }
3230 : }
3231 :
3232 16894 : Node* EffectControlLinearizer::LowerArgumentsFrame(Node* node) {
3233 33788 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
3234 :
3235 16894 : Node* frame = __ LoadFramePointer();
3236 : Node* parent_frame =
3237 16894 : __ Load(MachineType::Pointer(), frame,
3238 16894 : __ IntPtrConstant(StandardFrameConstants::kCallerFPOffset));
3239 16894 : Node* parent_frame_type = __ Load(
3240 : MachineType::AnyTagged(), parent_frame,
3241 16894 : __ IntPtrConstant(CommonFrameConstants::kContextOrFrameTypeOffset));
3242 16894 : __ GotoIf(__ WordEqual(parent_frame_type,
3243 : __ IntPtrConstant(StackFrame::TypeToMarker(
3244 : StackFrame::ARGUMENTS_ADAPTOR))),
3245 16894 : &done, parent_frame);
3246 : __ Goto(&done, frame);
3247 :
3248 : __ Bind(&done);
3249 16894 : return done.PhiAt(0);
3250 : }
3251 :
3252 28 : Node* EffectControlLinearizer::LowerNewDoubleElements(Node* node) {
3253 28 : AllocationType const allocation = AllocationTypeOf(node->op());
3254 : Node* length = node->InputAt(0);
3255 :
3256 56 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3257 28 : Node* zero_length = __ WordEqual(length, __ IntPtrConstant(0));
3258 56 : __ GotoIf(zero_length, &done,
3259 28 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3260 :
3261 : // Compute the effective size of the backing store.
3262 28 : Node* size = __ IntAdd(__ WordShl(length, __ IntPtrConstant(kDoubleSizeLog2)),
3263 28 : __ IntPtrConstant(FixedDoubleArray::kHeaderSize));
3264 :
3265 : // Allocate the result and initialize the header.
3266 28 : Node* result = __ Allocate(allocation, size);
3267 56 : __ StoreField(AccessBuilder::ForMap(), result,
3268 28 : __ FixedDoubleArrayMapConstant());
3269 56 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3270 28 : ChangeIntPtrToSmi(length));
3271 :
3272 : // Initialize the backing store with holes.
3273 : STATIC_ASSERT_FIELD_OFFSETS_EQUAL(HeapNumber::kValueOffset,
3274 : Oddball::kToNumberRawOffset);
3275 : Node* the_hole =
3276 28 : __ LoadField(AccessBuilder::ForHeapNumberValue(), __ TheHoleConstant());
3277 56 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3278 28 : __ Goto(&loop, __ IntPtrConstant(0));
3279 : __ Bind(&loop);
3280 : {
3281 : // Check if we've initialized everything.
3282 : Node* index = loop.PhiAt(0);
3283 28 : Node* check = __ UintLessThan(index, length);
3284 28 : __ GotoIfNot(check, &done, result);
3285 :
3286 : ElementAccess const access = {kTaggedBase, FixedDoubleArray::kHeaderSize,
3287 : Type::NumberOrHole(), MachineType::Float64(),
3288 : kNoWriteBarrier};
3289 28 : __ StoreElement(access, result, index, the_hole);
3290 :
3291 : // Advance the {index}.
3292 28 : index = __ IntAdd(index, __ IntPtrConstant(1));
3293 : __ Goto(&loop, index);
3294 : }
3295 :
3296 : __ Bind(&done);
3297 28 : return done.PhiAt(0);
3298 : }
3299 :
3300 464 : Node* EffectControlLinearizer::LowerNewSmiOrObjectElements(Node* node) {
3301 464 : AllocationType const allocation = AllocationTypeOf(node->op());
3302 : Node* length = node->InputAt(0);
3303 :
3304 928 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3305 464 : Node* zero_length = __ WordEqual(length, __ IntPtrConstant(0));
3306 928 : __ GotoIf(zero_length, &done,
3307 464 : jsgraph()->HeapConstant(factory()->empty_fixed_array()));
3308 :
3309 : // Compute the effective size of the backing store.
3310 464 : Node* size = __ IntAdd(__ WordShl(length, __ IntPtrConstant(kTaggedSizeLog2)),
3311 464 : __ IntPtrConstant(FixedArray::kHeaderSize));
3312 :
3313 : // Allocate the result and initialize the header.
3314 464 : Node* result = __ Allocate(allocation, size);
3315 464 : __ StoreField(AccessBuilder::ForMap(), result, __ FixedArrayMapConstant());
3316 928 : __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
3317 464 : ChangeIntPtrToSmi(length));
3318 :
3319 : // Initialize the backing store with holes.
3320 464 : Node* the_hole = __ TheHoleConstant();
3321 928 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
3322 464 : __ Goto(&loop, __ IntPtrConstant(0));
3323 : __ Bind(&loop);
3324 : {
3325 : // Check if we've initialized everything.
3326 : Node* index = loop.PhiAt(0);
3327 464 : Node* check = __ UintLessThan(index, length);
3328 464 : __ GotoIfNot(check, &done, result);
3329 :
3330 : // Storing "the_hole" doesn't need a write barrier.
3331 : ElementAccess const access = {kTaggedBase, FixedArray::kHeaderSize,
3332 : Type::Any(), MachineType::AnyTagged(),
3333 : kNoWriteBarrier};
3334 464 : __ StoreElement(access, result, index, the_hole);
3335 :
3336 : // Advance the {index}.
3337 464 : index = __ IntAdd(index, __ IntPtrConstant(1));
3338 : __ Goto(&loop, index);
3339 : }
3340 :
3341 : __ Bind(&done);
3342 464 : return done.PhiAt(0);
3343 : }
3344 :
3345 16418 : Node* EffectControlLinearizer::LowerNewArgumentsElements(Node* node) {
3346 16418 : Node* frame = NodeProperties::GetValueInput(node, 0);
3347 16418 : Node* length = NodeProperties::GetValueInput(node, 1);
3348 16418 : int mapped_count = NewArgumentsElementsMappedCountOf(node->op());
3349 :
3350 : Callable const callable =
3351 16418 : Builtins::CallableFor(isolate(), Builtins::kNewArgumentsElements);
3352 16418 : Operator::Properties const properties = node->op()->properties();
3353 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
3354 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3355 16418 : graph()->zone(), callable.descriptor(),
3356 16418 : callable.descriptor().GetStackParameterCount(), flags, properties);
3357 49254 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), frame,
3358 32836 : length, __ SmiConstant(mapped_count), __ NoContextConstant());
3359 : }
3360 :
3361 3813 : Node* EffectControlLinearizer::LowerNewConsOneByteString(Node* node) {
3362 3813 : Node* map = jsgraph()->HeapConstant(factory()->cons_one_byte_string_map());
3363 : Node* length = node->InputAt(0);
3364 : Node* first = node->InputAt(1);
3365 : Node* second = node->InputAt(2);
3366 :
3367 3813 : return AllocateConsString(map, length, first, second);
3368 : }
3369 :
3370 35 : Node* EffectControlLinearizer::LowerNewConsTwoByteString(Node* node) {
3371 35 : Node* map = jsgraph()->HeapConstant(factory()->cons_string_map());
3372 : Node* length = node->InputAt(0);
3373 : Node* first = node->InputAt(1);
3374 : Node* second = node->InputAt(2);
3375 :
3376 35 : return AllocateConsString(map, length, first, second);
3377 : }
3378 :
3379 1080 : Node* EffectControlLinearizer::LowerNewConsString(Node* node) {
3380 : Node* length = node->InputAt(0);
3381 : Node* first = node->InputAt(1);
3382 : Node* second = node->InputAt(2);
3383 :
3384 : // Determine the instance types of {first} and {second}.
3385 2160 : Node* first_map = __ LoadField(AccessBuilder::ForMap(), first);
3386 : Node* first_instance_type =
3387 1080 : __ LoadField(AccessBuilder::ForMapInstanceType(), first_map);
3388 1080 : Node* second_map = __ LoadField(AccessBuilder::ForMap(), second);
3389 : Node* second_instance_type =
3390 1080 : __ LoadField(AccessBuilder::ForMapInstanceType(), second_map);
3391 :
3392 : // Determine the proper map for the resulting ConsString.
3393 : // If both {first} and {second} are one-byte strings, we
3394 : // create a new ConsOneByteString, otherwise we create a
3395 : // new ConsString instead.
3396 1080 : auto if_onebyte = __ MakeLabel();
3397 1080 : auto if_twobyte = __ MakeLabel();
3398 2160 : auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
3399 : STATIC_ASSERT(kOneByteStringTag != 0);
3400 : STATIC_ASSERT(kTwoByteStringTag == 0);
3401 1080 : Node* instance_type = __ Word32And(first_instance_type, second_instance_type);
3402 : Node* encoding =
3403 1080 : __ Word32And(instance_type, __ Int32Constant(kStringEncodingMask));
3404 1080 : __ Branch(__ Word32Equal(encoding, __ Int32Constant(kTwoByteStringTag)),
3405 1080 : &if_twobyte, &if_onebyte);
3406 : __ Bind(&if_onebyte);
3407 2160 : __ Goto(&done,
3408 : jsgraph()->HeapConstant(factory()->cons_one_byte_string_map()));
3409 : __ Bind(&if_twobyte);
3410 2160 : __ Goto(&done, jsgraph()->HeapConstant(factory()->cons_string_map()));
3411 : __ Bind(&done);
3412 : Node* result_map = done.PhiAt(0);
3413 :
3414 : // Allocate the resulting ConsString.
3415 2160 : return AllocateConsString(result_map, length, first, second);
3416 : }
3417 :
3418 4928 : Node* EffectControlLinearizer::AllocateConsString(Node* map, Node* length,
3419 : Node* first, Node* second) {
3420 : Node* result =
3421 4928 : __ Allocate(AllocationType::kYoung, __ IntPtrConstant(ConsString::kSize));
3422 4928 : __ StoreField(AccessBuilder::ForMap(), result, map);
3423 9856 : __ StoreField(AccessBuilder::ForNameHashField(), result,
3424 4928 : __ Int32Constant(Name::kEmptyHashField));
3425 4928 : __ StoreField(AccessBuilder::ForStringLength(), result, length);
3426 4928 : __ StoreField(AccessBuilder::ForConsStringFirst(), result, first);
3427 4928 : __ StoreField(AccessBuilder::ForConsStringSecond(), result, second);
3428 4928 : return result;
3429 : }
3430 :
3431 1 : Node* EffectControlLinearizer::LowerSameValue(Node* node) {
3432 : Node* lhs = node->InputAt(0);
3433 : Node* rhs = node->InputAt(1);
3434 :
3435 : Callable const callable =
3436 1 : Builtins::CallableFor(isolate(), Builtins::kSameValue);
3437 : Operator::Properties properties = Operator::kEliminatable;
3438 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3439 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3440 1 : graph()->zone(), callable.descriptor(),
3441 1 : callable.descriptor().GetStackParameterCount(), flags, properties);
3442 3 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3443 2 : __ NoContextConstant());
3444 : }
3445 :
3446 77 : Node* EffectControlLinearizer::LowerNumberSameValue(Node* node) {
3447 : Node* lhs = node->InputAt(0);
3448 : Node* rhs = node->InputAt(1);
3449 :
3450 77 : auto is_float64_equal = __ MakeLabel();
3451 154 : auto done = __ MakeLabel(MachineRepresentation::kBit);
3452 :
3453 77 : __ GotoIf(__ Float64Equal(lhs, rhs), &is_float64_equal);
3454 :
3455 : // Return true iff both {lhs} and {rhs} are NaN.
3456 77 : __ GotoIf(__ Float64Equal(lhs, lhs), &done, __ Int32Constant(0));
3457 77 : __ GotoIf(__ Float64Equal(rhs, rhs), &done, __ Int32Constant(0));
3458 77 : __ Goto(&done, __ Int32Constant(1));
3459 :
3460 : __ Bind(&is_float64_equal);
3461 : // Even if the values are float64-equal, we still need to distinguish
3462 : // zero and minus zero.
3463 77 : Node* lhs_hi = __ Float64ExtractHighWord32(lhs);
3464 77 : Node* rhs_hi = __ Float64ExtractHighWord32(rhs);
3465 77 : __ Goto(&done, __ Word32Equal(lhs_hi, rhs_hi));
3466 :
3467 : __ Bind(&done);
3468 77 : return done.PhiAt(0);
3469 : }
3470 :
3471 1356 : Node* EffectControlLinearizer::LowerDeadValue(Node* node) {
3472 1356 : Node* input = NodeProperties::GetValueInput(node, 0);
3473 1356 : if (input->opcode() != IrOpcode::kUnreachable) {
3474 1024 : Node* unreachable = __ Unreachable();
3475 1024 : NodeProperties::ReplaceValueInput(node, unreachable, 0);
3476 : }
3477 1356 : return node;
3478 : }
3479 :
3480 331 : Node* EffectControlLinearizer::LowerStringToNumber(Node* node) {
3481 : Node* string = node->InputAt(0);
3482 :
3483 : Callable const callable =
3484 331 : Builtins::CallableFor(isolate(), Builtins::kStringToNumber);
3485 : Operator::Properties properties = Operator::kEliminatable;
3486 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3487 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3488 331 : graph()->zone(), callable.descriptor(),
3489 331 : callable.descriptor().GetStackParameterCount(), flags, properties);
3490 993 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), string,
3491 662 : __ NoContextConstant());
3492 : }
3493 :
3494 2247 : Node* EffectControlLinearizer::LowerStringCharCodeAt(Node* node) {
3495 : Node* receiver = node->InputAt(0);
3496 : Node* position = node->InputAt(1);
3497 :
3498 : // We need a loop here to properly deal with indirect strings
3499 : // (SlicedString, ConsString and ThinString).
3500 : auto loop = __ MakeLoopLabel(MachineRepresentation::kTagged,
3501 4494 : MachineType::PointerRepresentation());
3502 : auto loop_next = __ MakeLabel(MachineRepresentation::kTagged,
3503 4494 : MachineType::PointerRepresentation());
3504 4494 : auto loop_done = __ MakeLabel(MachineRepresentation::kWord32);
3505 : __ Goto(&loop, receiver, position);
3506 : __ Bind(&loop);
3507 : {
3508 : Node* receiver = loop.PhiAt(0);
3509 : Node* position = loop.PhiAt(1);
3510 2247 : Node* receiver_map = __ LoadField(AccessBuilder::ForMap(), receiver);
3511 : Node* receiver_instance_type =
3512 2247 : __ LoadField(AccessBuilder::ForMapInstanceType(), receiver_map);
3513 2247 : Node* receiver_representation = __ Word32And(
3514 2247 : receiver_instance_type, __ Int32Constant(kStringRepresentationMask));
3515 :
3516 : // Dispatch on the current {receiver}s string representation.
3517 2247 : auto if_seqstring = __ MakeLabel();
3518 2247 : auto if_consstring = __ MakeLabel();
3519 2247 : auto if_thinstring = __ MakeLabel();
3520 2247 : auto if_externalstring = __ MakeLabel();
3521 2247 : auto if_slicedstring = __ MakeLabel();
3522 2247 : auto if_runtime = __ MakeDeferredLabel();
3523 2247 : __ GotoIf(__ Word32Equal(receiver_representation,
3524 : __ Int32Constant(kSeqStringTag)),
3525 2247 : &if_seqstring);
3526 2247 : __ GotoIf(__ Word32Equal(receiver_representation,
3527 : __ Int32Constant(kConsStringTag)),
3528 2247 : &if_consstring);
3529 2247 : __ GotoIf(__ Word32Equal(receiver_representation,
3530 : __ Int32Constant(kThinStringTag)),
3531 2247 : &if_thinstring);
3532 2247 : __ GotoIf(__ Word32Equal(receiver_representation,
3533 : __ Int32Constant(kExternalStringTag)),
3534 2247 : &if_externalstring);
3535 2247 : __ Branch(__ Word32Equal(receiver_representation,
3536 : __ Int32Constant(kSlicedStringTag)),
3537 2247 : &if_slicedstring, &if_runtime);
3538 :
3539 : __ Bind(&if_seqstring);
3540 : {
3541 2247 : Node* receiver_is_onebyte = __ Word32Equal(
3542 : __ Word32Equal(__ Word32And(receiver_instance_type,
3543 : __ Int32Constant(kStringEncodingMask)),
3544 : __ Int32Constant(kTwoByteStringTag)),
3545 2247 : __ Int32Constant(0));
3546 2247 : Node* result = LoadFromSeqString(receiver, position, receiver_is_onebyte);
3547 : __ Goto(&loop_done, result);
3548 : }
3549 :
3550 : __ Bind(&if_thinstring);
3551 : {
3552 : Node* receiver_actual =
3553 2247 : __ LoadField(AccessBuilder::ForThinStringActual(), receiver);
3554 : __ Goto(&loop_next, receiver_actual, position);
3555 : }
3556 :
3557 : __ Bind(&if_consstring);
3558 : {
3559 : Node* receiver_second =
3560 2247 : __ LoadField(AccessBuilder::ForConsStringSecond(), receiver);
3561 2247 : __ GotoIfNot(__ WordEqual(receiver_second, __ EmptyStringConstant()),
3562 2247 : &if_runtime);
3563 : Node* receiver_first =
3564 2247 : __ LoadField(AccessBuilder::ForConsStringFirst(), receiver);
3565 : __ Goto(&loop_next, receiver_first, position);
3566 : }
3567 :
3568 : __ Bind(&if_externalstring);
3569 : {
3570 : // We need to bailout to the runtime for uncached external strings.
3571 2247 : __ GotoIf(__ Word32Equal(
3572 : __ Word32And(receiver_instance_type,
3573 : __ Int32Constant(kUncachedExternalStringMask)),
3574 : __ Int32Constant(kUncachedExternalStringTag)),
3575 2247 : &if_runtime);
3576 :
3577 : Node* receiver_data = __ LoadField(
3578 2247 : AccessBuilder::ForExternalStringResourceData(), receiver);
3579 :
3580 2247 : auto if_onebyte = __ MakeLabel();
3581 2247 : auto if_twobyte = __ MakeLabel();
3582 2247 : __ Branch(
3583 : __ Word32Equal(__ Word32And(receiver_instance_type,
3584 : __ Int32Constant(kStringEncodingMask)),
3585 : __ Int32Constant(kTwoByteStringTag)),
3586 2247 : &if_twobyte, &if_onebyte);
3587 :
3588 : __ Bind(&if_onebyte);
3589 : {
3590 2247 : Node* result = __ Load(MachineType::Uint8(), receiver_data, position);
3591 : __ Goto(&loop_done, result);
3592 : }
3593 :
3594 : __ Bind(&if_twobyte);
3595 : {
3596 2247 : Node* result = __ Load(MachineType::Uint16(), receiver_data,
3597 2247 : __ WordShl(position, __ IntPtrConstant(1)));
3598 : __ Goto(&loop_done, result);
3599 : }
3600 : }
3601 :
3602 : __ Bind(&if_slicedstring);
3603 : {
3604 : Node* receiver_offset =
3605 2247 : __ LoadField(AccessBuilder::ForSlicedStringOffset(), receiver);
3606 : Node* receiver_parent =
3607 2247 : __ LoadField(AccessBuilder::ForSlicedStringParent(), receiver);
3608 2247 : __ Goto(&loop_next, receiver_parent,
3609 : __ IntAdd(position, ChangeSmiToIntPtr(receiver_offset)));
3610 : }
3611 :
3612 : __ Bind(&if_runtime);
3613 : {
3614 2247 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3615 : Runtime::FunctionId id = Runtime::kStringCharCodeAt;
3616 2247 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3617 2247 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
3618 2247 : Node* result = __ Call(call_descriptor, __ CEntryStubConstant(1),
3619 : receiver, ChangeIntPtrToSmi(position),
3620 : __ ExternalConstant(ExternalReference::Create(id)),
3621 2247 : __ Int32Constant(2), __ NoContextConstant());
3622 2247 : __ Goto(&loop_done, ChangeSmiToInt32(result));
3623 : }
3624 :
3625 : __ Bind(&loop_next);
3626 : __ Goto(&loop, loop_next.PhiAt(0), loop_next.PhiAt(1));
3627 : }
3628 : __ Bind(&loop_done);
3629 2247 : return loop_done.PhiAt(0);
3630 : }
3631 :
3632 223 : Node* EffectControlLinearizer::LowerStringCodePointAt(
3633 : Node* node, UnicodeEncoding encoding) {
3634 : Node* receiver = node->InputAt(0);
3635 : Node* position = node->InputAt(1);
3636 :
3637 : Builtins::Name builtin = encoding == UnicodeEncoding::UTF16
3638 : ? Builtins::kStringCodePointAtUTF16
3639 223 : : Builtins::kStringCodePointAtUTF32;
3640 :
3641 223 : Callable const callable = Builtins::CallableFor(isolate(), builtin);
3642 223 : Operator::Properties properties = Operator::kNoThrow | Operator::kNoWrite;
3643 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3644 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3645 223 : graph()->zone(), callable.descriptor(),
3646 223 : callable.descriptor().GetStackParameterCount(), flags, properties);
3647 669 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3648 446 : position, __ NoContextConstant());
3649 : }
3650 :
3651 2247 : Node* EffectControlLinearizer::LoadFromSeqString(Node* receiver, Node* position,
3652 : Node* is_one_byte) {
3653 2247 : auto one_byte_load = __ MakeLabel();
3654 4494 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
3655 2247 : __ GotoIf(is_one_byte, &one_byte_load);
3656 : Node* two_byte_result = __ LoadElement(
3657 2247 : AccessBuilder::ForSeqTwoByteStringCharacter(), receiver, position);
3658 : __ Goto(&done, two_byte_result);
3659 :
3660 : __ Bind(&one_byte_load);
3661 : Node* one_byte_element = __ LoadElement(
3662 2247 : AccessBuilder::ForSeqOneByteStringCharacter(), receiver, position);
3663 : __ Goto(&done, one_byte_element);
3664 :
3665 : __ Bind(&done);
3666 2247 : return done.PhiAt(0);
3667 : }
3668 :
3669 970 : Node* EffectControlLinearizer::LowerStringFromSingleCharCode(Node* node) {
3670 : Node* value = node->InputAt(0);
3671 970 : Node* code = __ Word32And(value, __ Uint32Constant(0xFFFF));
3672 :
3673 970 : auto if_not_one_byte = __ MakeDeferredLabel();
3674 970 : auto cache_miss = __ MakeDeferredLabel();
3675 1940 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3676 :
3677 : // Check if the {code} is a one byte character
3678 970 : Node* check1 = __ Uint32LessThanOrEqual(
3679 970 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3680 970 : __ GotoIfNot(check1, &if_not_one_byte);
3681 : {
3682 : // Load the isolate wide single character string cache.
3683 970 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3684 :
3685 : // Compute the {cache} index for {code}.
3686 970 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3687 :
3688 : // Check if we have an entry for the {code} in the single character string
3689 : // cache already.
3690 : Node* entry =
3691 970 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3692 :
3693 970 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3694 970 : __ GotoIf(check2, &cache_miss);
3695 :
3696 : // Use the {entry} from the {cache}.
3697 : __ Goto(&done, entry);
3698 :
3699 : __ Bind(&cache_miss);
3700 : {
3701 : // Allocate a new SeqOneByteString for {code}.
3702 : Node* vtrue2 =
3703 970 : __ Allocate(AllocationType::kYoung,
3704 970 : __ IntPtrConstant(SeqOneByteString::SizeFor(1)));
3705 1940 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3706 970 : __ HeapConstant(factory()->one_byte_string_map()));
3707 1940 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3708 970 : __ Int32Constant(Name::kEmptyHashField));
3709 1940 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3710 970 : __ Int32Constant(1));
3711 1940 : __ Store(
3712 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3713 : vtrue2,
3714 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3715 970 : code);
3716 :
3717 : // Remember it in the {cache}.
3718 1940 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3719 970 : vtrue2);
3720 : __ Goto(&done, vtrue2);
3721 : }
3722 : }
3723 :
3724 : __ Bind(&if_not_one_byte);
3725 : {
3726 : // Allocate a new SeqTwoByteString for {code}.
3727 : Node* vfalse1 =
3728 970 : __ Allocate(AllocationType::kYoung,
3729 970 : __ IntPtrConstant(SeqTwoByteString::SizeFor(1)));
3730 1940 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3731 970 : __ HeapConstant(factory()->string_map()));
3732 1940 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3733 970 : __ Int32Constant(Name::kEmptyHashField));
3734 1940 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3735 970 : __ Int32Constant(1));
3736 1940 : __ Store(
3737 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3738 : vfalse1,
3739 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3740 970 : code);
3741 : __ Goto(&done, vfalse1);
3742 : }
3743 :
3744 : __ Bind(&done);
3745 970 : return done.PhiAt(0);
3746 : }
3747 :
3748 : #ifdef V8_INTL_SUPPORT
3749 :
3750 103 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3751 : Node* receiver = node->InputAt(0);
3752 :
3753 : Callable callable =
3754 103 : Builtins::CallableFor(isolate(), Builtins::kStringToLowerCaseIntl);
3755 103 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3756 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3757 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3758 103 : graph()->zone(), callable.descriptor(),
3759 103 : callable.descriptor().GetStackParameterCount(), flags, properties);
3760 309 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3761 206 : __ NoContextConstant());
3762 : }
3763 :
3764 43 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3765 : Node* receiver = node->InputAt(0);
3766 43 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
3767 : Runtime::FunctionId id = Runtime::kStringToUpperCaseIntl;
3768 43 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3769 43 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
3770 43 : return __ Call(call_descriptor, __ CEntryStubConstant(1), receiver,
3771 : __ ExternalConstant(ExternalReference::Create(id)),
3772 43 : __ Int32Constant(1), __ NoContextConstant());
3773 : }
3774 :
3775 : #else
3776 :
3777 : Node* EffectControlLinearizer::LowerStringToLowerCaseIntl(Node* node) {
3778 : UNREACHABLE();
3779 : return nullptr;
3780 : }
3781 :
3782 : Node* EffectControlLinearizer::LowerStringToUpperCaseIntl(Node* node) {
3783 : UNREACHABLE();
3784 : return nullptr;
3785 : }
3786 :
3787 : #endif // V8_INTL_SUPPORT
3788 :
3789 207 : Node* EffectControlLinearizer::LowerStringFromSingleCodePoint(Node* node) {
3790 : Node* value = node->InputAt(0);
3791 : Node* code = value;
3792 :
3793 207 : auto if_not_single_code = __ MakeDeferredLabel();
3794 207 : auto if_not_one_byte = __ MakeDeferredLabel();
3795 207 : auto cache_miss = __ MakeDeferredLabel();
3796 414 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
3797 :
3798 : // Check if the {code} is a single code unit
3799 207 : Node* check0 = __ Uint32LessThanOrEqual(code, __ Uint32Constant(0xFFFF));
3800 207 : __ GotoIfNot(check0, &if_not_single_code);
3801 :
3802 : {
3803 : // Check if the {code} is a one byte character
3804 207 : Node* check1 = __ Uint32LessThanOrEqual(
3805 207 : code, __ Uint32Constant(String::kMaxOneByteCharCode));
3806 207 : __ GotoIfNot(check1, &if_not_one_byte);
3807 : {
3808 : // Load the isolate wide single character string cache.
3809 207 : Node* cache = __ HeapConstant(factory()->single_character_string_cache());
3810 :
3811 : // Compute the {cache} index for {code}.
3812 207 : Node* index = machine()->Is32() ? code : __ ChangeUint32ToUint64(code);
3813 :
3814 : // Check if we have an entry for the {code} in the single character string
3815 : // cache already.
3816 : Node* entry =
3817 207 : __ LoadElement(AccessBuilder::ForFixedArrayElement(), cache, index);
3818 :
3819 207 : Node* check2 = __ WordEqual(entry, __ UndefinedConstant());
3820 207 : __ GotoIf(check2, &cache_miss);
3821 :
3822 : // Use the {entry} from the {cache}.
3823 : __ Goto(&done, entry);
3824 :
3825 : __ Bind(&cache_miss);
3826 : {
3827 : // Allocate a new SeqOneByteString for {code}.
3828 : Node* vtrue2 =
3829 207 : __ Allocate(AllocationType::kYoung,
3830 207 : __ IntPtrConstant(SeqOneByteString::SizeFor(1)));
3831 414 : __ StoreField(AccessBuilder::ForMap(), vtrue2,
3832 207 : __ HeapConstant(factory()->one_byte_string_map()));
3833 414 : __ StoreField(AccessBuilder::ForNameHashField(), vtrue2,
3834 207 : __ Int32Constant(Name::kEmptyHashField));
3835 414 : __ StoreField(AccessBuilder::ForStringLength(), vtrue2,
3836 207 : __ Int32Constant(1));
3837 414 : __ Store(
3838 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier),
3839 : vtrue2,
3840 : __ IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag),
3841 207 : code);
3842 :
3843 : // Remember it in the {cache}.
3844 414 : __ StoreElement(AccessBuilder::ForFixedArrayElement(), cache, index,
3845 207 : vtrue2);
3846 : __ Goto(&done, vtrue2);
3847 : }
3848 : }
3849 :
3850 : __ Bind(&if_not_one_byte);
3851 : {
3852 : // Allocate a new SeqTwoByteString for {code}.
3853 : Node* vfalse1 =
3854 207 : __ Allocate(AllocationType::kYoung,
3855 207 : __ IntPtrConstant(SeqTwoByteString::SizeFor(1)));
3856 414 : __ StoreField(AccessBuilder::ForMap(), vfalse1,
3857 207 : __ HeapConstant(factory()->string_map()));
3858 414 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse1,
3859 207 : __ IntPtrConstant(Name::kEmptyHashField));
3860 414 : __ StoreField(AccessBuilder::ForStringLength(), vfalse1,
3861 207 : __ Int32Constant(1));
3862 414 : __ Store(
3863 : StoreRepresentation(MachineRepresentation::kWord16, kNoWriteBarrier),
3864 : vfalse1,
3865 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3866 207 : code);
3867 : __ Goto(&done, vfalse1);
3868 : }
3869 : }
3870 :
3871 : __ Bind(&if_not_single_code);
3872 : // Generate surrogate pair string
3873 : {
3874 207 : switch (UnicodeEncodingOf(node->op())) {
3875 : case UnicodeEncoding::UTF16:
3876 : break;
3877 :
3878 : case UnicodeEncoding::UTF32: {
3879 : // Convert UTF32 to UTF16 code units, and store as a 32 bit word.
3880 92 : Node* lead_offset = __ Int32Constant(0xD800 - (0x10000 >> 10));
3881 :
3882 : // lead = (codepoint >> 10) + LEAD_OFFSET
3883 : Node* lead =
3884 92 : __ Int32Add(__ Word32Shr(code, __ Int32Constant(10)), lead_offset);
3885 :
3886 : // trail = (codepoint & 0x3FF) + 0xDC00;
3887 92 : Node* trail = __ Int32Add(__ Word32And(code, __ Int32Constant(0x3FF)),
3888 92 : __ Int32Constant(0xDC00));
3889 :
3890 : // codpoint = (trail << 16) | lead;
3891 : #if V8_TARGET_BIG_ENDIAN
3892 : code = __ Word32Or(__ Word32Shl(lead, __ Int32Constant(16)), trail);
3893 : #else
3894 92 : code = __ Word32Or(__ Word32Shl(trail, __ Int32Constant(16)), lead);
3895 : #endif
3896 92 : break;
3897 : }
3898 : }
3899 :
3900 : // Allocate a new SeqTwoByteString for {code}.
3901 : Node* vfalse0 =
3902 207 : __ Allocate(AllocationType::kYoung,
3903 207 : __ IntPtrConstant(SeqTwoByteString::SizeFor(2)));
3904 414 : __ StoreField(AccessBuilder::ForMap(), vfalse0,
3905 207 : __ HeapConstant(factory()->string_map()));
3906 414 : __ StoreField(AccessBuilder::ForNameHashField(), vfalse0,
3907 207 : __ Int32Constant(Name::kEmptyHashField));
3908 414 : __ StoreField(AccessBuilder::ForStringLength(), vfalse0,
3909 207 : __ Int32Constant(2));
3910 414 : __ Store(
3911 : StoreRepresentation(MachineRepresentation::kWord32, kNoWriteBarrier),
3912 : vfalse0,
3913 : __ IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag),
3914 207 : code);
3915 : __ Goto(&done, vfalse0);
3916 : }
3917 :
3918 : __ Bind(&done);
3919 207 : return done.PhiAt(0);
3920 : }
3921 :
3922 258 : Node* EffectControlLinearizer::LowerStringIndexOf(Node* node) {
3923 : Node* subject = node->InputAt(0);
3924 : Node* search_string = node->InputAt(1);
3925 : Node* position = node->InputAt(2);
3926 :
3927 : Callable callable =
3928 258 : Builtins::CallableFor(isolate(), Builtins::kStringIndexOf);
3929 : Operator::Properties properties = Operator::kEliminatable;
3930 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3931 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3932 258 : graph()->zone(), callable.descriptor(),
3933 258 : callable.descriptor().GetStackParameterCount(), flags, properties);
3934 774 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), subject,
3935 516 : search_string, position, __ NoContextConstant());
3936 : }
3937 :
3938 23424 : Node* EffectControlLinearizer::LowerStringLength(Node* node) {
3939 : Node* subject = node->InputAt(0);
3940 :
3941 46848 : return __ LoadField(AccessBuilder::ForStringLength(), subject);
3942 : }
3943 :
3944 11207 : Node* EffectControlLinearizer::LowerStringComparison(Callable const& callable,
3945 : Node* node) {
3946 : Node* lhs = node->InputAt(0);
3947 : Node* rhs = node->InputAt(1);
3948 :
3949 : Operator::Properties properties = Operator::kEliminatable;
3950 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3951 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3952 11207 : graph()->zone(), callable.descriptor(),
3953 11207 : callable.descriptor().GetStackParameterCount(), flags, properties);
3954 33621 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), lhs, rhs,
3955 22414 : __ NoContextConstant());
3956 : }
3957 :
3958 1080 : Node* EffectControlLinearizer::LowerStringSubstring(Node* node) {
3959 : Node* receiver = node->InputAt(0);
3960 : Node* start = ChangeInt32ToIntPtr(node->InputAt(1));
3961 : Node* end = ChangeInt32ToIntPtr(node->InputAt(2));
3962 :
3963 : Callable callable =
3964 1080 : Builtins::CallableFor(isolate(), Builtins::kStringSubstring);
3965 : Operator::Properties properties = Operator::kEliminatable;
3966 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
3967 : auto call_descriptor = Linkage::GetStubCallDescriptor(
3968 1080 : graph()->zone(), callable.descriptor(),
3969 1080 : callable.descriptor().GetStackParameterCount(), flags, properties);
3970 3240 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), receiver,
3971 2160 : start, end, __ NoContextConstant());
3972 : }
3973 :
3974 10379 : Node* EffectControlLinearizer::LowerStringEqual(Node* node) {
3975 : return LowerStringComparison(
3976 20758 : Builtins::CallableFor(isolate(), Builtins::kStringEqual), node);
3977 : }
3978 :
3979 121 : Node* EffectControlLinearizer::LowerStringLessThan(Node* node) {
3980 : return LowerStringComparison(
3981 242 : Builtins::CallableFor(isolate(), Builtins::kStringLessThan), node);
3982 : }
3983 :
3984 707 : Node* EffectControlLinearizer::LowerStringLessThanOrEqual(Node* node) {
3985 : return LowerStringComparison(
3986 1414 : Builtins::CallableFor(isolate(), Builtins::kStringLessThanOrEqual), node);
3987 : }
3988 :
3989 470 : Node* EffectControlLinearizer::LowerCheckFloat64Hole(Node* node,
3990 : Node* frame_state) {
3991 : // If we reach this point w/o eliminating the {node} that's marked
3992 : // with allow-return-hole, we cannot do anything, so just deoptimize
3993 : // in case of the hole NaN.
3994 : CheckFloat64HoleParameters const& params =
3995 470 : CheckFloat64HoleParametersOf(node->op());
3996 : Node* value = node->InputAt(0);
3997 :
3998 470 : auto if_nan = __ MakeDeferredLabel();
3999 470 : auto done = __ MakeLabel();
4000 :
4001 : // First check whether {value} is a NaN at all...
4002 470 : __ Branch(__ Float64Equal(value, value), &done, &if_nan);
4003 :
4004 : __ Bind(&if_nan);
4005 : {
4006 : // ...and only if {value} is a NaN, perform the expensive bit
4007 : // check. See http://crbug.com/v8/8264 for details.
4008 470 : Node* check = __ Word32Equal(__ Float64ExtractHighWord32(value),
4009 470 : __ Int32Constant(kHoleNanUpper32));
4010 : __ DeoptimizeIf(DeoptimizeReason::kHole, params.feedback(), check,
4011 470 : frame_state);
4012 : __ Goto(&done);
4013 : }
4014 :
4015 : __ Bind(&done);
4016 470 : return value;
4017 : }
4018 :
4019 104 : Node* EffectControlLinearizer::LowerCheckNotTaggedHole(Node* node,
4020 : Node* frame_state) {
4021 : Node* value = node->InputAt(0);
4022 104 : Node* check = __ WordEqual(value, __ TheHoleConstant());
4023 208 : __ DeoptimizeIf(DeoptimizeReason::kHole, VectorSlotPair(), check,
4024 104 : frame_state);
4025 104 : return value;
4026 : }
4027 :
4028 1667 : Node* EffectControlLinearizer::LowerConvertTaggedHoleToUndefined(Node* node) {
4029 : Node* value = node->InputAt(0);
4030 :
4031 1667 : auto if_is_hole = __ MakeDeferredLabel();
4032 3334 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4033 :
4034 1667 : Node* check = __ WordEqual(value, __ TheHoleConstant());
4035 1667 : __ GotoIf(check, &if_is_hole);
4036 : __ Goto(&done, value);
4037 :
4038 : __ Bind(&if_is_hole);
4039 1667 : __ Goto(&done, __ UndefinedConstant());
4040 :
4041 : __ Bind(&done);
4042 1667 : return done.PhiAt(0);
4043 : }
4044 :
4045 109 : void EffectControlLinearizer::LowerCheckEqualsInternalizedString(
4046 : Node* node, Node* frame_state) {
4047 : Node* exp = node->InputAt(0);
4048 : Node* val = node->InputAt(1);
4049 :
4050 109 : auto if_same = __ MakeLabel();
4051 109 : auto if_notsame = __ MakeDeferredLabel();
4052 109 : auto if_thinstring = __ MakeLabel();
4053 109 : auto if_notthinstring = __ MakeLabel();
4054 :
4055 : // Check if {exp} and {val} are the same, which is the likely case.
4056 109 : __ Branch(__ WordEqual(exp, val), &if_same, &if_notsame);
4057 :
4058 : __ Bind(&if_notsame);
4059 : {
4060 : // Now {val} could still be a non-internalized String that matches {exp}.
4061 218 : __ DeoptimizeIf(DeoptimizeReason::kWrongName, VectorSlotPair(),
4062 109 : ObjectIsSmi(val), frame_state);
4063 109 : Node* val_map = __ LoadField(AccessBuilder::ForMap(), val);
4064 : Node* val_instance_type =
4065 109 : __ LoadField(AccessBuilder::ForMapInstanceType(), val_map);
4066 :
4067 : // Check for the common case of ThinString first.
4068 109 : __ GotoIf(__ Word32Equal(val_instance_type,
4069 : __ Int32Constant(THIN_ONE_BYTE_STRING_TYPE)),
4070 109 : &if_thinstring);
4071 109 : __ Branch(
4072 : __ Word32Equal(val_instance_type, __ Int32Constant(THIN_STRING_TYPE)),
4073 109 : &if_thinstring, &if_notthinstring);
4074 :
4075 : __ Bind(&if_notthinstring);
4076 : {
4077 : // Check that the {val} is a non-internalized String, if it's anything
4078 : // else it cannot match the recorded feedback {exp} anyways.
4079 218 : __ DeoptimizeIfNot(
4080 : DeoptimizeReason::kWrongName, VectorSlotPair(),
4081 : __ Word32Equal(__ Word32And(val_instance_type,
4082 : __ Int32Constant(kIsNotStringMask |
4083 : kIsNotInternalizedMask)),
4084 : __ Int32Constant(kStringTag | kNotInternalizedTag)),
4085 109 : frame_state);
4086 :
4087 : // Try to find the {val} in the string table.
4088 : MachineSignature::Builder builder(graph()->zone(), 1, 2);
4089 : builder.AddReturn(MachineType::AnyTagged());
4090 : builder.AddParam(MachineType::Pointer());
4091 : builder.AddParam(MachineType::AnyTagged());
4092 109 : Node* try_internalize_string_function = __ ExternalConstant(
4093 109 : ExternalReference::try_internalize_string_function());
4094 : Node* const isolate_ptr =
4095 109 : __ ExternalConstant(ExternalReference::isolate_address(isolate()));
4096 : auto call_descriptor =
4097 109 : Linkage::GetSimplifiedCDescriptor(graph()->zone(), builder.Build());
4098 : Node* val_internalized =
4099 109 : __ Call(common()->Call(call_descriptor),
4100 109 : try_internalize_string_function, isolate_ptr, val);
4101 :
4102 : // Now see if the results match.
4103 218 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
4104 109 : __ WordEqual(exp, val_internalized), frame_state);
4105 : __ Goto(&if_same);
4106 : }
4107 :
4108 : __ Bind(&if_thinstring);
4109 : {
4110 : // The {val} is a ThinString, let's check the actual value.
4111 : Node* val_actual =
4112 109 : __ LoadField(AccessBuilder::ForThinStringActual(), val);
4113 218 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(),
4114 109 : __ WordEqual(exp, val_actual), frame_state);
4115 : __ Goto(&if_same);
4116 : }
4117 : }
4118 :
4119 : __ Bind(&if_same);
4120 109 : }
4121 :
4122 18 : void EffectControlLinearizer::LowerCheckEqualsSymbol(Node* node,
4123 : Node* frame_state) {
4124 : Node* exp = node->InputAt(0);
4125 : Node* val = node->InputAt(1);
4126 18 : Node* check = __ WordEqual(exp, val);
4127 36 : __ DeoptimizeIfNot(DeoptimizeReason::kWrongName, VectorSlotPair(), check,
4128 18 : frame_state);
4129 18 : }
4130 :
4131 54808 : Node* EffectControlLinearizer::AllocateHeapNumberWithValue(Node* value) {
4132 : Node* result =
4133 54808 : __ Allocate(AllocationType::kYoung, __ IntPtrConstant(HeapNumber::kSize));
4134 54810 : __ StoreField(AccessBuilder::ForMap(), result, __ HeapNumberMapConstant());
4135 54812 : __ StoreField(AccessBuilder::ForHeapNumberValue(), result, value);
4136 54812 : return result;
4137 : }
4138 :
4139 135241 : Node* EffectControlLinearizer::ChangeIntPtrToSmi(Node* value) {
4140 : // Do shift on 32bit values if Smis are stored in the lower word.
4141 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4142 : return __ ChangeInt32ToInt64(
4143 : __ Word32Shl(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
4144 : }
4145 135241 : return __ WordShl(value, SmiShiftBitsConstant());
4146 : }
4147 :
4148 0 : Node* EffectControlLinearizer::ChangeInt32ToIntPtr(Node* value) {
4149 134409 : if (machine()->Is64()) {
4150 134410 : value = __ ChangeInt32ToInt64(value);
4151 : }
4152 0 : return value;
4153 : }
4154 :
4155 0 : Node* EffectControlLinearizer::ChangeIntPtrToInt32(Node* value) {
4156 0 : if (machine()->Is64()) {
4157 0 : value = __ TruncateInt64ToInt32(value);
4158 : }
4159 0 : return value;
4160 : }
4161 :
4162 132249 : Node* EffectControlLinearizer::ChangeInt32ToSmi(Node* value) {
4163 : // Do shift on 32bit values if Smis are stored in the lower word.
4164 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4165 : return __ ChangeInt32ToInt64(__ Word32Shl(value, SmiShiftBitsConstant()));
4166 : }
4167 132252 : return ChangeIntPtrToSmi(ChangeInt32ToIntPtr(value));
4168 : }
4169 :
4170 0 : Node* EffectControlLinearizer::ChangeInt64ToSmi(Node* value) {
4171 : DCHECK(machine()->Is64());
4172 249 : return ChangeIntPtrToSmi(value);
4173 : }
4174 :
4175 0 : Node* EffectControlLinearizer::ChangeUint32ToUintPtr(Node* value) {
4176 676 : if (machine()->Is64()) {
4177 676 : value = __ ChangeUint32ToUint64(value);
4178 : }
4179 0 : return value;
4180 : }
4181 :
4182 660 : Node* EffectControlLinearizer::ChangeUint32ToSmi(Node* value) {
4183 : // Do shift on 32bit values if Smis are stored in the lower word.
4184 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4185 : return __ ChangeUint32ToUint64(__ Word32Shl(value, SmiShiftBitsConstant()));
4186 : } else {
4187 660 : return __ WordShl(ChangeUint32ToUintPtr(value), SmiShiftBitsConstant());
4188 : }
4189 : }
4190 :
4191 148908 : Node* EffectControlLinearizer::ChangeSmiToIntPtr(Node* value) {
4192 : // Do shift on 32bit values if Smis are stored in the lower word.
4193 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4194 : return __ ChangeInt32ToInt64(
4195 : __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant()));
4196 : }
4197 148908 : return __ WordSar(value, SmiShiftBitsConstant());
4198 : }
4199 :
4200 146227 : Node* EffectControlLinearizer::ChangeSmiToInt32(Node* value) {
4201 : // Do shift on 32bit values if Smis are stored in the lower word.
4202 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4203 : return __ Word32Sar(__ TruncateInt64ToInt32(value), SmiShiftBitsConstant());
4204 : }
4205 146227 : if (machine()->Is64()) {
4206 292454 : return __ TruncateInt64ToInt32(ChangeSmiToIntPtr(value));
4207 : }
4208 0 : return ChangeSmiToIntPtr(value);
4209 : }
4210 :
4211 386 : Node* EffectControlLinearizer::ChangeSmiToInt64(Node* value) {
4212 386 : CHECK(machine()->Is64());
4213 386 : return ChangeSmiToIntPtr(value);
4214 : }
4215 :
4216 256818 : Node* EffectControlLinearizer::ObjectIsSmi(Node* value) {
4217 256818 : return __ WordEqual(__ WordAnd(value, __ IntPtrConstant(kSmiTagMask)),
4218 256818 : __ IntPtrConstant(kSmiTag));
4219 : }
4220 :
4221 0 : Node* EffectControlLinearizer::SmiMaxValueConstant() {
4222 660 : return __ Int32Constant(Smi::kMaxValue);
4223 : }
4224 :
4225 0 : Node* EffectControlLinearizer::SmiShiftBitsConstant() {
4226 : if (machine()->Is64() && SmiValuesAre31Bits()) {
4227 : return __ Int32Constant(kSmiShiftSize + kSmiTagSize);
4228 : }
4229 284809 : return __ IntPtrConstant(kSmiShiftSize + kSmiTagSize);
4230 : }
4231 :
4232 20 : Node* EffectControlLinearizer::LowerPlainPrimitiveToNumber(Node* node) {
4233 : Node* value = node->InputAt(0);
4234 20 : return __ ToNumber(value);
4235 : }
4236 :
4237 9 : Node* EffectControlLinearizer::LowerPlainPrimitiveToWord32(Node* node) {
4238 : Node* value = node->InputAt(0);
4239 :
4240 9 : auto if_not_smi = __ MakeDeferredLabel();
4241 9 : auto if_to_number_smi = __ MakeLabel();
4242 18 : auto done = __ MakeLabel(MachineRepresentation::kWord32);
4243 :
4244 9 : Node* check0 = ObjectIsSmi(value);
4245 9 : __ GotoIfNot(check0, &if_not_smi);
4246 9 : __ Goto(&done, ChangeSmiToInt32(value));
4247 :
4248 : __ Bind(&if_not_smi);
4249 9 : Node* to_number = __ ToNumber(value);
4250 :
4251 9 : Node* check1 = ObjectIsSmi(to_number);
4252 9 : __ GotoIf(check1, &if_to_number_smi);
4253 9 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
4254 9 : __ Goto(&done, __ TruncateFloat64ToWord32(number));
4255 :
4256 : __ Bind(&if_to_number_smi);
4257 9 : __ Goto(&done, ChangeSmiToInt32(to_number));
4258 :
4259 : __ Bind(&done);
4260 9 : return done.PhiAt(0);
4261 : }
4262 :
4263 0 : Node* EffectControlLinearizer::LowerPlainPrimitiveToFloat64(Node* node) {
4264 : Node* value = node->InputAt(0);
4265 :
4266 0 : auto if_not_smi = __ MakeDeferredLabel();
4267 0 : auto if_to_number_smi = __ MakeLabel();
4268 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
4269 :
4270 0 : Node* check0 = ObjectIsSmi(value);
4271 0 : __ GotoIfNot(check0, &if_not_smi);
4272 0 : Node* from_smi = ChangeSmiToInt32(value);
4273 0 : __ Goto(&done, __ ChangeInt32ToFloat64(from_smi));
4274 :
4275 : __ Bind(&if_not_smi);
4276 0 : Node* to_number = __ ToNumber(value);
4277 0 : Node* check1 = ObjectIsSmi(to_number);
4278 0 : __ GotoIf(check1, &if_to_number_smi);
4279 :
4280 0 : Node* number = __ LoadField(AccessBuilder::ForHeapNumberValue(), to_number);
4281 : __ Goto(&done, number);
4282 :
4283 : __ Bind(&if_to_number_smi);
4284 0 : Node* number_from_smi = ChangeSmiToInt32(to_number);
4285 0 : number_from_smi = __ ChangeInt32ToFloat64(number_from_smi);
4286 : __ Goto(&done, number_from_smi);
4287 :
4288 : __ Bind(&done);
4289 0 : return done.PhiAt(0);
4290 : }
4291 :
4292 1669 : Node* EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node) {
4293 : Node* object = node->InputAt(0);
4294 : Node* elements = node->InputAt(1);
4295 :
4296 1669 : auto if_not_fixed_array = __ MakeDeferredLabel();
4297 3338 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4298 :
4299 : // Load the current map of {elements}.
4300 3338 : Node* elements_map = __ LoadField(AccessBuilder::ForMap(), elements);
4301 :
4302 : // Check if {elements} is not a copy-on-write FixedArray.
4303 1669 : Node* check = __ WordEqual(elements_map, __ FixedArrayMapConstant());
4304 1669 : __ GotoIfNot(check, &if_not_fixed_array);
4305 : // Nothing to do if the {elements} are not copy-on-write.
4306 : __ Goto(&done, elements);
4307 :
4308 : __ Bind(&if_not_fixed_array);
4309 : // We need to take a copy of the {elements} and set them up for {object}.
4310 : Operator::Properties properties = Operator::kEliminatable;
4311 : Callable callable =
4312 1669 : Builtins::CallableFor(isolate(), Builtins::kCopyFastSmiOrObjectElements);
4313 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
4314 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4315 1669 : graph()->zone(), callable.descriptor(),
4316 1669 : callable.descriptor().GetStackParameterCount(), flags, properties);
4317 5007 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
4318 1669 : object, __ NoContextConstant());
4319 : __ Goto(&done, result);
4320 :
4321 : __ Bind(&done);
4322 1669 : return done.PhiAt(0);
4323 : }
4324 :
4325 3485 : Node* EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node,
4326 : Node* frame_state) {
4327 3485 : GrowFastElementsParameters params = GrowFastElementsParametersOf(node->op());
4328 : Node* object = node->InputAt(0);
4329 : Node* elements = node->InputAt(1);
4330 : Node* index = node->InputAt(2);
4331 : Node* elements_length = node->InputAt(3);
4332 :
4333 6970 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4334 3485 : auto if_grow = __ MakeDeferredLabel();
4335 : auto if_not_grow = __ MakeLabel();
4336 :
4337 : // Check if we need to grow the {elements} backing store.
4338 3485 : Node* check = __ Uint32LessThan(index, elements_length);
4339 3485 : __ GotoIfNot(check, &if_grow);
4340 : __ Goto(&done, elements);
4341 :
4342 : __ Bind(&if_grow);
4343 : // We need to grow the {elements} for {object}.
4344 : Operator::Properties properties = Operator::kEliminatable;
4345 : Callable callable =
4346 : (params.mode() == GrowFastElementsMode::kDoubleElements)
4347 : ? Builtins::CallableFor(isolate(), Builtins::kGrowFastDoubleElements)
4348 : : Builtins::CallableFor(isolate(),
4349 3485 : Builtins::kGrowFastSmiOrObjectElements);
4350 : CallDescriptor::Flags call_flags = CallDescriptor::kNoFlags;
4351 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4352 3485 : graph()->zone(), callable.descriptor(),
4353 3485 : callable.descriptor().GetStackParameterCount(), call_flags, properties);
4354 : Node* new_elements =
4355 10455 : __ Call(call_descriptor, __ HeapConstant(callable.code()), object,
4356 3485 : ChangeInt32ToSmi(index), __ NoContextConstant());
4357 :
4358 : // Ensure that we were able to grow the {elements}.
4359 3485 : __ DeoptimizeIf(DeoptimizeReason::kCouldNotGrowElements, params.feedback(),
4360 3485 : ObjectIsSmi(new_elements), frame_state);
4361 : __ Goto(&done, new_elements);
4362 :
4363 : __ Bind(&done);
4364 3485 : return done.PhiAt(0);
4365 : }
4366 :
4367 618 : void EffectControlLinearizer::LowerTransitionElementsKind(Node* node) {
4368 618 : ElementsTransition const transition = ElementsTransitionOf(node->op());
4369 : Node* object = node->InputAt(0);
4370 :
4371 618 : auto if_map_same = __ MakeDeferredLabel();
4372 618 : auto done = __ MakeLabel();
4373 :
4374 618 : Node* source_map = __ HeapConstant(transition.source());
4375 618 : Node* target_map = __ HeapConstant(transition.target());
4376 :
4377 : // Load the current map of {object}.
4378 618 : Node* object_map = __ LoadField(AccessBuilder::ForMap(), object);
4379 :
4380 : // Check if {object_map} is the same as {source_map}.
4381 618 : Node* check = __ WordEqual(object_map, source_map);
4382 618 : __ GotoIf(check, &if_map_same);
4383 : __ Goto(&done);
4384 :
4385 : __ Bind(&if_map_same);
4386 618 : switch (transition.mode()) {
4387 : case ElementsTransition::kFastTransition:
4388 : // In-place migration of {object}, just store the {target_map}.
4389 228 : __ StoreField(AccessBuilder::ForMap(), object, target_map);
4390 228 : break;
4391 : case ElementsTransition::kSlowTransition: {
4392 : // Instance migration, call out to the runtime for {object}.
4393 390 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4394 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4395 390 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4396 390 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4397 390 : __ Call(call_descriptor, __ CEntryStubConstant(1), object, target_map,
4398 : __ ExternalConstant(ExternalReference::Create(id)),
4399 390 : __ Int32Constant(2), __ NoContextConstant());
4400 : break;
4401 : }
4402 : }
4403 : __ Goto(&done);
4404 :
4405 : __ Bind(&done);
4406 618 : }
4407 :
4408 7217 : Node* EffectControlLinearizer::LowerLoadMessage(Node* node) {
4409 : Node* offset = node->InputAt(0);
4410 : Node* object_pattern =
4411 14434 : __ LoadField(AccessBuilder::ForExternalIntPtr(), offset);
4412 7217 : return __ BitcastWordToTagged(object_pattern);
4413 : }
4414 :
4415 29702 : void EffectControlLinearizer::LowerStoreMessage(Node* node) {
4416 : Node* offset = node->InputAt(0);
4417 : Node* object = node->InputAt(1);
4418 29702 : Node* object_pattern = __ BitcastTaggedToWord(object);
4419 29702 : __ StoreField(AccessBuilder::ForExternalIntPtr(), offset, object_pattern);
4420 29702 : }
4421 :
4422 582 : Node* EffectControlLinearizer::LowerLoadFieldByIndex(Node* node) {
4423 : Node* object = node->InputAt(0);
4424 : Node* index = node->InputAt(1);
4425 582 : Node* zero = __ IntPtrConstant(0);
4426 582 : Node* one = __ IntPtrConstant(1);
4427 :
4428 : // Sign-extend the {index} on 64-bit architectures.
4429 582 : if (machine()->Is64()) {
4430 582 : index = __ ChangeInt32ToInt64(index);
4431 : }
4432 :
4433 582 : auto if_double = __ MakeDeferredLabel();
4434 1164 : auto done = __ MakeLabel(MachineRepresentation::kTagged);
4435 :
4436 : // Check if field is a mutable double field.
4437 582 : __ GotoIfNot(__ WordEqual(__ WordAnd(index, one), zero), &if_double);
4438 :
4439 : // The field is a proper Tagged field on {object}. The {index} is shifted
4440 : // to the left by one in the code below.
4441 : {
4442 : // Check if field is in-object or out-of-object.
4443 582 : auto if_outofobject = __ MakeLabel();
4444 582 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4445 :
4446 : // The field is located in the {object} itself.
4447 : {
4448 : Node* offset =
4449 582 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4450 582 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4451 582 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4452 : __ Goto(&done, result);
4453 : }
4454 :
4455 : // The field is located in the properties backing store of {object}.
4456 : // The {index} is equal to the negated out of property index plus 1.
4457 : __ Bind(&if_outofobject);
4458 : {
4459 : Node* properties =
4460 582 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4461 : Node* offset =
4462 582 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4463 : __ IntPtrConstant(kTaggedSizeLog2 - 1)),
4464 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4465 582 : kHeapObjectTag));
4466 582 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4467 : __ Goto(&done, result);
4468 : }
4469 : }
4470 :
4471 : // The field is a Double field, either unboxed in the object on 64-bit
4472 : // architectures, or as MutableHeapNumber.
4473 : __ Bind(&if_double);
4474 : {
4475 1164 : auto done_double = __ MakeLabel(MachineRepresentation::kFloat64);
4476 :
4477 582 : index = __ WordSar(index, one);
4478 :
4479 : // Check if field is in-object or out-of-object.
4480 582 : auto if_outofobject = __ MakeLabel();
4481 582 : __ GotoIf(__ IntLessThan(index, zero), &if_outofobject);
4482 :
4483 : // The field is located in the {object} itself.
4484 : {
4485 : Node* offset =
4486 582 : __ IntAdd(__ WordShl(index, __ IntPtrConstant(kTaggedSizeLog2)),
4487 582 : __ IntPtrConstant(JSObject::kHeaderSize - kHeapObjectTag));
4488 : if (FLAG_unbox_double_fields) {
4489 582 : Node* result = __ Load(MachineType::Float64(), object, offset);
4490 : __ Goto(&done_double, result);
4491 : } else {
4492 : Node* result = __ Load(MachineType::AnyTagged(), object, offset);
4493 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4494 : __ Goto(&done_double, result);
4495 : }
4496 : }
4497 :
4498 : __ Bind(&if_outofobject);
4499 : {
4500 : Node* properties =
4501 582 : __ LoadField(AccessBuilder::ForJSObjectPropertiesOrHash(), object);
4502 : Node* offset =
4503 582 : __ IntAdd(__ WordShl(__ IntSub(zero, index),
4504 : __ IntPtrConstant(kTaggedSizeLog2)),
4505 : __ IntPtrConstant((FixedArray::kHeaderSize - kTaggedSize) -
4506 582 : kHeapObjectTag));
4507 582 : Node* result = __ Load(MachineType::AnyTagged(), properties, offset);
4508 582 : result = __ LoadField(AccessBuilder::ForHeapNumberValue(), result);
4509 : __ Goto(&done_double, result);
4510 : }
4511 :
4512 : __ Bind(&done_double);
4513 : {
4514 582 : Node* result = AllocateHeapNumberWithValue(done_double.PhiAt(0));
4515 : __ Goto(&done, result);
4516 : }
4517 : }
4518 :
4519 : __ Bind(&done);
4520 582 : return done.PhiAt(0);
4521 : }
4522 :
4523 464 : Node* EffectControlLinearizer::BuildReverseBytes(ExternalArrayType type,
4524 : Node* value) {
4525 464 : switch (type) {
4526 : case kExternalInt8Array:
4527 : case kExternalUint8Array:
4528 : case kExternalUint8ClampedArray:
4529 : return value;
4530 :
4531 : case kExternalInt16Array: {
4532 55 : Node* result = __ Word32ReverseBytes(value);
4533 55 : result = __ Word32Sar(result, __ Int32Constant(16));
4534 55 : return result;
4535 : }
4536 :
4537 : case kExternalUint16Array: {
4538 55 : Node* result = __ Word32ReverseBytes(value);
4539 55 : result = __ Word32Shr(result, __ Int32Constant(16));
4540 55 : return result;
4541 : }
4542 :
4543 : case kExternalInt32Array: // Fall through.
4544 : case kExternalUint32Array:
4545 96 : return __ Word32ReverseBytes(value);
4546 :
4547 : case kExternalFloat32Array: {
4548 56 : Node* result = __ BitcastFloat32ToInt32(value);
4549 56 : result = __ Word32ReverseBytes(result);
4550 56 : result = __ BitcastInt32ToFloat32(result);
4551 56 : return result;
4552 : }
4553 :
4554 : case kExternalFloat64Array: {
4555 56 : if (machine()->Is64()) {
4556 56 : Node* result = __ BitcastFloat64ToInt64(value);
4557 56 : result = __ Word64ReverseBytes(result);
4558 56 : result = __ BitcastInt64ToFloat64(result);
4559 56 : return result;
4560 : } else {
4561 0 : Node* lo = __ Word32ReverseBytes(__ Float64ExtractLowWord32(value));
4562 0 : Node* hi = __ Word32ReverseBytes(__ Float64ExtractHighWord32(value));
4563 0 : Node* result = __ Float64Constant(0.0);
4564 0 : result = __ Float64InsertLowWord32(result, hi);
4565 0 : result = __ Float64InsertHighWord32(result, lo);
4566 0 : return result;
4567 : }
4568 : }
4569 :
4570 : case kExternalBigInt64Array:
4571 : case kExternalBigUint64Array:
4572 0 : UNREACHABLE();
4573 : }
4574 0 : }
4575 :
4576 268 : Node* EffectControlLinearizer::LowerLoadDataViewElement(Node* node) {
4577 268 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4578 : Node* buffer = node->InputAt(0);
4579 : Node* storage = node->InputAt(1);
4580 : Node* byte_offset = node->InputAt(2);
4581 : Node* index = node->InputAt(3);
4582 : Node* is_little_endian = node->InputAt(4);
4583 :
4584 : // We need to keep the {buffer} alive so that the GC will not release the
4585 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4586 268 : __ Retain(buffer);
4587 :
4588 : // Compute the effective offset.
4589 268 : Node* offset = __ IntAdd(byte_offset, index);
4590 :
4591 : MachineType const machine_type =
4592 268 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4593 :
4594 268 : Node* value = __ LoadUnaligned(machine_type, storage, offset);
4595 268 : auto big_endian = __ MakeLabel();
4596 536 : auto done = __ MakeLabel(machine_type.representation());
4597 :
4598 268 : __ GotoIfNot(is_little_endian, &big_endian);
4599 : { // Little-endian load.
4600 : #if V8_TARGET_LITTLE_ENDIAN
4601 : __ Goto(&done, value);
4602 : #else
4603 : __ Goto(&done, BuildReverseBytes(element_type, value));
4604 : #endif // V8_TARGET_LITTLE_ENDIAN
4605 : }
4606 :
4607 : __ Bind(&big_endian);
4608 : { // Big-endian load.
4609 : #if V8_TARGET_LITTLE_ENDIAN
4610 268 : __ Goto(&done, BuildReverseBytes(element_type, value));
4611 : #else
4612 : __ Goto(&done, value);
4613 : #endif // V8_TARGET_LITTLE_ENDIAN
4614 : }
4615 :
4616 : // We're done, return {result}.
4617 : __ Bind(&done);
4618 268 : return done.PhiAt(0);
4619 : }
4620 :
4621 571 : Node* EffectControlLinearizer::LowerLoadStackArgument(Node* node) {
4622 : Node* base = node->InputAt(0);
4623 : Node* index = node->InputAt(1);
4624 :
4625 : Node* argument =
4626 1142 : __ LoadElement(AccessBuilder::ForStackArgument(), base, index);
4627 :
4628 571 : return __ BitcastWordToTagged(argument);
4629 : }
4630 :
4631 196 : void EffectControlLinearizer::LowerStoreDataViewElement(Node* node) {
4632 196 : ExternalArrayType element_type = ExternalArrayTypeOf(node->op());
4633 : Node* buffer = node->InputAt(0);
4634 : Node* storage = node->InputAt(1);
4635 : Node* byte_offset = node->InputAt(2);
4636 : Node* index = node->InputAt(3);
4637 : Node* value = node->InputAt(4);
4638 : Node* is_little_endian = node->InputAt(5);
4639 :
4640 : // We need to keep the {buffer} alive so that the GC will not release the
4641 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4642 196 : __ Retain(buffer);
4643 :
4644 : // Compute the effective offset.
4645 196 : Node* offset = __ IntAdd(byte_offset, index);
4646 :
4647 : MachineType const machine_type =
4648 196 : AccessBuilder::ForTypedArrayElement(element_type, true).machine_type;
4649 :
4650 196 : auto big_endian = __ MakeLabel();
4651 392 : auto done = __ MakeLabel(machine_type.representation());
4652 :
4653 196 : __ GotoIfNot(is_little_endian, &big_endian);
4654 : { // Little-endian store.
4655 : #if V8_TARGET_LITTLE_ENDIAN
4656 : __ Goto(&done, value);
4657 : #else
4658 : __ Goto(&done, BuildReverseBytes(element_type, value));
4659 : #endif // V8_TARGET_LITTLE_ENDIAN
4660 : }
4661 :
4662 : __ Bind(&big_endian);
4663 : { // Big-endian store.
4664 : #if V8_TARGET_LITTLE_ENDIAN
4665 196 : __ Goto(&done, BuildReverseBytes(element_type, value));
4666 : #else
4667 : __ Goto(&done, value);
4668 : #endif // V8_TARGET_LITTLE_ENDIAN
4669 : }
4670 :
4671 : __ Bind(&done);
4672 : __ StoreUnaligned(machine_type.representation(), storage, offset,
4673 196 : done.PhiAt(0));
4674 196 : }
4675 :
4676 4510 : Node* EffectControlLinearizer::LowerLoadTypedElement(Node* node) {
4677 4510 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4678 : Node* buffer = node->InputAt(0);
4679 : Node* base = node->InputAt(1);
4680 : Node* external = node->InputAt(2);
4681 : Node* index = node->InputAt(3);
4682 :
4683 : // We need to keep the {buffer} alive so that the GC will not release the
4684 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4685 4510 : __ Retain(buffer);
4686 :
4687 : // Compute the effective storage pointer, handling the case where the
4688 : // {external} pointer is the effective storage pointer (i.e. the {base}
4689 : // is Smi zero).
4690 : Node* storage = IntPtrMatcher(base).Is(0)
4691 : ? external
4692 4510 : : __ UnsafePointerAdd(base, external);
4693 :
4694 : // Perform the actual typed element access.
4695 9020 : return __ LoadElement(AccessBuilder::ForTypedArrayElement(
4696 : array_type, true, LoadSensitivity::kCritical),
4697 9020 : storage, index);
4698 : }
4699 :
4700 4342 : void EffectControlLinearizer::LowerStoreTypedElement(Node* node) {
4701 4342 : ExternalArrayType array_type = ExternalArrayTypeOf(node->op());
4702 : Node* buffer = node->InputAt(0);
4703 : Node* base = node->InputAt(1);
4704 : Node* external = node->InputAt(2);
4705 : Node* index = node->InputAt(3);
4706 : Node* value = node->InputAt(4);
4707 :
4708 : // We need to keep the {buffer} alive so that the GC will not release the
4709 : // ArrayBuffer (if there's any) as long as we are still operating on it.
4710 4342 : __ Retain(buffer);
4711 :
4712 : // Compute the effective storage pointer, handling the case where the
4713 : // {external} pointer is the effective storage pointer (i.e. the {base}
4714 : // is Smi zero).
4715 : Node* storage = IntPtrMatcher(base).Is(0)
4716 : ? external
4717 4342 : : __ UnsafePointerAdd(base, external);
4718 :
4719 : // Perform the actual typed element access.
4720 8684 : __ StoreElement(AccessBuilder::ForTypedArrayElement(array_type, true),
4721 4342 : storage, index, value);
4722 4342 : }
4723 :
4724 326 : void EffectControlLinearizer::TransitionElementsTo(Node* node, Node* array,
4725 : ElementsKind from,
4726 : ElementsKind to) {
4727 : DCHECK(IsMoreGeneralElementsKindTransition(from, to));
4728 : DCHECK(to == HOLEY_ELEMENTS || to == HOLEY_DOUBLE_ELEMENTS);
4729 :
4730 : Handle<Map> target(to == HOLEY_ELEMENTS ? FastMapParameterOf(node->op())
4731 326 : : DoubleMapParameterOf(node->op()));
4732 326 : Node* target_map = __ HeapConstant(target);
4733 :
4734 326 : if (IsSimpleMapChangeTransition(from, to)) {
4735 106 : __ StoreField(AccessBuilder::ForMap(), array, target_map);
4736 : } else {
4737 : // Instance migration, call out to the runtime for {array}.
4738 220 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
4739 : Runtime::FunctionId id = Runtime::kTransitionElementsKind;
4740 220 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
4741 220 : graph()->zone(), id, 2, properties, CallDescriptor::kNoFlags);
4742 220 : __ Call(call_descriptor, __ CEntryStubConstant(1), array, target_map,
4743 : __ ExternalConstant(ExternalReference::Create(id)),
4744 220 : __ Int32Constant(2), __ NoContextConstant());
4745 : }
4746 326 : }
4747 :
4748 0 : Node* EffectControlLinearizer::IsElementsKindGreaterThan(
4749 : Node* kind, ElementsKind reference_kind) {
4750 396 : Node* ref_kind = __ Int32Constant(reference_kind);
4751 396 : Node* ret = __ Int32LessThan(ref_kind, kind);
4752 0 : return ret;
4753 : }
4754 :
4755 86 : void EffectControlLinearizer::LowerTransitionAndStoreElement(Node* node) {
4756 : Node* array = node->InputAt(0);
4757 : Node* index = node->InputAt(1);
4758 : Node* value = node->InputAt(2);
4759 :
4760 : // Possibly transition array based on input and store.
4761 : //
4762 : // -- TRANSITION PHASE -----------------
4763 : // kind = ElementsKind(array)
4764 : // if value is not smi {
4765 : // if kind == HOLEY_SMI_ELEMENTS {
4766 : // if value is heap number {
4767 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4768 : // kind = HOLEY_DOUBLE_ELEMENTS
4769 : // } else {
4770 : // Transition array to HOLEY_ELEMENTS
4771 : // kind = HOLEY_ELEMENTS
4772 : // }
4773 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4774 : // if value is not heap number {
4775 : // Transition array to HOLEY_ELEMENTS
4776 : // kind = HOLEY_ELEMENTS
4777 : // }
4778 : // }
4779 : // }
4780 : //
4781 : // -- STORE PHASE ----------------------
4782 : // [make sure {kind} is up-to-date]
4783 : // if kind == HOLEY_DOUBLE_ELEMENTS {
4784 : // if value is smi {
4785 : // float_value = convert smi to float
4786 : // Store array[index] = float_value
4787 : // } else {
4788 : // float_value = value
4789 : // Store array[index] = float_value
4790 : // }
4791 : // } else {
4792 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
4793 : // Store array[index] = value
4794 : // }
4795 : //
4796 172 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4797 : Node* kind;
4798 : {
4799 86 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4800 86 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4801 86 : Node* andit = __ Word32And(bit_field2, mask);
4802 86 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4803 86 : kind = __ Word32Shr(andit, shift);
4804 : }
4805 :
4806 172 : auto do_store = __ MakeLabel(MachineRepresentation::kWord32);
4807 : // We can store a smi anywhere.
4808 86 : __ GotoIf(ObjectIsSmi(value), &do_store, kind);
4809 :
4810 : // {value} is a HeapObject.
4811 86 : auto transition_smi_array = __ MakeDeferredLabel();
4812 86 : auto transition_double_to_fast = __ MakeDeferredLabel();
4813 : {
4814 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4815 86 : &transition_smi_array);
4816 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS), &do_store,
4817 86 : kind);
4818 :
4819 : // We have double elements kind. Only a HeapNumber can be stored
4820 : // without effecting a transition.
4821 86 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4822 86 : Node* heap_number_map = __ HeapNumberMapConstant();
4823 86 : Node* check = __ WordEqual(value_map, heap_number_map);
4824 86 : __ GotoIfNot(check, &transition_double_to_fast);
4825 : __ Goto(&do_store, kind);
4826 : }
4827 :
4828 : __ Bind(&transition_smi_array); // deferred code.
4829 : {
4830 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS or
4831 : // to HOLEY_ELEMENTS.
4832 86 : auto if_value_not_heap_number = __ MakeLabel();
4833 86 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
4834 86 : Node* heap_number_map = __ HeapNumberMapConstant();
4835 86 : Node* check = __ WordEqual(value_map, heap_number_map);
4836 86 : __ GotoIfNot(check, &if_value_not_heap_number);
4837 : {
4838 : // {value} is a HeapNumber.
4839 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4840 86 : HOLEY_DOUBLE_ELEMENTS);
4841 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS));
4842 : }
4843 : __ Bind(&if_value_not_heap_number);
4844 : {
4845 86 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
4846 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4847 : }
4848 : }
4849 :
4850 : __ Bind(&transition_double_to_fast); // deferred code.
4851 : {
4852 86 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
4853 86 : __ Goto(&do_store, __ Int32Constant(HOLEY_ELEMENTS));
4854 : }
4855 :
4856 : // Make sure kind is up-to-date.
4857 : __ Bind(&do_store);
4858 : kind = do_store.PhiAt(0);
4859 :
4860 86 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4861 86 : auto if_kind_is_double = __ MakeLabel();
4862 86 : auto done = __ MakeLabel();
4863 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4864 86 : &if_kind_is_double);
4865 : {
4866 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
4867 172 : __ StoreElement(AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS),
4868 86 : elements, index, value);
4869 : __ Goto(&done);
4870 : }
4871 : __ Bind(&if_kind_is_double);
4872 : {
4873 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
4874 86 : auto do_double_store = __ MakeLabel();
4875 86 : __ GotoIfNot(ObjectIsSmi(value), &do_double_store);
4876 : {
4877 86 : Node* int_value = ChangeSmiToInt32(value);
4878 86 : Node* float_value = __ ChangeInt32ToFloat64(int_value);
4879 172 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4880 86 : index, float_value);
4881 : __ Goto(&done);
4882 : }
4883 : __ Bind(&do_double_store);
4884 : {
4885 : Node* float_value =
4886 86 : __ LoadField(AccessBuilder::ForHeapNumberValue(), value);
4887 172 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
4888 86 : index, __ Float64SilenceNaN(float_value));
4889 : __ Goto(&done);
4890 : }
4891 : }
4892 :
4893 : __ Bind(&done);
4894 86 : }
4895 :
4896 28 : void EffectControlLinearizer::LowerTransitionAndStoreNumberElement(Node* node) {
4897 : Node* array = node->InputAt(0);
4898 : Node* index = node->InputAt(1);
4899 : Node* value = node->InputAt(2); // This is a Float64, not tagged.
4900 :
4901 : // Possibly transition array based on input and store.
4902 : //
4903 : // -- TRANSITION PHASE -----------------
4904 : // kind = ElementsKind(array)
4905 : // if kind == HOLEY_SMI_ELEMENTS {
4906 : // Transition array to HOLEY_DOUBLE_ELEMENTS
4907 : // } else if kind != HOLEY_DOUBLE_ELEMENTS {
4908 : // This is UNREACHABLE, execute a debug break.
4909 : // }
4910 : //
4911 : // -- STORE PHASE ----------------------
4912 : // Store array[index] = value (it's a float)
4913 : //
4914 56 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4915 : Node* kind;
4916 : {
4917 28 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4918 28 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4919 28 : Node* andit = __ Word32And(bit_field2, mask);
4920 28 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4921 28 : kind = __ Word32Shr(andit, shift);
4922 : }
4923 :
4924 28 : auto do_store = __ MakeLabel();
4925 :
4926 : // {value} is a float64.
4927 28 : auto transition_smi_array = __ MakeDeferredLabel();
4928 : {
4929 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4930 28 : &transition_smi_array);
4931 : // We expect that our input array started at HOLEY_SMI_ELEMENTS, and
4932 : // climbs the lattice up to HOLEY_DOUBLE_ELEMENTS. Force a debug break
4933 : // if this assumption is broken. It also would be the case that
4934 : // loop peeling can break this assumption.
4935 28 : __ GotoIf(__ Word32Equal(kind, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)),
4936 28 : &do_store);
4937 : // TODO(turbofan): It would be good to have an "Unreachable()" node type.
4938 28 : __ DebugBreak();
4939 : __ Goto(&do_store);
4940 : }
4941 :
4942 : __ Bind(&transition_smi_array); // deferred code.
4943 : {
4944 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS.
4945 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
4946 28 : HOLEY_DOUBLE_ELEMENTS);
4947 : __ Goto(&do_store);
4948 : }
4949 :
4950 : __ Bind(&do_store);
4951 :
4952 28 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
4953 56 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, index,
4954 28 : __ Float64SilenceNaN(value));
4955 28 : }
4956 :
4957 20 : void EffectControlLinearizer::LowerTransitionAndStoreNonNumberElement(
4958 : Node* node) {
4959 : Node* array = node->InputAt(0);
4960 : Node* index = node->InputAt(1);
4961 : Node* value = node->InputAt(2);
4962 :
4963 : // Possibly transition array based on input and store.
4964 : //
4965 : // -- TRANSITION PHASE -----------------
4966 : // kind = ElementsKind(array)
4967 : // if kind == HOLEY_SMI_ELEMENTS {
4968 : // Transition array to HOLEY_ELEMENTS
4969 : // } else if kind == HOLEY_DOUBLE_ELEMENTS {
4970 : // Transition array to HOLEY_ELEMENTS
4971 : // }
4972 : //
4973 : // -- STORE PHASE ----------------------
4974 : // // kind is HOLEY_ELEMENTS
4975 : // Store array[index] = value
4976 : //
4977 40 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
4978 : Node* kind;
4979 : {
4980 20 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
4981 20 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
4982 20 : Node* andit = __ Word32And(bit_field2, mask);
4983 20 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
4984 20 : kind = __ Word32Shr(andit, shift);
4985 : }
4986 :
4987 20 : auto do_store = __ MakeLabel();
4988 :
4989 20 : auto transition_smi_array = __ MakeDeferredLabel();
4990 20 : auto transition_double_to_fast = __ MakeDeferredLabel();
4991 : {
4992 : __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
4993 20 : &transition_smi_array);
4994 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
4995 20 : &transition_double_to_fast);
4996 : __ Goto(&do_store);
4997 : }
4998 :
4999 : __ Bind(&transition_smi_array); // deferred code.
5000 : {
5001 : // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_ELEMENTS.
5002 20 : TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
5003 : __ Goto(&do_store);
5004 : }
5005 :
5006 : __ Bind(&transition_double_to_fast); // deferred code.
5007 : {
5008 20 : TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
5009 : __ Goto(&do_store);
5010 : }
5011 :
5012 : __ Bind(&do_store);
5013 :
5014 20 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
5015 : // Our ElementsKind is HOLEY_ELEMENTS.
5016 20 : ElementAccess access = AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS);
5017 20 : Type value_type = ValueTypeParameterOf(node->op());
5018 20 : if (value_type.Is(Type::BooleanOrNullOrUndefined())) {
5019 16 : access.type = value_type;
5020 16 : access.write_barrier_kind = kNoWriteBarrier;
5021 : }
5022 20 : __ StoreElement(access, elements, index, value);
5023 20 : }
5024 :
5025 70 : void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
5026 : Node* array = node->InputAt(0);
5027 : Node* index = node->InputAt(1);
5028 : Node* value = node->InputAt(2); // int32
5029 :
5030 : // Store a signed small in an output array.
5031 : //
5032 : // kind = ElementsKind(array)
5033 : //
5034 : // -- STORE PHASE ----------------------
5035 : // if kind == HOLEY_DOUBLE_ELEMENTS {
5036 : // float_value = convert int32 to float
5037 : // Store array[index] = float_value
5038 : // } else {
5039 : // // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
5040 : // smi_value = convert int32 to smi
5041 : // Store array[index] = smi_value
5042 : // }
5043 : //
5044 140 : Node* map = __ LoadField(AccessBuilder::ForMap(), array);
5045 : Node* kind;
5046 : {
5047 70 : Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
5048 70 : Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
5049 70 : Node* andit = __ Word32And(bit_field2, mask);
5050 70 : Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
5051 70 : kind = __ Word32Shr(andit, shift);
5052 : }
5053 :
5054 70 : Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
5055 70 : auto if_kind_is_double = __ MakeLabel();
5056 70 : auto done = __ MakeLabel();
5057 : __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
5058 70 : &if_kind_is_double);
5059 : {
5060 : // Our ElementsKind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS.
5061 : // In this case, we know our value is a signed small, and we can optimize
5062 : // the ElementAccess information.
5063 70 : ElementAccess access = AccessBuilder::ForFixedArrayElement();
5064 70 : access.type = Type::SignedSmall();
5065 70 : access.machine_type = MachineType::TaggedSigned();
5066 70 : access.write_barrier_kind = kNoWriteBarrier;
5067 70 : Node* smi_value = ChangeInt32ToSmi(value);
5068 70 : __ StoreElement(access, elements, index, smi_value);
5069 : __ Goto(&done);
5070 : }
5071 : __ Bind(&if_kind_is_double);
5072 : {
5073 : // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
5074 70 : Node* float_value = __ ChangeInt32ToFloat64(value);
5075 140 : __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
5076 70 : index, float_value);
5077 : __ Goto(&done);
5078 : }
5079 :
5080 : __ Bind(&done);
5081 70 : }
5082 :
5083 2557 : void EffectControlLinearizer::LowerRuntimeAbort(Node* node) {
5084 2557 : AbortReason reason = AbortReasonOf(node->op());
5085 2557 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
5086 : Runtime::FunctionId id = Runtime::kAbort;
5087 2557 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
5088 2557 : graph()->zone(), id, 1, properties, CallDescriptor::kNoFlags);
5089 5114 : __ Call(call_descriptor, __ CEntryStubConstant(1),
5090 : jsgraph()->SmiConstant(static_cast<int>(reason)),
5091 : __ ExternalConstant(ExternalReference::Create(id)),
5092 2557 : __ Int32Constant(1), __ NoContextConstant());
5093 2557 : }
5094 :
5095 876 : Node* EffectControlLinearizer::LowerConvertReceiver(Node* node) {
5096 876 : ConvertReceiverMode const mode = ConvertReceiverModeOf(node->op());
5097 : Node* value = node->InputAt(0);
5098 : Node* global_proxy = node->InputAt(1);
5099 :
5100 876 : switch (mode) {
5101 : case ConvertReceiverMode::kNullOrUndefined: {
5102 : return global_proxy;
5103 : }
5104 : case ConvertReceiverMode::kNotNullOrUndefined: {
5105 809 : auto convert_to_object = __ MakeDeferredLabel();
5106 1618 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
5107 :
5108 : // Check if {value} is already a JSReceiver.
5109 1618 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
5110 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
5111 809 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
5112 : Node* value_instance_type =
5113 809 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
5114 809 : Node* check = __ Uint32LessThan(
5115 809 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
5116 809 : __ GotoIf(check, &convert_to_object);
5117 : __ Goto(&done_convert, value);
5118 :
5119 : // Wrap the primitive {value} into a JSValue.
5120 : __ Bind(&convert_to_object);
5121 : Operator::Properties properties = Operator::kEliminatable;
5122 809 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
5123 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5124 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5125 809 : graph()->zone(), callable.descriptor(),
5126 809 : callable.descriptor().GetStackParameterCount(), flags, properties);
5127 : Node* native_context = __ LoadField(
5128 809 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
5129 1618 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
5130 809 : value, native_context);
5131 : __ Goto(&done_convert, result);
5132 :
5133 : __ Bind(&done_convert);
5134 : return done_convert.PhiAt(0);
5135 : }
5136 : case ConvertReceiverMode::kAny: {
5137 67 : auto convert_to_object = __ MakeDeferredLabel();
5138 67 : auto convert_global_proxy = __ MakeDeferredLabel();
5139 134 : auto done_convert = __ MakeLabel(MachineRepresentation::kTagged);
5140 :
5141 : // Check if {value} is already a JSReceiver, or null/undefined.
5142 134 : __ GotoIf(ObjectIsSmi(value), &convert_to_object);
5143 : STATIC_ASSERT(LAST_TYPE == LAST_JS_RECEIVER_TYPE);
5144 67 : Node* value_map = __ LoadField(AccessBuilder::ForMap(), value);
5145 : Node* value_instance_type =
5146 67 : __ LoadField(AccessBuilder::ForMapInstanceType(), value_map);
5147 67 : Node* check = __ Uint32LessThan(
5148 67 : value_instance_type, __ Uint32Constant(FIRST_JS_RECEIVER_TYPE));
5149 67 : __ GotoIf(check, &convert_to_object);
5150 : __ Goto(&done_convert, value);
5151 :
5152 : // Wrap the primitive {value} into a JSValue.
5153 : __ Bind(&convert_to_object);
5154 67 : __ GotoIf(__ WordEqual(value, __ UndefinedConstant()),
5155 67 : &convert_global_proxy);
5156 67 : __ GotoIf(__ WordEqual(value, __ NullConstant()), &convert_global_proxy);
5157 : Operator::Properties properties = Operator::kEliminatable;
5158 67 : Callable callable = Builtins::CallableFor(isolate(), Builtins::kToObject);
5159 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5160 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5161 67 : graph()->zone(), callable.descriptor(),
5162 67 : callable.descriptor().GetStackParameterCount(), flags, properties);
5163 : Node* native_context = __ LoadField(
5164 67 : AccessBuilder::ForJSGlobalProxyNativeContext(), global_proxy);
5165 134 : Node* result = __ Call(call_descriptor, __ HeapConstant(callable.code()),
5166 67 : value, native_context);
5167 : __ Goto(&done_convert, result);
5168 :
5169 : // Replace the {value} with the {global_proxy}.
5170 : __ Bind(&convert_global_proxy);
5171 : __ Goto(&done_convert, global_proxy);
5172 :
5173 : __ Bind(&done_convert);
5174 : return done_convert.PhiAt(0);
5175 : }
5176 : }
5177 :
5178 0 : UNREACHABLE();
5179 : return nullptr;
5180 : }
5181 :
5182 8222 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundUp(Node* node) {
5183 : // Nothing to be done if a fast hardware instruction is available.
5184 8222 : if (machine()->Float64RoundUp().IsSupported()) {
5185 : return Nothing<Node*>();
5186 : }
5187 :
5188 : Node* const input = node->InputAt(0);
5189 :
5190 : // General case for ceil.
5191 : //
5192 : // if 0.0 < input then
5193 : // if 2^52 <= input then
5194 : // input
5195 : // else
5196 : // let temp1 = (2^52 + input) - 2^52 in
5197 : // if temp1 < input then
5198 : // temp1 + 1
5199 : // else
5200 : // temp1
5201 : // else
5202 : // if input == 0 then
5203 : // input
5204 : // else
5205 : // if input <= -2^52 then
5206 : // input
5207 : // else
5208 : // let temp1 = -0 - input in
5209 : // let temp2 = (2^52 + temp1) - 2^52 in
5210 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
5211 : // -0 - temp3
5212 :
5213 0 : auto if_not_positive = __ MakeDeferredLabel();
5214 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5215 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5216 0 : auto if_zero = __ MakeDeferredLabel();
5217 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
5218 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5219 :
5220 0 : Node* const zero = __ Float64Constant(0.0);
5221 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5222 0 : Node* const one = __ Float64Constant(1.0);
5223 :
5224 0 : Node* check0 = __ Float64LessThan(zero, input);
5225 0 : __ GotoIfNot(check0, &if_not_positive);
5226 : {
5227 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5228 0 : __ GotoIf(check1, &if_greater_than_two_52);
5229 : {
5230 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5231 0 : __ GotoIfNot(__ Float64LessThan(temp1, input), &done, temp1);
5232 0 : __ Goto(&done, __ Float64Add(temp1, one));
5233 : }
5234 :
5235 : __ Bind(&if_greater_than_two_52);
5236 : __ Goto(&done, input);
5237 : }
5238 :
5239 : __ Bind(&if_not_positive);
5240 : {
5241 0 : Node* check1 = __ Float64Equal(input, zero);
5242 0 : __ GotoIf(check1, &if_zero);
5243 :
5244 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5245 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5246 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5247 :
5248 : {
5249 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5250 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5251 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5252 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
5253 0 : __ GotoIfNot(check3, &done_temp3, temp2);
5254 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
5255 :
5256 : __ Bind(&done_temp3);
5257 : Node* temp3 = done_temp3.PhiAt(0);
5258 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
5259 : }
5260 : __ Bind(&if_less_than_minus_two_52);
5261 : __ Goto(&done, input);
5262 :
5263 : __ Bind(&if_zero);
5264 : __ Goto(&done, input);
5265 : }
5266 : __ Bind(&done);
5267 : return Just(done.PhiAt(0));
5268 : }
5269 :
5270 0 : Node* EffectControlLinearizer::BuildFloat64RoundDown(Node* value) {
5271 0 : if (machine()->Float64RoundDown().IsSupported()) {
5272 0 : return __ Float64RoundDown(value);
5273 : }
5274 :
5275 : Node* const input = value;
5276 :
5277 : // General case for floor.
5278 : //
5279 : // if 0.0 < input then
5280 : // if 2^52 <= input then
5281 : // input
5282 : // else
5283 : // let temp1 = (2^52 + input) - 2^52 in
5284 : // if input < temp1 then
5285 : // temp1 - 1
5286 : // else
5287 : // temp1
5288 : // else
5289 : // if input == 0 then
5290 : // input
5291 : // else
5292 : // if input <= -2^52 then
5293 : // input
5294 : // else
5295 : // let temp1 = -0 - input in
5296 : // let temp2 = (2^52 + temp1) - 2^52 in
5297 : // if temp2 < temp1 then
5298 : // -1 - temp2
5299 : // else
5300 : // -0 - temp2
5301 :
5302 0 : auto if_not_positive = __ MakeDeferredLabel();
5303 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5304 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5305 0 : auto if_temp2_lt_temp1 = __ MakeLabel();
5306 0 : auto if_zero = __ MakeDeferredLabel();
5307 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5308 :
5309 0 : Node* const zero = __ Float64Constant(0.0);
5310 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5311 :
5312 0 : Node* check0 = __ Float64LessThan(zero, input);
5313 0 : __ GotoIfNot(check0, &if_not_positive);
5314 : {
5315 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5316 0 : __ GotoIf(check1, &if_greater_than_two_52);
5317 : {
5318 0 : Node* const one = __ Float64Constant(1.0);
5319 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5320 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5321 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5322 : }
5323 :
5324 : __ Bind(&if_greater_than_two_52);
5325 : __ Goto(&done, input);
5326 : }
5327 :
5328 : __ Bind(&if_not_positive);
5329 : {
5330 0 : Node* check1 = __ Float64Equal(input, zero);
5331 0 : __ GotoIf(check1, &if_zero);
5332 :
5333 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5334 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5335 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5336 :
5337 : {
5338 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5339 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5340 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5341 0 : Node* check3 = __ Float64LessThan(temp2, temp1);
5342 0 : __ GotoIf(check3, &if_temp2_lt_temp1);
5343 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp2));
5344 :
5345 : __ Bind(&if_temp2_lt_temp1);
5346 0 : __ Goto(&done, __ Float64Sub(__ Float64Constant(-1.0), temp2));
5347 : }
5348 : __ Bind(&if_less_than_minus_two_52);
5349 : __ Goto(&done, input);
5350 :
5351 : __ Bind(&if_zero);
5352 : __ Goto(&done, input);
5353 : }
5354 : __ Bind(&done);
5355 : return done.PhiAt(0);
5356 : }
5357 :
5358 26793 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundDown(Node* node) {
5359 : // Nothing to be done if a fast hardware instruction is available.
5360 26793 : if (machine()->Float64RoundDown().IsSupported()) {
5361 : return Nothing<Node*>();
5362 : }
5363 :
5364 : Node* const input = node->InputAt(0);
5365 0 : return Just(BuildFloat64RoundDown(input));
5366 : }
5367 :
5368 266 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTiesEven(Node* node) {
5369 : // Nothing to be done if a fast hardware instruction is available.
5370 266 : if (machine()->Float64RoundTiesEven().IsSupported()) {
5371 : return Nothing<Node*>();
5372 : }
5373 :
5374 : Node* const input = node->InputAt(0);
5375 :
5376 : // Generate case for round ties to even:
5377 : //
5378 : // let value = floor(input) in
5379 : // let temp1 = input - value in
5380 : // if temp1 < 0.5 then
5381 : // value
5382 : // else if 0.5 < temp1 then
5383 : // value + 1.0
5384 : // else
5385 : // let temp2 = value % 2.0 in
5386 : // if temp2 == 0.0 then
5387 : // value
5388 : // else
5389 : // value + 1.0
5390 :
5391 0 : auto if_is_half = __ MakeLabel();
5392 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5393 :
5394 0 : Node* value = BuildFloat64RoundDown(input);
5395 0 : Node* temp1 = __ Float64Sub(input, value);
5396 :
5397 0 : Node* const half = __ Float64Constant(0.5);
5398 0 : Node* check0 = __ Float64LessThan(temp1, half);
5399 0 : __ GotoIf(check0, &done, value);
5400 :
5401 0 : Node* const one = __ Float64Constant(1.0);
5402 0 : Node* check1 = __ Float64LessThan(half, temp1);
5403 0 : __ GotoIfNot(check1, &if_is_half);
5404 0 : __ Goto(&done, __ Float64Add(value, one));
5405 :
5406 : __ Bind(&if_is_half);
5407 0 : Node* temp2 = __ Float64Mod(value, __ Float64Constant(2.0));
5408 0 : Node* check2 = __ Float64Equal(temp2, __ Float64Constant(0.0));
5409 0 : __ GotoIf(check2, &done, value);
5410 0 : __ Goto(&done, __ Float64Add(value, one));
5411 :
5412 : __ Bind(&done);
5413 : return Just(done.PhiAt(0));
5414 : }
5415 :
5416 140 : Node* EffectControlLinearizer::BuildFloat64RoundTruncate(Node* input) {
5417 140 : if (machine()->Float64RoundTruncate().IsSupported()) {
5418 140 : return __ Float64RoundTruncate(input);
5419 : }
5420 : // General case for trunc.
5421 : //
5422 : // if 0.0 < input then
5423 : // if 2^52 <= input then
5424 : // input
5425 : // else
5426 : // let temp1 = (2^52 + input) - 2^52 in
5427 : // if input < temp1 then
5428 : // temp1 - 1
5429 : // else
5430 : // temp1
5431 : // else
5432 : // if input == 0 then
5433 : // input
5434 : // else
5435 : // if input <= -2^52 then
5436 : // input
5437 : // else
5438 : // let temp1 = -0 - input in
5439 : // let temp2 = (2^52 + temp1) - 2^52 in
5440 : // let temp3 = (if temp1 < temp2 then temp2 - 1 else temp2) in
5441 : // -0 - temp3
5442 : //
5443 : // Note: We do not use the Diamond helper class here, because it really hurts
5444 : // readability with nested diamonds.
5445 :
5446 0 : auto if_not_positive = __ MakeDeferredLabel();
5447 0 : auto if_greater_than_two_52 = __ MakeDeferredLabel();
5448 0 : auto if_less_than_minus_two_52 = __ MakeDeferredLabel();
5449 0 : auto if_zero = __ MakeDeferredLabel();
5450 0 : auto done_temp3 = __ MakeLabel(MachineRepresentation::kFloat64);
5451 0 : auto done = __ MakeLabel(MachineRepresentation::kFloat64);
5452 :
5453 0 : Node* const zero = __ Float64Constant(0.0);
5454 0 : Node* const two_52 = __ Float64Constant(4503599627370496.0E0);
5455 0 : Node* const one = __ Float64Constant(1.0);
5456 :
5457 0 : Node* check0 = __ Float64LessThan(zero, input);
5458 0 : __ GotoIfNot(check0, &if_not_positive);
5459 : {
5460 0 : Node* check1 = __ Float64LessThanOrEqual(two_52, input);
5461 0 : __ GotoIf(check1, &if_greater_than_two_52);
5462 : {
5463 0 : Node* temp1 = __ Float64Sub(__ Float64Add(two_52, input), two_52);
5464 0 : __ GotoIfNot(__ Float64LessThan(input, temp1), &done, temp1);
5465 0 : __ Goto(&done, __ Float64Sub(temp1, one));
5466 : }
5467 :
5468 : __ Bind(&if_greater_than_two_52);
5469 : __ Goto(&done, input);
5470 : }
5471 :
5472 : __ Bind(&if_not_positive);
5473 : {
5474 0 : Node* check1 = __ Float64Equal(input, zero);
5475 0 : __ GotoIf(check1, &if_zero);
5476 :
5477 0 : Node* const minus_two_52 = __ Float64Constant(-4503599627370496.0E0);
5478 0 : Node* check2 = __ Float64LessThanOrEqual(input, minus_two_52);
5479 0 : __ GotoIf(check2, &if_less_than_minus_two_52);
5480 :
5481 : {
5482 0 : Node* const minus_zero = __ Float64Constant(-0.0);
5483 0 : Node* temp1 = __ Float64Sub(minus_zero, input);
5484 0 : Node* temp2 = __ Float64Sub(__ Float64Add(two_52, temp1), two_52);
5485 0 : Node* check3 = __ Float64LessThan(temp1, temp2);
5486 0 : __ GotoIfNot(check3, &done_temp3, temp2);
5487 0 : __ Goto(&done_temp3, __ Float64Sub(temp2, one));
5488 :
5489 : __ Bind(&done_temp3);
5490 : Node* temp3 = done_temp3.PhiAt(0);
5491 0 : __ Goto(&done, __ Float64Sub(minus_zero, temp3));
5492 : }
5493 : __ Bind(&if_less_than_minus_two_52);
5494 : __ Goto(&done, input);
5495 :
5496 : __ Bind(&if_zero);
5497 : __ Goto(&done, input);
5498 : }
5499 : __ Bind(&done);
5500 : return done.PhiAt(0);
5501 : }
5502 :
5503 6914 : Maybe<Node*> EffectControlLinearizer::LowerFloat64RoundTruncate(Node* node) {
5504 : // Nothing to be done if a fast hardware instruction is available.
5505 6914 : if (machine()->Float64RoundTruncate().IsSupported()) {
5506 : return Nothing<Node*>();
5507 : }
5508 :
5509 : Node* const input = node->InputAt(0);
5510 0 : return Just(BuildFloat64RoundTruncate(input));
5511 : }
5512 :
5513 176 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntry(Node* node) {
5514 176 : Node* table = NodeProperties::GetValueInput(node, 0);
5515 176 : Node* key = NodeProperties::GetValueInput(node, 1);
5516 :
5517 : {
5518 : Callable const callable =
5519 176 : Builtins::CallableFor(isolate(), Builtins::kFindOrderedHashMapEntry);
5520 176 : Operator::Properties const properties = node->op()->properties();
5521 : CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
5522 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5523 176 : graph()->zone(), callable.descriptor(),
5524 176 : callable.descriptor().GetStackParameterCount(), flags, properties);
5525 528 : return __ Call(call_descriptor, __ HeapConstant(callable.code()), table,
5526 352 : key, __ NoContextConstant());
5527 : }
5528 : }
5529 :
5530 16 : Node* EffectControlLinearizer::ComputeUnseededHash(Node* value) {
5531 : // See v8::internal::ComputeUnseededHash()
5532 16 : value = __ Int32Add(__ Word32Xor(value, __ Int32Constant(0xFFFFFFFF)),
5533 16 : __ Word32Shl(value, __ Int32Constant(15)));
5534 16 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(12)));
5535 16 : value = __ Int32Add(value, __ Word32Shl(value, __ Int32Constant(2)));
5536 16 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(4)));
5537 16 : value = __ Int32Mul(value, __ Int32Constant(2057));
5538 16 : value = __ Word32Xor(value, __ Word32Shr(value, __ Int32Constant(16)));
5539 16 : value = __ Word32And(value, __ Int32Constant(0x3FFFFFFF));
5540 16 : return value;
5541 : }
5542 :
5543 16 : Node* EffectControlLinearizer::LowerFindOrderedHashMapEntryForInt32Key(
5544 : Node* node) {
5545 16 : Node* table = NodeProperties::GetValueInput(node, 0);
5546 16 : Node* key = NodeProperties::GetValueInput(node, 1);
5547 :
5548 : // Compute the integer hash code.
5549 16 : Node* hash = ChangeUint32ToUintPtr(ComputeUnseededHash(key));
5550 :
5551 16 : Node* number_of_buckets = ChangeSmiToIntPtr(__ LoadField(
5552 32 : AccessBuilder::ForOrderedHashMapOrSetNumberOfBuckets(), table));
5553 16 : hash = __ WordAnd(hash, __ IntSub(number_of_buckets, __ IntPtrConstant(1)));
5554 16 : Node* first_entry = ChangeSmiToIntPtr(__ Load(
5555 : MachineType::TaggedSigned(), table,
5556 : __ IntAdd(__ WordShl(hash, __ IntPtrConstant(kTaggedSizeLog2)),
5557 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5558 16 : kHeapObjectTag))));
5559 :
5560 32 : auto loop = __ MakeLoopLabel(MachineType::PointerRepresentation());
5561 32 : auto done = __ MakeLabel(MachineType::PointerRepresentation());
5562 : __ Goto(&loop, first_entry);
5563 : __ Bind(&loop);
5564 : {
5565 : Node* entry = loop.PhiAt(0);
5566 : Node* check =
5567 16 : __ WordEqual(entry, __ IntPtrConstant(OrderedHashMap::kNotFound));
5568 16 : __ GotoIf(check, &done, entry);
5569 16 : entry = __ IntAdd(
5570 : __ IntMul(entry, __ IntPtrConstant(OrderedHashMap::kEntrySize)),
5571 16 : number_of_buckets);
5572 :
5573 16 : Node* candidate_key = __ Load(
5574 : MachineType::AnyTagged(), table,
5575 : __ IntAdd(__ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5576 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() -
5577 16 : kHeapObjectTag)));
5578 :
5579 16 : auto if_match = __ MakeLabel();
5580 16 : auto if_notmatch = __ MakeLabel();
5581 16 : auto if_notsmi = __ MakeDeferredLabel();
5582 16 : __ GotoIfNot(ObjectIsSmi(candidate_key), &if_notsmi);
5583 16 : __ Branch(__ Word32Equal(ChangeSmiToInt32(candidate_key), key), &if_match,
5584 16 : &if_notmatch);
5585 :
5586 : __ Bind(&if_notsmi);
5587 32 : __ GotoIfNot(
5588 32 : __ WordEqual(__ LoadField(AccessBuilder::ForMap(), candidate_key),
5589 : __ HeapNumberMapConstant()),
5590 16 : &if_notmatch);
5591 32 : __ Branch(__ Float64Equal(__ LoadField(AccessBuilder::ForHeapNumberValue(),
5592 : candidate_key),
5593 : __ ChangeInt32ToFloat64(key)),
5594 16 : &if_match, &if_notmatch);
5595 :
5596 : __ Bind(&if_match);
5597 : __ Goto(&done, entry);
5598 :
5599 : __ Bind(&if_notmatch);
5600 : {
5601 16 : Node* next_entry = ChangeSmiToIntPtr(__ Load(
5602 : MachineType::TaggedSigned(), table,
5603 : __ IntAdd(
5604 : __ WordShl(entry, __ IntPtrConstant(kTaggedSizeLog2)),
5605 : __ IntPtrConstant(OrderedHashMap::HashTableStartOffset() +
5606 : OrderedHashMap::kChainOffset * kTaggedSize -
5607 16 : kHeapObjectTag))));
5608 : __ Goto(&loop, next_entry);
5609 : }
5610 : }
5611 :
5612 : __ Bind(&done);
5613 16 : return done.PhiAt(0);
5614 : }
5615 :
5616 13 : Node* EffectControlLinearizer::LowerDateNow(Node* node) {
5617 13 : Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
5618 : Runtime::FunctionId id = Runtime::kDateCurrentTime;
5619 13 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
5620 13 : graph()->zone(), id, 0, properties, CallDescriptor::kNoFlags);
5621 13 : return __ Call(call_descriptor, __ CEntryStubConstant(1),
5622 : __ ExternalConstant(ExternalReference::Create(id)),
5623 13 : __ Int32Constant(0), __ NoContextConstant());
5624 : }
5625 :
5626 : #undef __
5627 :
5628 0 : Factory* EffectControlLinearizer::factory() const {
5629 0 : return isolate()->factory();
5630 : }
5631 :
5632 0 : Isolate* EffectControlLinearizer::isolate() const {
5633 0 : return jsgraph()->isolate();
5634 : }
5635 :
5636 : } // namespace compiler
5637 : } // namespace internal
5638 182994 : } // namespace v8
|
