Line data Source code
1 : // Copyright 2015 the V8 project authors. All rights reserved.
2 : // Use of this source code is governed by a BSD-style license that can be
3 : // found in the LICENSE file.
4 :
5 : #include "src/compiler/wasm-compiler.h"
6 :
7 : #include <memory>
8 :
9 : #include "src/assembler-inl.h"
10 : #include "src/assembler.h"
11 : #include "src/base/optional.h"
12 : #include "src/base/platform/elapsed-timer.h"
13 : #include "src/base/platform/platform.h"
14 : #include "src/base/v8-fallthrough.h"
15 : #include "src/builtins/builtins.h"
16 : #include "src/code-factory.h"
17 : #include "src/compiler/backend/code-generator.h"
18 : #include "src/compiler/backend/instruction-selector.h"
19 : #include "src/compiler/common-operator.h"
20 : #include "src/compiler/compiler-source-position-table.h"
21 : #include "src/compiler/diamond.h"
22 : #include "src/compiler/graph-visualizer.h"
23 : #include "src/compiler/graph.h"
24 : #include "src/compiler/int64-lowering.h"
25 : #include "src/compiler/js-graph.h"
26 : #include "src/compiler/js-operator.h"
27 : #include "src/compiler/linkage.h"
28 : #include "src/compiler/machine-operator.h"
29 : #include "src/compiler/node-matchers.h"
30 : #include "src/compiler/node-origin-table.h"
31 : #include "src/compiler/pipeline.h"
32 : #include "src/compiler/simd-scalar-lowering.h"
33 : #include "src/compiler/zone-stats.h"
34 : #include "src/counters.h"
35 : #include "src/heap/factory.h"
36 : #include "src/interface-descriptors.h"
37 : #include "src/isolate-inl.h"
38 : #include "src/log.h"
39 : #include "src/objects/heap-number.h"
40 : #include "src/optimized-compilation-info.h"
41 : #include "src/tracing/trace-event.h"
42 : #include "src/trap-handler/trap-handler.h"
43 : #include "src/wasm/function-body-decoder.h"
44 : #include "src/wasm/function-compiler.h"
45 : #include "src/wasm/graph-builder-interface.h"
46 : #include "src/wasm/jump-table-assembler.h"
47 : #include "src/wasm/memory-tracing.h"
48 : #include "src/wasm/object-access.h"
49 : #include "src/wasm/wasm-code-manager.h"
50 : #include "src/wasm/wasm-limits.h"
51 : #include "src/wasm/wasm-linkage.h"
52 : #include "src/wasm/wasm-module.h"
53 : #include "src/wasm/wasm-objects-inl.h"
54 : #include "src/wasm/wasm-opcodes.h"
55 : #include "src/wasm/wasm-text.h"
56 :
57 : namespace v8 {
58 : namespace internal {
59 : namespace compiler {
60 :
61 : namespace {
62 :
63 : // TODO(titzer): pull WASM_64 up to a common header.
64 : #if !V8_TARGET_ARCH_32_BIT || V8_TARGET_ARCH_X64
65 : #define WASM_64 1
66 : #else
67 : #define WASM_64 0
68 : #endif
69 :
70 : #define FATAL_UNSUPPORTED_OPCODE(opcode) \
71 : FATAL("Unsupported opcode 0x%x:%s", (opcode), \
72 : wasm::WasmOpcodes::OpcodeName(opcode));
73 :
74 : MachineType assert_size(int expected_size, MachineType type) {
75 : DCHECK_EQ(expected_size, ElementSizeInBytes(type.representation()));
76 : return type;
77 : }
78 :
79 : #define WASM_INSTANCE_OBJECT_SIZE(name) \
80 : (WasmInstanceObject::k##name##OffsetEnd - \
81 : WasmInstanceObject::k##name##Offset + 1) // NOLINT(whitespace/indent)
82 :
83 : #define WASM_INSTANCE_OBJECT_OFFSET(name) \
84 : wasm::ObjectAccess::ToTagged(WasmInstanceObject::k##name##Offset)
85 :
86 : #define LOAD_RAW(base_pointer, byte_offset, type) \
87 : SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type), base_pointer, \
88 : mcgraph()->Int32Constant(byte_offset), Effect(), \
89 : Control()))
90 :
91 : #define LOAD_INSTANCE_FIELD(name, type) \
92 : LOAD_RAW(instance_node_.get(), WASM_INSTANCE_OBJECT_OFFSET(name), \
93 : assert_size(WASM_INSTANCE_OBJECT_SIZE(name), type))
94 :
95 : #define LOAD_TAGGED_POINTER(base_pointer, byte_offset) \
96 : LOAD_RAW(base_pointer, byte_offset, MachineType::TaggedPointer())
97 :
98 : #define LOAD_TAGGED_ANY(base_pointer, byte_offset) \
99 : LOAD_RAW(base_pointer, byte_offset, MachineType::AnyTagged())
100 :
101 : #define LOAD_FIXED_ARRAY_SLOT(array_node, index, type) \
102 : LOAD_RAW(array_node, \
103 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index), type)
104 :
105 : #define LOAD_FIXED_ARRAY_SLOT_SMI(array_node, index) \
106 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedSigned())
107 :
108 : #define LOAD_FIXED_ARRAY_SLOT_PTR(array_node, index) \
109 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::TaggedPointer())
110 :
111 : #define LOAD_FIXED_ARRAY_SLOT_ANY(array_node, index) \
112 : LOAD_FIXED_ARRAY_SLOT(array_node, index, MachineType::AnyTagged())
113 :
114 : // This can be used to store tagged Smi values only.
115 : #define STORE_FIXED_ARRAY_SLOT_SMI(array_node, index, value) \
116 : SetEffect(graph()->NewNode( \
117 : mcgraph()->machine()->Store(StoreRepresentation( \
118 : MachineRepresentation::kTaggedSigned, kNoWriteBarrier)), \
119 : array_node, \
120 : mcgraph()->Int32Constant( \
121 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
122 : value, Effect(), Control()))
123 :
124 : // This can be used to store any tagged (Smi and HeapObject) value.
125 : #define STORE_FIXED_ARRAY_SLOT_ANY(array_node, index, value) \
126 : SetEffect(graph()->NewNode( \
127 : mcgraph()->machine()->Store(StoreRepresentation( \
128 : MachineRepresentation::kTagged, kFullWriteBarrier)), \
129 : array_node, \
130 : mcgraph()->Int32Constant( \
131 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(index)), \
132 : value, Effect(), Control()))
133 :
134 1404362 : void MergeControlToEnd(MachineGraph* mcgraph, Node* node) {
135 702181 : Graph* g = mcgraph->graph();
136 702181 : if (g->end()) {
137 54742 : NodeProperties::MergeControlToEnd(g, mcgraph->common(), node);
138 : } else {
139 647439 : g->SetEnd(g->NewNode(mcgraph->common()->End(1), node));
140 : }
141 702101 : }
142 :
143 : bool ContainsSimd(wasm::FunctionSig* sig) {
144 1674360 : for (auto type : sig->all()) {
145 837159 : if (type == wasm::kWasmS128) return true;
146 : }
147 : return false;
148 : }
149 :
150 : bool ContainsInt64(wasm::FunctionSig* sig) {
151 22524 : for (auto type : sig->all()) {
152 11280 : if (type == wasm::kWasmI64) return true;
153 : }
154 : return false;
155 : }
156 : } // namespace
157 :
158 653067 : WasmGraphBuilder::WasmGraphBuilder(
159 : wasm::CompilationEnv* env, Zone* zone, MachineGraph* mcgraph,
160 : wasm::FunctionSig* sig,
161 : compiler::SourcePositionTable* source_position_table)
162 : : zone_(zone),
163 : mcgraph_(mcgraph),
164 : env_(env),
165 : cur_buffer_(def_buffer_),
166 : cur_bufsize_(kDefaultBufferSize),
167 : has_simd_(ContainsSimd(sig)),
168 : untrusted_code_mitigations_(FLAG_untrusted_code_mitigations),
169 : sig_(sig),
170 1306134 : source_position_table_(source_position_table) {
171 : DCHECK_IMPLIES(use_trap_handler(), trap_handler::IsTrapHandlerEnabled());
172 : DCHECK_NOT_NULL(mcgraph_);
173 653067 : }
174 :
175 0 : Node* WasmGraphBuilder::Error() { return mcgraph()->Dead(); }
176 :
177 1306380 : Node* WasmGraphBuilder::Start(unsigned params) {
178 1306380 : Node* start = graph()->NewNode(mcgraph()->common()->Start(params));
179 : graph()->SetStart(start);
180 653116 : return start;
181 : }
182 :
183 849643 : Node* WasmGraphBuilder::Param(unsigned index) {
184 : return graph()->NewNode(mcgraph()->common()->Parameter(index),
185 1699323 : graph()->start());
186 : }
187 :
188 9091 : Node* WasmGraphBuilder::Loop(Node* entry) {
189 18180 : return graph()->NewNode(mcgraph()->common()->Loop(1), entry);
190 : }
191 :
192 18181 : Node* WasmGraphBuilder::TerminateLoop(Node* effect, Node* control) {
193 : Node* terminate =
194 9091 : graph()->NewNode(mcgraph()->common()->Terminate(), effect, control);
195 9090 : MergeControlToEnd(mcgraph(), terminate);
196 9090 : return terminate;
197 : }
198 :
199 1853 : Node* WasmGraphBuilder::TerminateThrow(Node* effect, Node* control) {
200 : Node* terminate =
201 927 : graph()->NewNode(mcgraph()->common()->Throw(), effect, control);
202 926 : MergeControlToEnd(mcgraph(), terminate);
203 926 : return terminate;
204 : }
205 :
206 2412941 : bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) {
207 2639633 : return phi && IrOpcode::IsPhiOpcode(phi->opcode()) &&
208 1433162 : NodeProperties::GetControlInput(phi) == merge;
209 : }
210 :
211 1271 : bool WasmGraphBuilder::ThrowsException(Node* node, Node** if_success,
212 1295 : Node** if_exception) {
213 1271 : if (node->op()->HasProperty(compiler::Operator::kNoThrow)) {
214 : return false;
215 : }
216 :
217 1295 : *if_success = graph()->NewNode(mcgraph()->common()->IfSuccess(), node);
218 : *if_exception =
219 1295 : graph()->NewNode(mcgraph()->common()->IfException(), node, node);
220 :
221 648 : return true;
222 : }
223 :
224 427059 : void WasmGraphBuilder::AppendToMerge(Node* merge, Node* from) {
225 : DCHECK(IrOpcode::IsMergeOpcode(merge->opcode()));
226 213529 : merge->AppendInput(mcgraph()->zone(), from);
227 : int new_size = merge->InputCount();
228 : NodeProperties::ChangeOp(
229 213530 : merge, mcgraph()->common()->ResizeMergeOrPhi(merge->op(), new_size));
230 213530 : }
231 :
232 323543 : void WasmGraphBuilder::AppendToPhi(Node* phi, Node* from) {
233 : DCHECK(IrOpcode::IsPhiOpcode(phi->opcode()));
234 : int new_size = phi->InputCount();
235 215694 : phi->InsertInput(mcgraph()->zone(), phi->InputCount() - 1, from);
236 : NodeProperties::ChangeOp(
237 107849 : phi, mcgraph()->common()->ResizeMergeOrPhi(phi->op(), new_size));
238 107847 : }
239 :
240 70572 : Node* WasmGraphBuilder::Merge(unsigned count, Node** controls) {
241 105858 : return graph()->NewNode(mcgraph()->common()->Merge(count), count, controls);
242 : }
243 :
244 98091 : Node* WasmGraphBuilder::Phi(wasm::ValueType type, unsigned count, Node** vals,
245 98089 : Node* control) {
246 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
247 98091 : Node** buf = Realloc(vals, count, count + 1);
248 98090 : buf[count] = control;
249 : return graph()->NewNode(
250 : mcgraph()->common()->Phi(wasm::ValueTypes::MachineRepresentationFor(type),
251 : count),
252 294268 : count + 1, buf);
253 : }
254 :
255 31184 : Node* WasmGraphBuilder::EffectPhi(unsigned count, Node** effects,
256 31184 : Node* control) {
257 : DCHECK(IrOpcode::IsMergeOpcode(control->opcode()));
258 31184 : Node** buf = Realloc(effects, count, count + 1);
259 31184 : buf[count] = control;
260 : return graph()->NewNode(mcgraph()->common()->EffectPhi(count), count + 1,
261 93551 : buf);
262 : }
263 :
264 1160 : Node* WasmGraphBuilder::RefNull() {
265 696 : return LOAD_INSTANCE_FIELD(NullValue, MachineType::TaggedPointer());
266 : }
267 :
268 4356 : Node* WasmGraphBuilder::NoContextConstant() {
269 : // TODO(titzer): avoiding a dependency on JSGraph here. Refactor.
270 4356 : return mcgraph()->IntPtrConstant(0);
271 : }
272 :
273 1463 : Node* WasmGraphBuilder::Uint32Constant(uint32_t value) {
274 0 : return mcgraph()->Uint32Constant(value);
275 : }
276 :
277 1685366 : Node* WasmGraphBuilder::Int32Constant(int32_t value) {
278 1685678 : return mcgraph()->Int32Constant(value);
279 : }
280 :
281 27500 : Node* WasmGraphBuilder::Int64Constant(int64_t value) {
282 27500 : return mcgraph()->Int64Constant(value);
283 : }
284 :
285 37817 : Node* WasmGraphBuilder::IntPtrConstant(intptr_t value) {
286 37817 : return mcgraph()->IntPtrConstant(value);
287 : }
288 :
289 22427 : void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position,
290 171317 : Node** effect, Node** control) {
291 : DCHECK_NOT_NULL(env_); // Wrappers don't get stack checks.
292 22427 : if (FLAG_wasm_no_stack_checks || !env_->runtime_exception_support) {
293 3056 : return;
294 : }
295 19371 : if (effect == nullptr) effect = effect_;
296 19371 : if (control == nullptr) control = control_;
297 :
298 : // This instruction sequence is matched in the instruction selector to
299 : // load the stack pointer directly on some platforms. Hence, when modifying
300 : // please also fix WasmStackCheckMatcher in node-matchers.h
301 :
302 : Node* limit_address = graph()->NewNode(
303 : mcgraph()->machine()->Load(MachineType::Pointer()), instance_node_.get(),
304 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(StackLimitAddress)),
305 58116 : *effect, *control);
306 : Node* limit = graph()->NewNode(
307 : mcgraph()->machine()->Load(MachineType::Pointer()), limit_address,
308 58136 : mcgraph()->IntPtrConstant(0), limit_address, *control);
309 19378 : *effect = limit;
310 19378 : Node* pointer = graph()->NewNode(mcgraph()->machine()->LoadStackPointer());
311 :
312 : Node* check =
313 19377 : graph()->NewNode(mcgraph()->machine()->UintLessThan(), limit, pointer);
314 :
315 38754 : Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue);
316 19378 : stack_check.Chain(*control);
317 :
318 19371 : if (stack_check_call_operator_ == nullptr) {
319 : // Build and cache the stack check call operator and the constant
320 : // representing the stack check code.
321 : auto call_descriptor = Linkage::GetStubCallDescriptor(
322 : mcgraph()->zone(), // zone
323 : NoContextDescriptor{}, // descriptor
324 : 0, // stack parameter count
325 : CallDescriptor::kNoFlags, // flags
326 : Operator::kNoProperties, // properties
327 23784 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
328 : // A direct call to a wasm runtime stub defined in this module.
329 : // Just encode the stub index. This will be patched at relocation.
330 : stack_check_code_node_.set(mcgraph()->RelocatableIntPtrConstant(
331 11892 : wasm::WasmCode::kWasmStackGuard, RelocInfo::WASM_STUB_CALL));
332 11899 : stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor);
333 : }
334 :
335 : Node* call = graph()->NewNode(stack_check_call_operator_.get(),
336 : stack_check_code_node_.get(), *effect,
337 19379 : stack_check.if_false);
338 :
339 19380 : SetSourcePosition(call, position);
340 :
341 19379 : Node* ephi = stack_check.EffectPhi(*effect, call);
342 :
343 19377 : *control = stack_check.merge;
344 19377 : *effect = ephi;
345 : }
346 :
347 496799 : void WasmGraphBuilder::PatchInStackCheckIfNeeded() {
348 983153 : if (!needs_stack_check_) return;
349 :
350 13346 : Node* start = graph()->start();
351 : // Place a stack check which uses a dummy node as control and effect.
352 13346 : Node* dummy = graph()->NewNode(mcgraph()->common()->Dead());
353 13351 : Node* control = dummy;
354 13351 : Node* effect = dummy;
355 : // The function-prologue stack check is associated with position 0, which
356 : // is never a position of any instruction in the function.
357 13351 : StackCheck(0, &effect, &control);
358 :
359 : // In testing, no steck checks were emitted. Nothing to rewire then.
360 13353 : if (effect == dummy) return;
361 :
362 : // Now patch all control uses of {start} to use {control} and all effect uses
363 : // to use {effect} instead. Then rewire the dummy node to use start instead.
364 10452 : NodeProperties::ReplaceUses(start, start, effect, control);
365 10453 : NodeProperties::ReplaceUses(dummy, nullptr, start, start);
366 : }
367 :
368 804958 : Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left, Node* right,
369 804958 : wasm::WasmCodePosition position) {
370 : const Operator* op;
371 804958 : MachineOperatorBuilder* m = mcgraph()->machine();
372 804958 : switch (opcode) {
373 : case wasm::kExprI32Add:
374 105960 : op = m->Int32Add();
375 105960 : break;
376 : case wasm::kExprI32Sub:
377 27910 : op = m->Int32Sub();
378 27908 : break;
379 : case wasm::kExprI32Mul:
380 18030 : op = m->Int32Mul();
381 18030 : break;
382 : case wasm::kExprI32DivS:
383 14180 : return BuildI32DivS(left, right, position);
384 : case wasm::kExprI32DivU:
385 14124 : return BuildI32DivU(left, right, position);
386 : case wasm::kExprI32RemS:
387 14104 : return BuildI32RemS(left, right, position);
388 : case wasm::kExprI32RemU:
389 14084 : return BuildI32RemU(left, right, position);
390 : case wasm::kExprI32And:
391 111339 : op = m->Word32And();
392 111339 : break;
393 : case wasm::kExprI32Ior:
394 19051 : op = m->Word32Or();
395 19051 : break;
396 : case wasm::kExprI32Xor:
397 14367 : op = m->Word32Xor();
398 14367 : break;
399 : case wasm::kExprI32Shl:
400 25741 : op = m->Word32Shl();
401 25741 : right = MaskShiftCount32(right);
402 25741 : break;
403 : case wasm::kExprI32ShrU:
404 29468 : op = m->Word32Shr();
405 29468 : right = MaskShiftCount32(right);
406 29468 : break;
407 : case wasm::kExprI32ShrS:
408 15615 : op = m->Word32Sar();
409 15615 : right = MaskShiftCount32(right);
410 15615 : break;
411 : case wasm::kExprI32Ror:
412 26944 : op = m->Word32Ror();
413 26944 : right = MaskShiftCount32(right);
414 26944 : break;
415 : case wasm::kExprI32Rol:
416 13472 : right = MaskShiftCount32(right);
417 13472 : return BuildI32Rol(left, right);
418 : case wasm::kExprI32Eq:
419 86387 : op = m->Word32Equal();
420 86387 : break;
421 : case wasm::kExprI32Ne:
422 38200 : return Invert(Binop(wasm::kExprI32Eq, left, right));
423 : case wasm::kExprI32LtS:
424 15470 : op = m->Int32LessThan();
425 15470 : break;
426 : case wasm::kExprI32LeS:
427 14337 : op = m->Int32LessThanOrEqual();
428 14337 : break;
429 : case wasm::kExprI32LtU:
430 17819 : op = m->Uint32LessThan();
431 17819 : break;
432 : case wasm::kExprI32LeU:
433 13632 : op = m->Uint32LessThanOrEqual();
434 13632 : break;
435 : case wasm::kExprI32GtS:
436 15677 : op = m->Int32LessThan();
437 : std::swap(left, right);
438 : break;
439 : case wasm::kExprI32GeS:
440 14060 : op = m->Int32LessThanOrEqual();
441 : std::swap(left, right);
442 : break;
443 : case wasm::kExprI32GtU:
444 15198 : op = m->Uint32LessThan();
445 : std::swap(left, right);
446 : break;
447 : case wasm::kExprI32GeU:
448 13632 : op = m->Uint32LessThanOrEqual();
449 : std::swap(left, right);
450 : break;
451 : case wasm::kExprI64And:
452 167 : op = m->Word64And();
453 167 : break;
454 : case wasm::kExprI64Add:
455 676 : op = m->Int64Add();
456 676 : break;
457 : case wasm::kExprI64Sub:
458 831 : op = m->Int64Sub();
459 831 : break;
460 : case wasm::kExprI64Mul:
461 807 : op = m->Int64Mul();
462 807 : break;
463 : case wasm::kExprI64DivS:
464 720 : return BuildI64DivS(left, right, position);
465 : case wasm::kExprI64DivU:
466 660 : return BuildI64DivU(left, right, position);
467 : case wasm::kExprI64RemS:
468 644 : return BuildI64RemS(left, right, position);
469 : case wasm::kExprI64RemU:
470 644 : return BuildI64RemU(left, right, position);
471 : case wasm::kExprI64Ior:
472 12706 : op = m->Word64Or();
473 12670 : break;
474 : case wasm::kExprI64Xor:
475 56 : op = m->Word64Xor();
476 56 : break;
477 : case wasm::kExprI64Shl:
478 13312 : op = m->Word64Shl();
479 13316 : right = MaskShiftCount64(right);
480 13318 : break;
481 : case wasm::kExprI64ShrU:
482 684 : op = m->Word64Shr();
483 684 : right = MaskShiftCount64(right);
484 684 : break;
485 : case wasm::kExprI64ShrS:
486 628 : op = m->Word64Sar();
487 628 : right = MaskShiftCount64(right);
488 628 : break;
489 : case wasm::kExprI64Eq:
490 287 : op = m->Word64Equal();
491 287 : break;
492 : case wasm::kExprI64Ne:
493 68 : return Invert(Binop(wasm::kExprI64Eq, left, right));
494 : case wasm::kExprI64LtS:
495 56 : op = m->Int64LessThan();
496 56 : break;
497 : case wasm::kExprI64LeS:
498 32 : op = m->Int64LessThanOrEqual();
499 32 : break;
500 : case wasm::kExprI64LtU:
501 40 : op = m->Uint64LessThan();
502 40 : break;
503 : case wasm::kExprI64LeU:
504 48 : op = m->Uint64LessThanOrEqual();
505 48 : break;
506 : case wasm::kExprI64GtS:
507 40 : op = m->Int64LessThan();
508 : std::swap(left, right);
509 : break;
510 : case wasm::kExprI64GeS:
511 32 : op = m->Int64LessThanOrEqual();
512 : std::swap(left, right);
513 : break;
514 : case wasm::kExprI64GtU:
515 40 : op = m->Uint64LessThan();
516 : std::swap(left, right);
517 : break;
518 : case wasm::kExprI64GeU:
519 32 : op = m->Uint64LessThanOrEqual();
520 : std::swap(left, right);
521 : break;
522 : case wasm::kExprI64Ror:
523 120 : op = m->Word64Ror();
524 120 : right = MaskShiftCount64(right);
525 120 : break;
526 : case wasm::kExprI64Rol:
527 60 : return BuildI64Rol(left, right);
528 : case wasm::kExprF32CopySign:
529 32 : return BuildF32CopySign(left, right);
530 : case wasm::kExprF64CopySign:
531 32 : return BuildF64CopySign(left, right);
532 : case wasm::kExprF32Add:
533 461 : op = m->Float32Add();
534 461 : break;
535 : case wasm::kExprF32Sub:
536 305 : op = m->Float32Sub();
537 305 : break;
538 : case wasm::kExprF32Mul:
539 425 : op = m->Float32Mul();
540 425 : break;
541 : case wasm::kExprF32Div:
542 349 : op = m->Float32Div();
543 349 : break;
544 : case wasm::kExprF32Eq:
545 821 : op = m->Float32Equal();
546 821 : break;
547 : case wasm::kExprF32Ne:
548 717 : return Invert(Binop(wasm::kExprF32Eq, left, right));
549 : case wasm::kExprF32Lt:
550 281 : op = m->Float32LessThan();
551 281 : break;
552 : case wasm::kExprF32Ge:
553 105 : op = m->Float32LessThanOrEqual();
554 : std::swap(left, right);
555 : break;
556 : case wasm::kExprF32Gt:
557 257 : op = m->Float32LessThan();
558 : std::swap(left, right);
559 : break;
560 : case wasm::kExprF32Le:
561 121 : op = m->Float32LessThanOrEqual();
562 121 : break;
563 : case wasm::kExprF64Add:
564 4173 : op = m->Float64Add();
565 4173 : break;
566 : case wasm::kExprF64Sub:
567 2902 : op = m->Float64Sub();
568 2902 : break;
569 : case wasm::kExprF64Mul:
570 6488 : op = m->Float64Mul();
571 6488 : break;
572 : case wasm::kExprF64Div:
573 639 : op = m->Float64Div();
574 639 : break;
575 : case wasm::kExprF64Eq:
576 727 : op = m->Float64Equal();
577 726 : break;
578 : case wasm::kExprF64Ne:
579 409 : return Invert(Binop(wasm::kExprF64Eq, left, right));
580 : case wasm::kExprF64Lt:
581 742 : op = m->Float64LessThan();
582 742 : break;
583 : case wasm::kExprF64Le:
584 303 : op = m->Float64LessThanOrEqual();
585 303 : break;
586 : case wasm::kExprF64Gt:
587 696 : op = m->Float64LessThan();
588 : std::swap(left, right);
589 : break;
590 : case wasm::kExprF64Ge:
591 301 : op = m->Float64LessThanOrEqual();
592 : std::swap(left, right);
593 : break;
594 : case wasm::kExprF32Min:
595 70 : op = m->Float32Min();
596 70 : break;
597 : case wasm::kExprF64Min:
598 80 : op = m->Float64Min();
599 80 : break;
600 : case wasm::kExprF32Max:
601 70 : op = m->Float32Max();
602 70 : break;
603 : case wasm::kExprF64Max:
604 72 : op = m->Float64Max();
605 72 : break;
606 : case wasm::kExprF64Pow:
607 9 : return BuildF64Pow(left, right);
608 : case wasm::kExprF64Atan2:
609 17 : op = m->Float64Atan2();
610 17 : break;
611 : case wasm::kExprF64Mod:
612 20 : return BuildF64Mod(left, right);
613 : case wasm::kExprI32AsmjsDivS:
614 385 : return BuildI32AsmjsDivS(left, right);
615 : case wasm::kExprI32AsmjsDivU:
616 137 : return BuildI32AsmjsDivU(left, right);
617 : case wasm::kExprI32AsmjsRemS:
618 421 : return BuildI32AsmjsRemS(left, right);
619 : case wasm::kExprI32AsmjsRemU:
620 186 : return BuildI32AsmjsRemU(left, right);
621 : case wasm::kExprI32AsmjsStoreMem8:
622 3041 : return BuildAsmjsStoreMem(MachineType::Int8(), left, right);
623 : case wasm::kExprI32AsmjsStoreMem16:
624 1385 : return BuildAsmjsStoreMem(MachineType::Int16(), left, right);
625 : case wasm::kExprI32AsmjsStoreMem:
626 31530 : return BuildAsmjsStoreMem(MachineType::Int32(), left, right);
627 : case wasm::kExprF32AsmjsStoreMem:
628 3299 : return BuildAsmjsStoreMem(MachineType::Float32(), left, right);
629 : case wasm::kExprF64AsmjsStoreMem:
630 759 : return BuildAsmjsStoreMem(MachineType::Float64(), left, right);
631 : default:
632 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
633 : }
634 651584 : return graph()->NewNode(op, left, right);
635 : }
636 :
637 149303 : Node* WasmGraphBuilder::Unop(wasm::WasmOpcode opcode, Node* input,
638 199168 : wasm::WasmCodePosition position) {
639 : const Operator* op;
640 149303 : MachineOperatorBuilder* m = mcgraph()->machine();
641 149303 : switch (opcode) {
642 : case wasm::kExprI32Eqz:
643 49713 : op = m->Word32Equal();
644 99426 : return graph()->NewNode(op, input, mcgraph()->Int32Constant(0));
645 : case wasm::kExprF32Abs:
646 62 : op = m->Float32Abs();
647 62 : break;
648 : case wasm::kExprF32Neg: {
649 221 : op = m->Float32Neg();
650 221 : break;
651 : }
652 : case wasm::kExprF32Sqrt:
653 174 : op = m->Float32Sqrt();
654 174 : break;
655 : case wasm::kExprF64Abs:
656 87 : op = m->Float64Abs();
657 87 : break;
658 : case wasm::kExprF64Neg: {
659 1051 : op = m->Float64Neg();
660 1051 : break;
661 : }
662 : case wasm::kExprF64Sqrt:
663 256 : op = m->Float64Sqrt();
664 256 : break;
665 : case wasm::kExprI32SConvertF32:
666 : case wasm::kExprI32UConvertF32:
667 : case wasm::kExprI32SConvertF64:
668 : case wasm::kExprI32UConvertF64:
669 : case wasm::kExprI32SConvertSatF64:
670 : case wasm::kExprI32UConvertSatF64:
671 : case wasm::kExprI32SConvertSatF32:
672 : case wasm::kExprI32UConvertSatF32:
673 352 : return BuildIntConvertFloat(input, position, opcode);
674 : case wasm::kExprI32AsmjsSConvertF64:
675 108 : return BuildI32AsmjsSConvertF64(input);
676 : case wasm::kExprI32AsmjsUConvertF64:
677 8 : return BuildI32AsmjsUConvertF64(input);
678 : case wasm::kExprF32ConvertF64:
679 3435 : op = m->TruncateFloat64ToFloat32();
680 3435 : break;
681 : case wasm::kExprF64SConvertI32:
682 592 : op = m->ChangeInt32ToFloat64();
683 592 : break;
684 : case wasm::kExprF64UConvertI32:
685 278 : op = m->ChangeUint32ToFloat64();
686 278 : break;
687 : case wasm::kExprF32SConvertI32:
688 188 : op = m->RoundInt32ToFloat32();
689 188 : break;
690 : case wasm::kExprF32UConvertI32:
691 96 : op = m->RoundUint32ToFloat32();
692 96 : break;
693 : case wasm::kExprI32AsmjsSConvertF32:
694 8 : return BuildI32AsmjsSConvertF32(input);
695 : case wasm::kExprI32AsmjsUConvertF32:
696 8 : return BuildI32AsmjsUConvertF32(input);
697 : case wasm::kExprF64ConvertF32:
698 5544 : op = m->ChangeFloat32ToFloat64();
699 5544 : break;
700 : case wasm::kExprF32ReinterpretI32:
701 184 : op = m->BitcastInt32ToFloat32();
702 184 : break;
703 : case wasm::kExprI32ReinterpretF32:
704 444 : op = m->BitcastFloat32ToInt32();
705 444 : break;
706 : case wasm::kExprI32Clz:
707 300 : op = m->Word32Clz();
708 300 : break;
709 : case wasm::kExprI32Ctz: {
710 340 : if (m->Word32Ctz().IsSupported()) {
711 340 : op = m->Word32Ctz().op();
712 340 : break;
713 0 : } else if (m->Word32ReverseBits().IsSupported()) {
714 0 : Node* reversed = graph()->NewNode(m->Word32ReverseBits().op(), input);
715 0 : Node* result = graph()->NewNode(m->Word32Clz(), reversed);
716 0 : return result;
717 : } else {
718 0 : return BuildI32Ctz(input);
719 : }
720 : }
721 : case wasm::kExprI32Popcnt: {
722 64 : if (m->Word32Popcnt().IsSupported()) {
723 64 : op = m->Word32Popcnt().op();
724 64 : break;
725 : } else {
726 0 : return BuildI32Popcnt(input);
727 : }
728 : }
729 : case wasm::kExprF32Floor: {
730 50 : if (!m->Float32RoundDown().IsSupported()) return BuildF32Floor(input);
731 50 : op = m->Float32RoundDown().op();
732 50 : break;
733 : }
734 : case wasm::kExprF32Ceil: {
735 50 : if (!m->Float32RoundUp().IsSupported()) return BuildF32Ceil(input);
736 50 : op = m->Float32RoundUp().op();
737 50 : break;
738 : }
739 : case wasm::kExprF32Trunc: {
740 204 : if (!m->Float32RoundTruncate().IsSupported()) return BuildF32Trunc(input);
741 204 : op = m->Float32RoundTruncate().op();
742 204 : break;
743 : }
744 : case wasm::kExprF32NearestInt: {
745 24 : if (!m->Float32RoundTiesEven().IsSupported())
746 0 : return BuildF32NearestInt(input);
747 24 : op = m->Float32RoundTiesEven().op();
748 24 : break;
749 : }
750 : case wasm::kExprF64Floor: {
751 98 : if (!m->Float64RoundDown().IsSupported()) return BuildF64Floor(input);
752 98 : op = m->Float64RoundDown().op();
753 98 : break;
754 : }
755 : case wasm::kExprF64Ceil: {
756 66 : if (!m->Float64RoundUp().IsSupported()) return BuildF64Ceil(input);
757 66 : op = m->Float64RoundUp().op();
758 66 : break;
759 : }
760 : case wasm::kExprF64Trunc: {
761 196 : if (!m->Float64RoundTruncate().IsSupported()) return BuildF64Trunc(input);
762 195 : op = m->Float64RoundTruncate().op();
763 195 : break;
764 : }
765 : case wasm::kExprF64NearestInt: {
766 24 : if (!m->Float64RoundTiesEven().IsSupported())
767 0 : return BuildF64NearestInt(input);
768 24 : op = m->Float64RoundTiesEven().op();
769 24 : break;
770 : }
771 : case wasm::kExprF64Acos: {
772 21 : return BuildF64Acos(input);
773 : }
774 : case wasm::kExprF64Asin: {
775 21 : return BuildF64Asin(input);
776 : }
777 : case wasm::kExprF64Atan:
778 21 : op = m->Float64Atan();
779 21 : break;
780 : case wasm::kExprF64Cos: {
781 146 : op = m->Float64Cos();
782 146 : break;
783 : }
784 : case wasm::kExprF64Sin: {
785 146 : op = m->Float64Sin();
786 146 : break;
787 : }
788 : case wasm::kExprF64Tan: {
789 21 : op = m->Float64Tan();
790 21 : break;
791 : }
792 : case wasm::kExprF64Exp: {
793 21 : op = m->Float64Exp();
794 21 : break;
795 : }
796 : case wasm::kExprF64Log:
797 21 : op = m->Float64Log();
798 21 : break;
799 : case wasm::kExprI32ConvertI64:
800 764 : op = m->TruncateInt64ToInt32();
801 764 : break;
802 : case wasm::kExprI64SConvertI32:
803 44 : op = m->ChangeInt32ToInt64();
804 44 : break;
805 : case wasm::kExprI64UConvertI32:
806 25289 : op = m->ChangeUint32ToUint64();
807 25292 : break;
808 : case wasm::kExprF64ReinterpretI64:
809 160 : op = m->BitcastInt64ToFloat64();
810 160 : break;
811 : case wasm::kExprI64ReinterpretF64:
812 412 : op = m->BitcastFloat64ToInt64();
813 412 : break;
814 : case wasm::kExprI64Clz:
815 36 : op = m->Word64Clz();
816 36 : break;
817 : case wasm::kExprI64Ctz: {
818 188 : OptionalOperator ctz64 = m->Word64Ctz();
819 188 : if (ctz64.IsSupported()) {
820 : op = ctz64.op();
821 188 : break;
822 0 : } else if (m->Is32() && m->Word32Ctz().IsSupported()) {
823 : op = ctz64.placeholder();
824 : break;
825 0 : } else if (m->Word64ReverseBits().IsSupported()) {
826 0 : Node* reversed = graph()->NewNode(m->Word64ReverseBits().op(), input);
827 0 : Node* result = graph()->NewNode(m->Word64Clz(), reversed);
828 0 : return result;
829 : } else {
830 0 : return BuildI64Ctz(input);
831 : }
832 : }
833 : case wasm::kExprI64Popcnt: {
834 44 : OptionalOperator popcnt64 = m->Word64Popcnt();
835 44 : if (popcnt64.IsSupported()) {
836 : op = popcnt64.op();
837 0 : } else if (m->Is32() && m->Word32Popcnt().IsSupported()) {
838 : op = popcnt64.placeholder();
839 : } else {
840 0 : return BuildI64Popcnt(input);
841 : }
842 44 : break;
843 : }
844 : case wasm::kExprI64Eqz:
845 152 : op = m->Word64Equal();
846 304 : return graph()->NewNode(op, input, mcgraph()->Int64Constant(0));
847 : case wasm::kExprF32SConvertI64:
848 64 : if (m->Is32()) {
849 0 : return BuildF32SConvertI64(input);
850 : }
851 64 : op = m->RoundInt64ToFloat32();
852 64 : break;
853 : case wasm::kExprF32UConvertI64:
854 32 : if (m->Is32()) {
855 0 : return BuildF32UConvertI64(input);
856 : }
857 32 : op = m->RoundUint64ToFloat32();
858 32 : break;
859 : case wasm::kExprF64SConvertI64:
860 87 : if (m->Is32()) {
861 0 : return BuildF64SConvertI64(input);
862 : }
863 87 : op = m->RoundInt64ToFloat64();
864 87 : break;
865 : case wasm::kExprF64UConvertI64:
866 135 : if (m->Is32()) {
867 0 : return BuildF64UConvertI64(input);
868 : }
869 135 : op = m->RoundUint64ToFloat64();
870 135 : break;
871 : case wasm::kExprI32SExtendI8:
872 4 : op = m->SignExtendWord8ToInt32();
873 4 : break;
874 : case wasm::kExprI32SExtendI16:
875 4 : op = m->SignExtendWord16ToInt32();
876 4 : break;
877 : case wasm::kExprI64SExtendI8:
878 4 : op = m->SignExtendWord8ToInt64();
879 4 : break;
880 : case wasm::kExprI64SExtendI16:
881 4 : op = m->SignExtendWord16ToInt64();
882 4 : break;
883 : case wasm::kExprI64SExtendI32:
884 4 : op = m->SignExtendWord32ToInt64();
885 4 : break;
886 : case wasm::kExprI64SConvertF32:
887 : case wasm::kExprI64UConvertF32:
888 : case wasm::kExprI64SConvertF64:
889 : case wasm::kExprI64UConvertF64:
890 : case wasm::kExprI64SConvertSatF32:
891 : case wasm::kExprI64UConvertSatF32:
892 : case wasm::kExprI64SConvertSatF64:
893 : case wasm::kExprI64UConvertSatF64:
894 : return mcgraph()->machine()->Is32()
895 : ? BuildCcallConvertFloat(input, position, opcode)
896 811 : : BuildIntConvertFloat(input, position, opcode);
897 : case wasm::kExprRefIsNull:
898 64 : return graph()->NewNode(m->WordEqual(), input, RefNull());
899 : case wasm::kExprI32AsmjsLoadMem8S:
900 2804 : return BuildAsmjsLoadMem(MachineType::Int8(), input);
901 : case wasm::kExprI32AsmjsLoadMem8U:
902 2082 : return BuildAsmjsLoadMem(MachineType::Uint8(), input);
903 : case wasm::kExprI32AsmjsLoadMem16S:
904 958 : return BuildAsmjsLoadMem(MachineType::Int16(), input);
905 : case wasm::kExprI32AsmjsLoadMem16U:
906 531 : return BuildAsmjsLoadMem(MachineType::Uint16(), input);
907 : case wasm::kExprI32AsmjsLoadMem:
908 43774 : return BuildAsmjsLoadMem(MachineType::Int32(), input);
909 : case wasm::kExprF32AsmjsLoadMem:
910 5396 : return BuildAsmjsLoadMem(MachineType::Float32(), input);
911 : case wasm::kExprF64AsmjsLoadMem:
912 845 : return BuildAsmjsLoadMem(MachineType::Float64(), input);
913 : default:
914 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
915 : }
916 41667 : return graph()->NewNode(op, input);
917 : }
918 :
919 3651 : Node* WasmGraphBuilder::Float32Constant(float value) {
920 3651 : return mcgraph()->Float32Constant(value);
921 : }
922 :
923 10443 : Node* WasmGraphBuilder::Float64Constant(double value) {
924 10443 : return mcgraph()->Float64Constant(value);
925 : }
926 :
927 : namespace {
928 592714 : Node* Branch(MachineGraph* mcgraph, Node* cond, Node** true_node,
929 : Node** false_node, Node* control, BranchHint hint) {
930 : DCHECK_NOT_NULL(cond);
931 : DCHECK_NOT_NULL(control);
932 : Node* branch =
933 98783 : mcgraph->graph()->NewNode(mcgraph->common()->Branch(hint), cond, control);
934 197574 : *true_node = mcgraph->graph()->NewNode(mcgraph->common()->IfTrue(), branch);
935 197575 : *false_node = mcgraph->graph()->NewNode(mcgraph->common()->IfFalse(), branch);
936 98788 : return branch;
937 : }
938 : } // namespace
939 :
940 83881 : Node* WasmGraphBuilder::BranchNoHint(Node* cond, Node** true_node,
941 167762 : Node** false_node) {
942 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
943 83881 : BranchHint::kNone);
944 : }
945 :
946 0 : Node* WasmGraphBuilder::BranchExpectTrue(Node* cond, Node** true_node,
947 0 : Node** false_node) {
948 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
949 0 : BranchHint::kTrue);
950 : }
951 :
952 0 : Node* WasmGraphBuilder::BranchExpectFalse(Node* cond, Node** true_node,
953 29800 : Node** false_node) {
954 : return Branch(mcgraph(), cond, true_node, false_node, Control(),
955 14900 : BranchHint::kFalse);
956 : }
957 :
958 34166 : TrapId WasmGraphBuilder::GetTrapIdForTrap(wasm::TrapReason reason) {
959 : // TODO(wasm): "!env_" should not happen when compiling an actual wasm
960 : // function.
961 34166 : if (!env_ || !env_->runtime_exception_support) {
962 : // We use TrapId::kInvalid as a marker to tell the code generator
963 : // to generate a call to a testing c-function instead of a runtime
964 : // stub. This code should only be called from a cctest.
965 : return TrapId::kInvalid;
966 : }
967 :
968 23860 : switch (reason) {
969 : #define TRAPREASON_TO_TRAPID(name) \
970 : case wasm::k##name: \
971 : static_assert( \
972 : static_cast<int>(TrapId::k##name) == wasm::WasmCode::kThrowWasm##name, \
973 : "trap id mismatch"); \
974 : return TrapId::k##name;
975 10509 : FOREACH_WASM_TRAPREASON(TRAPREASON_TO_TRAPID)
976 : #undef TRAPREASON_TO_TRAPID
977 : default:
978 0 : UNREACHABLE();
979 : }
980 : }
981 :
982 5338 : Node* WasmGraphBuilder::TrapIfTrue(wasm::TrapReason reason, Node* cond,
983 21353 : wasm::WasmCodePosition position) {
984 5338 : TrapId trap_id = GetTrapIdForTrap(reason);
985 : Node* node = SetControl(graph()->NewNode(mcgraph()->common()->TrapIf(trap_id),
986 5338 : cond, Effect(), Control()));
987 5339 : SetSourcePosition(node, position);
988 5338 : return node;
989 : }
990 :
991 28828 : Node* WasmGraphBuilder::TrapIfFalse(wasm::TrapReason reason, Node* cond,
992 115307 : wasm::WasmCodePosition position) {
993 28828 : TrapId trap_id = GetTrapIdForTrap(reason);
994 : Node* node = SetControl(graph()->NewNode(
995 28827 : mcgraph()->common()->TrapUnless(trap_id), cond, Effect(), Control()));
996 28826 : SetSourcePosition(node, position);
997 28828 : return node;
998 : }
999 :
1000 : // Add a check that traps if {node} is equal to {val}.
1001 70688 : Node* WasmGraphBuilder::TrapIfEq32(wasm::TrapReason reason, Node* node,
1002 : int32_t val,
1003 1912 : wasm::WasmCodePosition position) {
1004 : Int32Matcher m(node);
1005 204508 : if (m.HasValue() && !m.Is(val)) return graph()->start();
1006 4380 : if (val == 0) {
1007 3424 : return TrapIfFalse(reason, node, position);
1008 : } else {
1009 : return TrapIfTrue(reason,
1010 : graph()->NewNode(mcgraph()->machine()->Word32Equal(),
1011 : node, mcgraph()->Int32Constant(val)),
1012 2868 : position);
1013 : }
1014 : }
1015 :
1016 : // Add a check that traps if {node} is zero.
1017 0 : Node* WasmGraphBuilder::ZeroCheck32(wasm::TrapReason reason, Node* node,
1018 : wasm::WasmCodePosition position) {
1019 56492 : return TrapIfEq32(reason, node, 0, position);
1020 : }
1021 :
1022 : // Add a check that traps if {node} is equal to {val}.
1023 4168 : Node* WasmGraphBuilder::TrapIfEq64(wasm::TrapReason reason, Node* node,
1024 : int64_t val,
1025 7904 : wasm::WasmCodePosition position) {
1026 : Int64Matcher m(node);
1027 4620 : if (m.HasValue() && !m.Is(val)) return graph()->start();
1028 : return TrapIfTrue(reason,
1029 : graph()->NewNode(mcgraph()->machine()->Word64Equal(), node,
1030 : mcgraph()->Int64Constant(val)),
1031 11857 : position);
1032 : }
1033 :
1034 : // Add a check that traps if {node} is zero.
1035 0 : Node* WasmGraphBuilder::ZeroCheck64(wasm::TrapReason reason, Node* node,
1036 : wasm::WasmCodePosition position) {
1037 3451 : return TrapIfEq64(reason, node, 0, position);
1038 : }
1039 :
1040 9704 : Node* WasmGraphBuilder::Switch(unsigned count, Node* key) {
1041 14556 : return graph()->NewNode(mcgraph()->common()->Switch(count), key, Control());
1042 : }
1043 :
1044 210672 : Node* WasmGraphBuilder::IfValue(int32_t value, Node* sw) {
1045 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1046 421345 : return graph()->NewNode(mcgraph()->common()->IfValue(value), sw);
1047 : }
1048 :
1049 4852 : Node* WasmGraphBuilder::IfDefault(Node* sw) {
1050 : DCHECK_EQ(IrOpcode::kSwitch, sw->opcode());
1051 9704 : return graph()->NewNode(mcgraph()->common()->IfDefault(), sw);
1052 : }
1053 :
1054 4152617 : Node* WasmGraphBuilder::Return(unsigned count, Node** vals) {
1055 : static const int kStackAllocatedNodeBufferSize = 8;
1056 : Node* stack_buffer[kStackAllocatedNodeBufferSize];
1057 : std::vector<Node*> heap_buffer;
1058 :
1059 : Node** buf = stack_buffer;
1060 692026 : if (count + 3 > kStackAllocatedNodeBufferSize) {
1061 0 : heap_buffer.resize(count + 3);
1062 : buf = heap_buffer.data();
1063 : }
1064 :
1065 692026 : buf[0] = mcgraph()->Int32Constant(0);
1066 692126 : if (count > 0) {
1067 664860 : memcpy(buf + 1, vals, sizeof(void*) * count);
1068 : }
1069 1384252 : buf[count + 1] = Effect();
1070 1384252 : buf[count + 2] = Control();
1071 : Node* ret =
1072 2076303 : graph()->NewNode(mcgraph()->common()->Return(count), count + 3, buf);
1073 :
1074 692187 : MergeControlToEnd(mcgraph(), ret);
1075 692088 : return ret;
1076 : }
1077 :
1078 1617 : Node* WasmGraphBuilder::ReturnVoid() { return Return(0, nullptr); }
1079 :
1080 1541 : Node* WasmGraphBuilder::Unreachable(wasm::WasmCodePosition position) {
1081 1541 : TrapIfFalse(wasm::TrapReason::kTrapUnreachable, Int32Constant(0), position);
1082 : ReturnVoid();
1083 1541 : return nullptr;
1084 : }
1085 :
1086 111264 : Node* WasmGraphBuilder::MaskShiftCount32(Node* node) {
1087 : static const int32_t kMask32 = 0x1F;
1088 222480 : if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1089 : // Shifts by constants are so common we pattern-match them here.
1090 : Int32Matcher match(node);
1091 24 : if (match.HasValue()) {
1092 0 : int32_t masked = (match.Value() & kMask32);
1093 0 : if (match.Value() != masked) node = mcgraph()->Int32Constant(masked);
1094 : } else {
1095 : node = graph()->NewNode(mcgraph()->machine()->Word32And(), node,
1096 48 : mcgraph()->Int32Constant(kMask32));
1097 : }
1098 : }
1099 111240 : return node;
1100 : }
1101 :
1102 14763 : Node* WasmGraphBuilder::MaskShiftCount64(Node* node) {
1103 : static const int64_t kMask64 = 0x3F;
1104 29486 : if (!mcgraph()->machine()->Word32ShiftIsSafe()) {
1105 : // Shifts by constants are so common we pattern-match them here.
1106 : Int64Matcher match(node);
1107 20 : if (match.HasValue()) {
1108 0 : int64_t masked = (match.Value() & kMask64);
1109 0 : if (match.Value() != masked) node = mcgraph()->Int64Constant(masked);
1110 : } else {
1111 : node = graph()->NewNode(mcgraph()->machine()->Word64And(), node,
1112 40 : mcgraph()->Int64Constant(kMask64));
1113 : }
1114 : }
1115 14743 : return node;
1116 : }
1117 :
1118 : static bool ReverseBytesSupported(MachineOperatorBuilder* m,
1119 : size_t size_in_bytes) {
1120 0 : switch (size_in_bytes) {
1121 : case 4:
1122 : case 16:
1123 : return true;
1124 : case 8:
1125 : return m->Is64();
1126 : default:
1127 : break;
1128 : }
1129 : return false;
1130 : }
1131 :
1132 0 : Node* WasmGraphBuilder::BuildChangeEndiannessStore(
1133 0 : Node* node, MachineRepresentation mem_rep, wasm::ValueType wasmtype) {
1134 : Node* result;
1135 : Node* value = node;
1136 0 : MachineOperatorBuilder* m = mcgraph()->machine();
1137 0 : int valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasmtype);
1138 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1139 : bool isFloat = false;
1140 :
1141 0 : switch (wasmtype) {
1142 : case wasm::kWasmF64:
1143 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1144 : isFloat = true;
1145 : V8_FALLTHROUGH;
1146 : case wasm::kWasmI64:
1147 0 : result = mcgraph()->Int64Constant(0);
1148 0 : break;
1149 : case wasm::kWasmF32:
1150 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1151 : isFloat = true;
1152 : V8_FALLTHROUGH;
1153 : case wasm::kWasmI32:
1154 0 : result = mcgraph()->Int32Constant(0);
1155 0 : break;
1156 : case wasm::kWasmS128:
1157 : DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1158 : break;
1159 : default:
1160 0 : UNREACHABLE();
1161 : break;
1162 : }
1163 :
1164 0 : if (mem_rep == MachineRepresentation::kWord8) {
1165 : // No need to change endianness for byte size, return original node
1166 : return node;
1167 : }
1168 0 : if (wasmtype == wasm::kWasmI64 && mem_rep < MachineRepresentation::kWord64) {
1169 : // In case we store lower part of WasmI64 expression, we can truncate
1170 : // upper 32bits
1171 0 : value = graph()->NewNode(m->TruncateInt64ToInt32(), value);
1172 0 : valueSizeInBytes = wasm::ValueTypes::ElementSizeInBytes(wasm::kWasmI32);
1173 0 : valueSizeInBits = 8 * valueSizeInBytes;
1174 0 : if (mem_rep == MachineRepresentation::kWord16) {
1175 : value =
1176 0 : graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1177 : }
1178 0 : } else if (wasmtype == wasm::kWasmI32 &&
1179 0 : mem_rep == MachineRepresentation::kWord16) {
1180 : value =
1181 0 : graph()->NewNode(m->Word32Shl(), value, mcgraph()->Int32Constant(16));
1182 : }
1183 :
1184 : int i;
1185 : uint32_t shiftCount;
1186 :
1187 0 : if (ReverseBytesSupported(m, valueSizeInBytes)) {
1188 0 : switch (valueSizeInBytes) {
1189 : case 4:
1190 0 : result = graph()->NewNode(m->Word32ReverseBytes(), value);
1191 0 : break;
1192 : case 8:
1193 0 : result = graph()->NewNode(m->Word64ReverseBytes(), value);
1194 0 : break;
1195 : case 16: {
1196 : Node* byte_reversed_lanes[4];
1197 0 : for (int lane = 0; lane < 4; lane++) {
1198 : byte_reversed_lanes[lane] = graph()->NewNode(
1199 : m->Word32ReverseBytes(),
1200 : graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1201 0 : value));
1202 : }
1203 :
1204 : // This is making a copy of the value.
1205 : result =
1206 0 : graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1207 :
1208 0 : for (int lane = 0; lane < 4; lane++) {
1209 : result =
1210 : graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1211 0 : result, byte_reversed_lanes[lane]);
1212 : }
1213 :
1214 : break;
1215 : }
1216 : default:
1217 0 : UNREACHABLE();
1218 : break;
1219 : }
1220 : } else {
1221 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1222 : i += 8, shiftCount -= 16) {
1223 : Node* shiftLower;
1224 : Node* shiftHigher;
1225 : Node* lowerByte;
1226 : Node* higherByte;
1227 :
1228 : DCHECK_LT(0, shiftCount);
1229 : DCHECK_EQ(0, (shiftCount + 8) % 16);
1230 :
1231 0 : if (valueSizeInBits > 32) {
1232 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1233 0 : mcgraph()->Int64Constant(shiftCount));
1234 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1235 0 : mcgraph()->Int64Constant(shiftCount));
1236 : lowerByte = graph()->NewNode(
1237 : m->Word64And(), shiftLower,
1238 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1239 0 : << (valueSizeInBits - 8 - i)));
1240 : higherByte = graph()->NewNode(
1241 : m->Word64And(), shiftHigher,
1242 0 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1243 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1244 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1245 : } else {
1246 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1247 0 : mcgraph()->Int32Constant(shiftCount));
1248 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1249 0 : mcgraph()->Int32Constant(shiftCount));
1250 : lowerByte = graph()->NewNode(
1251 : m->Word32And(), shiftLower,
1252 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1253 0 : << (valueSizeInBits - 8 - i)));
1254 : higherByte = graph()->NewNode(
1255 : m->Word32And(), shiftHigher,
1256 0 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1257 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1258 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1259 : }
1260 : }
1261 : }
1262 :
1263 0 : if (isFloat) {
1264 0 : switch (wasmtype) {
1265 : case wasm::kWasmF64:
1266 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1267 0 : break;
1268 : case wasm::kWasmF32:
1269 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1270 0 : break;
1271 : default:
1272 0 : UNREACHABLE();
1273 : break;
1274 : }
1275 : }
1276 :
1277 0 : return result;
1278 : }
1279 :
1280 0 : Node* WasmGraphBuilder::BuildChangeEndiannessLoad(Node* node,
1281 : MachineType memtype,
1282 0 : wasm::ValueType wasmtype) {
1283 : Node* result;
1284 : Node* value = node;
1285 0 : MachineOperatorBuilder* m = mcgraph()->machine();
1286 : int valueSizeInBytes = ElementSizeInBytes(memtype.representation());
1287 0 : int valueSizeInBits = 8 * valueSizeInBytes;
1288 : bool isFloat = false;
1289 :
1290 0 : switch (memtype.representation()) {
1291 : case MachineRepresentation::kFloat64:
1292 0 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), node);
1293 : isFloat = true;
1294 : V8_FALLTHROUGH;
1295 : case MachineRepresentation::kWord64:
1296 0 : result = mcgraph()->Int64Constant(0);
1297 0 : break;
1298 : case MachineRepresentation::kFloat32:
1299 0 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), node);
1300 : isFloat = true;
1301 : V8_FALLTHROUGH;
1302 : case MachineRepresentation::kWord32:
1303 : case MachineRepresentation::kWord16:
1304 0 : result = mcgraph()->Int32Constant(0);
1305 0 : break;
1306 : case MachineRepresentation::kWord8:
1307 : // No need to change endianness for byte size, return original node
1308 : return node;
1309 : break;
1310 : case MachineRepresentation::kSimd128:
1311 : DCHECK(ReverseBytesSupported(m, valueSizeInBytes));
1312 : break;
1313 : default:
1314 0 : UNREACHABLE();
1315 : break;
1316 : }
1317 :
1318 : int i;
1319 : uint32_t shiftCount;
1320 :
1321 0 : if (ReverseBytesSupported(m, valueSizeInBytes < 4 ? 4 : valueSizeInBytes)) {
1322 0 : switch (valueSizeInBytes) {
1323 : case 2:
1324 : result =
1325 : graph()->NewNode(m->Word32ReverseBytes(),
1326 : graph()->NewNode(m->Word32Shl(), value,
1327 0 : mcgraph()->Int32Constant(16)));
1328 0 : break;
1329 : case 4:
1330 0 : result = graph()->NewNode(m->Word32ReverseBytes(), value);
1331 0 : break;
1332 : case 8:
1333 0 : result = graph()->NewNode(m->Word64ReverseBytes(), value);
1334 0 : break;
1335 : case 16: {
1336 : Node* byte_reversed_lanes[4];
1337 0 : for (int lane = 0; lane < 4; lane++) {
1338 : byte_reversed_lanes[lane] = graph()->NewNode(
1339 : m->Word32ReverseBytes(),
1340 : graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
1341 0 : value));
1342 : }
1343 :
1344 : // This is making a copy of the value.
1345 : result =
1346 0 : graph()->NewNode(mcgraph()->machine()->S128And(), value, value);
1347 :
1348 0 : for (int lane = 0; lane < 4; lane++) {
1349 : result =
1350 : graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(3 - lane),
1351 0 : result, byte_reversed_lanes[lane]);
1352 : }
1353 :
1354 : break;
1355 : }
1356 : default:
1357 0 : UNREACHABLE();
1358 : }
1359 : } else {
1360 0 : for (i = 0, shiftCount = valueSizeInBits - 8; i < valueSizeInBits / 2;
1361 : i += 8, shiftCount -= 16) {
1362 : Node* shiftLower;
1363 : Node* shiftHigher;
1364 : Node* lowerByte;
1365 : Node* higherByte;
1366 :
1367 : DCHECK_LT(0, shiftCount);
1368 : DCHECK_EQ(0, (shiftCount + 8) % 16);
1369 :
1370 0 : if (valueSizeInBits > 32) {
1371 : shiftLower = graph()->NewNode(m->Word64Shl(), value,
1372 0 : mcgraph()->Int64Constant(shiftCount));
1373 : shiftHigher = graph()->NewNode(m->Word64Shr(), value,
1374 0 : mcgraph()->Int64Constant(shiftCount));
1375 : lowerByte = graph()->NewNode(
1376 : m->Word64And(), shiftLower,
1377 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF)
1378 0 : << (valueSizeInBits - 8 - i)));
1379 : higherByte = graph()->NewNode(
1380 : m->Word64And(), shiftHigher,
1381 0 : mcgraph()->Int64Constant(static_cast<uint64_t>(0xFF) << i));
1382 0 : result = graph()->NewNode(m->Word64Or(), result, lowerByte);
1383 0 : result = graph()->NewNode(m->Word64Or(), result, higherByte);
1384 : } else {
1385 : shiftLower = graph()->NewNode(m->Word32Shl(), value,
1386 0 : mcgraph()->Int32Constant(shiftCount));
1387 : shiftHigher = graph()->NewNode(m->Word32Shr(), value,
1388 0 : mcgraph()->Int32Constant(shiftCount));
1389 : lowerByte = graph()->NewNode(
1390 : m->Word32And(), shiftLower,
1391 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF)
1392 0 : << (valueSizeInBits - 8 - i)));
1393 : higherByte = graph()->NewNode(
1394 : m->Word32And(), shiftHigher,
1395 0 : mcgraph()->Int32Constant(static_cast<uint32_t>(0xFF) << i));
1396 0 : result = graph()->NewNode(m->Word32Or(), result, lowerByte);
1397 0 : result = graph()->NewNode(m->Word32Or(), result, higherByte);
1398 : }
1399 : }
1400 : }
1401 :
1402 0 : if (isFloat) {
1403 0 : switch (memtype.representation()) {
1404 : case MachineRepresentation::kFloat64:
1405 0 : result = graph()->NewNode(m->BitcastInt64ToFloat64(), result);
1406 0 : break;
1407 : case MachineRepresentation::kFloat32:
1408 0 : result = graph()->NewNode(m->BitcastInt32ToFloat32(), result);
1409 0 : break;
1410 : default:
1411 0 : UNREACHABLE();
1412 : break;
1413 : }
1414 : }
1415 :
1416 : // We need to sign extend the value
1417 0 : if (memtype.IsSigned()) {
1418 : DCHECK(!isFloat);
1419 0 : if (valueSizeInBits < 32) {
1420 : Node* shiftBitCount;
1421 : // Perform sign extension using following trick
1422 : // result = (x << machine_width - type_width) >> (machine_width -
1423 : // type_width)
1424 0 : if (wasmtype == wasm::kWasmI64) {
1425 0 : shiftBitCount = mcgraph()->Int32Constant(64 - valueSizeInBits);
1426 : result = graph()->NewNode(
1427 : m->Word64Sar(),
1428 : graph()->NewNode(m->Word64Shl(),
1429 : graph()->NewNode(m->ChangeInt32ToInt64(), result),
1430 : shiftBitCount),
1431 0 : shiftBitCount);
1432 0 : } else if (wasmtype == wasm::kWasmI32) {
1433 0 : shiftBitCount = mcgraph()->Int32Constant(32 - valueSizeInBits);
1434 : result = graph()->NewNode(
1435 : m->Word32Sar(),
1436 : graph()->NewNode(m->Word32Shl(), result, shiftBitCount),
1437 0 : shiftBitCount);
1438 : }
1439 : }
1440 : }
1441 :
1442 0 : return result;
1443 : }
1444 :
1445 64 : Node* WasmGraphBuilder::BuildF32CopySign(Node* left, Node* right) {
1446 : Node* result = Unop(
1447 : wasm::kExprF32ReinterpretI32,
1448 : Binop(wasm::kExprI32Ior,
1449 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, left),
1450 : mcgraph()->Int32Constant(0x7FFFFFFF)),
1451 : Binop(wasm::kExprI32And, Unop(wasm::kExprI32ReinterpretF32, right),
1452 64 : mcgraph()->Int32Constant(0x80000000))));
1453 :
1454 32 : return result;
1455 : }
1456 :
1457 64 : Node* WasmGraphBuilder::BuildF64CopySign(Node* left, Node* right) {
1458 : #if WASM_64
1459 : Node* result = Unop(
1460 : wasm::kExprF64ReinterpretI64,
1461 : Binop(wasm::kExprI64Ior,
1462 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, left),
1463 : mcgraph()->Int64Constant(0x7FFFFFFFFFFFFFFF)),
1464 : Binop(wasm::kExprI64And, Unop(wasm::kExprI64ReinterpretF64, right),
1465 64 : mcgraph()->Int64Constant(0x8000000000000000))));
1466 :
1467 32 : return result;
1468 : #else
1469 : MachineOperatorBuilder* m = mcgraph()->machine();
1470 :
1471 : Node* high_word_left = graph()->NewNode(m->Float64ExtractHighWord32(), left);
1472 : Node* high_word_right =
1473 : graph()->NewNode(m->Float64ExtractHighWord32(), right);
1474 :
1475 : Node* new_high_word = Binop(wasm::kExprI32Ior,
1476 : Binop(wasm::kExprI32And, high_word_left,
1477 : mcgraph()->Int32Constant(0x7FFFFFFF)),
1478 : Binop(wasm::kExprI32And, high_word_right,
1479 : mcgraph()->Int32Constant(0x80000000)));
1480 :
1481 : return graph()->NewNode(m->Float64InsertHighWord32(), left, new_high_word);
1482 : #endif
1483 : }
1484 :
1485 : namespace {
1486 :
1487 1164 : MachineType IntConvertType(wasm::WasmOpcode opcode) {
1488 1164 : switch (opcode) {
1489 : case wasm::kExprI32SConvertF32:
1490 : case wasm::kExprI32SConvertF64:
1491 : case wasm::kExprI32SConvertSatF32:
1492 : case wasm::kExprI32SConvertSatF64:
1493 : return MachineType::Int32();
1494 : case wasm::kExprI32UConvertF32:
1495 : case wasm::kExprI32UConvertF64:
1496 : case wasm::kExprI32UConvertSatF32:
1497 : case wasm::kExprI32UConvertSatF64:
1498 : return MachineType::Uint32();
1499 : case wasm::kExprI64SConvertF32:
1500 : case wasm::kExprI64SConvertF64:
1501 : case wasm::kExprI64SConvertSatF32:
1502 : case wasm::kExprI64SConvertSatF64:
1503 : return MachineType::Int64();
1504 : case wasm::kExprI64UConvertF32:
1505 : case wasm::kExprI64UConvertF64:
1506 : case wasm::kExprI64UConvertSatF32:
1507 : case wasm::kExprI64UConvertSatF64:
1508 : return MachineType::Uint64();
1509 : default:
1510 0 : UNREACHABLE();
1511 : }
1512 : }
1513 :
1514 1165 : MachineType FloatConvertType(wasm::WasmOpcode opcode) {
1515 1165 : switch (opcode) {
1516 : case wasm::kExprI32SConvertF32:
1517 : case wasm::kExprI32UConvertF32:
1518 : case wasm::kExprI32SConvertSatF32:
1519 : case wasm::kExprI64SConvertF32:
1520 : case wasm::kExprI64UConvertF32:
1521 : case wasm::kExprI32UConvertSatF32:
1522 : case wasm::kExprI64SConvertSatF32:
1523 : case wasm::kExprI64UConvertSatF32:
1524 : return MachineType::Float32();
1525 : case wasm::kExprI32SConvertF64:
1526 : case wasm::kExprI32UConvertF64:
1527 : case wasm::kExprI64SConvertF64:
1528 : case wasm::kExprI64UConvertF64:
1529 : case wasm::kExprI32SConvertSatF64:
1530 : case wasm::kExprI32UConvertSatF64:
1531 : case wasm::kExprI64SConvertSatF64:
1532 : case wasm::kExprI64UConvertSatF64:
1533 : return MachineType::Float64();
1534 : default:
1535 0 : UNREACHABLE();
1536 : }
1537 : }
1538 :
1539 2330 : const Operator* ConvertOp(WasmGraphBuilder* builder, wasm::WasmOpcode opcode) {
1540 1165 : switch (opcode) {
1541 : case wasm::kExprI32SConvertF32:
1542 : case wasm::kExprI32SConvertSatF32:
1543 124 : return builder->mcgraph()->machine()->TruncateFloat32ToInt32();
1544 : case wasm::kExprI32UConvertF32:
1545 : case wasm::kExprI32UConvertSatF32:
1546 56 : return builder->mcgraph()->machine()->TruncateFloat32ToUint32();
1547 : case wasm::kExprI32SConvertF64:
1548 : case wasm::kExprI32SConvertSatF64:
1549 124 : return builder->mcgraph()->machine()->ChangeFloat64ToInt32();
1550 : case wasm::kExprI32UConvertF64:
1551 : case wasm::kExprI32UConvertSatF64:
1552 48 : return builder->mcgraph()->machine()->TruncateFloat64ToUint32();
1553 : case wasm::kExprI64SConvertF32:
1554 : case wasm::kExprI64SConvertSatF32:
1555 48 : return builder->mcgraph()->machine()->TryTruncateFloat32ToInt64();
1556 : case wasm::kExprI64UConvertF32:
1557 : case wasm::kExprI64UConvertSatF32:
1558 48 : return builder->mcgraph()->machine()->TryTruncateFloat32ToUint64();
1559 : case wasm::kExprI64SConvertF64:
1560 : case wasm::kExprI64SConvertSatF64:
1561 661 : return builder->mcgraph()->machine()->TryTruncateFloat64ToInt64();
1562 : case wasm::kExprI64UConvertF64:
1563 : case wasm::kExprI64UConvertSatF64:
1564 56 : return builder->mcgraph()->machine()->TryTruncateFloat64ToUint64();
1565 : default:
1566 0 : UNREACHABLE();
1567 : }
1568 : }
1569 :
1570 352 : wasm::WasmOpcode ConvertBackOp(wasm::WasmOpcode opcode) {
1571 352 : switch (opcode) {
1572 : case wasm::kExprI32SConvertF32:
1573 : case wasm::kExprI32SConvertSatF32:
1574 : return wasm::kExprF32SConvertI32;
1575 : case wasm::kExprI32UConvertF32:
1576 : case wasm::kExprI32UConvertSatF32:
1577 56 : return wasm::kExprF32UConvertI32;
1578 : case wasm::kExprI32SConvertF64:
1579 : case wasm::kExprI32SConvertSatF64:
1580 124 : return wasm::kExprF64SConvertI32;
1581 : case wasm::kExprI32UConvertF64:
1582 : case wasm::kExprI32UConvertSatF64:
1583 48 : return wasm::kExprF64UConvertI32;
1584 : default:
1585 0 : UNREACHABLE();
1586 : }
1587 : }
1588 :
1589 1164 : bool IsTrappingConvertOp(wasm::WasmOpcode opcode) {
1590 1164 : switch (opcode) {
1591 : case wasm::kExprI32SConvertF32:
1592 : case wasm::kExprI32UConvertF32:
1593 : case wasm::kExprI32SConvertF64:
1594 : case wasm::kExprI32UConvertF64:
1595 : case wasm::kExprI64SConvertF32:
1596 : case wasm::kExprI64UConvertF32:
1597 : case wasm::kExprI64SConvertF64:
1598 : case wasm::kExprI64UConvertF64:
1599 : return true;
1600 : case wasm::kExprI32SConvertSatF64:
1601 : case wasm::kExprI32UConvertSatF64:
1602 : case wasm::kExprI32SConvertSatF32:
1603 : case wasm::kExprI32UConvertSatF32:
1604 : case wasm::kExprI64SConvertSatF32:
1605 : case wasm::kExprI64UConvertSatF32:
1606 : case wasm::kExprI64SConvertSatF64:
1607 : case wasm::kExprI64UConvertSatF64:
1608 64 : return false;
1609 : default:
1610 0 : UNREACHABLE();
1611 : }
1612 : }
1613 :
1614 128 : Node* Zero(WasmGraphBuilder* builder, const MachineType& ty) {
1615 128 : switch (ty.representation()) {
1616 : case MachineRepresentation::kWord32:
1617 32 : return builder->Int32Constant(0);
1618 : case MachineRepresentation::kWord64:
1619 32 : return builder->Int64Constant(0);
1620 : case MachineRepresentation::kFloat32:
1621 32 : return builder->Float32Constant(0.0);
1622 : case MachineRepresentation::kFloat64:
1623 32 : return builder->Float64Constant(0.0);
1624 : default:
1625 0 : UNREACHABLE();
1626 : }
1627 : }
1628 :
1629 64 : Node* Min(WasmGraphBuilder* builder, const MachineType& ty) {
1630 64 : switch (ty.semantic()) {
1631 : case MachineSemantic::kInt32:
1632 16 : return builder->Int32Constant(std::numeric_limits<int32_t>::min());
1633 : case MachineSemantic::kUint32:
1634 16 : return builder->Int32Constant(std::numeric_limits<uint32_t>::min());
1635 : case MachineSemantic::kInt64:
1636 16 : return builder->Int64Constant(std::numeric_limits<int64_t>::min());
1637 : case MachineSemantic::kUint64:
1638 16 : return builder->Int64Constant(std::numeric_limits<uint64_t>::min());
1639 : default:
1640 0 : UNREACHABLE();
1641 : }
1642 : }
1643 :
1644 64 : Node* Max(WasmGraphBuilder* builder, const MachineType& ty) {
1645 64 : switch (ty.semantic()) {
1646 : case MachineSemantic::kInt32:
1647 16 : return builder->Int32Constant(std::numeric_limits<int32_t>::max());
1648 : case MachineSemantic::kUint32:
1649 16 : return builder->Int32Constant(std::numeric_limits<uint32_t>::max());
1650 : case MachineSemantic::kInt64:
1651 16 : return builder->Int64Constant(std::numeric_limits<int64_t>::max());
1652 : case MachineSemantic::kUint64:
1653 16 : return builder->Int64Constant(std::numeric_limits<uint64_t>::max());
1654 : default:
1655 0 : UNREACHABLE();
1656 : }
1657 : }
1658 :
1659 352 : wasm::WasmOpcode TruncOp(const MachineType& ty) {
1660 352 : switch (ty.representation()) {
1661 : case MachineRepresentation::kFloat32:
1662 : return wasm::kExprF32Trunc;
1663 : case MachineRepresentation::kFloat64:
1664 172 : return wasm::kExprF64Trunc;
1665 : default:
1666 0 : UNREACHABLE();
1667 : }
1668 : }
1669 :
1670 415 : wasm::WasmOpcode NeOp(const MachineType& ty) {
1671 415 : switch (ty.representation()) {
1672 : case MachineRepresentation::kFloat32:
1673 : return wasm::kExprF32Ne;
1674 : case MachineRepresentation::kFloat64:
1675 203 : return wasm::kExprF64Ne;
1676 : default:
1677 0 : UNREACHABLE();
1678 : }
1679 : }
1680 :
1681 64 : wasm::WasmOpcode LtOp(const MachineType& ty) {
1682 64 : switch (ty.representation()) {
1683 : case MachineRepresentation::kFloat32:
1684 : return wasm::kExprF32Lt;
1685 : case MachineRepresentation::kFloat64:
1686 32 : return wasm::kExprF64Lt;
1687 : default:
1688 0 : UNREACHABLE();
1689 : }
1690 : }
1691 :
1692 1164 : Node* ConvertTrapTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1693 1164 : const MachineType& int_ty, const MachineType& float_ty,
1694 : Node* trunc, Node* converted_value) {
1695 1164 : if (int_ty.representation() == MachineRepresentation::kWord32) {
1696 352 : Node* check = builder->Unop(ConvertBackOp(opcode), converted_value);
1697 351 : return builder->Binop(NeOp(float_ty), trunc, check);
1698 : }
1699 : return builder->graph()->NewNode(builder->mcgraph()->common()->Projection(1),
1700 1625 : trunc, builder->graph()->start());
1701 : }
1702 :
1703 64 : Node* ConvertSaturateTest(WasmGraphBuilder* builder, wasm::WasmOpcode opcode,
1704 64 : const MachineType& int_ty,
1705 : const MachineType& float_ty, Node* trunc,
1706 : Node* converted_value) {
1707 : Node* test = ConvertTrapTest(builder, opcode, int_ty, float_ty, trunc,
1708 64 : converted_value);
1709 64 : if (int_ty.representation() == MachineRepresentation::kWord64) {
1710 32 : test = builder->Binop(wasm::kExprI64Eq, test, builder->Int64Constant(0));
1711 : }
1712 64 : return test;
1713 : }
1714 :
1715 : } // namespace
1716 :
1717 1161 : Node* WasmGraphBuilder::BuildIntConvertFloat(Node* input,
1718 : wasm::WasmCodePosition position,
1719 256 : wasm::WasmOpcode opcode) {
1720 1161 : const MachineType int_ty = IntConvertType(opcode);
1721 1164 : const MachineType float_ty = FloatConvertType(opcode);
1722 1167 : const Operator* conv_op = ConvertOp(this, opcode);
1723 : Node* trunc = nullptr;
1724 : Node* converted_value = nullptr;
1725 : const bool is_int32 =
1726 1164 : int_ty.representation() == MachineRepresentation::kWord32;
1727 1164 : if (is_int32) {
1728 352 : trunc = Unop(TruncOp(float_ty), input);
1729 : converted_value = graph()->NewNode(conv_op, trunc);
1730 : } else {
1731 : trunc = graph()->NewNode(conv_op, input);
1732 : converted_value = graph()->NewNode(mcgraph()->common()->Projection(0),
1733 811 : trunc, graph()->start());
1734 : }
1735 1167 : if (IsTrappingConvertOp(opcode)) {
1736 : Node* test =
1737 1104 : ConvertTrapTest(this, opcode, int_ty, float_ty, trunc, converted_value);
1738 1102 : if (is_int32) {
1739 319 : TrapIfTrue(wasm::kTrapFloatUnrepresentable, test, position);
1740 : } else {
1741 : ZeroCheck64(wasm::kTrapFloatUnrepresentable, test, position);
1742 : }
1743 1097 : return converted_value;
1744 : }
1745 : Node* test = ConvertSaturateTest(this, opcode, int_ty, float_ty, trunc,
1746 64 : converted_value);
1747 128 : Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
1748 64 : tl_d.Chain(Control());
1749 64 : Node* nan_test = Binop(NeOp(float_ty), input, input);
1750 128 : Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
1751 64 : nan_d.Nest(tl_d, true);
1752 64 : Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
1753 128 : Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
1754 64 : sat_d.Nest(nan_d, false);
1755 : Node* sat_val =
1756 64 : sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
1757 : Node* nan_val =
1758 64 : nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
1759 64 : return tl_d.Phi(int_ty.representation(), nan_val, converted_value);
1760 : }
1761 :
1762 8 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF32(Node* input) {
1763 8 : MachineOperatorBuilder* m = mcgraph()->machine();
1764 : // asm.js must use the wacky JS semantics.
1765 8 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1766 16 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1767 : }
1768 :
1769 108 : Node* WasmGraphBuilder::BuildI32AsmjsSConvertF64(Node* input) {
1770 108 : MachineOperatorBuilder* m = mcgraph()->machine();
1771 : // asm.js must use the wacky JS semantics.
1772 216 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1773 : }
1774 :
1775 8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF32(Node* input) {
1776 8 : MachineOperatorBuilder* m = mcgraph()->machine();
1777 : // asm.js must use the wacky JS semantics.
1778 8 : input = graph()->NewNode(m->ChangeFloat32ToFloat64(), input);
1779 16 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1780 : }
1781 :
1782 8 : Node* WasmGraphBuilder::BuildI32AsmjsUConvertF64(Node* input) {
1783 8 : MachineOperatorBuilder* m = mcgraph()->machine();
1784 : // asm.js must use the wacky JS semantics.
1785 16 : return graph()->NewNode(m->TruncateFloat64ToWord32(), input);
1786 : }
1787 :
1788 0 : Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
1789 0 : MachineRepresentation input_type) {
1790 : Node* stack_slot_param =
1791 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(input_type));
1792 :
1793 : const Operator* store_op = mcgraph()->machine()->Store(
1794 0 : StoreRepresentation(input_type, kNoWriteBarrier));
1795 : SetEffect(graph()->NewNode(store_op, stack_slot_param,
1796 : mcgraph()->Int32Constant(0), input, Effect(),
1797 0 : Control()));
1798 :
1799 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
1800 : MachineSignature sig(1, 1, sig_types);
1801 :
1802 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1803 :
1804 0 : return BuildCCall(&sig, function, stack_slot_param);
1805 : }
1806 :
1807 0 : Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
1808 : return BuildBitCountingCall(input, ExternalReference::wasm_word32_ctz(),
1809 0 : MachineRepresentation::kWord32);
1810 : }
1811 :
1812 0 : Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
1813 : return Unop(wasm::kExprI64UConvertI32,
1814 : BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(),
1815 0 : MachineRepresentation::kWord64));
1816 : }
1817 :
1818 0 : Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
1819 : return BuildBitCountingCall(input, ExternalReference::wasm_word32_popcnt(),
1820 0 : MachineRepresentation::kWord32);
1821 : }
1822 :
1823 0 : Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
1824 : return Unop(
1825 : wasm::kExprI64UConvertI32,
1826 : BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(),
1827 0 : MachineRepresentation::kWord64));
1828 : }
1829 :
1830 0 : Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {
1831 0 : MachineType type = MachineType::Float32();
1832 0 : ExternalReference ref = ExternalReference::wasm_f32_trunc();
1833 :
1834 0 : return BuildCFuncInstruction(ref, type, input);
1835 : }
1836 :
1837 0 : Node* WasmGraphBuilder::BuildF32Floor(Node* input) {
1838 0 : MachineType type = MachineType::Float32();
1839 0 : ExternalReference ref = ExternalReference::wasm_f32_floor();
1840 0 : return BuildCFuncInstruction(ref, type, input);
1841 : }
1842 :
1843 0 : Node* WasmGraphBuilder::BuildF32Ceil(Node* input) {
1844 0 : MachineType type = MachineType::Float32();
1845 0 : ExternalReference ref = ExternalReference::wasm_f32_ceil();
1846 0 : return BuildCFuncInstruction(ref, type, input);
1847 : }
1848 :
1849 0 : Node* WasmGraphBuilder::BuildF32NearestInt(Node* input) {
1850 0 : MachineType type = MachineType::Float32();
1851 0 : ExternalReference ref = ExternalReference::wasm_f32_nearest_int();
1852 0 : return BuildCFuncInstruction(ref, type, input);
1853 : }
1854 :
1855 0 : Node* WasmGraphBuilder::BuildF64Trunc(Node* input) {
1856 0 : MachineType type = MachineType::Float64();
1857 0 : ExternalReference ref = ExternalReference::wasm_f64_trunc();
1858 0 : return BuildCFuncInstruction(ref, type, input);
1859 : }
1860 :
1861 0 : Node* WasmGraphBuilder::BuildF64Floor(Node* input) {
1862 0 : MachineType type = MachineType::Float64();
1863 0 : ExternalReference ref = ExternalReference::wasm_f64_floor();
1864 0 : return BuildCFuncInstruction(ref, type, input);
1865 : }
1866 :
1867 0 : Node* WasmGraphBuilder::BuildF64Ceil(Node* input) {
1868 0 : MachineType type = MachineType::Float64();
1869 0 : ExternalReference ref = ExternalReference::wasm_f64_ceil();
1870 0 : return BuildCFuncInstruction(ref, type, input);
1871 : }
1872 :
1873 0 : Node* WasmGraphBuilder::BuildF64NearestInt(Node* input) {
1874 0 : MachineType type = MachineType::Float64();
1875 0 : ExternalReference ref = ExternalReference::wasm_f64_nearest_int();
1876 0 : return BuildCFuncInstruction(ref, type, input);
1877 : }
1878 :
1879 21 : Node* WasmGraphBuilder::BuildF64Acos(Node* input) {
1880 21 : MachineType type = MachineType::Float64();
1881 21 : ExternalReference ref = ExternalReference::f64_acos_wrapper_function();
1882 21 : return BuildCFuncInstruction(ref, type, input);
1883 : }
1884 :
1885 21 : Node* WasmGraphBuilder::BuildF64Asin(Node* input) {
1886 21 : MachineType type = MachineType::Float64();
1887 21 : ExternalReference ref = ExternalReference::f64_asin_wrapper_function();
1888 21 : return BuildCFuncInstruction(ref, type, input);
1889 : }
1890 :
1891 9 : Node* WasmGraphBuilder::BuildF64Pow(Node* left, Node* right) {
1892 9 : MachineType type = MachineType::Float64();
1893 9 : ExternalReference ref = ExternalReference::wasm_float64_pow();
1894 9 : return BuildCFuncInstruction(ref, type, left, right);
1895 : }
1896 :
1897 20 : Node* WasmGraphBuilder::BuildF64Mod(Node* left, Node* right) {
1898 20 : MachineType type = MachineType::Float64();
1899 20 : ExternalReference ref = ExternalReference::f64_mod_wrapper_function();
1900 20 : return BuildCFuncInstruction(ref, type, left, right);
1901 : }
1902 :
1903 71 : Node* WasmGraphBuilder::BuildCFuncInstruction(ExternalReference ref,
1904 : MachineType type, Node* input0,
1905 968 : Node* input1) {
1906 : // We do truncation by calling a C function which calculates the result.
1907 : // The input is passed to the C function as a byte buffer holding the two
1908 : // input doubles. We reserve this byte buffer as a stack slot, store the
1909 : // parameters in this buffer slots, pass a pointer to the buffer to the C
1910 : // function, and after calling the C function we collect the return value from
1911 : // the buffer.
1912 :
1913 : const int type_size = ElementSizeInBytes(type.representation());
1914 71 : const int stack_slot_bytes = (input1 == nullptr ? 1 : 2) * type_size;
1915 : Node* stack_slot =
1916 71 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_bytes));
1917 :
1918 : const Operator* store_op = mcgraph()->machine()->Store(
1919 71 : StoreRepresentation(type.representation(), kNoWriteBarrier));
1920 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1921 71 : input0, Effect(), Control()));
1922 :
1923 71 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1924 :
1925 71 : if (input1 != nullptr) {
1926 : SetEffect(graph()->NewNode(store_op, stack_slot,
1927 : mcgraph()->Int32Constant(type_size), input1,
1928 29 : Effect(), Control()));
1929 : }
1930 :
1931 71 : MachineType sig_types[] = {MachineType::Pointer()};
1932 : MachineSignature sig(0, 1, sig_types);
1933 71 : BuildCCall(&sig, function, stack_slot);
1934 :
1935 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(type),
1936 : stack_slot, mcgraph()->Int32Constant(0),
1937 213 : Effect(), Control()));
1938 : }
1939 :
1940 0 : Node* WasmGraphBuilder::BuildF32SConvertI64(Node* input) {
1941 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1942 : return BuildIntToFloatConversionInstruction(
1943 : input, ExternalReference::wasm_int64_to_float32(),
1944 0 : MachineRepresentation::kWord64, MachineType::Float32());
1945 : }
1946 0 : Node* WasmGraphBuilder::BuildF32UConvertI64(Node* input) {
1947 : // TODO(titzer/bradnelson): Check handlng of asm.js case.
1948 : return BuildIntToFloatConversionInstruction(
1949 : input, ExternalReference::wasm_uint64_to_float32(),
1950 0 : MachineRepresentation::kWord64, MachineType::Float32());
1951 : }
1952 0 : Node* WasmGraphBuilder::BuildF64SConvertI64(Node* input) {
1953 : return BuildIntToFloatConversionInstruction(
1954 : input, ExternalReference::wasm_int64_to_float64(),
1955 0 : MachineRepresentation::kWord64, MachineType::Float64());
1956 : }
1957 0 : Node* WasmGraphBuilder::BuildF64UConvertI64(Node* input) {
1958 : return BuildIntToFloatConversionInstruction(
1959 : input, ExternalReference::wasm_uint64_to_float64(),
1960 0 : MachineRepresentation::kWord64, MachineType::Float64());
1961 : }
1962 :
1963 0 : Node* WasmGraphBuilder::BuildIntToFloatConversionInstruction(
1964 : Node* input, ExternalReference ref,
1965 : MachineRepresentation parameter_representation,
1966 0 : const MachineType result_type) {
1967 : int stack_slot_size =
1968 : std::max(ElementSizeInBytes(parameter_representation),
1969 0 : ElementSizeInBytes(result_type.representation()));
1970 : Node* stack_slot =
1971 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
1972 : const Operator* store_op = mcgraph()->machine()->Store(
1973 0 : StoreRepresentation(parameter_representation, kNoWriteBarrier));
1974 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
1975 0 : input, Effect(), Control()));
1976 0 : MachineType sig_types[] = {MachineType::Pointer()};
1977 : MachineSignature sig(0, 1, sig_types);
1978 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
1979 0 : BuildCCall(&sig, function, stack_slot);
1980 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
1981 : stack_slot, mcgraph()->Int32Constant(0),
1982 0 : Effect(), Control()));
1983 : }
1984 :
1985 : namespace {
1986 :
1987 0 : ExternalReference convert_ccall_ref(WasmGraphBuilder* builder,
1988 : wasm::WasmOpcode opcode) {
1989 0 : switch (opcode) {
1990 : case wasm::kExprI64SConvertF32:
1991 : case wasm::kExprI64SConvertSatF32:
1992 0 : return ExternalReference::wasm_float32_to_int64();
1993 : case wasm::kExprI64UConvertF32:
1994 : case wasm::kExprI64UConvertSatF32:
1995 0 : return ExternalReference::wasm_float32_to_uint64();
1996 : case wasm::kExprI64SConvertF64:
1997 : case wasm::kExprI64SConvertSatF64:
1998 0 : return ExternalReference::wasm_float64_to_int64();
1999 : case wasm::kExprI64UConvertF64:
2000 : case wasm::kExprI64UConvertSatF64:
2001 0 : return ExternalReference::wasm_float64_to_uint64();
2002 : default:
2003 0 : UNREACHABLE();
2004 : }
2005 : }
2006 :
2007 : } // namespace
2008 :
2009 0 : Node* WasmGraphBuilder::BuildCcallConvertFloat(Node* input,
2010 : wasm::WasmCodePosition position,
2011 0 : wasm::WasmOpcode opcode) {
2012 0 : const MachineType int_ty = IntConvertType(opcode);
2013 0 : const MachineType float_ty = FloatConvertType(opcode);
2014 0 : ExternalReference call_ref = convert_ccall_ref(this, opcode);
2015 0 : int stack_slot_size = std::max(ElementSizeInBytes(int_ty.representation()),
2016 0 : ElementSizeInBytes(float_ty.representation()));
2017 : Node* stack_slot =
2018 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(stack_slot_size));
2019 : const Operator* store_op = mcgraph()->machine()->Store(
2020 0 : StoreRepresentation(float_ty.representation(), kNoWriteBarrier));
2021 : SetEffect(graph()->NewNode(store_op, stack_slot, Int32Constant(0), input,
2022 : Effect(), Control()));
2023 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2024 : MachineSignature sig(1, 1, sig_types);
2025 : Node* function =
2026 0 : graph()->NewNode(mcgraph()->common()->ExternalConstant(call_ref));
2027 0 : Node* overflow = BuildCCall(&sig, function, stack_slot);
2028 0 : if (IsTrappingConvertOp(opcode)) {
2029 : ZeroCheck32(wasm::kTrapFloatUnrepresentable, overflow, position);
2030 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty),
2031 : stack_slot, Int32Constant(0), Effect(),
2032 0 : Control()));
2033 : }
2034 0 : Node* test = Binop(wasm::kExprI32Eq, overflow, Int32Constant(0), position);
2035 0 : Diamond tl_d(graph(), mcgraph()->common(), test, BranchHint::kFalse);
2036 0 : tl_d.Chain(Control());
2037 0 : Node* nan_test = Binop(NeOp(float_ty), input, input);
2038 0 : Diamond nan_d(graph(), mcgraph()->common(), nan_test, BranchHint::kFalse);
2039 0 : nan_d.Nest(tl_d, true);
2040 0 : Node* neg_test = Binop(LtOp(float_ty), input, Zero(this, float_ty));
2041 0 : Diamond sat_d(graph(), mcgraph()->common(), neg_test, BranchHint::kNone);
2042 0 : sat_d.Nest(nan_d, false);
2043 : Node* sat_val =
2044 0 : sat_d.Phi(int_ty.representation(), Min(this, int_ty), Max(this, int_ty));
2045 : Node* load =
2046 : SetEffect(graph()->NewNode(mcgraph()->machine()->Load(int_ty), stack_slot,
2047 0 : Int32Constant(0), Effect(), Control()));
2048 : Node* nan_val =
2049 0 : nan_d.Phi(int_ty.representation(), Zero(this, int_ty), sat_val);
2050 0 : return tl_d.Phi(int_ty.representation(), nan_val, load);
2051 : }
2052 :
2053 8960 : Node* WasmGraphBuilder::MemoryGrow(Node* input) {
2054 1120 : needs_stack_check_ = true;
2055 :
2056 : WasmMemoryGrowDescriptor interface_descriptor;
2057 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2058 : mcgraph()->zone(), // zone
2059 : interface_descriptor, // descriptor
2060 : interface_descriptor.GetStackParameterCount(), // stack parameter count
2061 : CallDescriptor::kNoFlags, // flags
2062 : Operator::kNoProperties, // properties
2063 1120 : StubCallMode::kCallWasmRuntimeStub); // stub call mode
2064 : // A direct call to a wasm runtime stub defined in this module.
2065 : // Just encode the stub index. This will be patched at relocation.
2066 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2067 1120 : wasm::WasmCode::kWasmMemoryGrow, RelocInfo::WASM_STUB_CALL);
2068 : return SetEffect(
2069 : SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
2070 2240 : call_target, input, Effect(), Control())));
2071 : }
2072 :
2073 503 : Node* WasmGraphBuilder::Throw(uint32_t exception_index,
2074 : const wasm::WasmException* exception,
2075 4190 : const Vector<Node*> values) {
2076 503 : needs_stack_check_ = true;
2077 503 : uint32_t encoded_size = WasmExceptionPackage::GetEncodedSize(exception);
2078 : Node* create_parameters[] = {
2079 503 : LoadExceptionTagFromTable(exception_index),
2080 1008 : BuildChangeUint31ToSmi(Uint32Constant(encoded_size))};
2081 : Node* except_obj =
2082 : BuildCallToRuntime(Runtime::kWasmThrowCreate, create_parameters,
2083 504 : arraysize(create_parameters));
2084 : Node* values_array =
2085 504 : BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2086 503 : uint32_t index = 0;
2087 1534 : const wasm::WasmExceptionSig* sig = exception->sig;
2088 503 : MachineOperatorBuilder* m = mcgraph()->machine();
2089 1534 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2090 528 : Node* value = values[i];
2091 264 : switch (sig->GetParam(i)) {
2092 : case wasm::kWasmF32:
2093 16 : value = graph()->NewNode(m->BitcastFloat32ToInt32(), value);
2094 : V8_FALLTHROUGH;
2095 : case wasm::kWasmI32:
2096 184 : BuildEncodeException32BitValue(values_array, &index, value);
2097 184 : break;
2098 : case wasm::kWasmF64:
2099 16 : value = graph()->NewNode(m->BitcastFloat64ToInt64(), value);
2100 : V8_FALLTHROUGH;
2101 : case wasm::kWasmI64: {
2102 : Node* upper32 = graph()->NewNode(
2103 : m->TruncateInt64ToInt32(),
2104 32 : Binop(wasm::kExprI64ShrU, value, Int64Constant(32)));
2105 32 : BuildEncodeException32BitValue(values_array, &index, upper32);
2106 32 : Node* lower32 = graph()->NewNode(m->TruncateInt64ToInt32(), value);
2107 32 : BuildEncodeException32BitValue(values_array, &index, lower32);
2108 32 : break;
2109 : }
2110 : case wasm::kWasmS128:
2111 : BuildEncodeException32BitValue(
2112 : values_array, &index,
2113 32 : graph()->NewNode(m->I32x4ExtractLane(0), value));
2114 : BuildEncodeException32BitValue(
2115 : values_array, &index,
2116 32 : graph()->NewNode(m->I32x4ExtractLane(1), value));
2117 : BuildEncodeException32BitValue(
2118 : values_array, &index,
2119 32 : graph()->NewNode(m->I32x4ExtractLane(2), value));
2120 : BuildEncodeException32BitValue(
2121 : values_array, &index,
2122 32 : graph()->NewNode(m->I32x4ExtractLane(3), value));
2123 16 : break;
2124 : case wasm::kWasmAnyRef:
2125 128 : STORE_FIXED_ARRAY_SLOT_ANY(values_array, index, value);
2126 32 : ++index;
2127 32 : break;
2128 : default:
2129 0 : UNREACHABLE();
2130 : }
2131 : }
2132 : DCHECK_EQ(encoded_size, index);
2133 : WasmThrowDescriptor interface_descriptor;
2134 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2135 : mcgraph()->zone(), interface_descriptor,
2136 : interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2137 503 : Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2138 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2139 504 : wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2140 : return SetEffect(SetControl(
2141 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2142 1512 : except_obj, Effect(), Control())));
2143 : }
2144 :
2145 312 : void WasmGraphBuilder::BuildEncodeException32BitValue(Node* values_array,
2146 : uint32_t* index,
2147 3432 : Node* value) {
2148 312 : MachineOperatorBuilder* machine = mcgraph()->machine();
2149 : Node* upper_halfword_as_smi = BuildChangeUint31ToSmi(
2150 624 : graph()->NewNode(machine->Word32Shr(), value, Int32Constant(16)));
2151 1248 : STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, upper_halfword_as_smi);
2152 312 : ++(*index);
2153 : Node* lower_halfword_as_smi = BuildChangeUint31ToSmi(
2154 624 : graph()->NewNode(machine->Word32And(), value, Int32Constant(0xFFFFu)));
2155 1248 : STORE_FIXED_ARRAY_SLOT_SMI(values_array, *index, lower_halfword_as_smi);
2156 312 : ++(*index);
2157 312 : }
2158 :
2159 272 : Node* WasmGraphBuilder::BuildDecodeException32BitValue(Node* values_array,
2160 2720 : uint32_t* index) {
2161 272 : MachineOperatorBuilder* machine = mcgraph()->machine();
2162 : Node* upper =
2163 1088 : BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2164 272 : (*index)++;
2165 272 : upper = graph()->NewNode(machine->Word32Shl(), upper, Int32Constant(16));
2166 : Node* lower =
2167 1088 : BuildChangeSmiToInt32(LOAD_FIXED_ARRAY_SLOT_SMI(values_array, *index));
2168 272 : (*index)++;
2169 272 : Node* value = graph()->NewNode(machine->Word32Or(), upper, lower);
2170 272 : return value;
2171 : }
2172 :
2173 16 : Node* WasmGraphBuilder::BuildDecodeException64BitValue(Node* values_array,
2174 : uint32_t* index) {
2175 : Node* upper = Binop(wasm::kExprI64Shl,
2176 : Unop(wasm::kExprI64UConvertI32,
2177 : BuildDecodeException32BitValue(values_array, index)),
2178 16 : Int64Constant(32));
2179 : Node* lower = Unop(wasm::kExprI64UConvertI32,
2180 16 : BuildDecodeException32BitValue(values_array, index));
2181 16 : return Binop(wasm::kExprI64Ior, upper, lower);
2182 : }
2183 :
2184 3385 : Node* WasmGraphBuilder::Rethrow(Node* except_obj) {
2185 423 : needs_stack_check_ = true;
2186 : WasmThrowDescriptor interface_descriptor;
2187 : auto call_descriptor = Linkage::GetStubCallDescriptor(
2188 : mcgraph()->zone(), interface_descriptor,
2189 : interface_descriptor.GetStackParameterCount(), CallDescriptor::kNoFlags,
2190 423 : Operator::kNoProperties, StubCallMode::kCallWasmRuntimeStub);
2191 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
2192 422 : wasm::WasmCode::kWasmThrow, RelocInfo::WASM_STUB_CALL);
2193 : return SetEffect(SetControl(
2194 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), call_target,
2195 847 : except_obj, Effect(), Control())));
2196 : }
2197 :
2198 376 : Node* WasmGraphBuilder::ExceptionTagEqual(Node* caught_tag,
2199 376 : Node* expected_tag) {
2200 376 : MachineOperatorBuilder* machine = mcgraph()->machine();
2201 752 : return graph()->NewNode(machine->WordEqual(), caught_tag, expected_tag);
2202 : }
2203 :
2204 8796 : Node* WasmGraphBuilder::LoadExceptionTagFromTable(uint32_t exception_index) {
2205 : Node* exceptions_table =
2206 1758 : LOAD_INSTANCE_FIELD(ExceptionsTable, MachineType::TaggedPointer());
2207 2640 : Node* tag = LOAD_FIXED_ARRAY_SLOT_PTR(exceptions_table, exception_index);
2208 880 : return tag;
2209 : }
2210 :
2211 376 : Node* WasmGraphBuilder::GetExceptionTag(Node* except_obj) {
2212 376 : needs_stack_check_ = true;
2213 376 : return BuildCallToRuntime(Runtime::kWasmExceptionGetTag, &except_obj, 1);
2214 : }
2215 :
2216 376 : Node** WasmGraphBuilder::GetExceptionValues(
2217 112 : Node* except_obj, const wasm::WasmException* exception) {
2218 : Node* values_array =
2219 376 : BuildCallToRuntime(Runtime::kWasmExceptionGetValues, &except_obj, 1);
2220 375 : uint32_t index = 0;
2221 1246 : const wasm::WasmExceptionSig* sig = exception->sig;
2222 375 : Node** values = Buffer(sig->parameter_count());
2223 1246 : for (size_t i = 0; i < sig->parameter_count(); ++i) {
2224 : Node* value;
2225 248 : switch (sig->GetParam(i)) {
2226 : case wasm::kWasmI32:
2227 200 : value = BuildDecodeException32BitValue(values_array, &index);
2228 200 : break;
2229 : case wasm::kWasmI64:
2230 8 : value = BuildDecodeException64BitValue(values_array, &index);
2231 8 : break;
2232 : case wasm::kWasmF32: {
2233 : value = Unop(wasm::kExprF32ReinterpretI32,
2234 8 : BuildDecodeException32BitValue(values_array, &index));
2235 8 : break;
2236 : }
2237 : case wasm::kWasmF64: {
2238 : value = Unop(wasm::kExprF64ReinterpretI64,
2239 8 : BuildDecodeException64BitValue(values_array, &index));
2240 8 : break;
2241 : }
2242 : case wasm::kWasmS128:
2243 : value = graph()->NewNode(
2244 : mcgraph()->machine()->I32x4Splat(),
2245 16 : BuildDecodeException32BitValue(values_array, &index));
2246 : value = graph()->NewNode(
2247 : mcgraph()->machine()->I32x4ReplaceLane(1), value,
2248 16 : BuildDecodeException32BitValue(values_array, &index));
2249 : value = graph()->NewNode(
2250 : mcgraph()->machine()->I32x4ReplaceLane(2), value,
2251 16 : BuildDecodeException32BitValue(values_array, &index));
2252 : value = graph()->NewNode(
2253 : mcgraph()->machine()->I32x4ReplaceLane(3), value,
2254 16 : BuildDecodeException32BitValue(values_array, &index));
2255 8 : break;
2256 : case wasm::kWasmAnyRef:
2257 48 : value = LOAD_FIXED_ARRAY_SLOT_ANY(values_array, index);
2258 16 : ++index;
2259 16 : break;
2260 : default:
2261 0 : UNREACHABLE();
2262 : }
2263 248 : values[i] = value;
2264 : }
2265 : DCHECK_EQ(index, WasmExceptionPackage::GetEncodedSize(exception));
2266 375 : return values;
2267 : }
2268 :
2269 14180 : Node* WasmGraphBuilder::BuildI32DivS(Node* left, Node* right,
2270 86992 : wasm::WasmCodePosition position) {
2271 14180 : MachineOperatorBuilder* m = mcgraph()->machine();
2272 : ZeroCheck32(wasm::kTrapDivByZero, right, position);
2273 : Node* before = Control();
2274 : Node* denom_is_m1;
2275 : Node* denom_is_not_m1;
2276 : BranchExpectFalse(
2277 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2278 14180 : &denom_is_m1, &denom_is_not_m1);
2279 14180 : SetControl(denom_is_m1);
2280 14180 : TrapIfEq32(wasm::kTrapDivUnrepresentable, left, kMinInt, position);
2281 14180 : if (Control() != denom_is_m1) {
2282 : SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2283 1912 : Control()));
2284 : } else {
2285 : SetControl(before);
2286 : }
2287 28360 : return graph()->NewNode(m->Int32Div(), left, right, Control());
2288 : }
2289 :
2290 14104 : Node* WasmGraphBuilder::BuildI32RemS(Node* left, Node* right,
2291 70520 : wasm::WasmCodePosition position) {
2292 14104 : MachineOperatorBuilder* m = mcgraph()->machine();
2293 :
2294 : ZeroCheck32(wasm::kTrapRemByZero, right, position);
2295 :
2296 : Diamond d(
2297 : graph(), mcgraph()->common(),
2298 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2299 42312 : BranchHint::kFalse);
2300 14104 : d.Chain(Control());
2301 :
2302 : return d.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2303 28208 : graph()->NewNode(m->Int32Mod(), left, right, d.if_false));
2304 : }
2305 :
2306 14124 : Node* WasmGraphBuilder::BuildI32DivU(Node* left, Node* right,
2307 14124 : wasm::WasmCodePosition position) {
2308 14124 : MachineOperatorBuilder* m = mcgraph()->machine();
2309 : return graph()->NewNode(m->Uint32Div(), left, right,
2310 28248 : ZeroCheck32(wasm::kTrapDivByZero, right, position));
2311 : }
2312 :
2313 14084 : Node* WasmGraphBuilder::BuildI32RemU(Node* left, Node* right,
2314 14084 : wasm::WasmCodePosition position) {
2315 14084 : MachineOperatorBuilder* m = mcgraph()->machine();
2316 : return graph()->NewNode(m->Uint32Mod(), left, right,
2317 28168 : ZeroCheck32(wasm::kTrapRemByZero, right, position));
2318 : }
2319 :
2320 1016 : Node* WasmGraphBuilder::BuildI32AsmjsDivS(Node* left, Node* right) {
2321 385 : MachineOperatorBuilder* m = mcgraph()->machine();
2322 :
2323 : Int32Matcher mr(right);
2324 385 : if (mr.HasValue()) {
2325 317 : if (mr.Value() == 0) {
2326 13 : return mcgraph()->Int32Constant(0);
2327 304 : } else if (mr.Value() == -1) {
2328 : // The result is the negation of the left input.
2329 26 : return graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2330 : }
2331 582 : return graph()->NewNode(m->Int32Div(), left, right, Control());
2332 : }
2333 :
2334 : // asm.js semantics return 0 on divide or mod by zero.
2335 68 : if (m->Int32DivIsSafe()) {
2336 : // The hardware instruction does the right thing (e.g. arm).
2337 0 : return graph()->NewNode(m->Int32Div(), left, right, graph()->start());
2338 : }
2339 :
2340 : // Check denominator for zero.
2341 : Diamond z(
2342 : graph(), mcgraph()->common(),
2343 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2344 204 : BranchHint::kFalse);
2345 :
2346 : // Check numerator for -1. (avoid minint / -1 case).
2347 : Diamond n(
2348 : graph(), mcgraph()->common(),
2349 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(-1)),
2350 204 : BranchHint::kFalse);
2351 :
2352 68 : Node* div = graph()->NewNode(m->Int32Div(), left, right, z.if_false);
2353 : Node* neg =
2354 68 : graph()->NewNode(m->Int32Sub(), mcgraph()->Int32Constant(0), left);
2355 :
2356 : return n.Phi(
2357 : MachineRepresentation::kWord32, neg,
2358 68 : z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0), div));
2359 : }
2360 :
2361 816 : Node* WasmGraphBuilder::BuildI32AsmjsRemS(Node* left, Node* right) {
2362 421 : CommonOperatorBuilder* c = mcgraph()->common();
2363 421 : MachineOperatorBuilder* m = mcgraph()->machine();
2364 421 : Node* const zero = mcgraph()->Int32Constant(0);
2365 :
2366 : Int32Matcher mr(right);
2367 421 : if (mr.HasValue()) {
2368 360 : if (mr.Value() == 0 || mr.Value() == -1) {
2369 : return zero;
2370 : }
2371 668 : return graph()->NewNode(m->Int32Mod(), left, right, Control());
2372 : }
2373 :
2374 : // General case for signed integer modulus, with optimization for (unknown)
2375 : // power of 2 right hand side.
2376 : //
2377 : // if 0 < right then
2378 : // msk = right - 1
2379 : // if right & msk != 0 then
2380 : // left % right
2381 : // else
2382 : // if left < 0 then
2383 : // -(-left & msk)
2384 : // else
2385 : // left & msk
2386 : // else
2387 : // if right < -1 then
2388 : // left % right
2389 : // else
2390 : // zero
2391 : //
2392 : // Note: We do not use the Diamond helper class here, because it really hurts
2393 : // readability with nested diamonds.
2394 61 : Node* const minus_one = mcgraph()->Int32Constant(-1);
2395 :
2396 61 : const Operator* const merge_op = c->Merge(2);
2397 61 : const Operator* const phi_op = c->Phi(MachineRepresentation::kWord32, 2);
2398 :
2399 61 : Node* check0 = graph()->NewNode(m->Int32LessThan(), zero, right);
2400 : Node* branch0 =
2401 61 : graph()->NewNode(c->Branch(BranchHint::kTrue), check0, graph()->start());
2402 :
2403 61 : Node* if_true0 = graph()->NewNode(c->IfTrue(), branch0);
2404 : Node* true0;
2405 : {
2406 61 : Node* msk = graph()->NewNode(m->Int32Add(), right, minus_one);
2407 :
2408 61 : Node* check1 = graph()->NewNode(m->Word32And(), right, msk);
2409 61 : Node* branch1 = graph()->NewNode(c->Branch(), check1, if_true0);
2410 :
2411 61 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2412 61 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2413 :
2414 61 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2415 : Node* false1;
2416 : {
2417 61 : Node* check2 = graph()->NewNode(m->Int32LessThan(), left, zero);
2418 : Node* branch2 =
2419 61 : graph()->NewNode(c->Branch(BranchHint::kFalse), check2, if_false1);
2420 :
2421 61 : Node* if_true2 = graph()->NewNode(c->IfTrue(), branch2);
2422 : Node* true2 = graph()->NewNode(
2423 : m->Int32Sub(), zero,
2424 : graph()->NewNode(m->Word32And(),
2425 183 : graph()->NewNode(m->Int32Sub(), zero, left), msk));
2426 :
2427 61 : Node* if_false2 = graph()->NewNode(c->IfFalse(), branch2);
2428 61 : Node* false2 = graph()->NewNode(m->Word32And(), left, msk);
2429 :
2430 : if_false1 = graph()->NewNode(merge_op, if_true2, if_false2);
2431 : false1 = graph()->NewNode(phi_op, true2, false2, if_false1);
2432 : }
2433 :
2434 : if_true0 = graph()->NewNode(merge_op, if_true1, if_false1);
2435 : true0 = graph()->NewNode(phi_op, true1, false1, if_true0);
2436 : }
2437 :
2438 61 : Node* if_false0 = graph()->NewNode(c->IfFalse(), branch0);
2439 : Node* false0;
2440 : {
2441 61 : Node* check1 = graph()->NewNode(m->Int32LessThan(), right, minus_one);
2442 : Node* branch1 =
2443 61 : graph()->NewNode(c->Branch(BranchHint::kTrue), check1, if_false0);
2444 :
2445 61 : Node* if_true1 = graph()->NewNode(c->IfTrue(), branch1);
2446 61 : Node* true1 = graph()->NewNode(m->Int32Mod(), left, right, if_true1);
2447 :
2448 61 : Node* if_false1 = graph()->NewNode(c->IfFalse(), branch1);
2449 : Node* false1 = zero;
2450 :
2451 : if_false0 = graph()->NewNode(merge_op, if_true1, if_false1);
2452 : false0 = graph()->NewNode(phi_op, true1, false1, if_false0);
2453 : }
2454 :
2455 : Node* merge0 = graph()->NewNode(merge_op, if_true0, if_false0);
2456 61 : return graph()->NewNode(phi_op, true0, false0, merge0);
2457 : }
2458 :
2459 548 : Node* WasmGraphBuilder::BuildI32AsmjsDivU(Node* left, Node* right) {
2460 137 : MachineOperatorBuilder* m = mcgraph()->machine();
2461 : // asm.js semantics return 0 on divide or mod by zero.
2462 137 : if (m->Uint32DivIsSafe()) {
2463 : // The hardware instruction does the right thing (e.g. arm).
2464 0 : return graph()->NewNode(m->Uint32Div(), left, right, graph()->start());
2465 : }
2466 :
2467 : // Explicit check for x % 0.
2468 : Diamond z(
2469 : graph(), mcgraph()->common(),
2470 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2471 411 : BranchHint::kFalse);
2472 :
2473 : return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2474 : graph()->NewNode(mcgraph()->machine()->Uint32Div(), left, right,
2475 411 : z.if_false));
2476 : }
2477 :
2478 744 : Node* WasmGraphBuilder::BuildI32AsmjsRemU(Node* left, Node* right) {
2479 186 : MachineOperatorBuilder* m = mcgraph()->machine();
2480 : // asm.js semantics return 0 on divide or mod by zero.
2481 : // Explicit check for x % 0.
2482 : Diamond z(
2483 : graph(), mcgraph()->common(),
2484 : graph()->NewNode(m->Word32Equal(), right, mcgraph()->Int32Constant(0)),
2485 558 : BranchHint::kFalse);
2486 :
2487 : Node* rem = graph()->NewNode(mcgraph()->machine()->Uint32Mod(), left, right,
2488 372 : z.if_false);
2489 : return z.Phi(MachineRepresentation::kWord32, mcgraph()->Int32Constant(0),
2490 186 : rem);
2491 : }
2492 :
2493 720 : Node* WasmGraphBuilder::BuildI64DivS(Node* left, Node* right,
2494 7192 : wasm::WasmCodePosition position) {
2495 1440 : if (mcgraph()->machine()->Is32()) {
2496 : return BuildDiv64Call(left, right, ExternalReference::wasm_int64_div(),
2497 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2498 : }
2499 : ZeroCheck64(wasm::kTrapDivByZero, right, position);
2500 : Node* before = Control();
2501 : Node* denom_is_m1;
2502 : Node* denom_is_not_m1;
2503 : BranchExpectFalse(graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2504 : mcgraph()->Int64Constant(-1)),
2505 1440 : &denom_is_m1, &denom_is_not_m1);
2506 720 : SetControl(denom_is_m1);
2507 : TrapIfEq64(wasm::kTrapDivUnrepresentable, left,
2508 720 : std::numeric_limits<int64_t>::min(), position);
2509 720 : if (Control() != denom_is_m1) {
2510 : SetControl(graph()->NewNode(mcgraph()->common()->Merge(2), denom_is_not_m1,
2511 1432 : Control()));
2512 : } else {
2513 : SetControl(before);
2514 : }
2515 : return graph()->NewNode(mcgraph()->machine()->Int64Div(), left, right,
2516 1440 : Control());
2517 : }
2518 :
2519 644 : Node* WasmGraphBuilder::BuildI64RemS(Node* left, Node* right,
2520 4508 : wasm::WasmCodePosition position) {
2521 1288 : if (mcgraph()->machine()->Is32()) {
2522 : return BuildDiv64Call(left, right, ExternalReference::wasm_int64_mod(),
2523 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2524 : }
2525 : ZeroCheck64(wasm::kTrapRemByZero, right, position);
2526 : Diamond d(mcgraph()->graph(), mcgraph()->common(),
2527 : graph()->NewNode(mcgraph()->machine()->Word64Equal(), right,
2528 1932 : mcgraph()->Int64Constant(-1)));
2529 :
2530 644 : d.Chain(Control());
2531 :
2532 : Node* rem = graph()->NewNode(mcgraph()->machine()->Int64Mod(), left, right,
2533 1288 : d.if_false);
2534 :
2535 : return d.Phi(MachineRepresentation::kWord64, mcgraph()->Int64Constant(0),
2536 644 : rem);
2537 : }
2538 :
2539 660 : Node* WasmGraphBuilder::BuildI64DivU(Node* left, Node* right,
2540 1320 : wasm::WasmCodePosition position) {
2541 1320 : if (mcgraph()->machine()->Is32()) {
2542 : return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_div(),
2543 0 : MachineType::Int64(), wasm::kTrapDivByZero, position);
2544 : }
2545 : return graph()->NewNode(mcgraph()->machine()->Uint64Div(), left, right,
2546 1320 : ZeroCheck64(wasm::kTrapDivByZero, right, position));
2547 : }
2548 644 : Node* WasmGraphBuilder::BuildI64RemU(Node* left, Node* right,
2549 1288 : wasm::WasmCodePosition position) {
2550 1288 : if (mcgraph()->machine()->Is32()) {
2551 : return BuildDiv64Call(left, right, ExternalReference::wasm_uint64_mod(),
2552 0 : MachineType::Int64(), wasm::kTrapRemByZero, position);
2553 : }
2554 : return graph()->NewNode(mcgraph()->machine()->Uint64Mod(), left, right,
2555 1288 : ZeroCheck64(wasm::kTrapRemByZero, right, position));
2556 : }
2557 :
2558 0 : Node* WasmGraphBuilder::BuildDiv64Call(Node* left, Node* right,
2559 : ExternalReference ref,
2560 : MachineType result_type,
2561 : wasm::TrapReason trap_zero,
2562 0 : wasm::WasmCodePosition position) {
2563 : Node* stack_slot =
2564 0 : graph()->NewNode(mcgraph()->machine()->StackSlot(2 * sizeof(double)));
2565 :
2566 : const Operator* store_op = mcgraph()->machine()->Store(
2567 0 : StoreRepresentation(MachineRepresentation::kWord64, kNoWriteBarrier));
2568 : SetEffect(graph()->NewNode(store_op, stack_slot, mcgraph()->Int32Constant(0),
2569 0 : left, Effect(), Control()));
2570 : SetEffect(graph()->NewNode(store_op, stack_slot,
2571 : mcgraph()->Int32Constant(sizeof(double)), right,
2572 0 : Effect(), Control()));
2573 :
2574 0 : MachineType sig_types[] = {MachineType::Int32(), MachineType::Pointer()};
2575 : MachineSignature sig(1, 1, sig_types);
2576 :
2577 0 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(ref));
2578 0 : Node* call = BuildCCall(&sig, function, stack_slot);
2579 :
2580 : ZeroCheck32(trap_zero, call, position);
2581 0 : TrapIfEq32(wasm::kTrapDivUnrepresentable, call, -1, position);
2582 : return SetEffect(graph()->NewNode(mcgraph()->machine()->Load(result_type),
2583 : stack_slot, mcgraph()->Int32Constant(0),
2584 0 : Effect(), Control()));
2585 : }
2586 :
2587 : template <typename... Args>
2588 159 : Node* WasmGraphBuilder::BuildCCall(MachineSignature* sig, Node* function,
2589 440 : Args... args) {
2590 : DCHECK_LE(sig->return_count(), 1);
2591 : DCHECK_EQ(sizeof...(args), sig->parameter_count());
2592 477 : Node* const call_args[] = {function, args..., Effect(), Control()};
2593 :
2594 : auto call_descriptor =
2595 159 : Linkage::GetSimplifiedCDescriptor(mcgraph()->zone(), sig);
2596 :
2597 159 : const Operator* op = mcgraph()->common()->Call(call_descriptor);
2598 318 : return SetEffect(graph()->NewNode(op, arraysize(call_args), call_args));
2599 : }
2600 :
2601 512350 : Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args,
2602 : Node*** rets,
2603 : wasm::WasmCodePosition position,
2604 : Node* instance_node,
2605 853922 : UseRetpoline use_retpoline) {
2606 170783 : if (instance_node == nullptr) {
2607 : DCHECK_NOT_NULL(instance_node_);
2608 : instance_node = instance_node_.get();
2609 : }
2610 170783 : needs_stack_check_ = true;
2611 : const size_t params = sig->parameter_count();
2612 : const size_t extra = 3; // instance_node, effect, and control.
2613 170783 : const size_t count = 1 + params + extra;
2614 :
2615 : // Reallocate the buffer to make space for extra inputs.
2616 170783 : args = Realloc(args, 1 + params, count);
2617 :
2618 : // Make room for the instance_node parameter at index 1, just after code.
2619 170783 : memmove(&args[2], &args[1], params * sizeof(Node*));
2620 170783 : args[1] = instance_node;
2621 :
2622 : // Add effect and control inputs.
2623 341566 : args[params + 2] = Effect();
2624 341566 : args[params + 3] = Control();
2625 :
2626 : auto call_descriptor =
2627 341566 : GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline);
2628 170786 : const Operator* op = mcgraph()->common()->Call(call_descriptor);
2629 341570 : Node* call = SetEffect(graph()->NewNode(op, static_cast<int>(count), args));
2630 : DCHECK(position == wasm::kNoCodePosition || position > 0);
2631 170787 : if (position > 0) SetSourcePosition(call, position);
2632 :
2633 : size_t ret_count = sig->return_count();
2634 170784 : if (ret_count == 0) return call; // No return value.
2635 :
2636 37761 : *rets = Buffer(ret_count);
2637 37762 : if (ret_count == 1) {
2638 : // Only a single return value.
2639 36626 : (*rets)[0] = call;
2640 : } else {
2641 : // Create projections for all return values.
2642 2288 : for (size_t i = 0; i < ret_count; i++) {
2643 2288 : (*rets)[i] = graph()->NewNode(mcgraph()->common()->Projection(i), call,
2644 4576 : graph()->start());
2645 : }
2646 : }
2647 : return call;
2648 : }
2649 :
2650 6942 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2651 : Node*** rets,
2652 : wasm::WasmCodePosition position,
2653 138872 : int func_index) {
2654 : // Load the imported function refs array from the instance.
2655 : Node* imported_function_refs =
2656 13885 : LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2657 : Node* ref_node =
2658 13886 : LOAD_FIXED_ARRAY_SLOT_PTR(imported_function_refs, func_index);
2659 :
2660 : // Load the target from the imported_targets array at a known offset.
2661 : Node* imported_targets =
2662 13887 : LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2663 : Node* target_node = SetEffect(graph()->NewNode(
2664 : mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2665 : mcgraph()->Int32Constant(func_index * kSystemPointerSize), Effect(),
2666 20831 : Control()));
2667 6944 : args[0] = target_node;
2668 : return BuildWasmCall(sig, args, rets, position, ref_node,
2669 6944 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2670 : }
2671 :
2672 464 : Node* WasmGraphBuilder::BuildImportCall(wasm::FunctionSig* sig, Node** args,
2673 : Node*** rets,
2674 : wasm::WasmCodePosition position,
2675 10208 : Node* func_index) {
2676 : // Load the imported function refs array from the instance.
2677 : Node* imported_function_refs =
2678 928 : LOAD_INSTANCE_FIELD(ImportedFunctionRefs, MachineType::TaggedPointer());
2679 : // Access fixed array at {header_size - tag + func_index * kTaggedSize}.
2680 : Node* imported_instances_data = graph()->NewNode(
2681 : mcgraph()->machine()->IntAdd(), imported_function_refs,
2682 : mcgraph()->IntPtrConstant(
2683 928 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
2684 : Node* func_index_times_tagged_size = graph()->NewNode(
2685 : mcgraph()->machine()->IntMul(), Uint32ToUintptr(func_index),
2686 928 : mcgraph()->Int32Constant(kTaggedSize));
2687 : Node* ref_node = SetEffect(
2688 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::TaggedPointer()),
2689 : imported_instances_data, func_index_times_tagged_size,
2690 464 : Effect(), Control()));
2691 :
2692 : // Load the target from the imported_targets array at the offset of
2693 : // {func_index}.
2694 : STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
2695 : Node* func_index_times_pointersize = func_index_times_tagged_size;
2696 : Node* imported_targets =
2697 928 : LOAD_INSTANCE_FIELD(ImportedFunctionTargets, MachineType::Pointer());
2698 : Node* target_node = SetEffect(graph()->NewNode(
2699 : mcgraph()->machine()->Load(MachineType::Pointer()), imported_targets,
2700 464 : func_index_times_pointersize, Effect(), Control()));
2701 464 : args[0] = target_node;
2702 : return BuildWasmCall(sig, args, rets, position, ref_node,
2703 464 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2704 : }
2705 :
2706 29123 : Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets,
2707 22179 : wasm::WasmCodePosition position) {
2708 : DCHECK_NULL(args[0]);
2709 58246 : wasm::FunctionSig* sig = env_->module->functions[index].sig;
2710 :
2711 29123 : if (env_ && index < env_->module->num_imported_functions) {
2712 : // Call to an imported function.
2713 6944 : return BuildImportCall(sig, args, rets, position, index);
2714 : }
2715 :
2716 : // A direct call to a wasm function defined in this module.
2717 : // Just encode the function index. This will be patched at instantiation.
2718 : Address code = static_cast<Address>(index);
2719 44358 : args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL);
2720 :
2721 22183 : return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline);
2722 : }
2723 :
2724 3285 : Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args,
2725 : Node*** rets,
2726 98546 : wasm::WasmCodePosition position) {
2727 : DCHECK_NOT_NULL(args[0]);
2728 : DCHECK_NOT_NULL(env_);
2729 :
2730 : // Assume only one table for now.
2731 6570 : wasm::FunctionSig* sig = env_->module->signatures[sig_index];
2732 :
2733 : Node* ift_size =
2734 6570 : LOAD_INSTANCE_FIELD(IndirectFunctionTableSize, MachineType::Uint32());
2735 :
2736 3285 : MachineOperatorBuilder* machine = mcgraph()->machine();
2737 3285 : Node* key = args[0];
2738 :
2739 : // Bounds check against the table size.
2740 3285 : Node* in_bounds = graph()->NewNode(machine->Uint32LessThan(), key, ift_size);
2741 3285 : TrapIfFalse(wasm::kTrapFuncInvalid, in_bounds, position);
2742 :
2743 : // Mask the key to prevent SSCA.
2744 3285 : if (untrusted_code_mitigations_) {
2745 : // mask = ((key - size) & ~key) >> 31
2746 : Node* neg_key =
2747 0 : graph()->NewNode(machine->Word32Xor(), key, Int32Constant(-1));
2748 : Node* masked_diff = graph()->NewNode(
2749 : machine->Word32And(),
2750 0 : graph()->NewNode(machine->Int32Sub(), key, ift_size), neg_key);
2751 : Node* mask =
2752 0 : graph()->NewNode(machine->Word32Sar(), masked_diff, Int32Constant(31));
2753 0 : key = graph()->NewNode(machine->Word32And(), key, mask);
2754 : }
2755 :
2756 : // Load signature from the table and check.
2757 : Node* ift_sig_ids =
2758 6569 : LOAD_INSTANCE_FIELD(IndirectFunctionTableSigIds, MachineType::Pointer());
2759 :
2760 6570 : int32_t expected_sig_id = env_->module->signature_ids[sig_index];
2761 : Node* int32_scaled_key = Uint32ToUintptr(
2762 6570 : graph()->NewNode(machine->Word32Shl(), key, Int32Constant(2)));
2763 :
2764 : Node* loaded_sig = SetEffect(
2765 : graph()->NewNode(machine->Load(MachineType::Int32()), ift_sig_ids,
2766 3285 : int32_scaled_key, Effect(), Control()));
2767 : Node* sig_match = graph()->NewNode(machine->WordEqual(), loaded_sig,
2768 3285 : Int32Constant(expected_sig_id));
2769 :
2770 3285 : TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position);
2771 :
2772 : Node* ift_targets =
2773 6569 : LOAD_INSTANCE_FIELD(IndirectFunctionTableTargets, MachineType::Pointer());
2774 6570 : Node* ift_instances = LOAD_INSTANCE_FIELD(IndirectFunctionTableRefs,
2775 : MachineType::TaggedPointer());
2776 :
2777 : Node* intptr_scaled_key = graph()->NewNode(
2778 3285 : machine->Word32Shl(), key, Int32Constant(kSystemPointerSizeLog2));
2779 :
2780 : Node* target = SetEffect(
2781 : graph()->NewNode(machine->Load(MachineType::Pointer()), ift_targets,
2782 3285 : intptr_scaled_key, Effect(), Control()));
2783 :
2784 : STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
2785 : Node* tagged_scaled_key = intptr_scaled_key;
2786 :
2787 : Node* target_instance = SetEffect(graph()->NewNode(
2788 : machine->Load(MachineType::TaggedPointer()),
2789 : graph()->NewNode(machine->IntAdd(), ift_instances, tagged_scaled_key),
2790 : Int32Constant(wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)),
2791 6570 : Effect(), Control()));
2792 :
2793 3285 : args[0] = target;
2794 :
2795 : return BuildWasmCall(sig, args, rets, position, target_instance,
2796 3285 : untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline);
2797 : }
2798 :
2799 26944 : Node* WasmGraphBuilder::BuildI32Rol(Node* left, Node* right) {
2800 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2801 : // TODO(weiliang): support Word32Rol opcode in TurboFan.
2802 : Int32Matcher m(right);
2803 13472 : if (m.HasValue()) {
2804 : return Binop(wasm::kExprI32Ror, left,
2805 26912 : mcgraph()->Int32Constant(32 - (m.Value() & 0x1F)));
2806 : } else {
2807 : return Binop(wasm::kExprI32Ror, left,
2808 16 : Binop(wasm::kExprI32Sub, mcgraph()->Int32Constant(32), right));
2809 : }
2810 : }
2811 :
2812 120 : Node* WasmGraphBuilder::BuildI64Rol(Node* left, Node* right) {
2813 : // Implement Rol by Ror since TurboFan does not have Rol opcode.
2814 : // TODO(weiliang): support Word64Rol opcode in TurboFan.
2815 : Int64Matcher m(right);
2816 60 : if (m.HasValue()) {
2817 : return Binop(wasm::kExprI64Ror, left,
2818 32 : mcgraph()->Int64Constant(64 - (m.Value() & 0x3F)));
2819 : } else {
2820 : return Binop(wasm::kExprI64Ror, left,
2821 44 : Binop(wasm::kExprI64Sub, mcgraph()->Int64Constant(64), right));
2822 : }
2823 : }
2824 :
2825 39394 : Node* WasmGraphBuilder::Invert(Node* node) {
2826 39394 : return Unop(wasm::kExprI32Eqz, node);
2827 : }
2828 :
2829 0 : bool CanCover(Node* value, IrOpcode::Value opcode) {
2830 0 : if (value->opcode() != opcode) return false;
2831 : bool first = true;
2832 0 : for (Edge const edge : value->use_edges()) {
2833 0 : if (NodeProperties::IsControlEdge(edge)) continue;
2834 0 : if (NodeProperties::IsEffectEdge(edge)) continue;
2835 : DCHECK(NodeProperties::IsValueEdge(edge));
2836 0 : if (!first) return false;
2837 : first = false;
2838 : }
2839 0 : return true;
2840 : }
2841 :
2842 63398 : Node* WasmGraphBuilder::BuildChangeInt32ToIntPtr(Node* value) {
2843 126796 : if (mcgraph()->machine()->Is64()) {
2844 63398 : value = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), value);
2845 : }
2846 63398 : return value;
2847 : }
2848 :
2849 126797 : Node* WasmGraphBuilder::BuildChangeInt32ToSmi(Node* value) {
2850 63398 : value = BuildChangeInt32ToIntPtr(value);
2851 : return graph()->NewNode(mcgraph()->machine()->WordShl(), value,
2852 126797 : BuildSmiShiftBitsConstant());
2853 : }
2854 :
2855 2832 : Node* WasmGraphBuilder::BuildChangeUint31ToSmi(Node* value) {
2856 : return graph()->NewNode(mcgraph()->machine()->WordShl(),
2857 4248 : Uint32ToUintptr(value), BuildSmiShiftBitsConstant());
2858 : }
2859 :
2860 285187 : Node* WasmGraphBuilder::BuildSmiShiftBitsConstant() {
2861 285187 : return mcgraph()->IntPtrConstant(kSmiShiftSize + kSmiTagSize);
2862 : }
2863 :
2864 661121 : Node* WasmGraphBuilder::BuildChangeSmiToInt32(Node* value) {
2865 : value = graph()->NewNode(mcgraph()->machine()->WordSar(), value,
2866 220374 : BuildSmiShiftBitsConstant());
2867 440748 : if (mcgraph()->machine()->Is64()) {
2868 : value =
2869 220374 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), value);
2870 : }
2871 220373 : return value;
2872 : }
2873 :
2874 288 : Node* WasmGraphBuilder::BuildConvertUint32ToSmiWithSaturation(Node* value,
2875 1152 : uint32_t maxval) {
2876 : DCHECK(Smi::IsValid(maxval));
2877 : Node* max = Uint32Constant(maxval);
2878 : Node* check = graph()->NewNode(mcgraph()->machine()->Uint32LessThanOrEqual(),
2879 288 : value, max);
2880 288 : Node* valsmi = BuildChangeUint31ToSmi(value);
2881 576 : Node* maxsmi = graph()->NewNode(mcgraph()->common()->NumberConstant(maxval));
2882 576 : Diamond d(graph(), mcgraph()->common(), check, BranchHint::kTrue);
2883 288 : d.Chain(Control());
2884 288 : return d.Phi(MachineRepresentation::kTagged, valsmi, maxsmi);
2885 : }
2886 :
2887 537147 : void WasmGraphBuilder::InitInstanceCache(
2888 5372007 : WasmInstanceCacheNodes* instance_cache) {
2889 : DCHECK_NOT_NULL(instance_node_);
2890 :
2891 : // Load the memory start.
2892 : instance_cache->mem_start = SetEffect(graph()->NewNode(
2893 : mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2894 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemoryStart)),
2895 1611587 : Effect(), Control()));
2896 :
2897 : // Load the memory size.
2898 : instance_cache->mem_size = SetEffect(graph()->NewNode(
2899 : mcgraph()->machine()->Load(MachineType::UintPtr()), instance_node_.get(),
2900 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(MemorySize)),
2901 1611644 : Effect(), Control()));
2902 :
2903 537246 : if (untrusted_code_mitigations_) {
2904 : // Load the memory mask.
2905 : instance_cache->mem_mask =
2906 0 : LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr());
2907 : } else {
2908 : // Explicitly set to nullptr to ensure a SEGV when we try to use it.
2909 537246 : instance_cache->mem_mask = nullptr;
2910 : }
2911 537246 : }
2912 :
2913 4259 : void WasmGraphBuilder::PrepareInstanceCacheForLoop(
2914 8518 : WasmInstanceCacheNodes* instance_cache, Node* control) {
2915 : #define INTRODUCE_PHI(field, rep) \
2916 : instance_cache->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 1), \
2917 : instance_cache->field, control);
2918 :
2919 12777 : INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2920 12777 : INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation());
2921 4259 : if (untrusted_code_mitigations_) {
2922 0 : INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation());
2923 : }
2924 :
2925 : #undef INTRODUCE_PHI
2926 4259 : }
2927 :
2928 34467 : void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to,
2929 : WasmInstanceCacheNodes* from,
2930 25841 : Node* merge) {
2931 : #define INTRODUCE_PHI(field, rep) \
2932 : if (to->field != from->field) { \
2933 : Node* vals[] = {to->field, from->field, merge}; \
2934 : to->field = graph()->NewNode(mcgraph()->common()->Phi(rep, 2), 3, vals); \
2935 : }
2936 :
2937 60307 : INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation());
2938 60310 : INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32);
2939 34468 : if (untrusted_code_mitigations_) {
2940 0 : INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32);
2941 : }
2942 :
2943 : #undef INTRODUCE_PHI
2944 34468 : }
2945 :
2946 213528 : void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to,
2947 : WasmInstanceCacheNodes* from,
2948 : Node* merge) {
2949 : to->mem_size = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2950 213528 : merge, to->mem_size, from->mem_size);
2951 : to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2952 213529 : merge, to->mem_start, from->mem_start);
2953 213530 : if (untrusted_code_mitigations_) {
2954 : to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(),
2955 0 : merge, to->mem_mask, from->mem_mask);
2956 : }
2957 213530 : }
2958 :
2959 992944 : Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep,
2960 : Node* merge, Node* tnode,
2961 13302 : Node* fnode) {
2962 992944 : if (IsPhiWithMerge(tnode, merge)) {
2963 94402 : AppendToPhi(tnode, fnode);
2964 898544 : } else if (tnode != fnode) {
2965 13302 : uint32_t count = merge->InputCount();
2966 : // + 1 for the merge node.
2967 13302 : Node** vals = Buffer(count + 1);
2968 13302 : for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode;
2969 13302 : vals[count - 1] = fnode;
2970 13302 : vals[count] = merge;
2971 : return graph()->NewNode(mcgraph()->common()->Phi(rep, count), count + 1,
2972 39906 : vals);
2973 : }
2974 979645 : return tnode;
2975 : }
2976 :
2977 213529 : Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode,
2978 : Node* fnode) {
2979 213529 : if (IsPhiWithMerge(tnode, merge)) {
2980 13448 : AppendToPhi(tnode, fnode);
2981 200082 : } else if (tnode != fnode) {
2982 128 : uint32_t count = merge->InputCount();
2983 128 : Node** effects = Buffer(count);
2984 493 : for (uint32_t j = 0; j < count - 1; j++) {
2985 365 : effects[j] = tnode;
2986 : }
2987 128 : effects[count - 1] = fnode;
2988 128 : tnode = EffectPhi(count, effects, merge);
2989 : }
2990 213530 : return tnode;
2991 : }
2992 :
2993 479 : Node* WasmGraphBuilder::GetImportedMutableGlobals() {
2994 248 : if (imported_mutable_globals_ == nullptr) {
2995 : // Load imported_mutable_globals_ from the instance object at runtime.
2996 463 : imported_mutable_globals_ = graph()->NewNode(
2997 : mcgraph()->machine()->Load(MachineType::UintPtr()),
2998 : instance_node_.get(),
2999 : mcgraph()->Int32Constant(
3000 : WASM_INSTANCE_OBJECT_OFFSET(ImportedMutableGlobals)),
3001 : graph()->start(), graph()->start());
3002 : }
3003 248 : return imported_mutable_globals_.get();
3004 : }
3005 :
3006 51368 : void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type,
3007 : const wasm::WasmGlobal& global,
3008 : Node** base_node,
3009 82689 : Node** offset_node) {
3010 : DCHECK_NOT_NULL(instance_node_);
3011 51368 : if (global.mutability && global.imported) {
3012 : *base_node = SetEffect(graph()->NewNode(
3013 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3014 : GetImportedMutableGlobals(),
3015 : mcgraph()->Int32Constant(global.index * sizeof(Address)), Effect(),
3016 544 : Control()));
3017 136 : *offset_node = mcgraph()->Int32Constant(0);
3018 : } else {
3019 51232 : if (globals_start_ == nullptr) {
3020 : // Load globals_start from the instance object at runtime.
3021 : // TODO(wasm): we currently generate only one load of the {globals_start}
3022 : // start per graph, which means it can be placed anywhere by the
3023 : // scheduler. This is legal because the globals_start should never change.
3024 : // However, in some cases (e.g. if the instance object is already in a
3025 : // register), it is slightly more efficient to reload this value from the
3026 : // instance object. Since this depends on register allocation, it is not
3027 : // possible to express in the graph, and would essentially constitute a
3028 : // "mem2reg" optimization in TurboFan.
3029 19834 : globals_start_ = graph()->NewNode(
3030 : mcgraph()->machine()->Load(MachineType::UintPtr()),
3031 : instance_node_.get(),
3032 : mcgraph()->Int32Constant(WASM_INSTANCE_OBJECT_OFFSET(GlobalsStart)),
3033 : graph()->start(), graph()->start());
3034 : }
3035 51235 : *base_node = globals_start_.get();
3036 102470 : *offset_node = mcgraph()->Int32Constant(global.offset);
3037 :
3038 51231 : if (mem_type == MachineType::Simd128() && global.offset != 0) {
3039 : // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken.
3040 : *base_node = graph()->NewNode(mcgraph()->machine()->IntAdd(), *base_node,
3041 31080 : *offset_node);
3042 10360 : *offset_node = mcgraph()->Int32Constant(0);
3043 : }
3044 : }
3045 51367 : }
3046 :
3047 112 : void WasmGraphBuilder::GetBaseAndOffsetForImportedMutableAnyRefGlobal(
3048 2126 : const wasm::WasmGlobal& global, Node** base, Node** offset) {
3049 : // Load the base from the ImportedMutableGlobalsBuffer of the instance.
3050 224 : Node* buffers = LOAD_INSTANCE_FIELD(ImportedMutableGlobalsBuffers,
3051 : MachineType::TaggedPointer());
3052 448 : *base = LOAD_FIXED_ARRAY_SLOT_ANY(buffers, global.index);
3053 :
3054 : // For the offset we need the index of the global in the buffer, and then
3055 : // calculate the actual offset from the index. Load the index from the
3056 : // ImportedMutableGlobals array of the instance.
3057 : Node* index = SetEffect(
3058 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::UintPtr()),
3059 : GetImportedMutableGlobals(),
3060 : mcgraph()->Int32Constant(global.index * sizeof(Address)),
3061 336 : Effect(), Control()));
3062 :
3063 : // From the index, calculate the actual offset in the FixeArray. This
3064 : // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
3065 : // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
3066 : Node* index_times_tagged_size =
3067 : graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
3068 223 : mcgraph()->Int32Constant(kTaggedSize));
3069 : *offset = graph()->NewNode(
3070 : mcgraph()->machine()->IntAdd(), index_times_tagged_size,
3071 : mcgraph()->IntPtrConstant(
3072 336 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
3073 112 : }
3074 :
3075 365431 : Node* WasmGraphBuilder::MemBuffer(uint32_t offset) {
3076 : DCHECK_NOT_NULL(instance_cache_);
3077 275601 : Node* mem_start = instance_cache_->mem_start;
3078 : DCHECK_NOT_NULL(mem_start);
3079 275601 : if (offset == 0) return mem_start;
3080 : return graph()->NewNode(mcgraph()->machine()->IntAdd(), mem_start,
3081 179658 : mcgraph()->IntPtrConstant(offset));
3082 : }
3083 :
3084 1472 : Node* WasmGraphBuilder::CurrentMemoryPages() {
3085 : // CurrentMemoryPages can not be called from asm.js.
3086 : DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin);
3087 : DCHECK_NOT_NULL(instance_cache_);
3088 368 : Node* mem_size = instance_cache_->mem_size;
3089 : DCHECK_NOT_NULL(mem_size);
3090 : Node* result =
3091 : graph()->NewNode(mcgraph()->machine()->WordShr(), mem_size,
3092 736 : mcgraph()->Int32Constant(wasm::kWasmPageSizeLog2));
3093 736 : if (mcgraph()->machine()->Is64()) {
3094 : result =
3095 368 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), result);
3096 : }
3097 368 : return result;
3098 : }
3099 :
3100 6220 : Node* WasmGraphBuilder::BuildLoadBuiltinFromInstance(int builtin_index) {
3101 : DCHECK(Builtins::IsBuiltinId(builtin_index));
3102 1244 : Node* isolate_root = LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
3103 1866 : return LOAD_TAGGED_POINTER(isolate_root,
3104 : IsolateData::builtin_slot_offset(builtin_index));
3105 : }
3106 :
3107 : // Only call this function for code which is not reused across instantiations,
3108 : // as we do not patch the embedded js_context.
3109 5261 : Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(
3110 : Runtime::FunctionId f, Node* js_context, Node** parameters,
3111 52609 : int parameter_count, Node** effect, Node* control) {
3112 5261 : const Runtime::Function* fun = Runtime::FunctionForId(f);
3113 : auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
3114 : mcgraph()->zone(), f, fun->nargs, Operator::kNoProperties,
3115 10522 : CallDescriptor::kNoFlags);
3116 : // The CEntryStub is loaded from the instance_node so that generated code is
3117 : // Isolate independent. At the moment this is only done for CEntryStub(1).
3118 : DCHECK_EQ(1, fun->result_size);
3119 : Node* centry_stub =
3120 10522 : LOAD_INSTANCE_FIELD(CEntryStub, MachineType::TaggedPointer());
3121 : // TODO(titzer): allow arbitrary number of runtime arguments
3122 : // At the moment we only allow 5 parameters. If more parameters are needed,
3123 : // increase this constant accordingly.
3124 : static const int kMaxParams = 5;
3125 : DCHECK_GE(kMaxParams, parameter_count);
3126 : Node* inputs[kMaxParams + 6];
3127 : int count = 0;
3128 5259 : inputs[count++] = centry_stub;
3129 12990 : for (int i = 0; i < parameter_count; i++) {
3130 7731 : inputs[count++] = parameters[i];
3131 : }
3132 5259 : inputs[count++] =
3133 10518 : mcgraph()->ExternalConstant(ExternalReference::Create(f)); // ref
3134 10520 : inputs[count++] = mcgraph()->Int32Constant(fun->nargs); // arity
3135 5262 : inputs[count++] = js_context; // js_context
3136 5262 : inputs[count++] = *effect;
3137 5262 : inputs[count++] = control;
3138 :
3139 : Node* call = mcgraph()->graph()->NewNode(
3140 10522 : mcgraph()->common()->Call(call_descriptor), count, inputs);
3141 5263 : *effect = call;
3142 5263 : return call;
3143 : }
3144 :
3145 4340 : Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f,
3146 : Node** parameters,
3147 4340 : int parameter_count) {
3148 : return BuildCallToRuntimeWithContext(f, NoContextConstant(), parameters,
3149 8678 : parameter_count, effect_, Control());
3150 : }
3151 :
3152 182017 : Node* WasmGraphBuilder::GetGlobal(uint32_t index) {
3153 72292 : const wasm::WasmGlobal& global = env_->module->globals[index];
3154 36250 : if (global.type == wasm::ValueType::kWasmAnyRef) {
3155 206 : if (global.mutability && global.imported) {
3156 80 : Node* base = nullptr;
3157 80 : Node* offset = nullptr;
3158 80 : GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
3159 : return SetEffect(
3160 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
3161 160 : base, offset, Effect(), Control()));
3162 : }
3163 : Node* globals_buffer =
3164 254 : LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3165 510 : return LOAD_FIXED_ARRAY_SLOT_ANY(globals_buffer, global.offset);
3166 : }
3167 :
3168 : MachineType mem_type =
3169 36044 : wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3170 36042 : Node* base = nullptr;
3171 36042 : Node* offset = nullptr;
3172 36042 : GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3173 36042 : &offset);
3174 : Node* result = SetEffect(graph()->NewNode(
3175 72088 : mcgraph()->machine()->Load(mem_type), base, offset, Effect(), Control()));
3176 : #if defined(V8_TARGET_BIG_ENDIAN)
3177 : result = BuildChangeEndiannessLoad(result, mem_type,
3178 : env_->module->globals[index].type);
3179 : #endif
3180 36043 : return result;
3181 : }
3182 :
3183 77758 : Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) {
3184 30860 : const wasm::WasmGlobal& global = env_->module->globals[index];
3185 15446 : if (global.type == wasm::ValueType::kWasmAnyRef) {
3186 120 : if (global.mutability && global.imported) {
3187 32 : Node* base = nullptr;
3188 32 : Node* offset = nullptr;
3189 32 : GetBaseAndOffsetForImportedMutableAnyRefGlobal(global, &base, &offset);
3190 :
3191 : return SetEffect(graph()->NewNode(
3192 : mcgraph()->machine()->Store(StoreRepresentation(
3193 : MachineRepresentation::kTagged, kFullWriteBarrier)),
3194 96 : base, offset, val, Effect(), Control()));
3195 : }
3196 : Node* globals_buffer =
3197 176 : LOAD_INSTANCE_FIELD(TaggedGlobalsBuffer, MachineType::TaggedPointer());
3198 440 : return STORE_FIXED_ARRAY_SLOT_ANY(globals_buffer,
3199 : env_->module->globals[index].offset, val);
3200 : }
3201 :
3202 : MachineType mem_type =
3203 15326 : wasm::ValueTypes::MachineTypeFor(env_->module->globals[index].type);
3204 15326 : Node* base = nullptr;
3205 15326 : Node* offset = nullptr;
3206 15326 : GetGlobalBaseAndOffset(mem_type, env_->module->globals[index], &base,
3207 15326 : &offset);
3208 : const Operator* op = mcgraph()->machine()->Store(
3209 30652 : StoreRepresentation(mem_type.representation(), kNoWriteBarrier));
3210 : #if defined(V8_TARGET_BIG_ENDIAN)
3211 : val = BuildChangeEndiannessStore(val, mem_type.representation(),
3212 : env_->module->globals[index].type);
3213 : #endif
3214 : return SetEffect(
3215 30652 : graph()->NewNode(op, base, offset, val, Effect(), Control()));
3216 : }
3217 :
3218 0 : void WasmGraphBuilder::GetTableBaseAndOffset(uint32_t table_index, Node* index,
3219 : wasm::WasmCodePosition position,
3220 : Node** base_node,
3221 0 : Node** offset_node) {
3222 0 : Node* tables = LOAD_INSTANCE_FIELD(Tables, MachineType::TaggedPointer());
3223 0 : Node* table = LOAD_FIXED_ARRAY_SLOT_ANY(tables, table_index);
3224 :
3225 : int storage_field_size = WasmTableObject::kElementsOffsetEnd -
3226 : WasmTableObject::kElementsOffset + 1;
3227 0 : Node* storage = LOAD_RAW(
3228 : table, wasm::ObjectAccess::ToTagged(WasmTableObject::kElementsOffset),
3229 : assert_size(storage_field_size, MachineType::TaggedPointer()));
3230 :
3231 : int length_field_size =
3232 : FixedArray::kLengthOffsetEnd - FixedArray::kLengthOffset + 1;
3233 : Node* storage_size =
3234 0 : LOAD_RAW(storage, wasm::ObjectAccess::ToTagged(FixedArray::kLengthOffset),
3235 : assert_size(length_field_size, MachineType::TaggedSigned()));
3236 :
3237 0 : storage_size = BuildChangeSmiToInt32(storage_size);
3238 : // Bounds check against the table size.
3239 : Node* in_bounds = graph()->NewNode(mcgraph()->machine()->Uint32LessThan(),
3240 0 : index, storage_size);
3241 0 : TrapIfFalse(wasm::kTrapTableOutOfBounds, in_bounds, position);
3242 :
3243 : // From the index, calculate the actual offset in the FixeArray. This
3244 : // is kHeaderSize + (index * kTaggedSize). kHeaderSize can be acquired with
3245 : // wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0).
3246 : Node* index_times_tagged_size =
3247 : graph()->NewNode(mcgraph()->machine()->IntMul(), Uint32ToUintptr(index),
3248 0 : mcgraph()->Int32Constant(kTaggedSize));
3249 :
3250 : *offset_node = graph()->NewNode(
3251 : mcgraph()->machine()->IntAdd(), index_times_tagged_size,
3252 : mcgraph()->IntPtrConstant(
3253 0 : wasm::ObjectAccess::ElementOffsetInTaggedFixedArray(0)));
3254 :
3255 0 : *base_node = storage;
3256 0 : }
3257 :
3258 0 : Node* WasmGraphBuilder::GetTable(uint32_t table_index, Node* index,
3259 0 : wasm::WasmCodePosition position) {
3260 0 : Node* base = nullptr;
3261 0 : Node* offset = nullptr;
3262 0 : GetTableBaseAndOffset(table_index, index, position, &base, &offset);
3263 : return SetEffect(
3264 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::AnyTagged()),
3265 0 : base, offset, Effect(), Control()));
3266 : }
3267 :
3268 0 : Node* WasmGraphBuilder::SetTable(uint32_t table_index, Node* index, Node* val,
3269 0 : wasm::WasmCodePosition position) {
3270 0 : Node* base = nullptr;
3271 0 : Node* offset = nullptr;
3272 0 : GetTableBaseAndOffset(table_index, index, position, &base, &offset);
3273 :
3274 : const Operator* op = mcgraph()->machine()->Store(
3275 0 : StoreRepresentation(MachineRepresentation::kTagged, kFullWriteBarrier));
3276 :
3277 0 : Node* store = graph()->NewNode(op, base, offset, val, Effect(), Control());
3278 0 : return SetEffect(store);
3279 : }
3280 :
3281 32641 : Node* WasmGraphBuilder::CheckBoundsAndAlignment(
3282 : uint8_t access_size, Node* index, uint32_t offset,
3283 20663 : wasm::WasmCodePosition position) {
3284 : // Atomic operations need bounds checks until the backend can emit protected
3285 : // loads.
3286 : index =
3287 32641 : BoundsCheckMem(access_size, index, offset, position, kNeedsBoundsCheck);
3288 :
3289 32643 : const uintptr_t align_mask = access_size - 1;
3290 :
3291 : // Don't emit an alignment check if the index is a constant.
3292 : // TODO(wasm): a constant match is also done above in {BoundsCheckMem}.
3293 : UintPtrMatcher match(index);
3294 32643 : if (match.HasValue()) {
3295 27479 : uintptr_t effective_offset = match.Value() + offset;
3296 27479 : if ((effective_offset & align_mask) != 0) {
3297 : // statically known to be unaligned; trap.
3298 0 : TrapIfEq32(wasm::kTrapUnalignedAccess, Int32Constant(0), 0, position);
3299 : }
3300 : return index;
3301 : }
3302 :
3303 : // Unlike regular memory accesses, atomic memory accesses should trap if
3304 : // the effective offset is misaligned.
3305 : // TODO(wasm): this addition is redundant with one inserted by {MemBuffer}.
3306 : Node* effective_offset = graph()->NewNode(mcgraph()->machine()->IntAdd(),
3307 10327 : MemBuffer(offset), index);
3308 :
3309 : Node* cond = graph()->NewNode(mcgraph()->machine()->WordAnd(),
3310 10333 : effective_offset, IntPtrConstant(align_mask));
3311 : TrapIfFalse(wasm::kTrapUnalignedAccess,
3312 : graph()->NewNode(mcgraph()->machine()->Word32Equal(), cond,
3313 : mcgraph()->Int32Constant(0)),
3314 15500 : position);
3315 5167 : return index;
3316 : }
3317 :
3318 : // Insert code to bounds check a memory access if necessary. Return the
3319 : // bounds-checked index, which is guaranteed to have (the equivalent of)
3320 : // {uintptr_t} representation.
3321 270971 : Node* WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index,
3322 : uint32_t offset,
3323 : wasm::WasmCodePosition position,
3324 303696 : EnforceBoundsCheck enforce_check) {
3325 : DCHECK_LE(1, access_size);
3326 270971 : index = Uint32ToUintptr(index);
3327 271042 : if (FLAG_wasm_no_bounds_checks) return index;
3328 :
3329 271049 : if (use_trap_handler() && enforce_check == kCanOmitBoundsCheck) {
3330 : return index;
3331 : }
3332 :
3333 65334 : if (!IsInBounds(offset, access_size, env_->max_memory_size)) {
3334 : // The access will be out of bounds, even for the largest memory.
3335 16 : TrapIfEq32(wasm::kTrapMemOutOfBounds, Int32Constant(0), 0, position);
3336 16 : return mcgraph()->IntPtrConstant(0);
3337 : }
3338 32651 : uint64_t end_offset = uint64_t{offset} + access_size - 1u;
3339 32651 : Node* end_offset_node = IntPtrConstant(end_offset);
3340 :
3341 : // The accessed memory is [index + offset, index + end_offset].
3342 : // Check that the last read byte (at {index + end_offset}) is in bounds.
3343 : // 1) Check that {end_offset < mem_size}. This also ensures that we can safely
3344 : // compute {effective_size} as {mem_size - end_offset)}.
3345 : // {effective_size} is >= 1 if condition 1) holds.
3346 : // 2) Check that {index + end_offset < mem_size} by
3347 : // - computing {effective_size} as {mem_size - end_offset} and
3348 : // - checking that {index < effective_size}.
3349 :
3350 32631 : auto m = mcgraph()->machine();
3351 32631 : Node* mem_size = instance_cache_->mem_size;
3352 32631 : if (end_offset >= env_->min_memory_size) {
3353 : // The end offset is larger than the smallest memory.
3354 : // Dynamically check the end offset against the dynamic memory size.
3355 5903 : Node* cond = graph()->NewNode(m->UintLessThan(), end_offset_node, mem_size);
3356 5903 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3357 : } else {
3358 : // The end offset is smaller than the smallest memory, so only one check is
3359 : // required. Check to see if the index is also a constant.
3360 : UintPtrMatcher match(index);
3361 26728 : if (match.HasValue()) {
3362 : uintptr_t index_val = match.Value();
3363 26732 : if (index_val < env_->min_memory_size - end_offset) {
3364 : // The input index is a constant and everything is statically within
3365 : // bounds of the smallest possible memory.
3366 : return index;
3367 : }
3368 : }
3369 : }
3370 :
3371 : // This produces a positive number, since {end_offset < min_size <= mem_size}.
3372 : Node* effective_size =
3373 5920 : graph()->NewNode(m->IntSub(), mem_size, end_offset_node);
3374 :
3375 : // Introduce the actual bounds check.
3376 5919 : Node* cond = graph()->NewNode(m->UintLessThan(), index, effective_size);
3377 5920 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3378 :
3379 5918 : if (untrusted_code_mitigations_) {
3380 : // In the fallthrough case, condition the index with the memory mask.
3381 0 : Node* mem_mask = instance_cache_->mem_mask;
3382 : DCHECK_NOT_NULL(mem_mask);
3383 0 : index = graph()->NewNode(m->WordAnd(), index, mem_mask);
3384 : }
3385 5918 : return index;
3386 : }
3387 :
3388 : // Check that the range [start, start + size) is in the range [0, max).
3389 152 : void WasmGraphBuilder::BoundsCheckRange(Node* start, Node* size, Node* max,
3390 152 : wasm::WasmCodePosition position) {
3391 : // The accessed memory is [start, end), where {end} is {start + size}. We
3392 : // want to check that {start + size <= max}, making sure that {start + size}
3393 : // doesn't overflow. This can be expressed as {start <= max - size} as long
3394 : // as {max - size} isn't negative, which is true if {size <= max}.
3395 152 : auto m = mcgraph()->machine();
3396 152 : Node* cond = graph()->NewNode(m->Uint32LessThanOrEqual(), size, max);
3397 152 : TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position);
3398 :
3399 : // This produces a positive number, since {size <= max}.
3400 152 : Node* effective_size = graph()->NewNode(m->Int32Sub(), max, size);
3401 :
3402 : // Introduce the actual bounds check.
3403 : Node* check =
3404 152 : graph()->NewNode(m->Uint32LessThanOrEqual(), start, effective_size);
3405 152 : TrapIfFalse(wasm::kTrapMemOutOfBounds, check, position);
3406 :
3407 : // TODO(binji): Does this need addtional untrusted_code_mitigations_ mask
3408 : // like BoundsCheckMem above?
3409 152 : }
3410 :
3411 112 : Node* WasmGraphBuilder::BoundsCheckMemRange(Node* start, Node* size,
3412 112 : wasm::WasmCodePosition position) {
3413 : // TODO(binji): Support trap handler.
3414 112 : if (!FLAG_wasm_no_bounds_checks) {
3415 112 : BoundsCheckRange(start, size, instance_cache_->mem_size, position);
3416 : }
3417 : return graph()->NewNode(mcgraph()->machine()->IntAdd(), MemBuffer(0),
3418 336 : Uint32ToUintptr(start));
3419 : }
3420 :
3421 4676 : const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset,
3422 4680 : wasm::ValueType type) {
3423 4676 : int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3424 4676 : MachineType mach_type = wasm::ValueTypes::MachineTypeFor(type);
3425 4680 : if (alignment == 0 || mcgraph()->machine()->UnalignedLoadSupported(
3426 8 : wasm::ValueTypes::MachineRepresentationFor(type))) {
3427 4676 : return mcgraph()->machine()->Load(mach_type);
3428 : }
3429 0 : return mcgraph()->machine()->UnalignedLoad(mach_type);
3430 : }
3431 :
3432 9064 : const Operator* WasmGraphBuilder::GetSafeStoreOperator(int offset,
3433 9076 : wasm::ValueType type) {
3434 9064 : int alignment = offset % (wasm::ValueTypes::ElementSizeInBytes(type));
3435 9064 : MachineRepresentation rep = wasm::ValueTypes::MachineRepresentationFor(type);
3436 9088 : if (alignment == 0 || mcgraph()->machine()->UnalignedStoreSupported(rep)) {
3437 : StoreRepresentation store_rep(rep, WriteBarrierKind::kNoWriteBarrier);
3438 9064 : return mcgraph()->machine()->Store(store_rep);
3439 : }
3440 : UnalignedStoreRepresentation store_rep(rep);
3441 0 : return mcgraph()->machine()->UnalignedStore(store_rep);
3442 : }
3443 :
3444 40 : Node* WasmGraphBuilder::TraceMemoryOperation(bool is_store,
3445 : MachineRepresentation rep,
3446 : Node* index, uint32_t offset,
3447 160 : wasm::WasmCodePosition position) {
3448 : int kAlign = 4; // Ensure that the LSB is 0, such that this looks like a Smi.
3449 : Node* info = graph()->NewNode(
3450 80 : mcgraph()->machine()->StackSlot(sizeof(wasm::MemoryTracingInfo), kAlign));
3451 :
3452 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
3453 80 : Int32Constant(offset), index);
3454 120 : auto store = [&](int offset, MachineRepresentation rep, Node* data) {
3455 : SetEffect(graph()->NewNode(
3456 : mcgraph()->machine()->Store(StoreRepresentation(rep, kNoWriteBarrier)),
3457 720 : info, mcgraph()->Int32Constant(offset), data, Effect(), Control()));
3458 160 : };
3459 : // Store address, is_store, and mem_rep.
3460 : store(offsetof(wasm::MemoryTracingInfo, address),
3461 40 : MachineRepresentation::kWord32, address);
3462 : store(offsetof(wasm::MemoryTracingInfo, is_store),
3463 : MachineRepresentation::kWord8,
3464 80 : mcgraph()->Int32Constant(is_store ? 1 : 0));
3465 : store(offsetof(wasm::MemoryTracingInfo, mem_rep),
3466 : MachineRepresentation::kWord8,
3467 80 : mcgraph()->Int32Constant(static_cast<int>(rep)));
3468 :
3469 40 : Node* call = BuildCallToRuntime(Runtime::kWasmTraceMemory, &info, 1);
3470 40 : SetSourcePosition(call, position);
3471 40 : return call;
3472 : }
3473 :
3474 94247 : Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype,
3475 : Node* index, uint32_t offset,
3476 : uint32_t alignment,
3477 565076 : wasm::WasmCodePosition position) {
3478 : Node* load;
3479 :
3480 : // Wasm semantics throw on OOB. Introduce explicit bounds check and
3481 : // conditioning when not using the trap handler.
3482 : index = BoundsCheckMem(wasm::ValueTypes::MemSize(memtype), index, offset,
3483 94250 : position, kCanOmitBoundsCheck);
3484 :
3485 187339 : if (memtype.representation() == MachineRepresentation::kWord8 ||
3486 93079 : mcgraph()->machine()->UnalignedLoadSupported(memtype.representation())) {
3487 94260 : if (use_trap_handler()) {
3488 : load = graph()->NewNode(mcgraph()->machine()->ProtectedLoad(memtype),
3489 188524 : MemBuffer(offset), index, Effect(), Control());
3490 94269 : SetSourcePosition(load, position);
3491 : } else {
3492 : load = graph()->NewNode(mcgraph()->machine()->Load(memtype),
3493 2 : MemBuffer(offset), index, Effect(), Control());
3494 : }
3495 : } else {
3496 : // TODO(eholk): Support unaligned loads with trap handlers.
3497 : DCHECK(!use_trap_handler());
3498 : load = graph()->NewNode(mcgraph()->machine()->UnalignedLoad(memtype),
3499 0 : MemBuffer(offset), index, Effect(), Control());
3500 : }
3501 :
3502 : SetEffect(load);
3503 :
3504 : #if defined(V8_TARGET_BIG_ENDIAN)
3505 : load = BuildChangeEndiannessLoad(load, memtype, type);
3506 : #endif
3507 :
3508 135514 : if (type == wasm::kWasmI64 &&
3509 : ElementSizeInBytes(memtype.representation()) < 8) {
3510 : // TODO(titzer): TF zeroes the upper bits of 64-bit loads for subword sizes.
3511 689 : if (memtype.IsSigned()) {
3512 : // sign extend
3513 352 : load = graph()->NewNode(mcgraph()->machine()->ChangeInt32ToInt64(), load);
3514 : } else {
3515 : // zero extend
3516 : load =
3517 337 : graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), load);
3518 : }
3519 : }
3520 :
3521 94262 : if (FLAG_trace_wasm_memory) {
3522 : TraceMemoryOperation(false, memtype.representation(), index, offset,
3523 24 : position);
3524 : }
3525 :
3526 94262 : return load;
3527 : }
3528 :
3529 144131 : Node* WasmGraphBuilder::StoreMem(MachineRepresentation mem_rep, Node* index,
3530 : uint32_t offset, uint32_t alignment, Node* val,
3531 : wasm::WasmCodePosition position,
3532 863913 : wasm::ValueType type) {
3533 : Node* store;
3534 :
3535 : index = BoundsCheckMem(i::ElementSizeInBytes(mem_rep), index, offset,
3536 144140 : position, kCanOmitBoundsCheck);
3537 :
3538 : #if defined(V8_TARGET_BIG_ENDIAN)
3539 : val = BuildChangeEndiannessStore(val, mem_rep, type);
3540 : #endif
3541 :
3542 287281 : if (mem_rep == MachineRepresentation::kWord8 ||
3543 143141 : mcgraph()->machine()->UnalignedStoreSupported(mem_rep)) {
3544 144140 : if (use_trap_handler()) {
3545 : store =
3546 : graph()->NewNode(mcgraph()->machine()->ProtectedStore(mem_rep),
3547 288284 : MemBuffer(offset), index, val, Effect(), Control());
3548 144207 : SetSourcePosition(store, position);
3549 : } else {
3550 : StoreRepresentation rep(mem_rep, kNoWriteBarrier);
3551 : store =
3552 : graph()->NewNode(mcgraph()->machine()->Store(rep), MemBuffer(offset),
3553 0 : index, val, Effect(), Control());
3554 : }
3555 : } else {
3556 : // TODO(eholk): Support unaligned stores with trap handlers.
3557 : DCHECK(!use_trap_handler());
3558 : UnalignedStoreRepresentation rep(mem_rep);
3559 : store =
3560 : graph()->NewNode(mcgraph()->machine()->UnalignedStore(rep),
3561 0 : MemBuffer(offset), index, val, Effect(), Control());
3562 : }
3563 :
3564 : SetEffect(store);
3565 :
3566 144208 : if (FLAG_trace_wasm_memory) {
3567 16 : TraceMemoryOperation(true, mem_rep, index, offset, position);
3568 : }
3569 :
3570 144208 : return store;
3571 : }
3572 :
3573 : namespace {
3574 56390 : Node* GetAsmJsOOBValue(MachineRepresentation rep, MachineGraph* mcgraph) {
3575 56390 : switch (rep) {
3576 : case MachineRepresentation::kWord8:
3577 : case MachineRepresentation::kWord16:
3578 : case MachineRepresentation::kWord32:
3579 50149 : return mcgraph->Int32Constant(0);
3580 : case MachineRepresentation::kWord64:
3581 0 : return mcgraph->Int64Constant(0);
3582 : case MachineRepresentation::kFloat32:
3583 5396 : return mcgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN());
3584 : case MachineRepresentation::kFloat64:
3585 845 : return mcgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN());
3586 : default:
3587 0 : UNREACHABLE();
3588 : }
3589 : }
3590 : } // namespace
3591 :
3592 563900 : Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) {
3593 : DCHECK_NOT_NULL(instance_cache_);
3594 56390 : Node* mem_start = instance_cache_->mem_start;
3595 56390 : Node* mem_size = instance_cache_->mem_size;
3596 : DCHECK_NOT_NULL(mem_start);
3597 : DCHECK_NOT_NULL(mem_size);
3598 :
3599 : // Asm.js semantics are defined in terms of typed arrays, hence OOB
3600 : // reads return {undefined} coerced to the result type (0 for integers, NaN
3601 : // for float and double).
3602 : // Note that we check against the memory size ignoring the size of the
3603 : // stored value, which is conservative if misaligned. Technically, asm.js
3604 : // should never have misaligned accesses.
3605 56390 : index = Uint32ToUintptr(index);
3606 : Diamond bounds_check(
3607 : graph(), mcgraph()->common(),
3608 : graph()->NewNode(mcgraph()->machine()->UintLessThan(), index, mem_size),
3609 169170 : BranchHint::kTrue);
3610 56390 : bounds_check.Chain(Control());
3611 :
3612 56390 : if (untrusted_code_mitigations_) {
3613 : // Condition the index with the memory mask.
3614 0 : Node* mem_mask = instance_cache_->mem_mask;
3615 : DCHECK_NOT_NULL(mem_mask);
3616 0 : index = graph()->NewNode(mcgraph()->machine()->WordAnd(), index, mem_mask);
3617 : }
3618 :
3619 : Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start,
3620 112780 : index, Effect(), bounds_check.if_true);
3621 56390 : SetEffect(bounds_check.EffectPhi(load, Effect()));
3622 56390 : SetControl(bounds_check.merge);
3623 : return bounds_check.Phi(type.representation(), load,
3624 56390 : GetAsmJsOOBValue(type.representation(), mcgraph()));
3625 : }
3626 :
3627 372887 : Node* WasmGraphBuilder::Uint32ToUintptr(Node* node) {
3628 745774 : if (mcgraph()->machine()->Is32()) return node;
3629 : // Fold instances of ChangeUint32ToUint64(IntConstant) directly.
3630 : Uint32Matcher matcher(node);
3631 372920 : if (matcher.HasValue()) {
3632 258281 : uintptr_t value = matcher.Value();
3633 258281 : return mcgraph()->IntPtrConstant(bit_cast<intptr_t>(value));
3634 : }
3635 229279 : return graph()->NewNode(mcgraph()->machine()->ChangeUint32ToUint64(), node);
3636 : }
3637 :
3638 40014 : Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index,
3639 320112 : Node* val) {
3640 : DCHECK_NOT_NULL(instance_cache_);
3641 40014 : Node* mem_start = instance_cache_->mem_start;
3642 40014 : Node* mem_size = instance_cache_->mem_size;
3643 : DCHECK_NOT_NULL(mem_start);
3644 : DCHECK_NOT_NULL(mem_size);
3645 :
3646 : // Asm.js semantics are to ignore OOB writes.
3647 : // Note that we check against the memory size ignoring the size of the
3648 : // stored value, which is conservative if misaligned. Technically, asm.js
3649 : // should never have misaligned accesses.
3650 : Diamond bounds_check(
3651 : graph(), mcgraph()->common(),
3652 : graph()->NewNode(mcgraph()->machine()->Uint32LessThan(), index, mem_size),
3653 120042 : BranchHint::kTrue);
3654 40014 : bounds_check.Chain(Control());
3655 :
3656 40014 : if (untrusted_code_mitigations_) {
3657 : // Condition the index with the memory mask.
3658 0 : Node* mem_mask = instance_cache_->mem_mask;
3659 : DCHECK_NOT_NULL(mem_mask);
3660 : index =
3661 0 : graph()->NewNode(mcgraph()->machine()->Word32And(), index, mem_mask);
3662 : }
3663 :
3664 40014 : index = Uint32ToUintptr(index);
3665 : const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation(
3666 80028 : type.representation(), WriteBarrierKind::kNoWriteBarrier));
3667 : Node* store = graph()->NewNode(store_op, mem_start, index, val, Effect(),
3668 40014 : bounds_check.if_true);
3669 40014 : SetEffect(bounds_check.EffectPhi(store, Effect()));
3670 40014 : SetControl(bounds_check.merge);
3671 40014 : return val;
3672 : }
3673 :
3674 0 : void WasmGraphBuilder::PrintDebugName(Node* node) {
3675 0 : PrintF("#%d:%s", node->id(), node->op()->mnemonic());
3676 0 : }
3677 :
3678 11923125 : Graph* WasmGraphBuilder::graph() { return mcgraph()->graph(); }
3679 :
3680 : namespace {
3681 4068 : Signature<MachineRepresentation>* CreateMachineSignature(
3682 8136 : Zone* zone, wasm::FunctionSig* sig) {
3683 : Signature<MachineRepresentation>::Builder builder(zone, sig->return_count(),
3684 : sig->parameter_count());
3685 8136 : for (auto ret : sig->returns()) {
3686 4068 : builder.AddReturn(wasm::ValueTypes::MachineRepresentationFor(ret));
3687 : }
3688 :
3689 6464 : for (auto param : sig->parameters()) {
3690 2396 : builder.AddParam(wasm::ValueTypes::MachineRepresentationFor(param));
3691 : }
3692 4068 : return builder.Build();
3693 : }
3694 : } // namespace
3695 :
3696 496993 : void WasmGraphBuilder::LowerInt64() {
3697 1490979 : if (mcgraph()->machine()->Is64()) return;
3698 : Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(), mcgraph()->common(),
3699 : mcgraph()->zone(),
3700 0 : CreateMachineSignature(mcgraph()->zone(), sig_));
3701 0 : r.LowerGraph();
3702 : }
3703 :
3704 0 : void WasmGraphBuilder::SimdScalarLoweringForTesting() {
3705 : SimdScalarLowering(mcgraph(), CreateMachineSignature(mcgraph()->zone(), sig_))
3706 0 : .LowerGraph();
3707 0 : }
3708 :
3709 412095 : void WasmGraphBuilder::SetSourcePosition(Node* node,
3710 : wasm::WasmCodePosition position) {
3711 : DCHECK_NE(position, wasm::kNoCodePosition);
3712 412095 : if (source_position_table_)
3713 325101 : source_position_table_->SetSourcePosition(node, SourcePosition(position));
3714 412120 : }
3715 :
3716 4312 : Node* WasmGraphBuilder::S128Zero() {
3717 2156 : has_simd_ = true;
3718 4312 : return graph()->NewNode(mcgraph()->machine()->S128Zero());
3719 : }
3720 :
3721 8328 : Node* WasmGraphBuilder::SimdOp(wasm::WasmOpcode opcode, Node* const* inputs) {
3722 4164 : has_simd_ = true;
3723 4164 : switch (opcode) {
3724 : case wasm::kExprF32x4Splat:
3725 840 : return graph()->NewNode(mcgraph()->machine()->F32x4Splat(), inputs[0]);
3726 : case wasm::kExprF32x4SConvertI32x4:
3727 : return graph()->NewNode(mcgraph()->machine()->F32x4SConvertI32x4(),
3728 24 : inputs[0]);
3729 : case wasm::kExprF32x4UConvertI32x4:
3730 : return graph()->NewNode(mcgraph()->machine()->F32x4UConvertI32x4(),
3731 24 : inputs[0]);
3732 : case wasm::kExprF32x4Abs:
3733 24 : return graph()->NewNode(mcgraph()->machine()->F32x4Abs(), inputs[0]);
3734 : case wasm::kExprF32x4Neg:
3735 24 : return graph()->NewNode(mcgraph()->machine()->F32x4Neg(), inputs[0]);
3736 : case wasm::kExprF32x4RecipApprox:
3737 : return graph()->NewNode(mcgraph()->machine()->F32x4RecipApprox(),
3738 24 : inputs[0]);
3739 : case wasm::kExprF32x4RecipSqrtApprox:
3740 : return graph()->NewNode(mcgraph()->machine()->F32x4RecipSqrtApprox(),
3741 24 : inputs[0]);
3742 : case wasm::kExprF32x4Add:
3743 : return graph()->NewNode(mcgraph()->machine()->F32x4Add(), inputs[0],
3744 72 : inputs[1]);
3745 : case wasm::kExprF32x4AddHoriz:
3746 : return graph()->NewNode(mcgraph()->machine()->F32x4AddHoriz(), inputs[0],
3747 24 : inputs[1]);
3748 : case wasm::kExprF32x4Sub:
3749 : return graph()->NewNode(mcgraph()->machine()->F32x4Sub(), inputs[0],
3750 24 : inputs[1]);
3751 : case wasm::kExprF32x4Mul:
3752 : return graph()->NewNode(mcgraph()->machine()->F32x4Mul(), inputs[0],
3753 24 : inputs[1]);
3754 : case wasm::kExprF32x4Min:
3755 : return graph()->NewNode(mcgraph()->machine()->F32x4Min(), inputs[0],
3756 24 : inputs[1]);
3757 : case wasm::kExprF32x4Max:
3758 : return graph()->NewNode(mcgraph()->machine()->F32x4Max(), inputs[0],
3759 24 : inputs[1]);
3760 : case wasm::kExprF32x4Eq:
3761 : return graph()->NewNode(mcgraph()->machine()->F32x4Eq(), inputs[0],
3762 24 : inputs[1]);
3763 : case wasm::kExprF32x4Ne:
3764 : return graph()->NewNode(mcgraph()->machine()->F32x4Ne(), inputs[0],
3765 24 : inputs[1]);
3766 : case wasm::kExprF32x4Lt:
3767 : return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[0],
3768 24 : inputs[1]);
3769 : case wasm::kExprF32x4Le:
3770 : return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[0],
3771 24 : inputs[1]);
3772 : case wasm::kExprF32x4Gt:
3773 : return graph()->NewNode(mcgraph()->machine()->F32x4Lt(), inputs[1],
3774 24 : inputs[0]);
3775 : case wasm::kExprF32x4Ge:
3776 : return graph()->NewNode(mcgraph()->machine()->F32x4Le(), inputs[1],
3777 24 : inputs[0]);
3778 : case wasm::kExprI32x4Splat:
3779 3732 : return graph()->NewNode(mcgraph()->machine()->I32x4Splat(), inputs[0]);
3780 : case wasm::kExprI32x4SConvertF32x4:
3781 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertF32x4(),
3782 24 : inputs[0]);
3783 : case wasm::kExprI32x4UConvertF32x4:
3784 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertF32x4(),
3785 24 : inputs[0]);
3786 : case wasm::kExprI32x4SConvertI16x8Low:
3787 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8Low(),
3788 24 : inputs[0]);
3789 : case wasm::kExprI32x4SConvertI16x8High:
3790 : return graph()->NewNode(mcgraph()->machine()->I32x4SConvertI16x8High(),
3791 24 : inputs[0]);
3792 : case wasm::kExprI32x4Neg:
3793 24 : return graph()->NewNode(mcgraph()->machine()->I32x4Neg(), inputs[0]);
3794 : case wasm::kExprI32x4Add:
3795 : return graph()->NewNode(mcgraph()->machine()->I32x4Add(), inputs[0],
3796 72 : inputs[1]);
3797 : case wasm::kExprI32x4AddHoriz:
3798 : return graph()->NewNode(mcgraph()->machine()->I32x4AddHoriz(), inputs[0],
3799 24 : inputs[1]);
3800 : case wasm::kExprI32x4Sub:
3801 : return graph()->NewNode(mcgraph()->machine()->I32x4Sub(), inputs[0],
3802 24 : inputs[1]);
3803 : case wasm::kExprI32x4Mul:
3804 : return graph()->NewNode(mcgraph()->machine()->I32x4Mul(), inputs[0],
3805 24 : inputs[1]);
3806 : case wasm::kExprI32x4MinS:
3807 : return graph()->NewNode(mcgraph()->machine()->I32x4MinS(), inputs[0],
3808 24 : inputs[1]);
3809 : case wasm::kExprI32x4MaxS:
3810 : return graph()->NewNode(mcgraph()->machine()->I32x4MaxS(), inputs[0],
3811 24 : inputs[1]);
3812 : case wasm::kExprI32x4Eq:
3813 : return graph()->NewNode(mcgraph()->machine()->I32x4Eq(), inputs[0],
3814 120 : inputs[1]);
3815 : case wasm::kExprI32x4Ne:
3816 : return graph()->NewNode(mcgraph()->machine()->I32x4Ne(), inputs[0],
3817 132 : inputs[1]);
3818 : case wasm::kExprI32x4LtS:
3819 : return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[1],
3820 24 : inputs[0]);
3821 : case wasm::kExprI32x4LeS:
3822 : return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[1],
3823 24 : inputs[0]);
3824 : case wasm::kExprI32x4GtS:
3825 : return graph()->NewNode(mcgraph()->machine()->I32x4GtS(), inputs[0],
3826 24 : inputs[1]);
3827 : case wasm::kExprI32x4GeS:
3828 : return graph()->NewNode(mcgraph()->machine()->I32x4GeS(), inputs[0],
3829 24 : inputs[1]);
3830 : case wasm::kExprI32x4UConvertI16x8Low:
3831 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8Low(),
3832 24 : inputs[0]);
3833 : case wasm::kExprI32x4UConvertI16x8High:
3834 : return graph()->NewNode(mcgraph()->machine()->I32x4UConvertI16x8High(),
3835 24 : inputs[0]);
3836 : case wasm::kExprI32x4MinU:
3837 : return graph()->NewNode(mcgraph()->machine()->I32x4MinU(), inputs[0],
3838 24 : inputs[1]);
3839 : case wasm::kExprI32x4MaxU:
3840 : return graph()->NewNode(mcgraph()->machine()->I32x4MaxU(), inputs[0],
3841 24 : inputs[1]);
3842 : case wasm::kExprI32x4LtU:
3843 : return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[1],
3844 24 : inputs[0]);
3845 : case wasm::kExprI32x4LeU:
3846 : return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[1],
3847 24 : inputs[0]);
3848 : case wasm::kExprI32x4GtU:
3849 : return graph()->NewNode(mcgraph()->machine()->I32x4GtU(), inputs[0],
3850 24 : inputs[1]);
3851 : case wasm::kExprI32x4GeU:
3852 : return graph()->NewNode(mcgraph()->machine()->I32x4GeU(), inputs[0],
3853 24 : inputs[1]);
3854 : case wasm::kExprI16x8Splat:
3855 2352 : return graph()->NewNode(mcgraph()->machine()->I16x8Splat(), inputs[0]);
3856 : case wasm::kExprI16x8SConvertI8x16Low:
3857 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16Low(),
3858 24 : inputs[0]);
3859 : case wasm::kExprI16x8SConvertI8x16High:
3860 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI8x16High(),
3861 24 : inputs[0]);
3862 : case wasm::kExprI16x8Neg:
3863 24 : return graph()->NewNode(mcgraph()->machine()->I16x8Neg(), inputs[0]);
3864 : case wasm::kExprI16x8SConvertI32x4:
3865 : return graph()->NewNode(mcgraph()->machine()->I16x8SConvertI32x4(),
3866 24 : inputs[0], inputs[1]);
3867 : case wasm::kExprI16x8Add:
3868 : return graph()->NewNode(mcgraph()->machine()->I16x8Add(), inputs[0],
3869 24 : inputs[1]);
3870 : case wasm::kExprI16x8AddSaturateS:
3871 : return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateS(),
3872 24 : inputs[0], inputs[1]);
3873 : case wasm::kExprI16x8AddHoriz:
3874 : return graph()->NewNode(mcgraph()->machine()->I16x8AddHoriz(), inputs[0],
3875 24 : inputs[1]);
3876 : case wasm::kExprI16x8Sub:
3877 : return graph()->NewNode(mcgraph()->machine()->I16x8Sub(), inputs[0],
3878 24 : inputs[1]);
3879 : case wasm::kExprI16x8SubSaturateS:
3880 : return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateS(),
3881 24 : inputs[0], inputs[1]);
3882 : case wasm::kExprI16x8Mul:
3883 : return graph()->NewNode(mcgraph()->machine()->I16x8Mul(), inputs[0],
3884 24 : inputs[1]);
3885 : case wasm::kExprI16x8MinS:
3886 : return graph()->NewNode(mcgraph()->machine()->I16x8MinS(), inputs[0],
3887 24 : inputs[1]);
3888 : case wasm::kExprI16x8MaxS:
3889 : return graph()->NewNode(mcgraph()->machine()->I16x8MaxS(), inputs[0],
3890 24 : inputs[1]);
3891 : case wasm::kExprI16x8Eq:
3892 : return graph()->NewNode(mcgraph()->machine()->I16x8Eq(), inputs[0],
3893 120 : inputs[1]);
3894 : case wasm::kExprI16x8Ne:
3895 : return graph()->NewNode(mcgraph()->machine()->I16x8Ne(), inputs[0],
3896 132 : inputs[1]);
3897 : case wasm::kExprI16x8LtS:
3898 : return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[1],
3899 24 : inputs[0]);
3900 : case wasm::kExprI16x8LeS:
3901 : return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[1],
3902 24 : inputs[0]);
3903 : case wasm::kExprI16x8GtS:
3904 : return graph()->NewNode(mcgraph()->machine()->I16x8GtS(), inputs[0],
3905 24 : inputs[1]);
3906 : case wasm::kExprI16x8GeS:
3907 : return graph()->NewNode(mcgraph()->machine()->I16x8GeS(), inputs[0],
3908 24 : inputs[1]);
3909 : case wasm::kExprI16x8UConvertI8x16Low:
3910 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16Low(),
3911 24 : inputs[0]);
3912 : case wasm::kExprI16x8UConvertI8x16High:
3913 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI8x16High(),
3914 24 : inputs[0]);
3915 : case wasm::kExprI16x8UConvertI32x4:
3916 : return graph()->NewNode(mcgraph()->machine()->I16x8UConvertI32x4(),
3917 24 : inputs[0], inputs[1]);
3918 : case wasm::kExprI16x8AddSaturateU:
3919 : return graph()->NewNode(mcgraph()->machine()->I16x8AddSaturateU(),
3920 24 : inputs[0], inputs[1]);
3921 : case wasm::kExprI16x8SubSaturateU:
3922 : return graph()->NewNode(mcgraph()->machine()->I16x8SubSaturateU(),
3923 24 : inputs[0], inputs[1]);
3924 : case wasm::kExprI16x8MinU:
3925 : return graph()->NewNode(mcgraph()->machine()->I16x8MinU(), inputs[0],
3926 24 : inputs[1]);
3927 : case wasm::kExprI16x8MaxU:
3928 : return graph()->NewNode(mcgraph()->machine()->I16x8MaxU(), inputs[0],
3929 24 : inputs[1]);
3930 : case wasm::kExprI16x8LtU:
3931 : return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[1],
3932 24 : inputs[0]);
3933 : case wasm::kExprI16x8LeU:
3934 : return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[1],
3935 24 : inputs[0]);
3936 : case wasm::kExprI16x8GtU:
3937 : return graph()->NewNode(mcgraph()->machine()->I16x8GtU(), inputs[0],
3938 24 : inputs[1]);
3939 : case wasm::kExprI16x8GeU:
3940 : return graph()->NewNode(mcgraph()->machine()->I16x8GeU(), inputs[0],
3941 24 : inputs[1]);
3942 : case wasm::kExprI8x16Splat:
3943 1728 : return graph()->NewNode(mcgraph()->machine()->I8x16Splat(), inputs[0]);
3944 : case wasm::kExprI8x16Neg:
3945 24 : return graph()->NewNode(mcgraph()->machine()->I8x16Neg(), inputs[0]);
3946 : case wasm::kExprI8x16SConvertI16x8:
3947 : return graph()->NewNode(mcgraph()->machine()->I8x16SConvertI16x8(),
3948 24 : inputs[0], inputs[1]);
3949 : case wasm::kExprI8x16Add:
3950 : return graph()->NewNode(mcgraph()->machine()->I8x16Add(), inputs[0],
3951 24 : inputs[1]);
3952 : case wasm::kExprI8x16AddSaturateS:
3953 : return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateS(),
3954 24 : inputs[0], inputs[1]);
3955 : case wasm::kExprI8x16Sub:
3956 : return graph()->NewNode(mcgraph()->machine()->I8x16Sub(), inputs[0],
3957 24 : inputs[1]);
3958 : case wasm::kExprI8x16SubSaturateS:
3959 : return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateS(),
3960 24 : inputs[0], inputs[1]);
3961 : case wasm::kExprI8x16Mul:
3962 : return graph()->NewNode(mcgraph()->machine()->I8x16Mul(), inputs[0],
3963 24 : inputs[1]);
3964 : case wasm::kExprI8x16MinS:
3965 : return graph()->NewNode(mcgraph()->machine()->I8x16MinS(), inputs[0],
3966 24 : inputs[1]);
3967 : case wasm::kExprI8x16MaxS:
3968 : return graph()->NewNode(mcgraph()->machine()->I8x16MaxS(), inputs[0],
3969 24 : inputs[1]);
3970 : case wasm::kExprI8x16Eq:
3971 : return graph()->NewNode(mcgraph()->machine()->I8x16Eq(), inputs[0],
3972 120 : inputs[1]);
3973 : case wasm::kExprI8x16Ne:
3974 : return graph()->NewNode(mcgraph()->machine()->I8x16Ne(), inputs[0],
3975 132 : inputs[1]);
3976 : case wasm::kExprI8x16LtS:
3977 : return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[1],
3978 24 : inputs[0]);
3979 : case wasm::kExprI8x16LeS:
3980 : return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[1],
3981 24 : inputs[0]);
3982 : case wasm::kExprI8x16GtS:
3983 : return graph()->NewNode(mcgraph()->machine()->I8x16GtS(), inputs[0],
3984 24 : inputs[1]);
3985 : case wasm::kExprI8x16GeS:
3986 : return graph()->NewNode(mcgraph()->machine()->I8x16GeS(), inputs[0],
3987 24 : inputs[1]);
3988 : case wasm::kExprI8x16UConvertI16x8:
3989 : return graph()->NewNode(mcgraph()->machine()->I8x16UConvertI16x8(),
3990 24 : inputs[0], inputs[1]);
3991 : case wasm::kExprI8x16AddSaturateU:
3992 : return graph()->NewNode(mcgraph()->machine()->I8x16AddSaturateU(),
3993 24 : inputs[0], inputs[1]);
3994 : case wasm::kExprI8x16SubSaturateU:
3995 : return graph()->NewNode(mcgraph()->machine()->I8x16SubSaturateU(),
3996 24 : inputs[0], inputs[1]);
3997 : case wasm::kExprI8x16MinU:
3998 : return graph()->NewNode(mcgraph()->machine()->I8x16MinU(), inputs[0],
3999 24 : inputs[1]);
4000 : case wasm::kExprI8x16MaxU:
4001 : return graph()->NewNode(mcgraph()->machine()->I8x16MaxU(), inputs[0],
4002 24 : inputs[1]);
4003 : case wasm::kExprI8x16LtU:
4004 : return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[1],
4005 24 : inputs[0]);
4006 : case wasm::kExprI8x16LeU:
4007 : return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[1],
4008 24 : inputs[0]);
4009 : case wasm::kExprI8x16GtU:
4010 : return graph()->NewNode(mcgraph()->machine()->I8x16GtU(), inputs[0],
4011 24 : inputs[1]);
4012 : case wasm::kExprI8x16GeU:
4013 : return graph()->NewNode(mcgraph()->machine()->I8x16GeU(), inputs[0],
4014 24 : inputs[1]);
4015 : case wasm::kExprS128And:
4016 : return graph()->NewNode(mcgraph()->machine()->S128And(), inputs[0],
4017 24 : inputs[1]);
4018 : case wasm::kExprS128Or:
4019 : return graph()->NewNode(mcgraph()->machine()->S128Or(), inputs[0],
4020 24 : inputs[1]);
4021 : case wasm::kExprS128Xor:
4022 : return graph()->NewNode(mcgraph()->machine()->S128Xor(), inputs[0],
4023 24 : inputs[1]);
4024 : case wasm::kExprS128Not:
4025 24 : return graph()->NewNode(mcgraph()->machine()->S128Not(), inputs[0]);
4026 : case wasm::kExprS128Select:
4027 : return graph()->NewNode(mcgraph()->machine()->S128Select(), inputs[2],
4028 84 : inputs[0], inputs[1]);
4029 : case wasm::kExprS1x4AnyTrue:
4030 108 : return graph()->NewNode(mcgraph()->machine()->S1x4AnyTrue(), inputs[0]);
4031 : case wasm::kExprS1x4AllTrue:
4032 108 : return graph()->NewNode(mcgraph()->machine()->S1x4AllTrue(), inputs[0]);
4033 : case wasm::kExprS1x8AnyTrue:
4034 108 : return graph()->NewNode(mcgraph()->machine()->S1x8AnyTrue(), inputs[0]);
4035 : case wasm::kExprS1x8AllTrue:
4036 108 : return graph()->NewNode(mcgraph()->machine()->S1x8AllTrue(), inputs[0]);
4037 : case wasm::kExprS1x16AnyTrue:
4038 108 : return graph()->NewNode(mcgraph()->machine()->S1x16AnyTrue(), inputs[0]);
4039 : case wasm::kExprS1x16AllTrue:
4040 108 : return graph()->NewNode(mcgraph()->machine()->S1x16AllTrue(), inputs[0]);
4041 : default:
4042 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4043 : }
4044 : }
4045 :
4046 18676 : Node* WasmGraphBuilder::SimdLaneOp(wasm::WasmOpcode opcode, uint8_t lane,
4047 18676 : Node* const* inputs) {
4048 18676 : has_simd_ = true;
4049 18676 : switch (opcode) {
4050 : case wasm::kExprF32x4ExtractLane:
4051 : return graph()->NewNode(mcgraph()->machine()->F32x4ExtractLane(lane),
4052 2112 : inputs[0]);
4053 : case wasm::kExprF32x4ReplaceLane:
4054 : return graph()->NewNode(mcgraph()->machine()->F32x4ReplaceLane(lane),
4055 192 : inputs[0], inputs[1]);
4056 : case wasm::kExprI32x4ExtractLane:
4057 : return graph()->NewNode(mcgraph()->machine()->I32x4ExtractLane(lane),
4058 13116 : inputs[0]);
4059 : case wasm::kExprI32x4ReplaceLane:
4060 : return graph()->NewNode(mcgraph()->machine()->I32x4ReplaceLane(lane),
4061 288 : inputs[0], inputs[1]);
4062 : case wasm::kExprI16x8ExtractLane:
4063 : return graph()->NewNode(mcgraph()->machine()->I16x8ExtractLane(lane),
4064 15840 : inputs[0]);
4065 : case wasm::kExprI16x8ReplaceLane:
4066 : return graph()->NewNode(mcgraph()->machine()->I16x8ReplaceLane(lane),
4067 288 : inputs[0], inputs[1]);
4068 : case wasm::kExprI8x16ExtractLane:
4069 : return graph()->NewNode(mcgraph()->machine()->I8x16ExtractLane(lane),
4070 23712 : inputs[0]);
4071 : case wasm::kExprI8x16ReplaceLane:
4072 : return graph()->NewNode(mcgraph()->machine()->I8x16ReplaceLane(lane),
4073 480 : inputs[0], inputs[1]);
4074 : default:
4075 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4076 : }
4077 : }
4078 :
4079 1272 : Node* WasmGraphBuilder::SimdShiftOp(wasm::WasmOpcode opcode, uint8_t shift,
4080 1272 : Node* const* inputs) {
4081 1272 : has_simd_ = true;
4082 1272 : switch (opcode) {
4083 : case wasm::kExprI32x4Shl:
4084 744 : return graph()->NewNode(mcgraph()->machine()->I32x4Shl(shift), inputs[0]);
4085 : case wasm::kExprI32x4ShrS:
4086 : return graph()->NewNode(mcgraph()->machine()->I32x4ShrS(shift),
4087 744 : inputs[0]);
4088 : case wasm::kExprI32x4ShrU:
4089 : return graph()->NewNode(mcgraph()->machine()->I32x4ShrU(shift),
4090 744 : inputs[0]);
4091 : case wasm::kExprI16x8Shl:
4092 360 : return graph()->NewNode(mcgraph()->machine()->I16x8Shl(shift), inputs[0]);
4093 : case wasm::kExprI16x8ShrS:
4094 : return graph()->NewNode(mcgraph()->machine()->I16x8ShrS(shift),
4095 360 : inputs[0]);
4096 : case wasm::kExprI16x8ShrU:
4097 : return graph()->NewNode(mcgraph()->machine()->I16x8ShrU(shift),
4098 360 : inputs[0]);
4099 : case wasm::kExprI8x16Shl:
4100 168 : return graph()->NewNode(mcgraph()->machine()->I8x16Shl(shift), inputs[0]);
4101 : case wasm::kExprI8x16ShrS:
4102 : return graph()->NewNode(mcgraph()->machine()->I8x16ShrS(shift),
4103 168 : inputs[0]);
4104 : case wasm::kExprI8x16ShrU:
4105 : return graph()->NewNode(mcgraph()->machine()->I8x16ShrU(shift),
4106 168 : inputs[0]);
4107 : default:
4108 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4109 : }
4110 : }
4111 :
4112 14544 : Node* WasmGraphBuilder::Simd8x16ShuffleOp(const uint8_t shuffle[16],
4113 14544 : Node* const* inputs) {
4114 14544 : has_simd_ = true;
4115 : return graph()->NewNode(mcgraph()->machine()->S8x16Shuffle(shuffle),
4116 43632 : inputs[0], inputs[1]);
4117 : }
4118 :
4119 : #define ATOMIC_BINOP_LIST(V) \
4120 : V(I32AtomicAdd, Add, Uint32, Word32) \
4121 : V(I64AtomicAdd, Add, Uint64, Word64) \
4122 : V(I32AtomicAdd8U, Add, Uint8, Word32) \
4123 : V(I32AtomicAdd16U, Add, Uint16, Word32) \
4124 : V(I64AtomicAdd8U, Add, Uint8, Word64) \
4125 : V(I64AtomicAdd16U, Add, Uint16, Word64) \
4126 : V(I64AtomicAdd32U, Add, Uint32, Word64) \
4127 : V(I32AtomicSub, Sub, Uint32, Word32) \
4128 : V(I64AtomicSub, Sub, Uint64, Word64) \
4129 : V(I32AtomicSub8U, Sub, Uint8, Word32) \
4130 : V(I32AtomicSub16U, Sub, Uint16, Word32) \
4131 : V(I64AtomicSub8U, Sub, Uint8, Word64) \
4132 : V(I64AtomicSub16U, Sub, Uint16, Word64) \
4133 : V(I64AtomicSub32U, Sub, Uint32, Word64) \
4134 : V(I32AtomicAnd, And, Uint32, Word32) \
4135 : V(I64AtomicAnd, And, Uint64, Word64) \
4136 : V(I32AtomicAnd8U, And, Uint8, Word32) \
4137 : V(I64AtomicAnd16U, And, Uint16, Word64) \
4138 : V(I32AtomicAnd16U, And, Uint16, Word32) \
4139 : V(I64AtomicAnd8U, And, Uint8, Word64) \
4140 : V(I64AtomicAnd32U, And, Uint32, Word64) \
4141 : V(I32AtomicOr, Or, Uint32, Word32) \
4142 : V(I64AtomicOr, Or, Uint64, Word64) \
4143 : V(I32AtomicOr8U, Or, Uint8, Word32) \
4144 : V(I32AtomicOr16U, Or, Uint16, Word32) \
4145 : V(I64AtomicOr8U, Or, Uint8, Word64) \
4146 : V(I64AtomicOr16U, Or, Uint16, Word64) \
4147 : V(I64AtomicOr32U, Or, Uint32, Word64) \
4148 : V(I32AtomicXor, Xor, Uint32, Word32) \
4149 : V(I64AtomicXor, Xor, Uint64, Word64) \
4150 : V(I32AtomicXor8U, Xor, Uint8, Word32) \
4151 : V(I32AtomicXor16U, Xor, Uint16, Word32) \
4152 : V(I64AtomicXor8U, Xor, Uint8, Word64) \
4153 : V(I64AtomicXor16U, Xor, Uint16, Word64) \
4154 : V(I64AtomicXor32U, Xor, Uint32, Word64) \
4155 : V(I32AtomicExchange, Exchange, Uint32, Word32) \
4156 : V(I64AtomicExchange, Exchange, Uint64, Word64) \
4157 : V(I32AtomicExchange8U, Exchange, Uint8, Word32) \
4158 : V(I32AtomicExchange16U, Exchange, Uint16, Word32) \
4159 : V(I64AtomicExchange8U, Exchange, Uint8, Word64) \
4160 : V(I64AtomicExchange16U, Exchange, Uint16, Word64) \
4161 : V(I64AtomicExchange32U, Exchange, Uint32, Word64)
4162 :
4163 : #define ATOMIC_CMP_EXCHG_LIST(V) \
4164 : V(I32AtomicCompareExchange, Uint32, Word32) \
4165 : V(I64AtomicCompareExchange, Uint64, Word64) \
4166 : V(I32AtomicCompareExchange8U, Uint8, Word32) \
4167 : V(I32AtomicCompareExchange16U, Uint16, Word32) \
4168 : V(I64AtomicCompareExchange8U, Uint8, Word64) \
4169 : V(I64AtomicCompareExchange16U, Uint16, Word64) \
4170 : V(I64AtomicCompareExchange32U, Uint32, Word64)
4171 :
4172 : #define ATOMIC_LOAD_LIST(V) \
4173 : V(I32AtomicLoad, Uint32, Word32) \
4174 : V(I64AtomicLoad, Uint64, Word64) \
4175 : V(I32AtomicLoad8U, Uint8, Word32) \
4176 : V(I32AtomicLoad16U, Uint16, Word32) \
4177 : V(I64AtomicLoad8U, Uint8, Word64) \
4178 : V(I64AtomicLoad16U, Uint16, Word64) \
4179 : V(I64AtomicLoad32U, Uint32, Word64)
4180 :
4181 : #define ATOMIC_STORE_LIST(V) \
4182 : V(I32AtomicStore, Uint32, kWord32, Word32) \
4183 : V(I64AtomicStore, Uint64, kWord64, Word64) \
4184 : V(I32AtomicStore8U, Uint8, kWord8, Word32) \
4185 : V(I32AtomicStore16U, Uint16, kWord16, Word32) \
4186 : V(I64AtomicStore8U, Uint8, kWord8, Word64) \
4187 : V(I64AtomicStore16U, Uint16, kWord16, Word64) \
4188 : V(I64AtomicStore32U, Uint32, kWord32, Word64)
4189 :
4190 32678 : Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs,
4191 : uint32_t alignment, uint32_t offset,
4192 134036 : wasm::WasmCodePosition position) {
4193 : Node* node;
4194 32678 : switch (opcode) {
4195 : #define BUILD_ATOMIC_BINOP(Name, Operation, Type, Prefix) \
4196 : case wasm::kExpr##Name: { \
4197 : Node* index = CheckBoundsAndAlignment( \
4198 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4199 : position); \
4200 : node = graph()->NewNode( \
4201 : mcgraph()->machine()->Prefix##Atomic##Operation(MachineType::Type()), \
4202 : MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4203 : break; \
4204 : }
4205 96688 : ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP)
4206 : #undef BUILD_ATOMIC_BINOP
4207 :
4208 : #define BUILD_ATOMIC_CMP_EXCHG(Name, Type, Prefix) \
4209 : case wasm::kExpr##Name: { \
4210 : Node* index = CheckBoundsAndAlignment( \
4211 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4212 : position); \
4213 : node = graph()->NewNode( \
4214 : mcgraph()->machine()->Prefix##AtomicCompareExchange( \
4215 : MachineType::Type()), \
4216 : MemBuffer(offset), index, inputs[1], inputs[2], Effect(), Control()); \
4217 : break; \
4218 : }
4219 733 : ATOMIC_CMP_EXCHG_LIST(BUILD_ATOMIC_CMP_EXCHG)
4220 : #undef BUILD_ATOMIC_CMP_EXCHG
4221 :
4222 : #define BUILD_ATOMIC_LOAD_OP(Name, Type, Prefix) \
4223 : case wasm::kExpr##Name: { \
4224 : Node* index = CheckBoundsAndAlignment( \
4225 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4226 : position); \
4227 : node = graph()->NewNode( \
4228 : mcgraph()->machine()->Prefix##AtomicLoad(MachineType::Type()), \
4229 : MemBuffer(offset), index, Effect(), Control()); \
4230 : break; \
4231 : }
4232 16392 : ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP)
4233 : #undef BUILD_ATOMIC_LOAD_OP
4234 :
4235 : #define BUILD_ATOMIC_STORE_OP(Name, Type, Rep, Prefix) \
4236 : case wasm::kExpr##Name: { \
4237 : Node* index = CheckBoundsAndAlignment( \
4238 : wasm::ValueTypes::MemSize(MachineType::Type()), inputs[0], offset, \
4239 : position); \
4240 : node = graph()->NewNode( \
4241 : mcgraph()->machine()->Prefix##AtomicStore(MachineRepresentation::Rep), \
4242 : MemBuffer(offset), index, inputs[1], Effect(), Control()); \
4243 : break; \
4244 : }
4245 15808 : ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP)
4246 : #undef BUILD_ATOMIC_STORE_OP
4247 : case wasm::kExprAtomicWake: {
4248 : Node* index = CheckBoundsAndAlignment(
4249 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4250 320 : position);
4251 : // Now that we've bounds-checked, compute the effective address.
4252 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4253 160 : Uint32Constant(offset), index);
4254 : WasmAtomicWakeDescriptor interface_descriptor;
4255 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4256 : mcgraph()->zone(), interface_descriptor,
4257 : interface_descriptor.GetStackParameterCount(),
4258 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4259 160 : StubCallMode::kCallWasmRuntimeStub);
4260 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4261 160 : wasm::WasmCode::kWasmAtomicWake, RelocInfo::WASM_STUB_CALL);
4262 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4263 : call_target, address, inputs[1], Effect(),
4264 320 : Control());
4265 : break;
4266 : }
4267 :
4268 : case wasm::kExprI32AtomicWait: {
4269 : Node* index = CheckBoundsAndAlignment(
4270 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4271 508 : position);
4272 : // Now that we've bounds-checked, compute the effective address.
4273 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4274 256 : Uint32Constant(offset), index);
4275 : Node* timeout;
4276 512 : if (mcgraph()->machine()->Is32()) {
4277 0 : timeout = BuildF64SConvertI64(inputs[2]);
4278 : } else {
4279 : timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4280 256 : inputs[2]);
4281 : }
4282 : WasmI32AtomicWaitDescriptor interface_descriptor;
4283 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4284 : mcgraph()->zone(), interface_descriptor,
4285 : interface_descriptor.GetStackParameterCount(),
4286 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4287 256 : StubCallMode::kCallWasmRuntimeStub);
4288 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4289 254 : wasm::WasmCode::kWasmI32AtomicWait, RelocInfo::WASM_STUB_CALL);
4290 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4291 : call_target, address, inputs[1], timeout,
4292 512 : Effect(), Control());
4293 : break;
4294 : }
4295 :
4296 : case wasm::kExprI64AtomicWait: {
4297 : Node* index = CheckBoundsAndAlignment(
4298 : wasm::ValueTypes::MemSize(MachineType::Uint32()), inputs[0], offset,
4299 506 : position);
4300 : // Now that we've bounds-checked, compute the effective address.
4301 : Node* address = graph()->NewNode(mcgraph()->machine()->Int32Add(),
4302 256 : Uint32Constant(offset), index);
4303 : Node* timeout;
4304 510 : if (mcgraph()->machine()->Is32()) {
4305 0 : timeout = BuildF64SConvertI64(inputs[2]);
4306 : } else {
4307 : timeout = graph()->NewNode(mcgraph()->machine()->RoundInt64ToFloat64(),
4308 255 : inputs[2]);
4309 : }
4310 : Node* expected_value_low = graph()->NewNode(
4311 510 : mcgraph()->machine()->TruncateInt64ToInt32(), inputs[1]);
4312 : Node* tmp = graph()->NewNode(mcgraph()->machine()->Word64Shr(), inputs[1],
4313 512 : Int64Constant(32));
4314 : Node* expected_value_high =
4315 255 : graph()->NewNode(mcgraph()->machine()->TruncateInt64ToInt32(), tmp);
4316 : WasmI64AtomicWaitDescriptor interface_descriptor;
4317 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4318 : mcgraph()->zone(), interface_descriptor,
4319 : interface_descriptor.GetStackParameterCount(),
4320 : CallDescriptor::kNoFlags, Operator::kNoProperties,
4321 256 : StubCallMode::kCallWasmRuntimeStub);
4322 : Node* call_target = mcgraph()->RelocatableIntPtrConstant(
4323 254 : wasm::WasmCode::kWasmI64AtomicWait, RelocInfo::WASM_STUB_CALL);
4324 : node = graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4325 : call_target, address, expected_value_high,
4326 256 : expected_value_low, timeout, Effect(), Control());
4327 : break;
4328 : }
4329 :
4330 : default:
4331 0 : FATAL_UNSUPPORTED_OPCODE(opcode);
4332 : }
4333 32686 : return SetEffect(node);
4334 : }
4335 :
4336 : #undef ATOMIC_BINOP_LIST
4337 : #undef ATOMIC_CMP_EXCHG_LIST
4338 : #undef ATOMIC_LOAD_LIST
4339 : #undef ATOMIC_STORE_LIST
4340 :
4341 64 : Node* WasmGraphBuilder::CheckDataSegmentIsPassiveAndNotDropped(
4342 640 : uint32_t data_segment_index, wasm::WasmCodePosition position) {
4343 : // The data segment index must be in bounds since it is required by
4344 : // validation.
4345 : DCHECK_LT(data_segment_index, env_->module->num_declared_data_segments);
4346 :
4347 : Node* dropped_data_segments =
4348 128 : LOAD_INSTANCE_FIELD(DroppedDataSegments, MachineType::Pointer());
4349 : Node* is_segment_dropped = SetEffect(graph()->NewNode(
4350 : mcgraph()->machine()->Load(MachineType::Uint8()), dropped_data_segments,
4351 192 : mcgraph()->IntPtrConstant(data_segment_index), Effect(), Control()));
4352 64 : TrapIfTrue(wasm::kTrapDataSegmentDropped, is_segment_dropped, position);
4353 64 : return dropped_data_segments;
4354 : }
4355 :
4356 40 : Node* WasmGraphBuilder::MemoryInit(uint32_t data_segment_index, Node* dst,
4357 : Node* src, Node* size,
4358 720 : wasm::WasmCodePosition position) {
4359 40 : CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
4360 40 : dst = BoundsCheckMemRange(dst, size, position);
4361 40 : MachineOperatorBuilder* m = mcgraph()->machine();
4362 :
4363 : Node* seg_index = Uint32Constant(data_segment_index);
4364 :
4365 : {
4366 : // Load segment size from WasmInstanceObject::data_segment_sizes.
4367 : Node* seg_size_array =
4368 80 : LOAD_INSTANCE_FIELD(DataSegmentSizes, MachineType::Pointer());
4369 : STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <= kMaxUInt32 >> 2);
4370 : Node* scaled_index = Uint32ToUintptr(
4371 80 : graph()->NewNode(m->Word32Shl(), seg_index, Int32Constant(2)));
4372 : Node* seg_size = SetEffect(graph()->NewNode(m->Load(MachineType::Uint32()),
4373 : seg_size_array, scaled_index,
4374 40 : Effect(), Control()));
4375 :
4376 : // Bounds check the src index against the segment size.
4377 40 : BoundsCheckRange(src, size, seg_size, position);
4378 : }
4379 :
4380 : {
4381 : // Load segment's base pointer from WasmInstanceObject::data_segment_starts.
4382 : Node* seg_start_array =
4383 80 : LOAD_INSTANCE_FIELD(DataSegmentStarts, MachineType::Pointer());
4384 : STATIC_ASSERT(wasm::kV8MaxWasmDataSegments <=
4385 : kMaxUInt32 / kSystemPointerSize);
4386 : Node* scaled_index = Uint32ToUintptr(graph()->NewNode(
4387 80 : m->Word32Shl(), seg_index, Int32Constant(kSystemPointerSizeLog2)));
4388 : Node* seg_start = SetEffect(
4389 : graph()->NewNode(m->Load(MachineType::Pointer()), seg_start_array,
4390 40 : scaled_index, Effect(), Control()));
4391 :
4392 : // Convert src index to pointer.
4393 40 : src = graph()->NewNode(m->IntAdd(), seg_start, Uint32ToUintptr(src));
4394 : }
4395 :
4396 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4397 120 : ExternalReference::wasm_memory_copy()));
4398 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
4399 40 : MachineType::Uint32()};
4400 : MachineSignature sig(0, 3, sig_types);
4401 40 : return BuildCCall(&sig, function, dst, src, size);
4402 : }
4403 :
4404 24 : Node* WasmGraphBuilder::DataDrop(uint32_t data_segment_index,
4405 144 : wasm::WasmCodePosition position) {
4406 : Node* dropped_data_segments =
4407 24 : CheckDataSegmentIsPassiveAndNotDropped(data_segment_index, position);
4408 : const Operator* store_op = mcgraph()->machine()->Store(
4409 24 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
4410 : return SetEffect(
4411 : graph()->NewNode(store_op, dropped_data_segments,
4412 : mcgraph()->IntPtrConstant(data_segment_index),
4413 72 : mcgraph()->Int32Constant(1), Effect(), Control()));
4414 : }
4415 :
4416 24 : Node* WasmGraphBuilder::MemoryCopy(Node* dst, Node* src, Node* size,
4417 24 : wasm::WasmCodePosition position) {
4418 24 : dst = BoundsCheckMemRange(dst, size, position);
4419 24 : src = BoundsCheckMemRange(src, size, position);
4420 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4421 72 : ExternalReference::wasm_memory_copy()));
4422 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Pointer(),
4423 24 : MachineType::Uint32()};
4424 : MachineSignature sig(0, 3, sig_types);
4425 24 : return BuildCCall(&sig, function, dst, src, size);
4426 : }
4427 :
4428 24 : Node* WasmGraphBuilder::MemoryFill(Node* dst, Node* value, Node* size,
4429 24 : wasm::WasmCodePosition position) {
4430 24 : dst = BoundsCheckMemRange(dst, size, position);
4431 : Node* function = graph()->NewNode(mcgraph()->common()->ExternalConstant(
4432 72 : ExternalReference::wasm_memory_fill()));
4433 : MachineType sig_types[] = {MachineType::Pointer(), MachineType::Uint32(),
4434 24 : MachineType::Uint32()};
4435 : MachineSignature sig(0, 3, sig_types);
4436 24 : return BuildCCall(&sig, function, dst, value, size);
4437 : }
4438 :
4439 48 : Node* WasmGraphBuilder::CheckElemSegmentIsPassiveAndNotDropped(
4440 480 : uint32_t elem_segment_index, wasm::WasmCodePosition position) {
4441 : // The elem segment index must be in bounds since it is required by
4442 : // validation.
4443 : DCHECK_LT(elem_segment_index, env_->module->elem_segments.size());
4444 :
4445 : Node* dropped_elem_segments =
4446 96 : LOAD_INSTANCE_FIELD(DroppedElemSegments, MachineType::Pointer());
4447 : Node* is_segment_dropped = SetEffect(graph()->NewNode(
4448 : mcgraph()->machine()->Load(MachineType::Uint8()), dropped_elem_segments,
4449 144 : mcgraph()->IntPtrConstant(elem_segment_index), Effect(), Control()));
4450 48 : TrapIfTrue(wasm::kTrapElemSegmentDropped, is_segment_dropped, position);
4451 48 : return dropped_elem_segments;
4452 : }
4453 :
4454 31 : Node* WasmGraphBuilder::TableInit(uint32_t table_index,
4455 : uint32_t elem_segment_index, Node* dst,
4456 : Node* src, Node* size,
4457 64 : wasm::WasmCodePosition position) {
4458 31 : CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
4459 : Node* args[] = {
4460 64 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_index)),
4461 64 : graph()->NewNode(mcgraph()->common()->NumberConstant(elem_segment_index)),
4462 32 : BuildConvertUint32ToSmiWithSaturation(dst, wasm::kV8MaxWasmTableSize),
4463 32 : BuildConvertUint32ToSmiWithSaturation(src, wasm::kV8MaxWasmTableSize),
4464 128 : BuildConvertUint32ToSmiWithSaturation(size, wasm::kV8MaxWasmTableSize)};
4465 : Node* result =
4466 32 : BuildCallToRuntime(Runtime::kWasmTableInit, args, arraysize(args));
4467 :
4468 32 : return result;
4469 : }
4470 :
4471 16 : Node* WasmGraphBuilder::ElemDrop(uint32_t elem_segment_index,
4472 96 : wasm::WasmCodePosition position) {
4473 : Node* dropped_elem_segments =
4474 16 : CheckElemSegmentIsPassiveAndNotDropped(elem_segment_index, position);
4475 : const Operator* store_op = mcgraph()->machine()->Store(
4476 16 : StoreRepresentation(MachineRepresentation::kWord8, kNoWriteBarrier));
4477 : return SetEffect(
4478 : graph()->NewNode(store_op, dropped_elem_segments,
4479 : mcgraph()->IntPtrConstant(elem_segment_index),
4480 48 : mcgraph()->Int32Constant(1), Effect(), Control()));
4481 : }
4482 :
4483 64 : Node* WasmGraphBuilder::TableCopy(uint32_t table_src_index,
4484 : uint32_t table_dst_index, Node* dst,
4485 : Node* src, Node* size,
4486 128 : wasm::WasmCodePosition position) {
4487 : Node* args[] = {
4488 128 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_src_index)),
4489 128 : graph()->NewNode(mcgraph()->common()->NumberConstant(table_dst_index)),
4490 64 : BuildConvertUint32ToSmiWithSaturation(dst, wasm::kV8MaxWasmTableSize),
4491 64 : BuildConvertUint32ToSmiWithSaturation(src, wasm::kV8MaxWasmTableSize),
4492 256 : BuildConvertUint32ToSmiWithSaturation(size, wasm::kV8MaxWasmTableSize)};
4493 : Node* result =
4494 64 : BuildCallToRuntime(Runtime::kWasmTableCopy, args, arraysize(args));
4495 :
4496 64 : return result;
4497 : }
4498 :
4499 0 : class WasmDecorator final : public GraphDecorator {
4500 : public:
4501 : explicit WasmDecorator(NodeOriginTable* origins, wasm::Decoder* decoder)
4502 0 : : origins_(origins), decoder_(decoder) {}
4503 :
4504 0 : void Decorate(Node* node) final {
4505 : origins_->SetNodeOrigin(
4506 : node, NodeOrigin("wasm graph creation", "n/a",
4507 0 : NodeOrigin::kWasmBytecode, decoder_->position()));
4508 0 : }
4509 :
4510 : private:
4511 : compiler::NodeOriginTable* origins_;
4512 : wasm::Decoder* decoder_;
4513 : };
4514 :
4515 0 : void WasmGraphBuilder::AddBytecodePositionDecorator(
4516 : NodeOriginTable* node_origins, wasm::Decoder* decoder) {
4517 : DCHECK_NULL(decorator_);
4518 0 : decorator_ = new (graph()->zone()) WasmDecorator(node_origins, decoder);
4519 0 : graph()->AddDecorator(decorator_);
4520 0 : }
4521 :
4522 0 : void WasmGraphBuilder::RemoveBytecodePositionDecorator() {
4523 : DCHECK_NOT_NULL(decorator_);
4524 0 : graph()->RemoveDecorator(decorator_);
4525 0 : decorator_ = nullptr;
4526 0 : }
4527 :
4528 : namespace {
4529 278213 : bool must_record_function_compilation(Isolate* isolate) {
4530 278213 : return isolate->logger()->is_listening_to_code_events() ||
4531 : isolate->is_profiling();
4532 : }
4533 :
4534 : PRINTF_FORMAT(4, 5)
4535 0 : void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag,
4536 : Isolate* isolate, Handle<Code> code,
4537 : const char* format, ...) {
4538 : DCHECK(must_record_function_compilation(isolate));
4539 :
4540 : ScopedVector<char> buffer(128);
4541 : va_list arguments;
4542 0 : va_start(arguments, format);
4543 0 : int len = VSNPrintF(buffer, format, arguments);
4544 0 : CHECK_LT(0, len);
4545 0 : va_end(arguments);
4546 : Handle<String> name_str =
4547 0 : isolate->factory()->NewStringFromAsciiChecked(buffer.start());
4548 0 : PROFILE(isolate, CodeCreateEvent(tag, AbstractCode::cast(*code), *name_str));
4549 0 : }
4550 :
4551 : class WasmWrapperGraphBuilder : public WasmGraphBuilder {
4552 : public:
4553 149250 : WasmWrapperGraphBuilder(Zone* zone, JSGraph* jsgraph, wasm::FunctionSig* sig,
4554 : compiler::SourcePositionTable* spt,
4555 : StubCallMode stub_mode, wasm::WasmFeatures features)
4556 : : WasmGraphBuilder(nullptr, zone, jsgraph, sig, spt),
4557 : isolate_(jsgraph->isolate()),
4558 : jsgraph_(jsgraph),
4559 : stub_mode_(stub_mode),
4560 298500 : enabled_features_(features) {}
4561 :
4562 46970 : Node* BuildAllocateHeapNumberWithValue(Node* value, Node* control) {
4563 129366 : MachineOperatorBuilder* machine = mcgraph()->machine();
4564 39447 : CommonOperatorBuilder* common = mcgraph()->common();
4565 39447 : Node* target = (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4566 : ? mcgraph()->RelocatableIntPtrConstant(
4567 : wasm::WasmCode::kWasmAllocateHeapNumber,
4568 : RelocInfo::WASM_STUB_CALL)
4569 : : jsgraph()->HeapConstant(
4570 46970 : BUILTIN_CODE(isolate_, AllocateHeapNumber));
4571 39447 : if (!allocate_heap_number_operator_.is_set()) {
4572 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4573 : mcgraph()->zone(), AllocateHeapNumberDescriptor(), 0,
4574 33075 : CallDescriptor::kNoFlags, Operator::kNoThrow, stub_mode_);
4575 11025 : allocate_heap_number_operator_.set(common->Call(call_descriptor));
4576 : }
4577 : Node* heap_number = graph()->NewNode(allocate_heap_number_operator_.get(),
4578 : target, Effect(), control);
4579 : SetEffect(
4580 : graph()->NewNode(machine->Store(StoreRepresentation(
4581 : MachineRepresentation::kFloat64, kNoWriteBarrier)),
4582 : heap_number, BuildHeapNumberValueIndexConstant(),
4583 78894 : value, heap_number, control));
4584 39447 : return heap_number;
4585 : }
4586 :
4587 81410 : Node* BuildChangeSmiToFloat64(Node* value) {
4588 : return graph()->NewNode(mcgraph()->machine()->ChangeInt32ToFloat64(),
4589 162820 : BuildChangeSmiToInt32(value));
4590 : }
4591 :
4592 81409 : Node* BuildTestHeapObject(Node* value) {
4593 : return graph()->NewNode(mcgraph()->machine()->WordAnd(), value,
4594 162819 : mcgraph()->IntPtrConstant(kHeapObjectTag));
4595 : }
4596 :
4597 81409 : Node* BuildLoadHeapNumberValue(Node* value) {
4598 : return SetEffect(graph()->NewNode(
4599 : mcgraph()->machine()->Load(MachineType::Float64()), value,
4600 407048 : BuildHeapNumberValueIndexConstant(), Effect(), Control()));
4601 : }
4602 :
4603 : Node* BuildHeapNumberValueIndexConstant() {
4604 120856 : return mcgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag);
4605 : }
4606 :
4607 : Node* BuildChangeInt32ToTagged(Node* value) {
4608 : MachineOperatorBuilder* machine = mcgraph()->machine();
4609 : CommonOperatorBuilder* common = mcgraph()->common();
4610 :
4611 : if (SmiValuesAre32Bits()) {
4612 23952 : return BuildChangeInt32ToSmi(value);
4613 : }
4614 : DCHECK(SmiValuesAre31Bits());
4615 :
4616 : Node* effect = Effect();
4617 : Node* control = Control();
4618 : Node* add = graph()->NewNode(machine->Int32AddWithOverflow(), value, value,
4619 : graph()->start());
4620 :
4621 : Node* ovf = graph()->NewNode(common->Projection(1), add, graph()->start());
4622 : Node* branch =
4623 : graph()->NewNode(common->Branch(BranchHint::kFalse), ovf, control);
4624 :
4625 : Node* if_true = graph()->NewNode(common->IfTrue(), branch);
4626 : Node* vtrue = BuildAllocateHeapNumberWithValue(
4627 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value), if_true);
4628 : Node* etrue = Effect();
4629 :
4630 : Node* if_false = graph()->NewNode(common->IfFalse(), branch);
4631 : Node* vfalse = graph()->NewNode(common->Projection(0), add, if_false);
4632 : vfalse = BuildChangeInt32ToIntPtr(vfalse);
4633 :
4634 : Node* merge =
4635 : SetControl(graph()->NewNode(common->Merge(2), if_true, if_false));
4636 : SetEffect(graph()->NewNode(common->EffectPhi(2), etrue, effect, merge));
4637 : return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4638 : vtrue, vfalse, merge);
4639 : }
4640 :
4641 39447 : Node* BuildChangeFloat64ToTagged(Node* value) {
4642 315576 : MachineOperatorBuilder* machine = mcgraph()->machine();
4643 39447 : CommonOperatorBuilder* common = mcgraph()->common();
4644 :
4645 : // Check several conditions:
4646 : // i32?
4647 : // ├─ true: zero?
4648 : // │ ├─ true: negative?
4649 : // │ │ ├─ true: box
4650 : // │ │ └─ false: potentially Smi
4651 : // │ └─ false: potentially Smi
4652 : // └─ false: box
4653 : // For potential Smi values, depending on whether Smis are 31 or 32 bit, we
4654 : // still need to check whether the value fits in a Smi.
4655 :
4656 : Node* effect = Effect();
4657 : Node* control = Control();
4658 39447 : Node* value32 = graph()->NewNode(machine->RoundFloat64ToInt32(), value);
4659 : Node* check_i32 = graph()->NewNode(
4660 : machine->Float64Equal(), value,
4661 78894 : graph()->NewNode(machine->ChangeInt32ToFloat64(), value32));
4662 39447 : Node* branch_i32 = graph()->NewNode(common->Branch(), check_i32, control);
4663 :
4664 39447 : Node* if_i32 = graph()->NewNode(common->IfTrue(), branch_i32);
4665 39447 : Node* if_not_i32 = graph()->NewNode(common->IfFalse(), branch_i32);
4666 :
4667 : // We only need to check for -0 if the {value} can potentially contain -0.
4668 : Node* check_zero = graph()->NewNode(machine->Word32Equal(), value32,
4669 39447 : mcgraph()->Int32Constant(0));
4670 : Node* branch_zero = graph()->NewNode(common->Branch(BranchHint::kFalse),
4671 39447 : check_zero, if_i32);
4672 :
4673 39447 : Node* if_zero = graph()->NewNode(common->IfTrue(), branch_zero);
4674 39447 : Node* if_not_zero = graph()->NewNode(common->IfFalse(), branch_zero);
4675 :
4676 : // In case of 0, we need to check the high bits for the IEEE -0 pattern.
4677 : Node* check_negative = graph()->NewNode(
4678 : machine->Int32LessThan(),
4679 : graph()->NewNode(machine->Float64ExtractHighWord32(), value),
4680 78894 : mcgraph()->Int32Constant(0));
4681 : Node* branch_negative = graph()->NewNode(common->Branch(BranchHint::kFalse),
4682 39447 : check_negative, if_zero);
4683 :
4684 39447 : Node* if_negative = graph()->NewNode(common->IfTrue(), branch_negative);
4685 : Node* if_not_negative =
4686 39447 : graph()->NewNode(common->IfFalse(), branch_negative);
4687 :
4688 : // We need to create a box for negative 0.
4689 : Node* if_smi =
4690 39447 : graph()->NewNode(common->Merge(2), if_not_zero, if_not_negative);
4691 39447 : Node* if_box = graph()->NewNode(common->Merge(2), if_not_i32, if_negative);
4692 :
4693 : // On 64-bit machines we can just wrap the 32-bit integer in a smi, for
4694 : // 32-bit machines we need to deal with potential overflow and fallback to
4695 : // boxing.
4696 : Node* vsmi;
4697 : if (SmiValuesAre32Bits()) {
4698 39447 : vsmi = BuildChangeInt32ToSmi(value32);
4699 : } else {
4700 : DCHECK(SmiValuesAre31Bits());
4701 : Node* smi_tag = graph()->NewNode(machine->Int32AddWithOverflow(), value32,
4702 : value32, if_smi);
4703 :
4704 : Node* check_ovf =
4705 : graph()->NewNode(common->Projection(1), smi_tag, if_smi);
4706 : Node* branch_ovf = graph()->NewNode(common->Branch(BranchHint::kFalse),
4707 : check_ovf, if_smi);
4708 :
4709 : Node* if_ovf = graph()->NewNode(common->IfTrue(), branch_ovf);
4710 : if_box = graph()->NewNode(common->Merge(2), if_ovf, if_box);
4711 :
4712 : if_smi = graph()->NewNode(common->IfFalse(), branch_ovf);
4713 : vsmi = graph()->NewNode(common->Projection(0), smi_tag, if_smi);
4714 : vsmi = BuildChangeInt32ToIntPtr(vsmi);
4715 : }
4716 :
4717 : // Allocate the box for the {value}.
4718 39447 : Node* vbox = BuildAllocateHeapNumberWithValue(value, if_box);
4719 : Node* ebox = Effect();
4720 :
4721 : Node* merge =
4722 39447 : SetControl(graph()->NewNode(common->Merge(2), if_smi, if_box));
4723 39447 : SetEffect(graph()->NewNode(common->EffectPhi(2), effect, ebox, merge));
4724 : return graph()->NewNode(common->Phi(MachineRepresentation::kTagged, 2),
4725 78894 : vsmi, vbox, merge);
4726 : }
4727 :
4728 6266 : int AddArgumentNodes(Node** args, int pos, int param_count,
4729 43007 : wasm::FunctionSig* sig) {
4730 : // Convert wasm numbers to JS values.
4731 55539 : for (int i = 0; i < param_count; ++i) {
4732 : Node* param =
4733 43007 : Param(i + 1); // Start from index 1 to drop the instance_node.
4734 86014 : args[pos++] = ToJS(param, sig->GetParam(i));
4735 : }
4736 6266 : return pos;
4737 : }
4738 :
4739 159459 : Node* BuildJavaScriptToNumber(Node* node, Node* js_context) {
4740 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4741 : mcgraph()->zone(), TypeConversionDescriptor{}, 0,
4742 654637 : CallDescriptor::kNoFlags, Operator::kNoProperties, stub_mode_);
4743 : Node* stub_code =
4744 81410 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4745 : ? mcgraph()->RelocatableIntPtrConstant(
4746 : wasm::WasmCode::kWasmToNumber, RelocInfo::WASM_STUB_CALL)
4747 162820 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, ToNumber));
4748 :
4749 : Node* result = SetEffect(
4750 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), stub_code,
4751 81409 : node, js_context, Effect(), Control()));
4752 :
4753 81409 : SetSourcePosition(result, 1);
4754 :
4755 81410 : return result;
4756 : }
4757 :
4758 81410 : Node* BuildChangeTaggedToFloat64(Node* value) {
4759 1221141 : MachineOperatorBuilder* machine = mcgraph()->machine();
4760 81410 : CommonOperatorBuilder* common = mcgraph()->common();
4761 :
4762 : // Implement the following decision tree:
4763 : // heap object?
4764 : // ├─ true: undefined?
4765 : // │ ├─ true: f64 const
4766 : // │ └─ false: load heap number value
4767 : // └─ false: smi to float64
4768 :
4769 81410 : Node* check_heap_object = BuildTestHeapObject(value);
4770 : Diamond is_heap_object(graph(), common, check_heap_object,
4771 81410 : BranchHint::kFalse);
4772 81409 : is_heap_object.Chain(Control());
4773 :
4774 81409 : SetControl(is_heap_object.if_true);
4775 : Node* orig_effect = Effect();
4776 :
4777 : Node* undefined_node =
4778 162819 : LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
4779 : Node* check_undefined =
4780 81409 : graph()->NewNode(machine->WordEqual(), value, undefined_node);
4781 : Node* effect_tagged = Effect();
4782 :
4783 81409 : Diamond is_undefined(graph(), common, check_undefined, BranchHint::kFalse);
4784 81410 : is_undefined.Nest(is_heap_object, true);
4785 :
4786 81409 : SetControl(is_undefined.if_false);
4787 81409 : Node* vheap_number = BuildLoadHeapNumberValue(value);
4788 : Node* effect_undefined = Effect();
4789 :
4790 81410 : SetControl(is_undefined.merge);
4791 : Node* vundefined =
4792 81410 : mcgraph()->Float64Constant(std::numeric_limits<double>::quiet_NaN());
4793 : Node* vtagged = is_undefined.Phi(MachineRepresentation::kFloat64,
4794 81410 : vundefined, vheap_number);
4795 :
4796 81410 : effect_tagged = is_undefined.EffectPhi(effect_tagged, effect_undefined);
4797 :
4798 : // If input is Smi: just convert to float64.
4799 81410 : Node* vfrom_smi = BuildChangeSmiToFloat64(value);
4800 :
4801 81410 : SetControl(is_heap_object.merge);
4802 81410 : SetEffect(is_heap_object.EffectPhi(effect_tagged, orig_effect));
4803 : return is_heap_object.Phi(MachineRepresentation::kFloat64, vtagged,
4804 81409 : vfrom_smi);
4805 : }
4806 :
4807 63887 : Node* ToJS(Node* node, wasm::ValueType type) {
4808 63887 : switch (type) {
4809 : case wasm::kWasmI32:
4810 23951 : return BuildChangeInt32ToTagged(node);
4811 : case wasm::kWasmS128:
4812 0 : UNREACHABLE();
4813 : case wasm::kWasmI64: {
4814 : DCHECK(enabled_features_.bigint);
4815 32 : return BuildChangeInt64ToBigInt(node);
4816 : }
4817 : case wasm::kWasmF32:
4818 : node = graph()->NewNode(mcgraph()->machine()->ChangeFloat32ToFloat64(),
4819 11021 : node);
4820 11021 : return BuildChangeFloat64ToTagged(node);
4821 : case wasm::kWasmF64:
4822 28426 : return BuildChangeFloat64ToTagged(node);
4823 : case wasm::kWasmAnyRef:
4824 : case wasm::kWasmAnyFunc:
4825 : return node;
4826 : default:
4827 0 : UNREACHABLE();
4828 : }
4829 : }
4830 :
4831 56 : Node* BuildChangeInt64ToBigInt(Node* input) {
4832 : BigIntToWasmI64Descriptor interface_descriptor;
4833 :
4834 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4835 : mcgraph()->zone(), // zone
4836 : interface_descriptor, // descriptor
4837 : interface_descriptor.GetStackParameterCount(), // stack parameter count
4838 : CallDescriptor::kNoFlags, // flags
4839 : Operator::kNoProperties, // properties
4840 264 : stub_mode_); // stub call mode
4841 :
4842 : Node* target =
4843 32 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4844 : ? mcgraph()->RelocatableIntPtrConstant(
4845 : wasm::WasmCode::kBigIntToWasmI64, RelocInfo::WASM_STUB_CALL)
4846 64 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, I64ToBigInt));
4847 :
4848 : return SetEffect(
4849 : SetControl(graph()->NewNode(mcgraph()->common()->Call(call_descriptor),
4850 64 : target, input, Effect(), Control())));
4851 : }
4852 :
4853 24 : Node* BuildChangeBigIntToInt64(Node* input, Node* context) {
4854 : BigIntToI64Descriptor interface_descriptor;
4855 :
4856 : auto call_descriptor = Linkage::GetStubCallDescriptor(
4857 : mcgraph()->zone(), // zone
4858 : interface_descriptor, // descriptor
4859 : interface_descriptor.GetStackParameterCount(), // stack parameter count
4860 : CallDescriptor::kNoFlags, // flags
4861 : Operator::kNoProperties, // properties
4862 136 : stub_mode_); // stub call mode
4863 :
4864 : Node* target =
4865 16 : (stub_mode_ == StubCallMode::kCallWasmRuntimeStub)
4866 : ? mcgraph()->RelocatableIntPtrConstant(
4867 : wasm::WasmCode::kWasmBigIntToI64, RelocInfo::WASM_STUB_CALL)
4868 32 : : jsgraph()->HeapConstant(BUILTIN_CODE(isolate_, BigIntToI64));
4869 :
4870 : return SetEffect(SetControl(
4871 : graph()->NewNode(mcgraph()->common()->Call(call_descriptor), target,
4872 32 : input, context, Effect(), Control())));
4873 : }
4874 :
4875 83434 : Node* FromJS(Node* node, Node* js_context, wasm::ValueType type) {
4876 : DCHECK_NE(wasm::kWasmStmt, type);
4877 :
4878 : // The parameter is of type AnyRef, we take it as is.
4879 83434 : if (type == wasm::kWasmAnyRef) {
4880 1960 : return node;
4881 : }
4882 :
4883 81474 : if (type == wasm::kWasmAnyFunc) {
4884 : Node* check =
4885 : BuildChangeSmiToInt32(SetEffect(BuildCallToRuntimeWithContext(
4886 : Runtime::kWasmIsValidAnyFuncValue, js_context, &node, 1, effect_,
4887 41056 : Control())));
4888 :
4889 : Diamond type_check(graph(), mcgraph()->common(), check,
4890 96 : BranchHint::kTrue);
4891 48 : type_check.Chain(Control());
4892 :
4893 48 : Node* effect = Effect();
4894 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
4895 48 : nullptr, 0, &effect, type_check.if_false);
4896 :
4897 96 : SetEffect(type_check.EffectPhi(Effect(), effect));
4898 :
4899 48 : SetControl(type_check.merge);
4900 :
4901 48 : return node;
4902 : }
4903 : Node* num = nullptr;
4904 :
4905 81426 : if (type != wasm::kWasmI64) {
4906 : // Do a JavaScript ToNumber.
4907 81410 : num = BuildJavaScriptToNumber(node, js_context);
4908 :
4909 : // Change representation.
4910 81410 : num = BuildChangeTaggedToFloat64(num);
4911 : }
4912 :
4913 81426 : switch (type) {
4914 : case wasm::kWasmI32: {
4915 : num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToWord32(),
4916 28634 : num);
4917 28634 : break;
4918 : }
4919 : case wasm::kWasmI64: {
4920 : DCHECK(enabled_features_.bigint);
4921 16 : num = BuildChangeBigIntToInt64(node, js_context);
4922 16 : break;
4923 : }
4924 : case wasm::kWasmF32:
4925 : num = graph()->NewNode(mcgraph()->machine()->TruncateFloat64ToFloat32(),
4926 12038 : num);
4927 12038 : break;
4928 : case wasm::kWasmF64:
4929 : break;
4930 : case wasm::kWasmS128:
4931 0 : UNREACHABLE();
4932 : default:
4933 0 : UNREACHABLE();
4934 : }
4935 : DCHECK_NOT_NULL(num);
4936 :
4937 81426 : return num;
4938 : }
4939 :
4940 289292 : void BuildModifyThreadInWasmFlag(bool new_value) {
4941 578552 : if (!trap_handler::IsTrapHandlerEnabled()) return;
4942 : Node* isolate_root =
4943 4624944 : LOAD_INSTANCE_FIELD(IsolateRoot, MachineType::Pointer());
4944 :
4945 : Node* thread_in_wasm_flag_address =
4946 867138 : LOAD_RAW(isolate_root, Isolate::thread_in_wasm_flag_address_offset(),
4947 : MachineType::Pointer());
4948 :
4949 289046 : if (FLAG_debug_code) {
4950 : Node* flag_value = SetEffect(
4951 : graph()->NewNode(mcgraph()->machine()->Load(MachineType::Pointer()),
4952 : thread_in_wasm_flag_address,
4953 32 : mcgraph()->Int32Constant(0), Effect(), Control()));
4954 : Node* check =
4955 : graph()->NewNode(mcgraph()->machine()->Word32Equal(), flag_value,
4956 48 : mcgraph()->Int32Constant(new_value ? 0 : 1));
4957 :
4958 : Diamond flag_check(graph(), mcgraph()->common(), check,
4959 32 : BranchHint::kTrue);
4960 16 : flag_check.Chain(Control());
4961 : Node* message_id = jsgraph()->SmiConstant(static_cast<int32_t>(
4962 : new_value ? AbortReason::kUnexpectedThreadInWasmSet
4963 32 : : AbortReason::kUnexpectedThreadInWasmUnset));
4964 :
4965 16 : Node* effect = Effect();
4966 : BuildCallToRuntimeWithContext(Runtime::kAbort, NoContextConstant(),
4967 : &message_id, 1, &effect,
4968 32 : flag_check.if_false);
4969 :
4970 32 : SetEffect(flag_check.EffectPhi(Effect(), effect));
4971 :
4972 16 : SetControl(flag_check.merge);
4973 : }
4974 :
4975 : SetEffect(graph()->NewNode(
4976 : mcgraph()->machine()->Store(StoreRepresentation(
4977 : MachineRepresentation::kWord32, kNoWriteBarrier)),
4978 : thread_in_wasm_flag_address, mcgraph()->Int32Constant(0),
4979 1445230 : mcgraph()->Int32Constant(new_value ? 1 : 0), Effect(), Control()));
4980 : }
4981 :
4982 695540 : Node* BuildLoadFunctionDataFromExportedFunction(Node* closure) {
4983 : Node* shared = SetEffect(graph()->NewNode(
4984 : jsgraph()->machine()->Load(MachineType::AnyTagged()), closure,
4985 : jsgraph()->Int32Constant(
4986 : wasm::ObjectAccess::SharedFunctionInfoOffsetInTaggedJSFunction()),
4987 1112864 : Effect(), Control()));
4988 : return SetEffect(graph()->NewNode(
4989 : jsgraph()->machine()->Load(MachineType::AnyTagged()), shared,
4990 : jsgraph()->Int32Constant(SharedFunctionInfo::kFunctionDataOffset -
4991 : kHeapObjectTag),
4992 417324 : Effect(), Control()));
4993 : }
4994 :
4995 417324 : Node* BuildLoadInstanceFromExportedFunctionData(Node* function_data) {
4996 : return SetEffect(graph()->NewNode(
4997 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
4998 : jsgraph()->Int32Constant(WasmExportedFunctionData::kInstanceOffset -
4999 : kHeapObjectTag),
5000 834648 : Effect(), Control()));
5001 : }
5002 :
5003 1392 : Node* BuildLoadFunctionIndexFromExportedFunctionData(Node* function_data) {
5004 : Node* function_index_smi = SetEffect(graph()->NewNode(
5005 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
5006 : jsgraph()->Int32Constant(
5007 : WasmExportedFunctionData::kFunctionIndexOffset - kHeapObjectTag),
5008 2320 : Effect(), Control()));
5009 464 : Node* function_index = BuildChangeSmiToInt32(function_index_smi);
5010 464 : return function_index;
5011 : }
5012 :
5013 413722 : Node* BuildLoadJumpTableOffsetFromExportedFunctionData(Node* function_data) {
5014 : Node* jump_table_offset_smi = SetEffect(graph()->NewNode(
5015 : jsgraph()->machine()->Load(MachineType::AnyTagged()), function_data,
5016 : jsgraph()->Int32Constant(
5017 : WasmExportedFunctionData::kJumpTableOffsetOffset - kHeapObjectTag),
5018 689537 : Effect(), Control()));
5019 137908 : Node* jump_table_offset = BuildChangeSmiToInt32(jump_table_offset_smi);
5020 137908 : return jump_table_offset;
5021 : }
5022 :
5023 396444 : void BuildJSToWasmWrapper(bool is_import) {
5024 378425 : const int wasm_count = static_cast<int>(sig_->parameter_count());
5025 :
5026 : // Build the start and the JS parameter nodes.
5027 1106397 : SetEffect(SetControl(Start(wasm_count + 5)));
5028 :
5029 : // Create the js_closure and js_context parameters.
5030 : Node* js_closure =
5031 : graph()->NewNode(jsgraph()->common()->Parameter(
5032 : Linkage::kJSCallClosureParamIndex, "%closure"),
5033 278216 : graph()->start());
5034 : Node* js_context = graph()->NewNode(
5035 : mcgraph()->common()->Parameter(
5036 : Linkage::GetJSCallContextParamIndex(wasm_count + 1), "%context"),
5037 278216 : graph()->start());
5038 :
5039 : // Create the instance_node node to pass as parameter. It is loaded from
5040 : // an actual reference to an instance or a placeholder reference,
5041 : // called {WasmExportedFunction} via the {WasmExportedFunctionData}
5042 : // structure.
5043 139108 : Node* function_data = BuildLoadFunctionDataFromExportedFunction(js_closure);
5044 : instance_node_.set(
5045 139108 : BuildLoadInstanceFromExportedFunctionData(function_data));
5046 :
5047 139108 : if (!wasm::IsJSCompatibleSignature(sig_, enabled_features_.bigint)) {
5048 : // Throw a TypeError. Use the js_context of the calling javascript
5049 : // function (passed as a parameter), such that the generated code is
5050 : // js_context independent.
5051 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context,
5052 736 : nullptr, 0, effect_, Control());
5053 : Return(jsgraph()->SmiConstant(0));
5054 139843 : return;
5055 : }
5056 :
5057 138372 : const int args_count = wasm_count + 1; // +1 for wasm_code.
5058 138372 : Node** args = Buffer(args_count);
5059 : Node** rets;
5060 :
5061 : // Convert JS parameters to wasm numbers.
5062 356809 : for (int i = 0; i < wasm_count; ++i) {
5063 80065 : Node* param = Param(i + 1);
5064 160130 : Node* wasm_param = FromJS(param, js_context, sig_->GetParam(i));
5065 80065 : args[i + 1] = wasm_param;
5066 : }
5067 :
5068 : // Set the ThreadInWasm flag before we do the actual call.
5069 138372 : BuildModifyThreadInWasmFlag(true);
5070 :
5071 138371 : if (is_import) {
5072 : // Call to an imported function.
5073 : // Load function index from {WasmExportedFunctionData}.
5074 : Node* function_index =
5075 464 : BuildLoadFunctionIndexFromExportedFunctionData(function_data);
5076 464 : BuildImportCall(sig_, args, &rets, wasm::kNoCodePosition, function_index);
5077 : } else {
5078 : // Call to a wasm function defined in this module.
5079 : // The call target is the jump table slot for that function.
5080 : Node* jump_table_start =
5081 275815 : LOAD_INSTANCE_FIELD(JumpTableStart, MachineType::Pointer());
5082 : Node* jump_table_offset =
5083 137908 : BuildLoadJumpTableOffsetFromExportedFunctionData(function_data);
5084 : Node* jump_table_slot = graph()->NewNode(
5085 137908 : mcgraph()->machine()->IntAdd(), jump_table_start, jump_table_offset);
5086 137908 : args[0] = jump_table_slot;
5087 :
5088 : BuildWasmCall(sig_, args, &rets, wasm::kNoCodePosition, nullptr,
5089 137908 : kNoRetpoline);
5090 : }
5091 :
5092 : // Clear the ThreadInWasm flag.
5093 138372 : BuildModifyThreadInWasmFlag(false);
5094 :
5095 138372 : Node* jsval = sig_->return_count() == 0 ? jsgraph()->UndefinedConstant()
5096 276744 : : ToJS(rets[0], sig_->GetReturn());
5097 : Return(jsval);
5098 : }
5099 :
5100 6342 : bool BuildWasmImportCallWrapper(WasmImportCallKind kind) {
5101 15977 : int wasm_count = static_cast<int>(sig_->parameter_count());
5102 :
5103 : // Build the start and the parameter nodes.
5104 163871 : SetEffect(SetControl(Start(wasm_count + 4)));
5105 :
5106 6342 : instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
5107 :
5108 : Node* native_context =
5109 12684 : LOAD_INSTANCE_FIELD(NativeContext, MachineType::TaggedPointer());
5110 :
5111 6342 : if (kind == WasmImportCallKind::kRuntimeTypeError) {
5112 : // =======================================================================
5113 : // === Runtime TypeError =================================================
5114 : // =======================================================================
5115 : BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError,
5116 : native_context, nullptr, 0, effect_,
5117 76 : Control());
5118 : // We don't need to return a value here, as the runtime call will not
5119 : // return anyway (the c entry stub will trigger stack unwinding).
5120 : ReturnVoid();
5121 76 : return false;
5122 : }
5123 :
5124 : // The callable is passed as the last parameter, after WASM arguments.
5125 6266 : Node* callable_node = Param(wasm_count + 1);
5126 :
5127 : Node* undefined_node =
5128 12532 : LOAD_INSTANCE_FIELD(UndefinedValue, MachineType::TaggedPointer());
5129 :
5130 : Node* call = nullptr;
5131 : bool sloppy_receiver = true;
5132 :
5133 : // Clear the ThreadInWasm flag.
5134 6266 : BuildModifyThreadInWasmFlag(false);
5135 :
5136 6266 : switch (kind) {
5137 : // =======================================================================
5138 : // === JS Functions with matching arity ==================================
5139 : // =======================================================================
5140 : case WasmImportCallKind::kJSFunctionArityMatch:
5141 : sloppy_receiver = false;
5142 : V8_FALLTHROUGH; // fallthru
5143 : case WasmImportCallKind::kJSFunctionArityMatchSloppy: {
5144 3972 : Node** args = Buffer(wasm_count + 9);
5145 : int pos = 0;
5146 : Node* function_context = SetEffect(graph()->NewNode(
5147 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5148 : callable_node,
5149 : mcgraph()->Int32Constant(
5150 : wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
5151 7944 : Effect(), Control()));
5152 3972 : args[pos++] = callable_node; // target callable.
5153 : // Receiver.
5154 3972 : if (sloppy_receiver) {
5155 7800 : Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
5156 : native_context, Context::GLOBAL_PROXY_INDEX);
5157 3900 : args[pos++] = global_proxy;
5158 : } else {
5159 72 : args[pos++] = undefined_node;
5160 : }
5161 :
5162 : auto call_descriptor = Linkage::GetJSCallDescriptor(
5163 3972 : graph()->zone(), false, wasm_count + 1, CallDescriptor::kNoFlags);
5164 :
5165 : // Convert wasm numbers to JS values.
5166 3972 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5167 :
5168 3972 : args[pos++] = undefined_node; // new target
5169 7944 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5170 3972 : args[pos++] = function_context;
5171 7944 : args[pos++] = Effect();
5172 7944 : args[pos++] = Control();
5173 :
5174 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5175 7944 : args);
5176 3972 : break;
5177 : }
5178 : // =======================================================================
5179 : // === JS Functions with arguments adapter ===============================
5180 : // =======================================================================
5181 : case WasmImportCallKind::kJSFunctionArityMismatch:
5182 : sloppy_receiver = false;
5183 : V8_FALLTHROUGH; // fallthru
5184 : case WasmImportCallKind::kJSFunctionArityMismatchSloppy: {
5185 622 : Node** args = Buffer(wasm_count + 9);
5186 : int pos = 0;
5187 : Node* function_context = SetEffect(graph()->NewNode(
5188 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5189 : callable_node,
5190 : mcgraph()->Int32Constant(
5191 : wasm::ObjectAccess::ContextOffsetInTaggedJSFunction()),
5192 1244 : Effect(), Control()));
5193 : args[pos++] =
5194 622 : BuildLoadBuiltinFromInstance(Builtins::kArgumentsAdaptorTrampoline);
5195 622 : args[pos++] = callable_node; // target callable
5196 622 : args[pos++] = undefined_node; // new target
5197 622 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5198 :
5199 : // Load shared function info, and then the formal parameter count.
5200 : Node* shared_function_info = SetEffect(graph()->NewNode(
5201 : mcgraph()->machine()->Load(MachineType::TaggedPointer()),
5202 : callable_node,
5203 : mcgraph()->Int32Constant(
5204 : wasm::ObjectAccess::
5205 : SharedFunctionInfoOffsetInTaggedJSFunction()),
5206 1244 : Effect(), Control()));
5207 : Node* formal_param_count = SetEffect(graph()->NewNode(
5208 : mcgraph()->machine()->Load(MachineType::Uint16()),
5209 : shared_function_info,
5210 : mcgraph()->Int32Constant(
5211 : wasm::ObjectAccess::
5212 : FormalParameterCountOffsetInSharedFunctionInfo()),
5213 1244 : Effect(), Control()));
5214 622 : args[pos++] = formal_param_count;
5215 :
5216 : // Receiver.
5217 622 : if (sloppy_receiver) {
5218 1144 : Node* global_proxy = LOAD_FIXED_ARRAY_SLOT_PTR(
5219 : native_context, Context::GLOBAL_PROXY_INDEX);
5220 572 : args[pos++] = global_proxy;
5221 : } else {
5222 50 : args[pos++] = undefined_node;
5223 : }
5224 :
5225 : #ifdef V8_TARGET_ARCH_IA32
5226 : // TODO(v8:6666): Remove kAllowCallThroughSlot and use a pc-relative
5227 : // call instead once builtins are embedded in every build configuration.
5228 : CallDescriptor::Flags flags = CallDescriptor::kAllowCallThroughSlot;
5229 : #else
5230 : CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
5231 : #endif
5232 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5233 : mcgraph()->zone(), ArgumentsAdaptorDescriptor{}, 1 + wasm_count,
5234 1244 : flags, Operator::kNoProperties);
5235 :
5236 : // Convert wasm numbers to JS values.
5237 622 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5238 622 : args[pos++] = function_context;
5239 1244 : args[pos++] = Effect();
5240 1244 : args[pos++] = Control();
5241 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5242 1244 : args);
5243 : break;
5244 : }
5245 : // =======================================================================
5246 : // === General case of unknown callable ==================================
5247 : // =======================================================================
5248 : case WasmImportCallKind::kUseCallBuiltin: {
5249 1672 : Node** args = Buffer(wasm_count + 9);
5250 : int pos = 0;
5251 : args[pos++] = mcgraph()->RelocatableIntPtrConstant(
5252 1672 : wasm::WasmCode::kWasmCallJavaScript, RelocInfo::WASM_STUB_CALL);
5253 1672 : args[pos++] = callable_node;
5254 1672 : args[pos++] = mcgraph()->Int32Constant(wasm_count); // argument count
5255 1672 : args[pos++] = undefined_node; // receiver
5256 :
5257 : auto call_descriptor = Linkage::GetStubCallDescriptor(
5258 : graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1,
5259 : CallDescriptor::kNoFlags, Operator::kNoProperties,
5260 3344 : StubCallMode::kCallWasmRuntimeStub);
5261 :
5262 : // Convert wasm numbers to JS values.
5263 1672 : pos = AddArgumentNodes(args, pos, wasm_count, sig_);
5264 :
5265 : // The native_context is sufficient here, because all kind of callables
5266 : // which depend on the context provide their own context. The context
5267 : // here is only needed if the target is a constructor to throw a
5268 : // TypeError, if the target is a native function, or if the target is a
5269 : // callable JSObject, which can only be constructed by the runtime.
5270 1672 : args[pos++] = native_context;
5271 3344 : args[pos++] = Effect();
5272 3344 : args[pos++] = Control();
5273 :
5274 : call = graph()->NewNode(mcgraph()->common()->Call(call_descriptor), pos,
5275 3344 : args);
5276 1672 : break;
5277 : }
5278 : default:
5279 0 : UNREACHABLE();
5280 : }
5281 : DCHECK_NOT_NULL(call);
5282 :
5283 : SetEffect(call);
5284 6266 : SetSourcePosition(call, 0);
5285 :
5286 : // Convert the return value back.
5287 6266 : Node* val = sig_->return_count() == 0
5288 : ? mcgraph()->Int32Constant(0)
5289 12532 : : FromJS(call, native_context, sig_->GetReturn());
5290 :
5291 : // Set the ThreadInWasm flag again.
5292 6266 : BuildModifyThreadInWasmFlag(true);
5293 :
5294 : Return(val);
5295 6266 : return true;
5296 : }
5297 :
5298 4888 : void BuildWasmInterpreterEntry(uint32_t func_index) {
5299 16180 : int param_count = static_cast<int>(sig_->parameter_count());
5300 :
5301 : // Build the start and the parameter nodes.
5302 43436 : SetEffect(SetControl(Start(param_count + 3)));
5303 :
5304 : // Create the instance_node from the passed parameter.
5305 2444 : instance_node_.set(Param(wasm::kWasmInstanceParameterIndex));
5306 :
5307 : // Compute size for the argument buffer.
5308 2444 : int args_size_bytes = 0;
5309 13952 : for (wasm::ValueType type : sig_->parameters()) {
5310 9064 : args_size_bytes += wasm::ValueTypes::ElementSizeInBytes(type);
5311 : }
5312 :
5313 : // The return value is also passed via this buffer:
5314 : DCHECK_GE(wasm::kV8MaxWasmFunctionReturns, sig_->return_count());
5315 : // TODO(wasm): Handle multi-value returns.
5316 : DCHECK_EQ(1, wasm::kV8MaxWasmFunctionReturns);
5317 : int return_size_bytes =
5318 : sig_->return_count() == 0
5319 : ? 0
5320 4672 : : wasm::ValueTypes::ElementSizeInBytes(sig_->GetReturn());
5321 :
5322 : // Get a stack slot for the arguments.
5323 : Node* arg_buffer =
5324 2444 : args_size_bytes == 0 && return_size_bytes == 0
5325 : ? mcgraph()->IntPtrConstant(0)
5326 : : graph()->NewNode(mcgraph()->machine()->StackSlot(
5327 7220 : std::max(args_size_bytes, return_size_bytes), 8));
5328 :
5329 : // Now store all our arguments to the buffer.
5330 : int offset = 0;
5331 :
5332 13952 : for (int i = 0; i < param_count; ++i) {
5333 9064 : wasm::ValueType type = sig_->GetParam(i);
5334 : // Start from the parameter with index 1 to drop the instance_node.
5335 : SetEffect(graph()->NewNode(GetSafeStoreOperator(offset, type), arg_buffer,
5336 9064 : Int32Constant(offset), Param(i + 1), Effect(),
5337 18128 : Control()));
5338 9064 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5339 : }
5340 : DCHECK_EQ(args_size_bytes, offset);
5341 :
5342 : // We are passing the raw arg_buffer here. To the GC and other parts, it
5343 : // looks like a Smi (lowest bit not set). In the runtime function however,
5344 : // don't call Smi::value on it, but just cast it to a byte pointer.
5345 4888 : Node* parameters[] = {jsgraph()->SmiConstant(func_index), arg_buffer};
5346 : BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters,
5347 2444 : arraysize(parameters));
5348 :
5349 : // Read back the return value.
5350 4888 : if (sig_->return_count() == 0) {
5351 : Return(Int32Constant(0));
5352 : } else {
5353 : // TODO(wasm): Implement multi-return.
5354 : DCHECK_EQ(1, sig_->return_count());
5355 : MachineType load_rep =
5356 2228 : wasm::ValueTypes::MachineTypeFor(sig_->GetReturn());
5357 : Node* val = SetEffect(
5358 : graph()->NewNode(mcgraph()->machine()->Load(load_rep), arg_buffer,
5359 2228 : Int32Constant(0), Effect(), Control()));
5360 : Return(val);
5361 : }
5362 :
5363 4888 : if (ContainsInt64(sig_)) LowerInt64();
5364 2444 : }
5365 :
5366 2712 : void BuildCWasmEntry() {
5367 : // Build the start and the JS parameter nodes.
5368 30092 : SetEffect(SetControl(Start(CWasmEntryParameters::kNumParameters + 5)));
5369 :
5370 : // Create parameter nodes (offset by 1 for the receiver parameter).
5371 1356 : Node* code_entry = Param(CWasmEntryParameters::kCodeEntry + 1);
5372 1356 : Node* object_ref_node = Param(CWasmEntryParameters::kObjectRef + 1);
5373 1356 : Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1);
5374 :
5375 4016 : int wasm_arg_count = static_cast<int>(sig_->parameter_count());
5376 : int arg_count =
5377 1356 : wasm_arg_count + 4; // code, object_ref_node, control, effect
5378 1356 : Node** args = Buffer(arg_count);
5379 :
5380 : int pos = 0;
5381 1356 : args[pos++] = code_entry;
5382 1356 : args[pos++] = object_ref_node;
5383 :
5384 : int offset = 0;
5385 7388 : for (wasm::ValueType type : sig_->parameters()) {
5386 : Node* arg_load = SetEffect(
5387 : graph()->NewNode(GetSafeLoadOperator(offset, type), arg_buffer,
5388 4676 : Int32Constant(offset), Effect(), Control()));
5389 4676 : args[pos++] = arg_load;
5390 4676 : offset += wasm::ValueTypes::ElementSizeInBytes(type);
5391 : }
5392 :
5393 2712 : args[pos++] = Effect();
5394 2712 : args[pos++] = Control();
5395 : DCHECK_EQ(arg_count, pos);
5396 :
5397 : // Call the wasm code.
5398 2712 : auto call_descriptor = GetWasmCallDescriptor(mcgraph()->zone(), sig_);
5399 :
5400 : Node* call = SetEffect(graph()->NewNode(
5401 2712 : mcgraph()->common()->Call(call_descriptor), arg_count, args));
5402 :
5403 : // Store the return value.
5404 : DCHECK_GE(1, sig_->return_count());
5405 2712 : if (sig_->return_count() == 1) {
5406 : StoreRepresentation store_rep(
5407 : wasm::ValueTypes::MachineRepresentationFor(sig_->GetReturn()),
5408 1304 : kNoWriteBarrier);
5409 : SetEffect(graph()->NewNode(mcgraph()->machine()->Store(store_rep),
5410 : arg_buffer, Int32Constant(0), call, Effect(),
5411 1304 : Control()));
5412 : }
5413 : Return(jsgraph()->SmiConstant(0));
5414 :
5415 2712 : if (mcgraph()->machine()->Is32() && ContainsInt64(sig_)) {
5416 : MachineRepresentation sig_reps[] = {
5417 : MachineRepresentation::kWord32, // return value
5418 : MachineRepresentation::kTagged, // receiver
5419 : MachineRepresentation::kTagged, // arg0 (code)
5420 : MachineRepresentation::kTagged // arg1 (buffer)
5421 0 : };
5422 : Signature<MachineRepresentation> c_entry_sig(1, 2, sig_reps);
5423 : Int64Lowering r(mcgraph()->graph(), mcgraph()->machine(),
5424 0 : mcgraph()->common(), mcgraph()->zone(), &c_entry_sig);
5425 0 : r.LowerGraph();
5426 : }
5427 1356 : }
5428 :
5429 : JSGraph* jsgraph() { return jsgraph_; }
5430 :
5431 : private:
5432 : Isolate* const isolate_;
5433 : JSGraph* jsgraph_;
5434 : StubCallMode stub_mode_;
5435 : SetOncePointer<const Operator> allocate_heap_number_operator_;
5436 : wasm::WasmFeatures enabled_features_;
5437 : };
5438 :
5439 140464 : void AppendSignature(char* buffer, size_t max_name_len,
5440 140464 : wasm::FunctionSig* sig) {
5441 140464 : size_t name_len = strlen(buffer);
5442 : auto append_name_char = [&](char c) {
5443 249009 : if (name_len + 1 < max_name_len) buffer[name_len++] = c;
5444 : };
5445 226169 : for (wasm::ValueType t : sig->parameters()) {
5446 : append_name_char(wasm::ValueTypes::ShortNameOf(t));
5447 : }
5448 : append_name_char(':');
5449 163304 : for (wasm::ValueType t : sig->returns()) {
5450 : append_name_char(wasm::ValueTypes::ShortNameOf(t));
5451 : }
5452 140464 : buffer[name_len] = '\0';
5453 140464 : }
5454 :
5455 : } // namespace
5456 :
5457 278216 : MaybeHandle<Code> CompileJSToWasmWrapper(Isolate* isolate,
5458 139108 : wasm::FunctionSig* sig,
5459 : bool is_import) {
5460 278216 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5461 : "CompileJSToWasmWrapper");
5462 : //----------------------------------------------------------------------------
5463 : // Create the Graph.
5464 : //----------------------------------------------------------------------------
5465 278215 : Zone zone(isolate->allocator(), ZONE_NAME);
5466 139108 : Graph graph(&zone);
5467 139108 : CommonOperatorBuilder common(&zone);
5468 : MachineOperatorBuilder machine(
5469 : &zone, MachineType::PointerRepresentation(),
5470 : InstructionSelector::SupportedMachineOperatorFlags(),
5471 139108 : InstructionSelector::AlignmentRequirements());
5472 139108 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5473 :
5474 139108 : Node* control = nullptr;
5475 139108 : Node* effect = nullptr;
5476 :
5477 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5478 : StubCallMode::kCallCodeObject,
5479 139108 : wasm::WasmFeaturesFromIsolate(isolate));
5480 : builder.set_control_ptr(&control);
5481 : builder.set_effect_ptr(&effect);
5482 139108 : builder.BuildJSToWasmWrapper(is_import);
5483 :
5484 : //----------------------------------------------------------------------------
5485 : // Run the compilation pipeline.
5486 : //----------------------------------------------------------------------------
5487 : static constexpr size_t kMaxNameLen = 128;
5488 139107 : char debug_name[kMaxNameLen] = "js_to_wasm:";
5489 139107 : AppendSignature(debug_name, kMaxNameLen, sig);
5490 :
5491 : // Schedule and compile to machine code.
5492 139108 : int params = static_cast<int>(sig->parameter_count());
5493 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5494 139108 : &zone, false, params + 1, CallDescriptor::kNoFlags);
5495 :
5496 : MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5497 : isolate, incoming, &graph, Code::JS_TO_WASM_FUNCTION, debug_name,
5498 139108 : WasmAssemblerOptions());
5499 : Handle<Code> code;
5500 139106 : if (!maybe_code.ToHandle(&code)) {
5501 0 : return maybe_code;
5502 : }
5503 : #ifdef ENABLE_DISASSEMBLER
5504 : if (FLAG_print_opt_code) {
5505 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5506 : OFStream os(tracing_scope.file());
5507 : code->Disassemble(debug_name, os);
5508 : }
5509 : #endif
5510 :
5511 139108 : if (must_record_function_compilation(isolate)) {
5512 : RecordFunctionCompilation(CodeEventListener::STUB_TAG, isolate, code, "%s",
5513 0 : debug_name);
5514 : }
5515 :
5516 278214 : return code;
5517 : }
5518 :
5519 119577 : WasmImportCallKind GetWasmImportCallKind(Handle<JSReceiver> target,
5520 10229 : wasm::FunctionSig* expected_sig,
5521 : bool has_bigint_feature) {
5522 119577 : if (WasmExportedFunction::IsWasmExportedFunction(*target)) {
5523 107288 : auto imported_function = WasmExportedFunction::cast(*target);
5524 : wasm::FunctionSig* imported_sig =
5525 : imported_function->instance()
5526 214576 : ->module()
5527 214576 : ->functions[imported_function->function_index()]
5528 107288 : .sig;
5529 107288 : if (*imported_sig != *expected_sig) {
5530 : return WasmImportCallKind::kLinkError;
5531 : }
5532 107128 : return WasmImportCallKind::kWasmToWasm;
5533 : }
5534 : // Assuming we are calling to JS, check whether this would be a runtime error.
5535 12289 : if (!wasm::IsJSCompatibleSignature(expected_sig, has_bigint_feature)) {
5536 : return WasmImportCallKind::kRuntimeTypeError;
5537 : }
5538 : // For JavaScript calls, determine whether the target has an arity match
5539 : // and whether it has a sloppy receiver.
5540 24426 : if (target->IsJSFunction()) {
5541 10469 : Handle<JSFunction> function = Handle<JSFunction>::cast(target);
5542 10469 : SharedFunctionInfo shared = function->shared();
5543 :
5544 : // Check for math intrinsics.
5545 : #define COMPARE_SIG_FOR_BUILTIN(name) \
5546 : { \
5547 : wasm::FunctionSig* sig = wasm::WasmOpcodes::Signature(wasm::kExpr##name); \
5548 : if (!sig) sig = wasm::WasmOpcodes::AsmjsSignature(wasm::kExpr##name); \
5549 : DCHECK_NOT_NULL(sig); \
5550 : if (*expected_sig == *sig) return WasmImportCallKind::k##name; \
5551 : }
5552 : #define COMPARE_SIG_FOR_BUILTIN_F64(name) \
5553 : case Builtins::kMath##name: \
5554 : COMPARE_SIG_FOR_BUILTIN(F64##name); \
5555 : break;
5556 : #define COMPARE_SIG_FOR_BUILTIN_F32_F64(name) \
5557 : case Builtins::kMath##name: \
5558 : COMPARE_SIG_FOR_BUILTIN(F64##name); \
5559 : COMPARE_SIG_FOR_BUILTIN(F32##name); \
5560 : break;
5561 :
5562 10469 : if (FLAG_wasm_math_intrinsics && shared->HasBuiltinId()) {
5563 504 : switch (shared->builtin_id()) {
5564 18 : COMPARE_SIG_FOR_BUILTIN_F64(Acos);
5565 18 : COMPARE_SIG_FOR_BUILTIN_F64(Asin);
5566 17 : COMPARE_SIG_FOR_BUILTIN_F64(Atan);
5567 27 : COMPARE_SIG_FOR_BUILTIN_F64(Cos);
5568 27 : COMPARE_SIG_FOR_BUILTIN_F64(Sin);
5569 17 : COMPARE_SIG_FOR_BUILTIN_F64(Tan);
5570 17 : COMPARE_SIG_FOR_BUILTIN_F64(Exp);
5571 18 : COMPARE_SIG_FOR_BUILTIN_F64(Log);
5572 18 : COMPARE_SIG_FOR_BUILTIN_F64(Atan2);
5573 : //===========================================================
5574 : // TODO(8505): Math.pow for wasm does not match JS.
5575 : // COMPARE_SIG_FOR_BUILTIN_F64(Pow);
5576 : //===========================================================
5577 106 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Min);
5578 24 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Max);
5579 34 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Abs);
5580 34 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Ceil);
5581 43 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Floor);
5582 52 : COMPARE_SIG_FOR_BUILTIN_F32_F64(Sqrt);
5583 : case Builtins::kMathFround:
5584 8 : COMPARE_SIG_FOR_BUILTIN(F32ConvertF64);
5585 : break;
5586 : default:
5587 : break;
5588 : }
5589 : }
5590 :
5591 : #undef COMPARE_SIG_FOR_BUILTIN
5592 : #undef COMPARE_SIG_FOR_BUILTIN_F64
5593 : #undef COMPARE_SIG_FOR_BUILTIN_F32_F64
5594 :
5595 10245 : if (IsClassConstructor(shared->kind())) {
5596 : // Class constructor will throw anyway.
5597 : return WasmImportCallKind::kUseCallBuiltin;
5598 : }
5599 20146 : bool sloppy = is_sloppy(shared->language_mode()) && !shared->native();
5600 20458 : if (shared->internal_formal_parameter_count() ==
5601 : expected_sig->parameter_count()) {
5602 : return sloppy ? WasmImportCallKind::kJSFunctionArityMatchSloppy
5603 7651 : : WasmImportCallKind::kJSFunctionArityMatch;
5604 : }
5605 : return sloppy ? WasmImportCallKind::kJSFunctionArityMismatchSloppy
5606 2578 : : WasmImportCallKind::kJSFunctionArityMismatch;
5607 : }
5608 : // Unknown case. Use the call builtin.
5609 : return WasmImportCallKind::kUseCallBuiltin;
5610 : }
5611 :
5612 224 : wasm::WasmOpcode GetMathIntrinsicOpcode(WasmImportCallKind kind,
5613 : const char** name_ptr) {
5614 : #define CASE(name) \
5615 : case WasmImportCallKind::k##name: \
5616 : *name_ptr = "WasmMathIntrinsic:" #name; \
5617 : return wasm::kExpr##name
5618 224 : switch (kind) {
5619 8 : CASE(F64Acos);
5620 8 : CASE(F64Asin);
5621 8 : CASE(F64Atan);
5622 8 : CASE(F64Cos);
5623 8 : CASE(F64Sin);
5624 8 : CASE(F64Tan);
5625 8 : CASE(F64Exp);
5626 8 : CASE(F64Log);
5627 8 : CASE(F64Atan2);
5628 0 : CASE(F64Pow);
5629 16 : CASE(F64Ceil);
5630 16 : CASE(F64Floor);
5631 16 : CASE(F64Sqrt);
5632 16 : CASE(F64Min);
5633 16 : CASE(F64Max);
5634 16 : CASE(F64Abs);
5635 8 : CASE(F32Min);
5636 8 : CASE(F32Max);
5637 8 : CASE(F32Abs);
5638 8 : CASE(F32Ceil);
5639 8 : CASE(F32Floor);
5640 8 : CASE(F32Sqrt);
5641 8 : CASE(F32ConvertF64);
5642 : default:
5643 0 : UNREACHABLE();
5644 : return wasm::kExprUnreachable;
5645 : }
5646 : #undef CASE
5647 : }
5648 :
5649 224 : wasm::WasmCode* CompileWasmMathIntrinsic(wasm::WasmEngine* wasm_engine,
5650 : wasm::NativeModule* native_module,
5651 : WasmImportCallKind kind,
5652 448 : wasm::FunctionSig* sig) {
5653 : DCHECK_EQ(1, sig->return_count());
5654 :
5655 448 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5656 : "CompileWasmMathIntrinsic");
5657 :
5658 448 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5659 :
5660 : // Compile a WASM function with a single bytecode and let TurboFan
5661 : // generate either inlined machine code or a call to a helper.
5662 : SourcePositionTable* source_positions = nullptr;
5663 224 : MachineGraph* mcgraph = new (&zone) MachineGraph(
5664 448 : new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5665 : new (&zone) MachineOperatorBuilder(
5666 : &zone, MachineType::PointerRepresentation(),
5667 : InstructionSelector::SupportedMachineOperatorFlags(),
5668 672 : InstructionSelector::AlignmentRequirements()));
5669 :
5670 : wasm::CompilationEnv env(
5671 : native_module->module(), wasm::UseTrapHandler::kNoTrapHandler,
5672 : wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport,
5673 : wasm::kAllWasmFeatures, wasm::LowerSimd::kNoLowerSimd);
5674 :
5675 : WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig,
5676 224 : source_positions);
5677 :
5678 : // Set up the graph start.
5679 224 : Node* start = builder.Start(static_cast<int>(sig->parameter_count() + 1 + 1));
5680 224 : Node* effect = start;
5681 224 : Node* control = start;
5682 : builder.set_effect_ptr(&effect);
5683 : builder.set_control_ptr(&control);
5684 224 : builder.set_instance_node(builder.Param(wasm::kWasmInstanceParameterIndex));
5685 :
5686 : // Generate either a unop or a binop.
5687 : Node* result = nullptr;
5688 224 : const char* debug_name = "WasmMathIntrinsic";
5689 224 : auto opcode = GetMathIntrinsicOpcode(kind, &debug_name);
5690 224 : switch (sig->parameter_count()) {
5691 : case 1:
5692 168 : result = builder.Unop(opcode, builder.Param(1));
5693 168 : break;
5694 : case 2:
5695 56 : result = builder.Binop(opcode, builder.Param(1), builder.Param(2));
5696 56 : break;
5697 : default:
5698 0 : UNREACHABLE();
5699 : break;
5700 : }
5701 :
5702 : builder.Return(result);
5703 :
5704 : // Run the compiler pipeline to generate machine code.
5705 224 : auto call_descriptor = GetWasmCallDescriptor(&zone, sig);
5706 224 : if (mcgraph->machine()->Is32()) {
5707 : call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5708 : }
5709 :
5710 : wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5711 : wasm_engine, call_descriptor, mcgraph, Code::WASM_FUNCTION,
5712 : wasm::WasmCode::kFunction, debug_name, WasmStubAssemblerOptions(),
5713 224 : native_module, source_positions);
5714 224 : CHECK_NOT_NULL(wasm_code);
5715 : // TODO(titzer): add counters for math intrinsic code size / allocation
5716 :
5717 224 : return wasm_code;
5718 : }
5719 :
5720 6566 : wasm::WasmCode* CompileWasmImportCallWrapper(wasm::WasmEngine* wasm_engine,
5721 : wasm::NativeModule* native_module,
5722 : WasmImportCallKind kind,
5723 : wasm::FunctionSig* sig,
5724 : bool source_positions) {
5725 : DCHECK_NE(WasmImportCallKind::kLinkError, kind);
5726 : DCHECK_NE(WasmImportCallKind::kWasmToWasm, kind);
5727 :
5728 : // Check for math intrinsics first.
5729 6566 : if (FLAG_wasm_math_intrinsics &&
5730 6566 : kind >= WasmImportCallKind::kFirstMathIntrinsic &&
5731 : kind <= WasmImportCallKind::kLastMathIntrinsic) {
5732 224 : return CompileWasmMathIntrinsic(wasm_engine, native_module, kind, sig);
5733 : }
5734 :
5735 12684 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5736 : "CompileWasmImportCallWrapper");
5737 : //----------------------------------------------------------------------------
5738 : // Create the Graph
5739 : //----------------------------------------------------------------------------
5740 12684 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5741 6342 : Graph graph(&zone);
5742 6342 : CommonOperatorBuilder common(&zone);
5743 : MachineOperatorBuilder machine(
5744 : &zone, MachineType::PointerRepresentation(),
5745 : InstructionSelector::SupportedMachineOperatorFlags(),
5746 6342 : InstructionSelector::AlignmentRequirements());
5747 6342 : JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
5748 :
5749 6342 : Node* control = nullptr;
5750 6342 : Node* effect = nullptr;
5751 :
5752 : SourcePositionTable* source_position_table =
5753 6832 : source_positions ? new (&zone) SourcePositionTable(&graph) : nullptr;
5754 :
5755 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, source_position_table,
5756 : StubCallMode::kCallWasmRuntimeStub,
5757 : native_module->enabled_features());
5758 : builder.set_control_ptr(&control);
5759 : builder.set_effect_ptr(&effect);
5760 6342 : builder.BuildWasmImportCallWrapper(kind);
5761 :
5762 : const char* func_name = "wasm-to-js";
5763 :
5764 : // Schedule and compile to machine code.
5765 : CallDescriptor* incoming =
5766 : GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline,
5767 6342 : WasmGraphBuilder::kExtraCallableParam);
5768 6342 : if (machine.Is32()) {
5769 : incoming = GetI32WasmCallDescriptor(&zone, incoming);
5770 : }
5771 : wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5772 : wasm_engine, incoming, &jsgraph, Code::WASM_TO_JS_FUNCTION,
5773 : wasm::WasmCode::kWasmToJsWrapper, func_name, WasmStubAssemblerOptions(),
5774 6342 : native_module, source_position_table);
5775 6342 : CHECK_NOT_NULL(wasm_code);
5776 :
5777 6342 : return wasm_code;
5778 : }
5779 :
5780 2444 : wasm::WasmCode* CompileWasmInterpreterEntry(wasm::WasmEngine* wasm_engine,
5781 : wasm::NativeModule* native_module,
5782 : uint32_t func_index,
5783 : wasm::FunctionSig* sig) {
5784 : //----------------------------------------------------------------------------
5785 : // Create the Graph
5786 : //----------------------------------------------------------------------------
5787 2444 : Zone zone(wasm_engine->allocator(), ZONE_NAME);
5788 2444 : Graph graph(&zone);
5789 2444 : CommonOperatorBuilder common(&zone);
5790 : MachineOperatorBuilder machine(
5791 : &zone, MachineType::PointerRepresentation(),
5792 : InstructionSelector::SupportedMachineOperatorFlags(),
5793 2444 : InstructionSelector::AlignmentRequirements());
5794 2444 : JSGraph jsgraph(nullptr, &graph, &common, nullptr, nullptr, &machine);
5795 :
5796 2444 : Node* control = nullptr;
5797 2444 : Node* effect = nullptr;
5798 :
5799 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5800 : StubCallMode::kCallWasmRuntimeStub,
5801 : native_module->enabled_features());
5802 : builder.set_control_ptr(&control);
5803 : builder.set_effect_ptr(&effect);
5804 2444 : builder.BuildWasmInterpreterEntry(func_index);
5805 :
5806 : // Schedule and compile to machine code.
5807 2444 : CallDescriptor* incoming = GetWasmCallDescriptor(&zone, sig);
5808 2444 : if (machine.Is32()) {
5809 : incoming = GetI32WasmCallDescriptor(&zone, incoming);
5810 : }
5811 :
5812 : EmbeddedVector<char, 32> func_name;
5813 : func_name.Truncate(
5814 2444 : SNPrintF(func_name, "wasm-interpreter-entry#%d", func_index));
5815 :
5816 : wasm::WasmCode* wasm_code = Pipeline::GenerateCodeForWasmNativeStub(
5817 : wasm_engine, incoming, &jsgraph, Code::WASM_INTERPRETER_ENTRY,
5818 2444 : wasm::WasmCode::kInterpreterEntry, func_name.start(),
5819 2444 : WasmStubAssemblerOptions(), native_module);
5820 2444 : CHECK_NOT_NULL(wasm_code);
5821 :
5822 2444 : return wasm_code;
5823 : }
5824 :
5825 1356 : MaybeHandle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig) {
5826 1356 : Zone zone(isolate->allocator(), ZONE_NAME);
5827 1356 : Graph graph(&zone);
5828 1356 : CommonOperatorBuilder common(&zone);
5829 : MachineOperatorBuilder machine(
5830 : &zone, MachineType::PointerRepresentation(),
5831 : InstructionSelector::SupportedMachineOperatorFlags(),
5832 1356 : InstructionSelector::AlignmentRequirements());
5833 1356 : JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine);
5834 :
5835 1356 : Node* control = nullptr;
5836 1356 : Node* effect = nullptr;
5837 :
5838 : WasmWrapperGraphBuilder builder(&zone, &jsgraph, sig, nullptr,
5839 : StubCallMode::kCallCodeObject,
5840 1356 : wasm::WasmFeaturesFromIsolate(isolate));
5841 : builder.set_control_ptr(&control);
5842 : builder.set_effect_ptr(&effect);
5843 1356 : builder.BuildCWasmEntry();
5844 :
5845 : // Schedule and compile to machine code.
5846 : CallDescriptor* incoming = Linkage::GetJSCallDescriptor(
5847 : &zone, false, CWasmEntryParameters::kNumParameters + 1,
5848 1356 : CallDescriptor::kNoFlags);
5849 :
5850 : // Build a name in the form "c-wasm-entry:<params>:<returns>".
5851 : static constexpr size_t kMaxNameLen = 128;
5852 1356 : char debug_name[kMaxNameLen] = "c-wasm-entry:";
5853 1356 : AppendSignature(debug_name, kMaxNameLen, sig);
5854 :
5855 : MaybeHandle<Code> maybe_code = Pipeline::GenerateCodeForWasmHeapStub(
5856 : isolate, incoming, &graph, Code::C_WASM_ENTRY, debug_name,
5857 1356 : AssemblerOptions::Default(isolate));
5858 : Handle<Code> code;
5859 1356 : if (!maybe_code.ToHandle(&code)) {
5860 0 : return maybe_code;
5861 : }
5862 : #ifdef ENABLE_DISASSEMBLER
5863 : if (FLAG_print_opt_code) {
5864 : CodeTracer::Scope tracing_scope(isolate->GetCodeTracer());
5865 : OFStream os(tracing_scope.file());
5866 : code->Disassemble(debug_name, os);
5867 : }
5868 : #endif
5869 :
5870 1356 : return code;
5871 : }
5872 :
5873 504196 : TurbofanWasmCompilationUnit::TurbofanWasmCompilationUnit(
5874 : wasm::WasmCompilationUnit* wasm_unit)
5875 504196 : : wasm_unit_(wasm_unit) {}
5876 :
5877 : // Clears unique_ptrs, but (part of) the type is forward declared in the header.
5878 : TurbofanWasmCompilationUnit::~TurbofanWasmCompilationUnit() = default;
5879 :
5880 503096 : bool TurbofanWasmCompilationUnit::BuildGraphForWasmFunction(
5881 : wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
5882 : wasm::WasmFeatures* detected, double* decode_ms, MachineGraph* mcgraph,
5883 : NodeOriginTable* node_origins, SourcePositionTable* source_positions,
5884 : wasm::WasmError* error_out) {
5885 : base::ElapsedTimer decode_timer;
5886 : if (FLAG_trace_wasm_decode_time) {
5887 : decode_timer.Start();
5888 : }
5889 :
5890 : // Create a TF graph during decoding.
5891 : WasmGraphBuilder builder(env, mcgraph->zone(), mcgraph, func_body.sig,
5892 1006192 : source_positions);
5893 : wasm::VoidResult graph_construction_result = wasm::BuildTFGraph(
5894 : wasm_unit_->wasm_engine_->allocator(), env->enabled_features, env->module,
5895 1006370 : &builder, detected, func_body, node_origins);
5896 503220 : if (graph_construction_result.failed()) {
5897 : if (FLAG_trace_wasm_compiler) {
5898 : StdoutStream{} << "Compilation failed: "
5899 : << graph_construction_result.error().message()
5900 : << std::endl;
5901 : }
5902 : *error_out = graph_construction_result.error();
5903 6688 : return false;
5904 : }
5905 :
5906 496532 : builder.LowerInt64();
5907 :
5908 1001160 : if (builder.has_simd() &&
5909 8204 : (!CpuFeatures::SupportsWasmSimd128() || env->lower_simd)) {
5910 : SimdScalarLowering(mcgraph,
5911 : CreateMachineSignature(mcgraph->zone(), func_body.sig))
5912 12204 : .LowerGraph();
5913 : }
5914 :
5915 992970 : if (wasm_unit_->func_index_ >= FLAG_trace_wasm_ast_start &&
5916 496492 : wasm_unit_->func_index_ < FLAG_trace_wasm_ast_end) {
5917 : PrintRawWasmCode(wasm_unit_->wasm_engine_->allocator(), func_body,
5918 0 : env->module, wasm::kPrintLocals);
5919 : }
5920 : if (FLAG_trace_wasm_decode_time) {
5921 : *decode_ms = decode_timer.Elapsed().InMillisecondsF();
5922 : }
5923 : return true;
5924 : }
5925 :
5926 : namespace {
5927 503145 : Vector<const char> GetDebugName(Zone* zone, int index) {
5928 : // TODO(herhut): Use name from module if available.
5929 : constexpr int kBufferLength = 24;
5930 :
5931 : EmbeddedVector<char, kBufferLength> name_vector;
5932 503145 : int name_len = SNPrintF(name_vector, "wasm-function#%d", index);
5933 : DCHECK(name_len > 0 && name_len < name_vector.length());
5934 :
5935 503185 : char* index_name = zone->NewArray<char>(name_len);
5936 503198 : memcpy(index_name, name_vector.start(), name_len);
5937 503198 : return Vector<const char>(index_name, name_len);
5938 : }
5939 : } // namespace
5940 :
5941 503211 : wasm::WasmCompilationResult TurbofanWasmCompilationUnit::ExecuteCompilation(
5942 : wasm::CompilationEnv* env, const wasm::FunctionBody& func_body,
5943 : Counters* counters, wasm::WasmFeatures* detected) {
5944 1006423 : TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.wasm"),
5945 : "ExecuteTurbofanCompilation");
5946 503212 : double decode_ms = 0;
5947 : size_t node_count = 0;
5948 :
5949 1006178 : Zone zone(wasm_unit_->wasm_engine_->allocator(), ZONE_NAME);
5950 1496025 : MachineGraph* mcgraph = new (&zone) MachineGraph(
5951 1006452 : new (&zone) Graph(&zone), new (&zone) CommonOperatorBuilder(&zone),
5952 : new (&zone) MachineOperatorBuilder(
5953 : &zone, MachineType::PointerRepresentation(),
5954 : InstructionSelector::SupportedMachineOperatorFlags(),
5955 1509599 : InstructionSelector::AlignmentRequirements()));
5956 :
5957 : OptimizedCompilationInfo info(GetDebugName(&zone, wasm_unit_->func_index_),
5958 1006397 : &zone, Code::WASM_FUNCTION);
5959 503167 : if (env->runtime_exception_support) {
5960 : info.SetWasmRuntimeExceptionSupport();
5961 : }
5962 :
5963 503167 : if (info.trace_turbo_json_enabled()) {
5964 0 : TurboCfgFile tcf;
5965 0 : tcf << AsC1VCompilation(&info);
5966 : }
5967 :
5968 : NodeOriginTable* node_origins = info.trace_turbo_json_enabled()
5969 : ? new (&zone)
5970 0 : NodeOriginTable(mcgraph->graph())
5971 503167 : : nullptr;
5972 : SourcePositionTable* source_positions =
5973 503163 : new (mcgraph->zone()) SourcePositionTable(mcgraph->graph());
5974 : wasm::WasmError error;
5975 503196 : if (!BuildGraphForWasmFunction(env, func_body, detected, &decode_ms, mcgraph,
5976 503095 : node_origins, source_positions, &error)) {
5977 : DCHECK(!error.empty());
5978 13376 : return wasm::WasmCompilationResult{std::move(error)};
5979 : }
5980 :
5981 496508 : if (node_origins) {
5982 0 : node_origins->AddDecorator();
5983 : }
5984 :
5985 : base::ElapsedTimer pipeline_timer;
5986 : if (FLAG_trace_wasm_decode_time) {
5987 : node_count = mcgraph->graph()->NodeCount();
5988 : pipeline_timer.Start();
5989 : }
5990 :
5991 : // Run the compiler pipeline to generate machine code.
5992 496508 : auto call_descriptor = GetWasmCallDescriptor(&zone, func_body.sig);
5993 496441 : if (mcgraph->machine()->Is32()) {
5994 : call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor);
5995 : }
5996 :
5997 : Pipeline::GenerateCodeForWasmFunction(
5998 : &info, wasm_unit_->wasm_engine_, mcgraph, call_descriptor,
5999 : source_positions, node_origins, func_body, env->module,
6000 496441 : wasm_unit_->func_index_);
6001 :
6002 : if (FLAG_trace_wasm_decode_time) {
6003 : double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF();
6004 : PrintF(
6005 : "wasm-compilation phase 1 ok: %u bytes, %0.3f ms decode, %zu nodes, "
6006 : "%0.3f ms pipeline\n",
6007 : static_cast<unsigned>(func_body.end - func_body.start), decode_ms,
6008 : node_count, pipeline_ms);
6009 : }
6010 : // TODO(bradnelson): Improve histogram handling of size_t.
6011 : counters->wasm_compile_function_peak_memory_bytes()->AddSample(
6012 992834 : static_cast<int>(mcgraph->graph()->zone()->allocation_size()));
6013 1992691 : return std::move(*info.ReleaseWasmCompilationResult());
6014 : }
6015 :
6016 : namespace {
6017 : // Helper for allocating either an GP or FP reg, or the next stack slot.
6018 : class LinkageLocationAllocator {
6019 : public:
6020 : template <size_t kNumGpRegs, size_t kNumFpRegs>
6021 : constexpr LinkageLocationAllocator(const Register (&gp)[kNumGpRegs],
6022 : const DoubleRegister (&fp)[kNumFpRegs])
6023 : : allocator_(wasm::LinkageAllocator(gp, fp)) {}
6024 :
6025 4828551 : LinkageLocation Next(MachineRepresentation rep) {
6026 4828551 : MachineType type = MachineType::TypeForRepresentation(rep);
6027 4828601 : if (IsFloatingPoint(rep)) {
6028 9081147 : if (allocator_.CanAllocateFP(rep)) {
6029 : int reg_code = allocator_.NextFpReg(rep);
6030 : return LinkageLocation::ForRegister(reg_code, type);
6031 : }
6032 4317350 : } else if (allocator_.CanAllocateGP()) {
6033 : int reg_code = allocator_.NextGpReg();
6034 : return LinkageLocation::ForRegister(reg_code, type);
6035 : }
6036 : // Cannot use register; use stack slot.
6037 154699 : int index = -1 - allocator_.NextStackSlot(rep);
6038 : return LinkageLocation::ForCallerFrameSlot(index, type);
6039 : }
6040 :
6041 : void SetStackOffset(int offset) { allocator_.SetStackOffset(offset); }
6042 4251874 : int NumStackSlots() const { return allocator_.NumStackSlots(); }
6043 :
6044 : private:
6045 : wasm::LinkageAllocator allocator_;
6046 : };
6047 : } // namespace
6048 :
6049 : // General code uses the above configuration data.
6050 2125885 : CallDescriptor* GetWasmCallDescriptor(
6051 9938397 : Zone* zone, wasm::FunctionSig* fsig,
6052 : WasmGraphBuilder::UseRetpoline use_retpoline,
6053 : WasmGraphBuilder::ExtraCallableParam extra_callable_param) {
6054 : // The extra here is to accomodate the instance object as first parameter
6055 : // and, in the case of an import wrapper, the additional callable.
6056 2125885 : int extra_params = extra_callable_param ? 2 : 1;
6057 : LocationSignature::Builder locations(zone, fsig->return_count(),
6058 2125885 : fsig->parameter_count() + extra_params);
6059 :
6060 : // Add register and/or stack parameter(s).
6061 : LinkageLocationAllocator params(wasm::kGpParamRegisters,
6062 2125956 : wasm::kFpParamRegisters);
6063 :
6064 : // The instance object.
6065 2125956 : locations.AddParam(params.Next(MachineRepresentation::kTaggedPointer));
6066 : const size_t param_offset = 1; // Actual params start here.
6067 :
6068 : // Parameters are separated into two groups (first all untagged, then all
6069 : // tagged parameters). This allows for easy iteration of tagged parameters
6070 : // during frame iteration.
6071 : const size_t parameter_count = fsig->parameter_count();
6072 2983888 : for (size_t i = 0; i < parameter_count; i++) {
6073 : MachineRepresentation param =
6074 857972 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
6075 : // Skip tagged parameters (e.g. any-ref).
6076 864562 : if (IsAnyTagged(param)) continue;
6077 851314 : auto l = params.Next(param);
6078 851363 : locations.AddParamAt(i + param_offset, l);
6079 : }
6080 858012 : for (size_t i = 0; i < parameter_count; i++) {
6081 : MachineRepresentation param =
6082 857980 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetParam(i));
6083 : // Skip untagged parameters.
6084 1709376 : if (!IsAnyTagged(param)) continue;
6085 6624 : auto l = params.Next(param);
6086 6636 : locations.AddParamAt(i + param_offset, l);
6087 : }
6088 :
6089 : // Import call wrappers have an additional (implicit) parameter, the callable.
6090 : // For consistency with JS, we use the JSFunction register.
6091 2125948 : if (extra_callable_param) {
6092 : locations.AddParam(LinkageLocation::ForRegister(
6093 : kJSFunctionRegister.code(), MachineType::TaggedPointer()));
6094 : }
6095 :
6096 : // Add return location(s).
6097 : LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
6098 2125948 : wasm::kFpReturnRegisters);
6099 :
6100 : int parameter_slots = params.NumStackSlots();
6101 : if (kPadArguments) parameter_slots = RoundUp(parameter_slots, 2);
6102 :
6103 : rets.SetStackOffset(parameter_slots);
6104 :
6105 2125948 : const int return_count = static_cast<int>(locations.return_count_);
6106 3970727 : for (int i = 0; i < return_count; i++) {
6107 : MachineRepresentation ret =
6108 3689548 : wasm::ValueTypes::MachineRepresentationFor(fsig->GetReturn(i));
6109 1844774 : auto l = rets.Next(ret);
6110 1844779 : locations.AddReturn(l);
6111 : }
6112 :
6113 : const RegList kCalleeSaveRegisters = 0;
6114 : const RegList kCalleeSaveFPRegisters = 0;
6115 :
6116 : // The target for wasm calls is always a code object.
6117 : MachineType target_type = MachineType::Pointer();
6118 : LinkageLocation target_loc = LinkageLocation::ForAnyRegister(target_type);
6119 :
6120 : CallDescriptor::Kind kind = extra_callable_param
6121 : ? CallDescriptor::kCallWasmImportWrapper
6122 2125953 : : CallDescriptor::kCallWasmFunction;
6123 :
6124 : CallDescriptor::Flags flags =
6125 2125953 : use_retpoline ? CallDescriptor::kRetpoline : CallDescriptor::kNoFlags;
6126 : return new (zone) CallDescriptor( // --
6127 : kind, // kind
6128 : target_type, // target MachineType
6129 : target_loc, // target location
6130 : locations.Build(), // location_sig
6131 : parameter_slots, // stack_parameter_count
6132 : compiler::Operator::kNoProperties, // properties
6133 : kCalleeSaveRegisters, // callee-saved registers
6134 : kCalleeSaveFPRegisters, // callee-saved fp regs
6135 : flags, // flags
6136 : "wasm-call", // debug name
6137 : 0, // allocatable registers
6138 8503731 : rets.NumStackSlots() - parameter_slots); // stack_return_count
6139 : }
6140 :
6141 : namespace {
6142 0 : CallDescriptor* ReplaceTypeInCallDescriptorWith(
6143 0 : Zone* zone, CallDescriptor* call_descriptor, size_t num_replacements,
6144 : MachineType input_type, MachineRepresentation output_type) {
6145 : size_t parameter_count = call_descriptor->ParameterCount();
6146 : size_t return_count = call_descriptor->ReturnCount();
6147 0 : for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
6148 0 : if (call_descriptor->GetParameterType(i) == input_type) {
6149 0 : parameter_count += num_replacements - 1;
6150 : }
6151 : }
6152 0 : for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
6153 0 : if (call_descriptor->GetReturnType(i) == input_type) {
6154 0 : return_count += num_replacements - 1;
6155 : }
6156 : }
6157 0 : if (parameter_count == call_descriptor->ParameterCount() &&
6158 : return_count == call_descriptor->ReturnCount()) {
6159 : return call_descriptor;
6160 : }
6161 :
6162 : LocationSignature::Builder locations(zone, return_count, parameter_count);
6163 :
6164 : LinkageLocationAllocator params(wasm::kGpParamRegisters,
6165 0 : wasm::kFpParamRegisters);
6166 0 : for (size_t i = 0; i < call_descriptor->ParameterCount(); i++) {
6167 0 : if (call_descriptor->GetParameterType(i) == input_type) {
6168 0 : for (size_t j = 0; j < num_replacements; j++) {
6169 0 : locations.AddParam(params.Next(output_type));
6170 : }
6171 : } else {
6172 : locations.AddParam(
6173 0 : params.Next(call_descriptor->GetParameterType(i).representation()));
6174 : }
6175 : }
6176 :
6177 : LinkageLocationAllocator rets(wasm::kGpReturnRegisters,
6178 0 : wasm::kFpReturnRegisters);
6179 : rets.SetStackOffset(params.NumStackSlots());
6180 0 : for (size_t i = 0; i < call_descriptor->ReturnCount(); i++) {
6181 0 : if (call_descriptor->GetReturnType(i) == input_type) {
6182 0 : for (size_t j = 0; j < num_replacements; j++) {
6183 0 : locations.AddReturn(rets.Next(output_type));
6184 : }
6185 : } else {
6186 : locations.AddReturn(
6187 0 : rets.Next(call_descriptor->GetReturnType(i).representation()));
6188 : }
6189 : }
6190 :
6191 : return new (zone) CallDescriptor( // --
6192 : call_descriptor->kind(), // kind
6193 : call_descriptor->GetInputType(0), // target MachineType
6194 : call_descriptor->GetInputLocation(0), // target location
6195 : locations.Build(), // location_sig
6196 : params.NumStackSlots(), // stack_parameter_count
6197 : call_descriptor->properties(), // properties
6198 : call_descriptor->CalleeSavedRegisters(), // callee-saved registers
6199 : call_descriptor->CalleeSavedFPRegisters(), // callee-saved fp regs
6200 : call_descriptor->flags(), // flags
6201 : call_descriptor->debug_name(), // debug name
6202 : call_descriptor->AllocatableRegisters(), // allocatable registers
6203 0 : rets.NumStackSlots() - params.NumStackSlots()); // stack_return_count
6204 : }
6205 : } // namespace
6206 :
6207 0 : CallDescriptor* GetI32WasmCallDescriptor(Zone* zone,
6208 : CallDescriptor* call_descriptor) {
6209 : return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 2,
6210 : MachineType::Int64(),
6211 0 : MachineRepresentation::kWord32);
6212 : }
6213 :
6214 0 : CallDescriptor* GetI32WasmCallDescriptorForSimd(
6215 : Zone* zone, CallDescriptor* call_descriptor) {
6216 : return ReplaceTypeInCallDescriptorWith(zone, call_descriptor, 4,
6217 : MachineType::Simd128(),
6218 0 : MachineRepresentation::kWord32);
6219 : }
6220 :
6221 496467 : AssemblerOptions WasmAssemblerOptions() {
6222 635575 : AssemblerOptions options;
6223 : // Relocation info required to serialize {WasmCode} for proper functions.
6224 : options.record_reloc_info_for_serialization = true;
6225 : options.enable_root_array_delta_access = false;
6226 496467 : return options;
6227 : }
6228 :
6229 0 : AssemblerOptions WasmStubAssemblerOptions() {
6230 9010 : AssemblerOptions options;
6231 : // Relocation info not necessary because stubs are not serialized.
6232 9010 : options.record_reloc_info_for_serialization = false;
6233 : options.enable_root_array_delta_access = false;
6234 0 : return options;
6235 : }
6236 :
6237 : #undef WASM_64
6238 : #undef FATAL_UNSUPPORTED_OPCODE
6239 : #undef WASM_INSTANCE_OBJECT_SIZE
6240 : #undef WASM_INSTANCE_OBJECT_OFFSET
6241 : #undef LOAD_RAW
6242 : #undef LOAD_INSTANCE_FIELD
6243 : #undef LOAD_TAGGED_POINTER
6244 : #undef LOAD_TAGGED_ANY
6245 : #undef LOAD_FIXED_ARRAY_SLOT
6246 : #undef LOAD_FIXED_ARRAY_SLOT_SMI
6247 : #undef LOAD_FIXED_ARRAY_SLOT_PTR
6248 : #undef LOAD_FIXED_ARRAY_SLOT_ANY
6249 : #undef STORE_FIXED_ARRAY_SLOT_SMI
6250 : #undef STORE_FIXED_ARRAY_SLOT_ANY
6251 :
6252 : } // namespace compiler
6253 : } // namespace internal
6254 178779 : } // namespace v8
|
