/src/WasmEdge/include/executor/executor.h
Line | Count | Source |
1 | | // SPDX-License-Identifier: Apache-2.0 |
2 | | // SPDX-FileCopyrightText: Copyright The WasmEdge Authors |
3 | | |
4 | | //===-- wasmedge/executor/executor.h - Executor class definition ----------===// |
5 | | // |
6 | | // Part of the WasmEdge Project. |
7 | | // |
8 | | //===----------------------------------------------------------------------===// |
9 | | /// |
10 | | /// \file |
11 | | /// This file contains the declaration of the Executor class, which instantiates |
12 | | /// and runs Wasm modules. |
13 | | /// |
14 | | //===----------------------------------------------------------------------===// |
15 | | #pragma once |
16 | | |
17 | | #include "ast/component/component.h" |
18 | | #include "ast/module.h" |
19 | | #include "common/async.h" |
20 | | #include "common/configure.h" |
21 | | #include "common/defines.h" |
22 | | #include "common/errcode.h" |
23 | | #include "common/statistics.h" |
24 | | #include "common/types.h" |
25 | | #include "runtime/callingframe.h" |
26 | | #include "runtime/instance/component/component.h" |
27 | | #include "runtime/instance/module.h" |
28 | | #include "runtime/stackmgr.h" |
29 | | #include "runtime/storemgr.h" |
30 | | |
31 | | #include <atomic> |
32 | | #include <csignal> |
33 | | #include <cstdint> |
34 | | #include <functional> |
35 | | #include <memory> |
36 | | #include <mutex> |
37 | | #include <optional> |
38 | | #include <shared_mutex> |
39 | | #include <string_view> |
40 | | #include <type_traits> |
41 | | #include <utility> |
42 | | #include <vector> |
43 | | |
44 | | namespace WasmEdge { |
45 | | namespace Executor { |
46 | | |
47 | | namespace { |
48 | | |
49 | | // Template return type aliasing |
50 | | /// Accept unsigned integer types. (uint32_t, uint64_t) |
51 | | template <typename T> |
52 | | using TypeU = typename std::enable_if_t<IsWasmUnsignV<T>, Expect<void>>; |
53 | | /// Accept integer types. (uint32_t, int32_t, uint64_t, int64_t) |
54 | | template <typename T> |
55 | | using TypeI = typename std::enable_if_t<IsWasmIntV<T>, Expect<void>>; |
56 | | /// Accept floating types. (float, double) |
57 | | template <typename T> |
58 | | using TypeF = typename std::enable_if_t<IsWasmFloatV<T>, Expect<void>>; |
59 | | /// Accept all num types. (uint32_t, int32_t, uint64_t, int64_t, float, double) |
60 | | template <typename T> |
61 | | using TypeT = typename std::enable_if_t<IsWasmNumV<T>, Expect<void>>; |
62 | | /// Accept Wasm built-in num types. (uint32_t, uint64_t, float, double) |
63 | | template <typename T> |
64 | | using TypeN = typename std::enable_if_t<IsWasmNativeNumV<T>, Expect<void>>; |
65 | | |
66 | | /// Accept (unsigned integer types, unsigned integer types). |
67 | | template <typename T1, typename T2> |
68 | | using TypeUU = typename std::enable_if_t<IsWasmUnsignV<T1> && IsWasmUnsignV<T2>, |
69 | | Expect<void>>; |
70 | | /// Accept (integer types, unsigned integer types). |
71 | | template <typename T1, typename T2> |
72 | | using TypeIU = typename std::enable_if_t<IsWasmIntV<T1> && IsWasmUnsignV<T2>, |
73 | | Expect<void>>; |
74 | | /// Accept (floating types, floating types). |
75 | | template <typename T1, typename T2> |
76 | | using TypeFF = typename std::enable_if_t<IsWasmFloatV<T1> && IsWasmFloatV<T2>, |
77 | | Expect<void>>; |
78 | | /// Accept (integer types, floating types). |
79 | | template <typename T1, typename T2> |
80 | | using TypeIF = |
81 | | typename std::enable_if_t<IsWasmIntV<T1> && IsWasmFloatV<T2>, Expect<void>>; |
82 | | /// Accept (floating types, integer types). |
83 | | template <typename T1, typename T2> |
84 | | using TypeFI = |
85 | | typename std::enable_if_t<IsWasmFloatV<T1> && IsWasmIntV<T2>, Expect<void>>; |
86 | | /// Accept (Wasm built-in num types, Wasm built-in num types). |
87 | | template <typename T1, typename T2> |
88 | | using TypeNN = |
89 | | typename std::enable_if_t<IsWasmNativeNumV<T1> && IsWasmNativeNumV<T2> && |
90 | | sizeof(T1) == sizeof(T2), |
91 | | Expect<void>>; |
92 | | |
93 | | } // namespace |
94 | | |
95 | | /// Helper class for handling the pre- and post-host functions. |
96 | | class HostFuncHandler { |
97 | | public: |
98 | 0 | void setPreHost(void *HostData, std::function<void(void *)> HostFunc) { |
99 | 0 | std::unique_lock Lock(Mutex); |
100 | 0 | PreHostData = HostData; |
101 | 0 | PreHostFunc = HostFunc; |
102 | 0 | } |
103 | 0 | void setPostHost(void *HostData, std::function<void(void *)> HostFunc) { |
104 | 0 | std::unique_lock Lock(Mutex); |
105 | 0 | PostHostData = HostData; |
106 | 0 | PostHostFunc = HostFunc; |
107 | 0 | } |
108 | 0 | void invokePreHostFunc() { |
109 | 0 | std::function<void(void *)> FuncSnapshot; |
110 | 0 | void *DataSnapshot = nullptr; |
111 | 0 | { |
112 | 0 | std::shared_lock Lock(Mutex); |
113 | 0 | FuncSnapshot = PreHostFunc; |
114 | 0 | DataSnapshot = PreHostData; |
115 | 0 | } |
116 | 0 | if (FuncSnapshot.operator bool()) { |
117 | 0 | FuncSnapshot(DataSnapshot); |
118 | 0 | } |
119 | 0 | } |
120 | 0 | void invokePostHostFunc() { |
121 | 0 | std::function<void(void *)> FuncSnapshot; |
122 | 0 | void *DataSnapshot = nullptr; |
123 | 0 | { |
124 | 0 | std::shared_lock Lock(Mutex); |
125 | 0 | FuncSnapshot = PostHostFunc; |
126 | 0 | DataSnapshot = PostHostData; |
127 | 0 | } |
128 | 0 | if (FuncSnapshot.operator bool()) { |
129 | 0 | FuncSnapshot(DataSnapshot); |
130 | 0 | } |
131 | 0 | } |
132 | | |
133 | | private: |
134 | | void *PreHostData = nullptr; |
135 | | void *PostHostData = nullptr; |
136 | | std::function<void(void *)> PreHostFunc = {}; |
137 | | std::function<void(void *)> PostHostFunc = {}; |
138 | | mutable std::shared_mutex Mutex; |
139 | | }; |
140 | | |
141 | | /// Executor flow control class. |
142 | | class Executor { |
143 | | public: |
144 | | Executor(const Configure &Conf, Statistics::Statistics *S = nullptr) noexcept |
145 | 0 | : Conf(Conf) { |
146 | 0 | if (Conf.getStatisticsConfigure().isInstructionCounting() || |
147 | 0 | Conf.getStatisticsConfigure().isCostMeasuring() || |
148 | 0 | Conf.getStatisticsConfigure().isTimeMeasuring()) { |
149 | 0 | Stat = S; |
150 | 0 | } else { |
151 | 0 | Stat = nullptr; |
152 | 0 | } |
153 | 0 | if (Stat) { |
154 | 0 | Stat->setCostLimit(Conf.getStatisticsConfigure().getCostLimit()); |
155 | 0 | } |
156 | 0 | } |
157 | | |
158 | | /// Getter for configuration. |
159 | 0 | const Configure &getConfigure() const { return Conf; } |
160 | | |
161 | | /// Instantiate a WASM Module as an anonymous module instance. |
162 | | Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>> |
163 | | instantiateModule(Runtime::StoreManager &StoreMgr, const AST::Module &Mod); |
164 | | |
165 | | /// Instantiate and register a WASM module as a named module instance. |
166 | | Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>> |
167 | | registerModule(Runtime::StoreManager &StoreMgr, const AST::Module &Mod, |
168 | | std::string_view Name); |
169 | | |
170 | | /// Register an instantiated module as a named module instance. |
171 | | Expect<void> registerModule(Runtime::StoreManager &StoreMgr, |
172 | | const Runtime::Instance::ModuleInstance &ModInst); |
173 | | |
174 | | /// Register an instantiated module under the given alias name. |
175 | | Expect<void> registerModule(Runtime::StoreManager &StoreMgr, |
176 | | const Runtime::Instance::ModuleInstance &ModInst, |
177 | | std::string_view Name); |
178 | | |
179 | | /// Instantiate a Component as an anonymous component instance. |
180 | | Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>> |
181 | | instantiateComponent(Runtime::StoreManager &StoreMgr, |
182 | | const AST::Component::Component &Comp); |
183 | | |
184 | | /// Instantiate and register a Component as a named component instance. |
185 | | Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>> |
186 | | registerComponent(Runtime::StoreManager &StoreMgr, |
187 | | const AST::Component::Component &Comp, |
188 | | std::string_view Name); |
189 | | |
190 | | /// Register an instantiated component into a named component instance. |
191 | | Expect<void> |
192 | | registerComponent(Runtime::StoreManager &StoreMgr, |
193 | | const Runtime::Instance::ComponentInstance &CompInst); |
194 | | |
195 | | /// Register a host function which will be invoked before calling a |
196 | | /// host function. |
197 | | Expect<void> registerPreHostFunction(void *HostData, |
198 | | std::function<void(void *)> HostFunc); |
199 | | |
200 | | /// Register a host function which will be invoked after calling a |
201 | | /// host function. |
202 | | Expect<void> registerPostHostFunction(void *HostData, |
203 | | std::function<void(void *)> HostFunc); |
204 | | |
205 | | /// Register a callback for lazy function compilation |
206 | | void registerLazyCompilationCallback( |
207 | | std::function<Expect<void>(const Runtime::Instance::FunctionInstance *)> |
208 | 0 | Callback) { |
209 | 0 | LazyCompilationHandler = std::move(Callback); |
210 | 0 | } |
211 | | |
212 | | /// Invoke a WASM function by function instance. |
213 | | Expect<std::vector<std::pair<ValVariant, ValType>>> |
214 | | invoke(const Runtime::Instance::FunctionInstance *FuncInst, |
215 | | Span<const ValVariant> Params, Span<const ValType> ParamTypes); |
216 | | |
217 | | /// Invoke a Component function by function instance. |
218 | | Expect<std::vector<std::pair<ComponentValVariant, ComponentValType>>> |
219 | | invoke(const Runtime::Instance::Component::FunctionInstance *FuncInst, |
220 | | Span<const ComponentValVariant> Params, |
221 | | Span<const ComponentValType> ParamTypes); |
222 | | |
223 | | /// Asynchronous invoke a WASM function by function instance. |
224 | | Async<Expect<std::vector<std::pair<ValVariant, ValType>>>> |
225 | | asyncInvoke(const Runtime::Instance::FunctionInstance *FuncInst, |
226 | | Span<const ValVariant> Params, Span<const ValType> ParamTypes); |
227 | | |
228 | | /// Stop execution |
229 | 0 | void stop() noexcept { |
230 | 0 | StopToken.store(1, std::memory_order_relaxed); |
231 | 0 | atomicNotifyAll(); |
232 | 0 | } |
233 | | |
234 | | private: |
235 | | /// Run Wasm bytecode expression for initialization. |
236 | | Expect<void> runExpression(Runtime::StackManager &StackMgr, |
237 | | AST::InstrView Instrs); |
238 | | |
239 | | /// Run Wasm function. |
240 | | Expect<void> runFunction(Runtime::StackManager &StackMgr, |
241 | | const Runtime::Instance::FunctionInstance &Func, |
242 | | Span<const ValVariant> Params); |
243 | | |
244 | | /// Execute instructions. |
245 | | Expect<void> execute(Runtime::StackManager &StackMgr, |
246 | | const AST::InstrView::iterator Start, |
247 | | const AST::InstrView::iterator End); |
248 | | |
249 | | /// \name Functions for instantiation. |
250 | | /// @{ |
251 | | /// Instantiation of Module Instance. |
252 | | Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>> |
253 | | instantiate(Runtime::StoreManager &StoreMgr, const AST::Module &Mod, |
254 | | std::optional<std::string_view> Name = std::nullopt); |
255 | | |
256 | | /// Instantiation of Imports. |
257 | | Expect<void> instantiate( |
258 | | std::function<const Runtime::Instance::ModuleInstance *(std::string_view)> |
259 | | ModuleFinder, |
260 | | Runtime::Instance::ModuleInstance &ModInst, |
261 | | const AST::ImportSection &ImportSec); |
262 | | |
263 | | /// Instantiation of Function Instances. |
264 | | Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst, |
265 | | const AST::FunctionSection &FuncSec, |
266 | | const AST::CodeSection &CodeSec); |
267 | | |
268 | | /// Instantiation of Table Instances. |
269 | | Expect<void> instantiate(Runtime::StackManager &StackMgr, |
270 | | Runtime::Instance::ModuleInstance &ModInst, |
271 | | const AST::TableSection &TabSec); |
272 | | |
273 | | /// Instantiation of Memory Instances. |
274 | | Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst, |
275 | | const AST::MemorySection &MemSec); |
276 | | |
277 | | /// Instantiateion of Tag Instances. |
278 | | Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst, |
279 | | const AST::TagSection &TagSec); |
280 | | |
281 | | /// Instantiation of Global Instances. |
282 | | Expect<void> instantiate(Runtime::StackManager &StackMgr, |
283 | | Runtime::Instance::ModuleInstance &ModInst, |
284 | | const AST::GlobalSection &GlobSec); |
285 | | |
286 | | /// Instantiation of Element Instances. |
287 | | Expect<void> instantiate(Runtime::StackManager &StackMgr, |
288 | | Runtime::Instance::ModuleInstance &ModInst, |
289 | | const AST::ElementSection &ElemSec); |
290 | | |
291 | | /// Initialize table with Element Instances. |
292 | | Expect<void> initTable(Runtime::StackManager &StackMgr, |
293 | | const AST::ElementSection &ElemSec); |
294 | | |
295 | | /// Instantiation of Data Instances. |
296 | | Expect<void> instantiate(Runtime::StackManager &StackMgr, |
297 | | Runtime::Instance::ModuleInstance &ModInst, |
298 | | const AST::DataSection &DataSec); |
299 | | |
300 | | /// Initialize memory with Data Instances. |
301 | | Expect<void> initMemory(Runtime::StackManager &StackMgr, |
302 | | const AST::DataSection &DataSec); |
303 | | |
304 | | /// Instantiation of Exports. |
305 | | Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst, |
306 | | const AST::ExportSection &ExportSec); |
307 | | /// @} |
308 | | |
309 | | /// \name Functions for instantiation of component model. |
310 | | /// @{ |
311 | | /// Instantiation of Component Instance. |
312 | | Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>> |
313 | | instantiate(Runtime::StoreManager &StoreMgr, |
314 | | const AST::Component::Component &Comp, |
315 | | std::optional<std::string_view> Name = std::nullopt); |
316 | | |
317 | | /// Instantiation of Child Component Instance. |
318 | | Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>> |
319 | | instantiate(Runtime::Instance::ComponentImportManager &ImportMgr, |
320 | | const AST::Component::Component &Comp); |
321 | | |
322 | | /// Instantiation of Child Core Module Instance. |
323 | | Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>> |
324 | | instantiate(Runtime::Instance::ComponentImportManager &ImportMgr, |
325 | | const AST::Module &Mod); |
326 | | |
327 | | /// Instantiation of Core Module Section. |
328 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
329 | | const AST::Component::CoreModuleSection &CoreModSec); |
330 | | |
331 | | /// Instantiation of Core Instance Section. |
332 | | Expect<void> |
333 | | instantiate(Runtime::Instance::ComponentInstance &CompInst, |
334 | | const AST::Component::CoreInstanceSection &CoreInstSec); |
335 | | |
336 | | /// Instantiation of Core Type Section. |
337 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
338 | | const AST::Component::CoreTypeSection &CoreTypeSec); |
339 | | |
340 | | /// Instantiation of Component Section. |
341 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
342 | | const AST::Component::ComponentSection &CompSec); |
343 | | |
344 | | /// Instantiation of Instance Section. |
345 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
346 | | const AST::Component::InstanceSection &InstSec); |
347 | | |
348 | | /// Instantiation of Alias Section. |
349 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
350 | | const AST::Component::AliasSection &AliasSec); |
351 | | |
352 | | /// Instantiation of Type Section. |
353 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
354 | | const AST::Component::TypeSection &TypeSec); |
355 | | |
356 | | /// Instantiation of Canonical Section. |
357 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
358 | | const AST::Component::CanonSection &CanonSec); |
359 | | |
360 | | /// Instantiation of Start Section. |
361 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
362 | | const AST::Component::StartSection &StartSec); |
363 | | |
364 | | /// Instantiation of Import Section. |
365 | | Expect<void> instantiate(Runtime::StoreManager &StoreMgr, |
366 | | Runtime::Instance::ComponentInstance &CompInst, |
367 | | const AST::Component::ImportSection &ImportSec); |
368 | | |
369 | | /// Instantiation of Import Section in the child component instance. |
370 | | Expect<void> instantiate(Runtime::Instance::ComponentImportManager &ImportMgr, |
371 | | Runtime::Instance::ComponentInstance &CompInst, |
372 | | const AST::Component::ImportSection &ImportSec); |
373 | | |
374 | | /// Instantiation of Export Section. |
375 | | Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst, |
376 | | const AST::Component::ExportSection &ExportSec); |
377 | | /// @} |
378 | | |
379 | | /// \name Helper Functions for canonical ABI |
380 | | /// @{ |
381 | | std::vector<ValVariant> |
382 | | convValsToCoreWASM(Span<const ComponentValVariant> Vals, |
383 | | Span<const ComponentValType> ValTypes, |
384 | | Runtime::Instance::FunctionInstance *RFuncInst, |
385 | | Runtime::Instance::MemoryInstance *MemInst) noexcept; |
386 | | |
387 | | Expect<std::vector<std::pair<ComponentValVariant, ComponentValType>>> |
388 | | convValsToComponent(Span<const std::pair<ValVariant, ValType>> CoreVals, |
389 | | Span<const ComponentValType> ValTypes, |
390 | | Runtime::Instance::MemoryInstance *MemInst); |
391 | | /// @} |
392 | | |
393 | | /// \name Helper Functions for block controls. |
394 | | /// @{ |
395 | | /// Helper function for calling functions. Return the continuation iterator. |
396 | | /// Set `IsNativeEntry` when entering from the native code. |
397 | | Expect<AST::InstrView::iterator> |
398 | | enterFunction(Runtime::StackManager &StackMgr, |
399 | | const Runtime::Instance::FunctionInstance &Func, |
400 | | const AST::InstrView::iterator RetIt, bool IsTailCall = false, |
401 | | bool IsNativeEntry = false); |
402 | | |
403 | | /// Helper function for branching to label. |
404 | | Expect<void> branchToLabel(Runtime::StackManager &StackMgr, |
405 | | const AST::Instruction::JumpDescriptor &JumpDesc, |
406 | | AST::InstrView::iterator &PC) noexcept; |
407 | | |
408 | | /// Helper function for throwing an exception. Pass `ExnInst` on `throw_ref` |
409 | | /// to reuse the exception instance and preserve exnref identity. |
410 | | Expect<void> throwException( |
411 | | Runtime::StackManager &StackMgr, Runtime::Instance::TagInstance &TagInst, |
412 | | AST::InstrView::iterator &PC, |
413 | | const Runtime::Instance::ExceptionInstance *ExnInst = nullptr) noexcept; |
414 | | /// @} |
415 | | |
416 | | /// \name Helper Function for checking memory offset boundary. |
417 | | /// @{ |
418 | | Expect<void> |
419 | | checkOffsetOverflow(const Runtime::Instance::MemoryInstance &MemInst, |
420 | | const AST::Instruction &Instr, const uint64_t Val, |
421 | | const uint64_t Size) const noexcept; |
422 | | /// @} |
423 | | |
424 | | /// \name Helper Functions for GC instructions. |
425 | | /// @{ |
426 | | Expect<RefVariant> structNew(Runtime::StackManager &StackMgr, |
427 | | const uint32_t TypeIdx, |
428 | | Span<const ValVariant> Args = {}) const noexcept; |
429 | | Expect<ValVariant> structGet(Runtime::StackManager &StackMgr, |
430 | | const RefVariant Ref, const uint32_t TypeIdx, |
431 | | const uint32_t Off, |
432 | | const bool IsSigned = false) const noexcept; |
433 | | Expect<void> structSet(Runtime::StackManager &StackMgr, const RefVariant Ref, |
434 | | const ValVariant Val, const uint32_t TypeIdx, |
435 | | const uint32_t Off) const noexcept; |
436 | | Expect<RefVariant> arrayNew(Runtime::StackManager &StackMgr, |
437 | | const uint32_t TypeIdx, const uint32_t Length, |
438 | | Span<const ValVariant> Args = {}) const noexcept; |
439 | | Expect<RefVariant> arrayNewData(Runtime::StackManager &StackMgr, |
440 | | const uint32_t TypeIdx, |
441 | | const uint32_t DataIdx, const uint32_t Start, |
442 | | const uint32_t Length) const noexcept; |
443 | | Expect<RefVariant> arrayNewElem(Runtime::StackManager &StackMgr, |
444 | | const uint32_t TypeIdx, |
445 | | const uint32_t ElemIdx, const uint32_t Start, |
446 | | const uint32_t Length) const noexcept; |
447 | | Expect<ValVariant> arrayGet(Runtime::StackManager &StackMgr, |
448 | | const RefVariant &Ref, const uint32_t TypeIdx, |
449 | | const uint32_t Idx, |
450 | | const bool IsSigned = false) const noexcept; |
451 | | Expect<void> arraySet(Runtime::StackManager &StackMgr, const RefVariant &Ref, |
452 | | const ValVariant &Val, const uint32_t TypeIdx, |
453 | | const uint32_t Idx) const noexcept; |
454 | | Expect<void> arrayFill(Runtime::StackManager &StackMgr, const RefVariant &Ref, |
455 | | const ValVariant &Val, const uint32_t TypeIdx, |
456 | | const uint32_t Idx, const uint32_t Cnt) const noexcept; |
457 | | Expect<void> arrayInitData(Runtime::StackManager &StackMgr, |
458 | | const RefVariant &Ref, const uint32_t TypeIdx, |
459 | | const uint32_t DataIdx, const uint32_t DstIdx, |
460 | | const uint32_t SrcIdx, |
461 | | const uint32_t Cnt) const noexcept; |
462 | | Expect<void> arrayInitElem(Runtime::StackManager &StackMgr, |
463 | | const RefVariant &Ref, const uint32_t TypeIdx, |
464 | | const uint32_t ElemIdx, const uint32_t DstIdx, |
465 | | const uint32_t SrcIdx, |
466 | | const uint32_t Cnt) const noexcept; |
467 | | Expect<void> arrayCopy(Runtime::StackManager &StackMgr, |
468 | | const RefVariant &DstRef, const uint32_t DstTypeIdx, |
469 | | const uint32_t DstIdx, const RefVariant &SrcRef, |
470 | | const uint32_t SrcTypeIdx, const uint32_t SrcIdx, |
471 | | const uint32_t Cnt) const noexcept; |
472 | | /// @} |
473 | | |
474 | | /// \name Helper Functions for atomic operations. |
475 | | /// @{ |
476 | | template <typename T> |
477 | | Expect<uint64_t> atomicWait(Runtime::Instance::MemoryInstance &MemInst, |
478 | | uint64_t Address, EndianValue<T> Expected, |
479 | | int64_t Timeout) noexcept; |
480 | | Expect<uint64_t> atomicNotify(Runtime::Instance::MemoryInstance &MemInst, |
481 | | uint64_t Address, uint64_t Count) noexcept; |
482 | | void atomicNotifyAll() noexcept; |
483 | | /// @} |
484 | | |
485 | | /// \name Helper Functions for getting instances or types. |
486 | | /// @{ |
487 | | /// Helper function for getting defined type by index. |
488 | | const AST::SubType *getDefTypeByIdx(Runtime::StackManager &StackMgr, |
489 | | const uint32_t Idx) const; |
490 | | |
491 | | /// Helper function for getting composite type by index. Assuming validated. |
492 | | const WasmEdge::AST::CompositeType & |
493 | | getCompositeTypeByIdx(Runtime::StackManager &StackMgr, |
494 | | const uint32_t Idx) const noexcept; |
495 | | |
496 | | /// Helper function for getting struct storage type by index. |
497 | | const ValType &getStructStorageTypeByIdx(Runtime::StackManager &StackMgr, |
498 | | const uint32_t Idx, |
499 | | const uint32_t Off) const noexcept; |
500 | | |
501 | | /// Helper function for getting array storage type by index. |
502 | | const ValType &getArrayStorageTypeByIdx(Runtime::StackManager &StackMgr, |
503 | | const uint32_t Idx) const noexcept; |
504 | | |
505 | | /// Helper function for getting function instance by index. |
506 | | Runtime::Instance::FunctionInstance * |
507 | | getFuncInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
508 | | |
509 | | /// Helper function for getting table instance by index. |
510 | | Runtime::Instance::TableInstance * |
511 | | getTabInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
512 | | |
513 | | /// Helper function for getting memory instance by index. |
514 | | Runtime::Instance::MemoryInstance * |
515 | | getMemInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
516 | | |
517 | | /// Helper function for getting tag instance by index. |
518 | | Runtime::Instance::TagInstance * |
519 | | getTagInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
520 | | |
521 | | /// Helper function for getting global instance by index. |
522 | | Runtime::Instance::GlobalInstance * |
523 | | getGlobInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
524 | | |
525 | | /// Helper function for getting element instance by index. |
526 | | Runtime::Instance::ElementInstance * |
527 | | getElemInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
528 | | |
529 | | /// Helper function for getting data instance by index. |
530 | | Runtime::Instance::DataInstance * |
531 | | getDataInstByIdx(Runtime::StackManager &StackMgr, const uint32_t Idx) const; |
532 | | |
533 | | /// Helper function for converting into bottom abstract heap type. |
534 | | TypeCode toBottomType(Runtime::StackManager &StackMgr, |
535 | | const ValType &Type) const; |
536 | | |
537 | | /// Helper function for cleaning unused bits of numeric values in ValVariant. |
538 | | void cleanNumericVal(ValVariant &Val, const ValType &Type) const noexcept; |
539 | | |
540 | | /// Helper function for packing ValVariant for packed type. |
541 | | ValVariant packVal(const ValType &Type, const ValVariant &Val) const noexcept; |
542 | | |
543 | | /// Helper function for packing ValVariant vector for packed type. |
544 | | std::vector<ValVariant> |
545 | | packVals(const ValType &Type, std::vector<ValVariant> &&Vals) const noexcept; |
546 | | |
547 | | /// Helper function for unpacking ValVariant for packed type. |
548 | | ValVariant unpackVal(const ValType &Type, const ValVariant &Val, |
549 | | bool IsSigned = false) const noexcept; |
550 | | /// @} |
551 | | |
552 | | /// \name Interpreter - Run instructions functions |
553 | | /// @{ |
554 | | /// ======= Control instructions ======= |
555 | | Expect<void> runIfElseOp(Runtime::StackManager &StackMgr, |
556 | | const AST::Instruction &Instr, |
557 | | AST::InstrView::iterator &PC) noexcept; |
558 | | Expect<void> runThrowOp(Runtime::StackManager &StackMgr, |
559 | | const AST::Instruction &Instr, |
560 | | AST::InstrView::iterator &PC) noexcept; |
561 | | Expect<void> runThrowRefOp(Runtime::StackManager &StackMgr, |
562 | | const AST::Instruction &Instr, |
563 | | AST::InstrView::iterator &PC) noexcept; |
564 | | Expect<void> runBrOp(Runtime::StackManager &StackMgr, |
565 | | const AST::Instruction &Instr, |
566 | | AST::InstrView::iterator &PC) noexcept; |
567 | | Expect<void> runBrIfOp(Runtime::StackManager &StackMgr, |
568 | | const AST::Instruction &Instr, |
569 | | AST::InstrView::iterator &PC) noexcept; |
570 | | Expect<void> runBrOnNullOp(Runtime::StackManager &StackMgr, |
571 | | const AST::Instruction &Instr, |
572 | | AST::InstrView::iterator &PC) noexcept; |
573 | | Expect<void> runBrOnNonNullOp(Runtime::StackManager &StackMgr, |
574 | | const AST::Instruction &Instr, |
575 | | AST::InstrView::iterator &PC) noexcept; |
576 | | Expect<void> runBrTableOp(Runtime::StackManager &StackMgr, |
577 | | const AST::Instruction &Instr, |
578 | | AST::InstrView::iterator &PC) noexcept; |
579 | | Expect<void> runBrOnCastOp(Runtime::StackManager &StackMgr, |
580 | | const AST::Instruction &Instr, |
581 | | AST::InstrView::iterator &PC, |
582 | | bool IsReverse = false) noexcept; |
583 | | Expect<void> runReturnOp(Runtime::StackManager &StackMgr, |
584 | | AST::InstrView::iterator &PC) noexcept; |
585 | | Expect<void> runCallOp(Runtime::StackManager &StackMgr, |
586 | | const AST::Instruction &Instr, |
587 | | AST::InstrView::iterator &PC, |
588 | | bool IsTailCall = false) noexcept; |
589 | | Expect<void> runCallRefOp(Runtime::StackManager &StackMgr, |
590 | | const AST::Instruction &Instr, |
591 | | AST::InstrView::iterator &PC, |
592 | | bool IsTailCall = false) noexcept; |
593 | | Expect<void> runCallIndirectOp(Runtime::StackManager &StackMgr, |
594 | | const AST::Instruction &Instr, |
595 | | AST::InstrView::iterator &PC, |
596 | | bool IsTailCall = false) noexcept; |
597 | | Expect<void> runTryTableOp(Runtime::StackManager &StackMgr, |
598 | | const AST::Instruction &Instr, |
599 | | AST::InstrView::iterator &PC) noexcept; |
600 | | /// ======= Variable instructions ======= |
601 | | Expect<void> runLocalGetOp(Runtime::StackManager &StackMgr, |
602 | | uint32_t StackOffset) const noexcept; |
603 | | Expect<void> runLocalSetOp(Runtime::StackManager &StackMgr, |
604 | | uint32_t StackOffset) const noexcept; |
605 | | Expect<void> runLocalTeeOp(Runtime::StackManager &StackMgr, |
606 | | uint32_t StackOffset) const noexcept; |
607 | | Expect<void> runGlobalGetOp(Runtime::StackManager &StackMgr, |
608 | | uint32_t Idx) const noexcept; |
609 | | Expect<void> runGlobalSetOp(Runtime::StackManager &StackMgr, |
610 | | uint32_t Idx) const noexcept; |
611 | | /// ======= Reference instructions ======= |
612 | | Expect<void> runRefNullOp(Runtime::StackManager &StackMgr, |
613 | | const ValType &Type) const noexcept; |
614 | | Expect<void> runRefIsNullOp(ValVariant &Val) const noexcept; |
615 | | Expect<void> runRefFuncOp(Runtime::StackManager &StackMgr, |
616 | | uint32_t Idx) const noexcept; |
617 | | Expect<void> runRefEqOp(ValVariant &Val1, |
618 | | const ValVariant &Val2) const noexcept; |
619 | | Expect<void> runRefAsNonNullOp(RefVariant &Val, |
620 | | const AST::Instruction &Instr) const noexcept; |
621 | | Expect<void> runStructNewOp(Runtime::StackManager &StackMgr, |
622 | | const uint32_t TypeIdx, |
623 | | const bool IsDefault = false) const noexcept; |
624 | | Expect<void> runStructGetOp(Runtime::StackManager &StackMgr, |
625 | | const uint32_t TypeIdx, const uint32_t Off, |
626 | | const AST::Instruction &Instr, |
627 | | const bool IsSigned = false) const noexcept; |
628 | | Expect<void> runStructSetOp(Runtime::StackManager &StackMgr, |
629 | | const ValVariant &Val, const uint32_t TypeIdx, |
630 | | const uint32_t Off, |
631 | | const AST::Instruction &Instr) const noexcept; |
632 | | Expect<void> runArrayNewOp(Runtime::StackManager &StackMgr, |
633 | | const uint32_t TypeIdx, const uint32_t InitCnt, |
634 | | uint32_t Length) const noexcept; |
635 | | Expect<void> runArrayNewDataOp(Runtime::StackManager &StackMgr, |
636 | | const uint32_t TypeIdx, const uint32_t DataIdx, |
637 | | const AST::Instruction &Instr) const noexcept; |
638 | | Expect<void> runArrayNewElemOp(Runtime::StackManager &StackMgr, |
639 | | const uint32_t TypeIdx, const uint32_t ElemIdx, |
640 | | const AST::Instruction &Instr) const noexcept; |
641 | | Expect<void> runArrayGetOp(Runtime::StackManager &StackMgr, |
642 | | const uint32_t TypeIdx, |
643 | | const AST::Instruction &Instr, |
644 | | const bool IsSigned = false) const noexcept; |
645 | | Expect<void> runArraySetOp(Runtime::StackManager &StackMgr, |
646 | | const ValVariant &Val, const uint32_t TypeIdx, |
647 | | const AST::Instruction &Instr) const noexcept; |
648 | | Expect<void> runArrayLenOp(ValVariant &Val, |
649 | | const AST::Instruction &Instr) const noexcept; |
650 | | Expect<void> runArrayFillOp(Runtime::StackManager &StackMgr, |
651 | | const uint32_t Cnt, const ValVariant &Val, |
652 | | const uint32_t TypeIdx, |
653 | | const AST::Instruction &Instr) const noexcept; |
654 | | Expect<void> runArrayCopyOp(Runtime::StackManager &StackMgr, |
655 | | const uint32_t Cnt, const uint32_t DstTypeIdx, |
656 | | const uint32_t SrcTypeIdx, |
657 | | const AST::Instruction &Instr) const noexcept; |
658 | | Expect<void> runArrayInitDataOp(Runtime::StackManager &StackMgr, |
659 | | const uint32_t Cnt, const uint32_t TypeIdx, |
660 | | const uint32_t DataIdx, |
661 | | const AST::Instruction &Instr) const noexcept; |
662 | | Expect<void> runArrayInitElemOp(Runtime::StackManager &StackMgr, |
663 | | const uint32_t Cnt, const uint32_t TypeIdx, |
664 | | const uint32_t ElemIdx, |
665 | | const AST::Instruction &Instr) const noexcept; |
666 | | Expect<void> runRefTestOp(const Runtime::Instance::ModuleInstance *ModInst, |
667 | | ValVariant &Val, const AST::Instruction &Instr, |
668 | | const bool IsCast = false) const noexcept; |
669 | | Expect<void> runRefConvOp(RefVariant &Val, TypeCode TCode) const noexcept; |
670 | | Expect<void> runRefI31Op(ValVariant &Val) const noexcept; |
671 | | Expect<void> runI31GetOp(ValVariant &Val, const AST::Instruction &Instr, |
672 | | const bool IsSigned = false) const noexcept; |
673 | | /// ======= Table instructions ======= |
674 | | Expect<void> runTableGetOp(Runtime::StackManager &StackMgr, |
675 | | Runtime::Instance::TableInstance &TabInst, |
676 | | const AST::Instruction &Instr); |
677 | | Expect<void> runTableSetOp(Runtime::StackManager &StackMgr, |
678 | | Runtime::Instance::TableInstance &TabInst, |
679 | | const AST::Instruction &Instr); |
680 | | Expect<void> runTableInitOp(Runtime::StackManager &StackMgr, |
681 | | Runtime::Instance::TableInstance &TabInst, |
682 | | Runtime::Instance::ElementInstance &ElemInst, |
683 | | const AST::Instruction &Instr); |
684 | | Expect<void> runElemDropOp(Runtime::Instance::ElementInstance &ElemInst); |
685 | | Expect<void> runTableCopyOp(Runtime::StackManager &StackMgr, |
686 | | Runtime::Instance::TableInstance &TabInstDst, |
687 | | Runtime::Instance::TableInstance &TabInstSrc, |
688 | | const AST::Instruction &Instr); |
689 | | Expect<void> runTableGrowOp(Runtime::StackManager &StackMgr, |
690 | | Runtime::Instance::TableInstance &TabInst); |
691 | | Expect<void> runTableSizeOp(Runtime::StackManager &StackMgr, |
692 | | Runtime::Instance::TableInstance &TabInst); |
693 | | Expect<void> runTableFillOp(Runtime::StackManager &StackMgr, |
694 | | Runtime::Instance::TableInstance &TabInst, |
695 | | const AST::Instruction &Instr); |
696 | | /// ======= Memory instructions ======= |
697 | | template <typename T, uint32_t BitWidth = sizeof(T) * 8> |
698 | | TypeT<T> runLoadOp(Runtime::StackManager &StackMgr, |
699 | | Runtime::Instance::MemoryInstance &MemInst, |
700 | | const AST::Instruction &Instr); |
701 | | template <typename T, uint32_t BitWidth = sizeof(T) * 8> |
702 | | TypeN<T> runStoreOp(Runtime::StackManager &StackMgr, |
703 | | Runtime::Instance::MemoryInstance &MemInst, |
704 | | const AST::Instruction &Instr); |
705 | | Expect<void> runMemorySizeOp(Runtime::StackManager &StackMgr, |
706 | | Runtime::Instance::MemoryInstance &MemInst); |
707 | | Expect<void> runMemoryGrowOp(Runtime::StackManager &StackMgr, |
708 | | Runtime::Instance::MemoryInstance &MemInst); |
709 | | Expect<void> runMemoryInitOp(Runtime::StackManager &StackMgr, |
710 | | Runtime::Instance::MemoryInstance &MemInst, |
711 | | Runtime::Instance::DataInstance &DataInst, |
712 | | const AST::Instruction &Instr); |
713 | | Expect<void> runDataDropOp(Runtime::Instance::DataInstance &DataInst); |
714 | | Expect<void> runMemoryCopyOp(Runtime::StackManager &StackMgr, |
715 | | Runtime::Instance::MemoryInstance &MemInstDst, |
716 | | Runtime::Instance::MemoryInstance &MemInstSrc, |
717 | | const AST::Instruction &Instr); |
718 | | Expect<void> runMemoryFillOp(Runtime::StackManager &StackMgr, |
719 | | Runtime::Instance::MemoryInstance &MemInst, |
720 | | const AST::Instruction &Instr); |
721 | | /// ======= Test and Relation Numeric instructions ======= |
722 | | template <typename T> TypeU<T> runEqzOp(ValVariant &Val) const; |
723 | | template <typename T> |
724 | | TypeT<T> runEqOp(ValVariant &Val1, const ValVariant &Val2) const; |
725 | | template <typename T> |
726 | | TypeT<T> runNeOp(ValVariant &Val1, const ValVariant &Val2) const; |
727 | | template <typename T> |
728 | | TypeT<T> runLtOp(ValVariant &Val1, const ValVariant &Val2) const; |
729 | | template <typename T> |
730 | | TypeT<T> runGtOp(ValVariant &Val1, const ValVariant &Val2) const; |
731 | | template <typename T> |
732 | | TypeT<T> runLeOp(ValVariant &Val1, const ValVariant &Val2) const; |
733 | | template <typename T> |
734 | | TypeT<T> runGeOp(ValVariant &Val1, const ValVariant &Val2) const; |
735 | | /// ======= Unary Numeric instructions ======= |
736 | | template <typename T> TypeU<T> runClzOp(ValVariant &Val) const; |
737 | | template <typename T> TypeU<T> runCtzOp(ValVariant &Val) const; |
738 | | template <typename T> TypeU<T> runPopcntOp(ValVariant &Val) const; |
739 | | template <typename T> TypeF<T> runAbsOp(ValVariant &Val) const; |
740 | | template <typename T> TypeF<T> runNegOp(ValVariant &Val) const; |
741 | | template <typename T> TypeF<T> runCeilOp(ValVariant &Val) const; |
742 | | template <typename T> TypeF<T> runFloorOp(ValVariant &Val) const; |
743 | | template <typename T> TypeF<T> runTruncOp(ValVariant &Val) const; |
744 | | template <typename T> TypeF<T> runNearestOp(ValVariant &Val) const; |
745 | | template <typename T> TypeF<T> runSqrtOp(ValVariant &Val) const; |
746 | | /// ======= Binary Numeric instructions ======= |
747 | | template <typename T> |
748 | | TypeN<T> runAddOp(ValVariant &Val1, const ValVariant &Val2) const; |
749 | | template <typename T> |
750 | | TypeN<T> runSubOp(ValVariant &Val1, const ValVariant &Val2) const; |
751 | | template <typename T> |
752 | | TypeN<T> runMulOp(ValVariant &Val1, const ValVariant &Val2) const; |
753 | | template <typename T> |
754 | | TypeT<T> runDivOp(const AST::Instruction &Instr, ValVariant &Val1, |
755 | | const ValVariant &Val2) const; |
756 | | template <typename T> |
757 | | TypeI<T> runRemOp(const AST::Instruction &Instr, ValVariant &Val1, |
758 | | const ValVariant &Val2) const; |
759 | | template <typename T> |
760 | | TypeU<T> runAndOp(ValVariant &Val1, const ValVariant &Val2) const; |
761 | | template <typename T> |
762 | | TypeU<T> runOrOp(ValVariant &Val1, const ValVariant &Val2) const; |
763 | | template <typename T> |
764 | | TypeU<T> runXorOp(ValVariant &Val1, const ValVariant &Val2) const; |
765 | | template <typename T> |
766 | | TypeU<T> runShlOp(ValVariant &Val1, const ValVariant &Val2) const; |
767 | | template <typename T> |
768 | | TypeI<T> runShrOp(ValVariant &Val1, const ValVariant &Val2) const; |
769 | | template <typename T> |
770 | | TypeU<T> runRotlOp(ValVariant &Val1, const ValVariant &Val2) const; |
771 | | template <typename T> |
772 | | TypeU<T> runRotrOp(ValVariant &Val1, const ValVariant &Val2) const; |
773 | | template <typename T> |
774 | | TypeF<T> runMinOp(ValVariant &Val1, const ValVariant &Val2) const; |
775 | | template <typename T> |
776 | | TypeF<T> runMaxOp(ValVariant &Val1, const ValVariant &Val2) const; |
777 | | template <typename T> |
778 | | TypeF<T> runCopysignOp(ValVariant &Val1, const ValVariant &Val2) const; |
779 | | /// ======= Cast Numeric instructions ======= |
780 | | template <typename TIn, typename TOut> |
781 | | TypeUU<TIn, TOut> runWrapOp(ValVariant &Val) const; |
782 | | template <typename TIn, typename TOut> |
783 | | TypeFI<TIn, TOut> runTruncateOp(const AST::Instruction &Instr, |
784 | | ValVariant &Val) const; |
785 | | template <typename TIn, typename TOut> |
786 | | TypeFI<TIn, TOut> runTruncateSatOp(ValVariant &Val) const; |
787 | | template <typename TIn, typename TOut, size_t B = sizeof(TIn) * 8> |
788 | | TypeIU<TIn, TOut> runExtendOp(ValVariant &Val) const; |
789 | | template <typename TIn, typename TOut> |
790 | | TypeIF<TIn, TOut> runConvertOp(ValVariant &Val) const; |
791 | | template <typename TIn, typename TOut> |
792 | | TypeFF<TIn, TOut> runDemoteOp(ValVariant &Val) const; |
793 | | template <typename TIn, typename TOut> |
794 | | TypeFF<TIn, TOut> runPromoteOp(ValVariant &Val) const; |
795 | | template <typename TIn, typename TOut> |
796 | | TypeNN<TIn, TOut> runReinterpretOp(ValVariant &Val) const; |
797 | | /// ======= SIMD Memory instructions ======= |
798 | | template <typename TIn, typename TOut> |
799 | | Expect<void> runLoadExpandOp(Runtime::StackManager &StackMgr, |
800 | | Runtime::Instance::MemoryInstance &MemInst, |
801 | | const AST::Instruction &Instr); |
802 | | template <typename T> |
803 | | Expect<void> runLoadSplatOp(Runtime::StackManager &StackMgr, |
804 | | Runtime::Instance::MemoryInstance &MemInst, |
805 | | const AST::Instruction &Instr); |
806 | | template <typename T> |
807 | | Expect<void> runLoadLaneOp(Runtime::StackManager &StackMgr, |
808 | | Runtime::Instance::MemoryInstance &MemInst, |
809 | | const AST::Instruction &Instr); |
810 | | template <typename T> |
811 | | Expect<void> runStoreLaneOp(Runtime::StackManager &StackMgr, |
812 | | Runtime::Instance::MemoryInstance &MemInst, |
813 | | const AST::Instruction &Instr); |
814 | | /// ======= SIMD Lane instructions ======= |
815 | | template <typename TIn, typename TOut = TIn> |
816 | | Expect<void> runExtractLaneOp(ValVariant &Val, const uint8_t Index) const; |
817 | | template <typename TIn, typename TOut = TIn> |
818 | | Expect<void> runReplaceLaneOp(ValVariant &Val1, const ValVariant &Val2, |
819 | | const uint8_t Index) const; |
820 | | /// ======= SIMD Numeric instructions ======= |
821 | | template <typename TIn, typename TOut = TIn> |
822 | | Expect<void> runSplatOp(ValVariant &Val) const; |
823 | | template <typename T> |
824 | | Expect<void> runVectorEqOp(ValVariant &Val1, const ValVariant &Val2) const; |
825 | | template <typename T> |
826 | | Expect<void> runVectorNeOp(ValVariant &Val1, const ValVariant &Val2) const; |
827 | | template <typename T> |
828 | | Expect<void> runVectorLtOp(ValVariant &Val1, const ValVariant &Val2) const; |
829 | | template <typename T> |
830 | | Expect<void> runVectorGtOp(ValVariant &Val1, const ValVariant &Val2) const; |
831 | | template <typename T> |
832 | | Expect<void> runVectorLeOp(ValVariant &Val1, const ValVariant &Val2) const; |
833 | | template <typename T> |
834 | | Expect<void> runVectorGeOp(ValVariant &Val1, const ValVariant &Val2) const; |
835 | | template <typename T> Expect<void> runVectorAbsOp(ValVariant &Val) const; |
836 | | template <typename T> Expect<void> runVectorNegOp(ValVariant &Val) const; |
837 | | inline Expect<void> runVectorPopcntOp(ValVariant &Val) const; |
838 | | template <typename T> Expect<void> runVectorSqrtOp(ValVariant &Val) const; |
839 | | template <typename TIn, typename TOut> |
840 | | Expect<void> runVectorTruncSatOp(ValVariant &Val) const; |
841 | | template <typename TIn, typename TOut> |
842 | | Expect<void> runVectorConvertOp(ValVariant &Val) const; |
843 | | inline Expect<void> runVectorDemoteOp(ValVariant &Val) const; |
844 | | inline Expect<void> runVectorPromoteOp(ValVariant &Val) const; |
845 | | inline Expect<void> runVectorAnyTrueOp(ValVariant &Val) const; |
846 | | template <typename T> Expect<void> runVectorAllTrueOp(ValVariant &Val) const; |
847 | | template <typename T> Expect<void> runVectorBitMaskOp(ValVariant &Val) const; |
848 | | template <typename TIn, typename TOut> |
849 | | Expect<void> runVectorNarrowOp(ValVariant &Val1, |
850 | | const ValVariant &Val2) const; |
851 | | template <typename TIn, typename TOut> |
852 | | Expect<void> runVectorExtendLowOp(ValVariant &Val) const; |
853 | | template <typename TIn, typename TOut> |
854 | | Expect<void> runVectorExtendHighOp(ValVariant &Val) const; |
855 | | template <typename TIn, typename TOut> |
856 | | Expect<void> runVectorExtAddPairwiseOp(ValVariant &Val) const; |
857 | | template <typename TIn, typename TOut> |
858 | | Expect<void> runVectorExtMulLowOp(ValVariant &Val1, |
859 | | const ValVariant &Val2) const; |
860 | | template <typename TIn, typename TOut> |
861 | | Expect<void> runVectorExtMulHighOp(ValVariant &Val1, |
862 | | const ValVariant &Val2) const; |
863 | | inline Expect<void> runVectorQ15MulSatOp(ValVariant &Val1, |
864 | | const ValVariant &Val2) const; |
865 | | template <typename T> |
866 | | Expect<void> runVectorShlOp(ValVariant &Val1, const ValVariant &Val2) const; |
867 | | template <typename T> |
868 | | Expect<void> runVectorShrOp(ValVariant &Val1, const ValVariant &Val2) const; |
869 | | template <typename T> |
870 | | Expect<void> runVectorAddOp(ValVariant &Val1, const ValVariant &Val2) const; |
871 | | template <typename T> |
872 | | Expect<void> runVectorAddSatOp(ValVariant &Val1, |
873 | | const ValVariant &Val2) const; |
874 | | template <typename T> |
875 | | Expect<void> runVectorSubOp(ValVariant &Val1, const ValVariant &Val2) const; |
876 | | template <typename T> |
877 | | Expect<void> runVectorSubSatOp(ValVariant &Val1, |
878 | | const ValVariant &Val2) const; |
879 | | template <typename T> |
880 | | Expect<void> runVectorMulOp(ValVariant &Val1, const ValVariant &Val2) const; |
881 | | template <typename T> |
882 | | Expect<void> runVectorDivOp(ValVariant &Val1, const ValVariant &Val2) const; |
883 | | template <typename T> |
884 | | Expect<void> runVectorMinOp(ValVariant &Val1, const ValVariant &Val2) const; |
885 | | template <typename T> |
886 | | Expect<void> runVectorMaxOp(ValVariant &Val1, const ValVariant &Val2) const; |
887 | | template <typename T> |
888 | | Expect<void> runVectorFMinOp(ValVariant &Val1, const ValVariant &Val2) const; |
889 | | template <typename T> |
890 | | Expect<void> runVectorFMaxOp(ValVariant &Val1, const ValVariant &Val2) const; |
891 | | template <typename T, typename ET> |
892 | | Expect<void> runVectorAvgrOp(ValVariant &Val1, const ValVariant &Val2) const; |
893 | | template <typename T> Expect<void> runVectorCeilOp(ValVariant &Val) const; |
894 | | template <typename T> Expect<void> runVectorFloorOp(ValVariant &Val) const; |
895 | | template <typename T> Expect<void> runVectorTruncOp(ValVariant &Val) const; |
896 | | template <typename T> Expect<void> runVectorNearestOp(ValVariant &Val) const; |
897 | | /// ======= Relaxed SIMD instructions ======= |
898 | | template <typename T> |
899 | | Expect<void> runVectorRelaxedLaneselectOp(ValVariant &Val1, |
900 | | const ValVariant &Val2, |
901 | | const ValVariant &Mask) const; |
902 | | inline Expect<void> |
903 | | runVectorRelaxedIntegerDotProductOp(ValVariant &Val1, |
904 | | const ValVariant &Val2) const; |
905 | | inline Expect<void> runVectorRelaxedIntegerDotProductOpAdd( |
906 | | ValVariant &Val1, const ValVariant &Val2, const ValVariant &C) const; |
907 | | /// ======= Atomic instructions ======= |
908 | | Expect<void> runAtomicNotifyOp(Runtime::StackManager &StackMgr, |
909 | | Runtime::Instance::MemoryInstance &MemInst, |
910 | | const AST::Instruction &Instr); |
911 | | Expect<void> runMemoryFenceOp(); |
912 | | template <typename T> |
913 | | TypeT<T> runAtomicWaitOp(Runtime::StackManager &StackMgr, |
914 | | Runtime::Instance::MemoryInstance &MemInst, |
915 | | const AST::Instruction &Instr); |
916 | | template <typename T, typename I> |
917 | | TypeT<T> runAtomicLoadOp(Runtime::StackManager &StackMgr, |
918 | | Runtime::Instance::MemoryInstance &MemInst, |
919 | | const AST::Instruction &Instr); |
920 | | template <typename T, typename I> |
921 | | TypeT<T> runAtomicStoreOp(Runtime::StackManager &StackMgr, |
922 | | Runtime::Instance::MemoryInstance &MemInst, |
923 | | const AST::Instruction &Instr); |
924 | | template <typename T, typename I> |
925 | | TypeT<T> runAtomicAddOp(Runtime::StackManager &StackMgr, |
926 | | Runtime::Instance::MemoryInstance &MemInst, |
927 | | const AST::Instruction &Instr); |
928 | | template <typename T, typename I> |
929 | | TypeT<T> runAtomicSubOp(Runtime::StackManager &StackMgr, |
930 | | Runtime::Instance::MemoryInstance &MemInst, |
931 | | const AST::Instruction &Instr); |
932 | | template <typename T, typename I> |
933 | | TypeT<T> runAtomicOrOp(Runtime::StackManager &StackMgr, |
934 | | Runtime::Instance::MemoryInstance &MemInst, |
935 | | const AST::Instruction &Instr); |
936 | | template <typename T, typename I> |
937 | | TypeT<T> runAtomicAndOp(Runtime::StackManager &StackMgr, |
938 | | Runtime::Instance::MemoryInstance &MemInst, |
939 | | const AST::Instruction &Instr); |
940 | | template <typename T, typename I> |
941 | | TypeT<T> runAtomicXorOp(Runtime::StackManager &StackMgr, |
942 | | Runtime::Instance::MemoryInstance &MemInst, |
943 | | const AST::Instruction &Instr); |
944 | | template <typename T, typename I> |
945 | | TypeT<T> runAtomicExchangeOp(Runtime::StackManager &StackMgr, |
946 | | Runtime::Instance::MemoryInstance &MemInst, |
947 | | const AST::Instruction &Instr); |
948 | | template <typename T, typename I> |
949 | | TypeT<T> |
950 | | runAtomicCompareExchangeOp(Runtime::StackManager &StackMgr, |
951 | | Runtime::Instance::MemoryInstance &MemInst, |
952 | | const AST::Instruction &Instr); |
953 | | /// @} |
954 | | |
955 | | public: |
956 | | /// \name AOT/JIT - Run compiled functions |
957 | | /// @{ |
958 | | Expect<void> proxyTrap(Runtime::StackManager &StackMgr, |
959 | | const uint32_t Code) noexcept; |
960 | | Expect<void> proxyCall(Runtime::StackManager &StackMgr, |
961 | | const uint32_t FuncIdx, const ValVariant *Args, |
962 | | ValVariant *Rets) noexcept; |
963 | | Expect<void> proxyCallIndirect(Runtime::StackManager &StackMgr, |
964 | | const uint32_t TableIdx, |
965 | | const uint32_t FuncTypeIdx, |
966 | | const uint32_t FuncIdx, const ValVariant *Args, |
967 | | ValVariant *Rets) noexcept; |
968 | | Expect<void> proxyCallRef(Runtime::StackManager &StackMgr, |
969 | | const RefVariant Ref, const ValVariant *Args, |
970 | | ValVariant *Rets) noexcept; |
971 | | Expect<RefVariant> proxyRefFunc(Runtime::StackManager &StackMgr, |
972 | | const uint32_t FuncIdx) noexcept; |
973 | | Expect<RefVariant> proxyStructNew(Runtime::StackManager &StackMgr, |
974 | | const uint32_t TypeIdx, |
975 | | const ValVariant *Args, |
976 | | const uint32_t ArgSize) noexcept; |
977 | | Expect<void> proxyStructGet(Runtime::StackManager &StackMgr, |
978 | | const RefVariant Ref, const uint32_t TypeIdx, |
979 | | const uint32_t Off, const bool IsSigned, |
980 | | ValVariant *Ret) noexcept; |
981 | | Expect<void> proxyStructSet(Runtime::StackManager &StackMgr, |
982 | | const RefVariant Ref, const uint32_t TypeIdx, |
983 | | const uint32_t Off, |
984 | | const ValVariant *Val) noexcept; |
985 | | Expect<RefVariant> proxyArrayNew(Runtime::StackManager &StackMgr, |
986 | | const uint32_t TypeIdx, |
987 | | const uint32_t Length, |
988 | | const ValVariant *Args, |
989 | | const uint32_t ArgSize) noexcept; |
990 | | Expect<RefVariant> proxyArrayNewData(Runtime::StackManager &StackMgr, |
991 | | const uint32_t TypeIdx, |
992 | | const uint32_t DataIdx, |
993 | | const uint32_t Start, |
994 | | const uint32_t Length) noexcept; |
995 | | Expect<RefVariant> proxyArrayNewElem(Runtime::StackManager &StackMgr, |
996 | | const uint32_t TypeIdx, |
997 | | const uint32_t ElemIdx, |
998 | | const uint32_t Start, |
999 | | const uint32_t Length) noexcept; |
1000 | | Expect<void> proxyArrayGet(Runtime::StackManager &StackMgr, |
1001 | | const RefVariant Ref, const uint32_t TypeIdx, |
1002 | | const uint32_t Idx, const bool IsSigned, |
1003 | | ValVariant *Ret) noexcept; |
1004 | | Expect<void> proxyArraySet(Runtime::StackManager &StackMgr, |
1005 | | const RefVariant Ref, const uint32_t TypeIdx, |
1006 | | const uint32_t Idx, |
1007 | | const ValVariant *Val) noexcept; |
1008 | | Expect<uint32_t> proxyArrayLen(Runtime::StackManager &StackMgr, |
1009 | | const RefVariant Ref) noexcept; |
1010 | | Expect<void> proxyArrayFill(Runtime::StackManager &StackMgr, |
1011 | | const RefVariant Ref, const uint32_t TypeIdx, |
1012 | | const uint32_t Idx, const uint32_t Cnt, |
1013 | | const ValVariant *Val) noexcept; |
1014 | | Expect<void> proxyArrayCopy(Runtime::StackManager &StackMgr, |
1015 | | const RefVariant DstRef, |
1016 | | const uint32_t DstTypeIdx, const uint32_t DstIdx, |
1017 | | const RefVariant SrcRef, |
1018 | | const uint32_t SrcTypeIdx, const uint32_t SrcIdx, |
1019 | | const uint32_t Cnt) noexcept; |
1020 | | Expect<void> proxyArrayInitData(Runtime::StackManager &StackMgr, |
1021 | | const RefVariant Ref, const uint32_t TypeIdx, |
1022 | | const uint32_t DataIdx, const uint32_t DstIdx, |
1023 | | const uint32_t SrcIdx, |
1024 | | const uint32_t Cnt) noexcept; |
1025 | | Expect<void> proxyArrayInitElem(Runtime::StackManager &StackMgr, |
1026 | | const RefVariant Ref, const uint32_t TypeIdx, |
1027 | | const uint32_t ElemIdx, const uint32_t DstIdx, |
1028 | | const uint32_t SrcIdx, |
1029 | | const uint32_t Cnt) noexcept; |
1030 | | Expect<uint32_t> proxyRefTest(Runtime::StackManager &StackMgr, |
1031 | | const RefVariant Ref, ValType VTTest) noexcept; |
1032 | | Expect<RefVariant> proxyRefCast(Runtime::StackManager &StackMgr, |
1033 | | const RefVariant Ref, |
1034 | | ValType VTCast) noexcept; |
1035 | | Expect<void> proxyTableInit(Runtime::StackManager &StackMgr, |
1036 | | const uint32_t TableIdx, const uint32_t ElemIdx, |
1037 | | const uint64_t DstOff, const uint32_t SrcOff, |
1038 | | const uint32_t Len) noexcept; |
1039 | | Expect<void> proxyElemDrop(Runtime::StackManager &StackMgr, |
1040 | | const uint32_t ElemIdx) noexcept; |
1041 | | Expect<void> proxyTableCopy(Runtime::StackManager &StackMgr, |
1042 | | const uint32_t TableIdxDst, |
1043 | | const uint32_t TableIdxSrc, const uint64_t DstOff, |
1044 | | const uint64_t SrcOff, |
1045 | | const uint64_t Len) noexcept; |
1046 | | Expect<uint64_t> proxyTableGrow(Runtime::StackManager &StackMgr, |
1047 | | const uint32_t TableIdx, const RefVariant Val, |
1048 | | const uint64_t NewSize) noexcept; |
1049 | | Expect<void> proxyTableFill(Runtime::StackManager &StackMgr, |
1050 | | const uint32_t TableIdx, const uint64_t Off, |
1051 | | const RefVariant Ref, |
1052 | | const uint64_t Len) noexcept; |
1053 | | Expect<uint64_t> proxyMemGrow(Runtime::StackManager &StackMgr, |
1054 | | const uint32_t MemIdx, |
1055 | | const uint64_t NewSize) noexcept; |
1056 | | Expect<void> proxyMemInit(Runtime::StackManager &StackMgr, |
1057 | | const uint32_t MemIdx, const uint32_t DataIdx, |
1058 | | const uint64_t DstOff, const uint32_t SrcOff, |
1059 | | const uint32_t Len) noexcept; |
1060 | | Expect<void> proxyDataDrop(Runtime::StackManager &StackMgr, |
1061 | | const uint32_t DataIdx) noexcept; |
1062 | | Expect<void> proxyMemCopy(Runtime::StackManager &StackMgr, |
1063 | | const uint32_t DstMemIdx, const uint32_t SrcMemIdx, |
1064 | | const uint64_t DstOff, const uint64_t SrcOff, |
1065 | | const uint64_t Len) noexcept; |
1066 | | Expect<void> proxyMemFill(Runtime::StackManager &StackMgr, |
1067 | | const uint32_t MemIdx, const uint64_t Off, |
1068 | | const uint8_t Val, const uint64_t Len) noexcept; |
1069 | | Expect<uint64_t> proxyMemAtomicNotify(Runtime::StackManager &StackMgr, |
1070 | | const uint32_t MemIdx, |
1071 | | const uint64_t Offset, |
1072 | | const uint64_t Count) noexcept; |
1073 | | Expect<uint64_t> |
1074 | | proxyMemAtomicWait(Runtime::StackManager &StackMgr, const uint32_t MemIdx, |
1075 | | const uint64_t Offset, const uint64_t Expected, |
1076 | | const int64_t Timeout, const uint32_t BitWidth) noexcept; |
1077 | | Expect<void *> proxyTableGetFuncSymbol(Runtime::StackManager &StackMgr, |
1078 | | const uint32_t TableIdx, |
1079 | | const uint32_t FuncTypeIdx, |
1080 | | const uint32_t FuncIdx) noexcept; |
1081 | | Expect<void *> proxyRefGetFuncSymbol(Runtime::StackManager &StackMgr, |
1082 | | const RefVariant Ref) noexcept; |
1083 | | Expect<void *> proxyFuncGetFuncSymbol(Runtime::StackManager &StackMgr, |
1084 | | const uint32_t FuncIdx) noexcept; |
1085 | | Expect<void> proxyThrow(Runtime::StackManager &StackMgr, |
1086 | | const uint32_t TagIdx, const ValVariant *Vals, |
1087 | | const uint32_t Num) noexcept; |
1088 | | Expect<void> proxyThrowRef(Runtime::StackManager &StackMgr, |
1089 | | const RefVariant Ref) noexcept; |
1090 | | Expect<void> proxyCatchPop(Runtime::StackManager &StackMgr, ValVariant *Out, |
1091 | | const uint32_t PopPayload, |
1092 | | const uint32_t NeedRef) noexcept; |
1093 | | /// @} |
1094 | | |
1095 | | /// Callbacks for compiled modules |
1096 | | static const Executable::IntrinsicsTable Intrinsics; |
1097 | | /// Proxy helper template struct |
1098 | | template <typename FuncPtr> struct ProxyHelper; |
1099 | | |
1100 | | private: |
1101 | | /// Execution context for compiled functions. |
1102 | | struct ExecutionContextStruct { |
1103 | | #if WASMEDGE_ALLOCATOR_IS_STABLE |
1104 | | uint8_t *const *Memories; |
1105 | | #else |
1106 | | uint8_t **const *Memories; |
1107 | | #endif |
1108 | | const uint64_t *const *MemorySizes; |
1109 | | RefVariant **const *TableRefs; |
1110 | | const uint64_t *const *TableSizes; |
1111 | | ValVariant *const *Globals; |
1112 | | void *const *Tags; |
1113 | | void *const *PendingExnTagAddr; |
1114 | | std::atomic_uint64_t *InstrCount; |
1115 | | uint64_t *CostTable; |
1116 | | std::atomic_uint64_t *Gas; |
1117 | | uint64_t GasLimit; |
1118 | | std::atomic_uint32_t *StopToken; |
1119 | | const void *ModuleInst; |
1120 | | }; |
1121 | | |
1122 | | /// Restores thread local VM reference after overwriting it. |
1123 | | struct SavedThreadLocal { |
1124 | | SavedThreadLocal(Executor &Ex, Runtime::StackManager &StackMgr, |
1125 | | const Runtime::Instance::FunctionInstance &Func) noexcept; |
1126 | | |
1127 | | SavedThreadLocal(const SavedThreadLocal &) = delete; |
1128 | | SavedThreadLocal(SavedThreadLocal &&) = delete; |
1129 | | |
1130 | | ~SavedThreadLocal() noexcept; |
1131 | | |
1132 | | Executor *SavedThis; |
1133 | | Runtime::StackManager *SavedCurrentStack; |
1134 | | ExecutionContextStruct SavedExecutionContext; |
1135 | | }; |
1136 | | |
1137 | | /// Pending exception passing across the compiled and the native-entered |
1138 | | /// frames. A non-null tag means pending; the exception instance keeps the |
1139 | | /// throw_ref identity. |
1140 | | struct PendingExnStruct { |
1141 | | /// Tag instance of the pending exception. Null when no exception is |
1142 | | /// pending. |
1143 | | Runtime::Instance::TagInstance *TagInst = nullptr; |
1144 | | /// Exception instance identity. Null for a fresh throw; set on rethrowing |
1145 | | /// to keep the exnref identity. |
1146 | | const Runtime::Instance::ExceptionInstance *Inst = nullptr; |
1147 | | |
1148 | | /// Getter and setter of the payload values. |
1149 | 0 | const std::vector<ValVariant> &getPayload() const noexcept { |
1150 | 0 | return Payload; |
1151 | 0 | } |
1152 | 0 | void setPayload(Span<const ValVariant> Vals) noexcept { |
1153 | 0 | Payload.assign(Vals.begin(), Vals.end()); |
1154 | 0 | } |
1155 | | |
1156 | | private: |
1157 | | std::vector<ValVariant> Payload; |
1158 | | }; |
1159 | | |
1160 | | /// Pointer to current object. |
1161 | | static thread_local Executor *This; |
1162 | | /// Stack passed into compiled functions |
1163 | | static thread_local Runtime::StackManager *CurrentStack; |
1164 | | /// Execution context for compiled functions |
1165 | | static thread_local ExecutionContextStruct ExecutionContext; |
1166 | | /// Pending exception for compiled functions |
1167 | | static thread_local PendingExnStruct PendingExn; |
1168 | | /// Record stack trace on error |
1169 | | static thread_local std::array<uint32_t, 256> StackTrace; |
1170 | | static thread_local size_t StackTraceSize; |
1171 | | |
1172 | | /// WasmEdge configuration |
1173 | | const Configure Conf; |
1174 | | /// Executor statistics |
1175 | | Statistics::Statistics *Stat; |
1176 | | /// Stop execution |
1177 | | std::atomic_uint32_t StopToken = 0; |
1178 | | /// Memory instance this Executor is currently waiting on (for stop()). |
1179 | | std::atomic<Runtime::Instance::MemoryInstance *> WaitingMemory = nullptr; |
1180 | | /// Executor Host Function Handler |
1181 | | HostFuncHandler HostFuncHelper = {}; |
1182 | | /// Callback for lazy function compilation |
1183 | | std::function<Expect<void>(const Runtime::Instance::FunctionInstance *)> |
1184 | | LazyCompilationHandler; |
1185 | | |
1186 | | /// Helper function for triggering lazy compilation. |
1187 | | /// XXX: Calling checkLazyCompilation in one thread while another thread calls |
1188 | | /// unsafeUpgradeToCompiled on the same FuncInst could result in a race |
1189 | | /// condition if checking FuncInst->isCompiledFunction() directly here. As a |
1190 | | /// temporary workaround, checks for compilation state are deferred to the |
1191 | | /// LazyCompilationHandler, which must serialize them against compiled-state |
1192 | | /// upgrades (the lazy JIT engine does so under its internal lock). |
1193 | | Expect<void> checkLazyCompilation( |
1194 | 0 | const Runtime::Instance::FunctionInstance *FuncInst) const noexcept { |
1195 | 0 | if (unlikely(LazyCompilationHandler != nullptr)) { |
1196 | 0 | return LazyCompilationHandler(FuncInst); |
1197 | 0 | } |
1198 | 0 | return {}; |
1199 | 0 | } |
1200 | | }; |
1201 | | |
1202 | | } // namespace Executor |
1203 | | } // namespace WasmEdge |
1204 | | |
1205 | | #include "engine/atomic.ipp" |
1206 | | #include "engine/binary_numeric.ipp" |
1207 | | #include "engine/cast_numeric.ipp" |
1208 | | #include "engine/memory.ipp" |
1209 | | #include "engine/relation_numeric.ipp" |
1210 | | #include "engine/unary_numeric.ipp" |