Coverage Report

Created: 2026-11-12 06:35

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/WasmEdge/include/executor/executor.h
Line
Count
Source
1
// SPDX-License-Identifier: Apache-2.0
2
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4
//===-- wasmedge/executor/executor.h - Executor class definition ----------===//
5
//
6
// Part of the WasmEdge Project.
7
//
8
//===----------------------------------------------------------------------===//
9
///
10
/// \file
11
/// This file contains the declaration of the Executor class, which instantiates
12
/// and runs Wasm modules.
13
///
14
//===----------------------------------------------------------------------===//
15
#pragma once
16
17
#include "ast/component/component.h"
18
#include "ast/module.h"
19
#include "common/async.h"
20
#include "common/configure.h"
21
#include "common/defines.h"
22
#include "common/errcode.h"
23
#include "common/statistics.h"
24
#include "common/types.h"
25
#include "runtime/callingframe.h"
26
#include "runtime/instance/component/component.h"
27
#include "runtime/instance/module.h"
28
#include "runtime/stackmgr.h"
29
#include "runtime/storemgr.h"
30
#include "system/stacktrace.h"
31
32
#include <atomic>
33
#include <csignal>
34
#include <cstddef>
35
#include <cstdint>
36
#include <functional>
37
#include <memory>
38
#include <mutex>
39
#include <optional>
40
#include <shared_mutex>
41
#include <string_view>
42
#include <type_traits>
43
#include <utility>
44
#include <vector>
45
46
namespace WasmEdge {
47
namespace Executor {
48
49
namespace {
50
51
// Template return type aliasing
52
/// Accept unsigned integer types. (uint32_t, uint64_t)
53
template <typename T>
54
using TypeU = typename std::enable_if_t<IsWasmUnsignV<T>, Expect<void>>;
55
/// Accept integer types. (uint32_t, int32_t, uint64_t, int64_t)
56
template <typename T>
57
using TypeI = typename std::enable_if_t<IsWasmIntV<T>, Expect<void>>;
58
/// Accept floating types. (float, double)
59
template <typename T>
60
using TypeF = typename std::enable_if_t<IsWasmFloatV<T>, Expect<void>>;
61
/// Accept all num types. (uint32_t, int32_t, uint64_t, int64_t, float, double)
62
template <typename T>
63
using TypeT = typename std::enable_if_t<IsWasmNumV<T>, Expect<void>>;
64
/// Accept Wasm built-in num types. (uint32_t, uint64_t, float, double)
65
template <typename T>
66
using TypeN = typename std::enable_if_t<IsWasmNativeNumV<T>, Expect<void>>;
67
68
/// Accept (unsigned integer types, unsigned integer types).
69
template <typename T1, typename T2>
70
using TypeUU = typename std::enable_if_t<IsWasmUnsignV<T1> && IsWasmUnsignV<T2>,
71
                                         Expect<void>>;
72
/// Accept (integer types, unsigned integer types).
73
template <typename T1, typename T2>
74
using TypeIU = typename std::enable_if_t<IsWasmIntV<T1> && IsWasmUnsignV<T2>,
75
                                         Expect<void>>;
76
/// Accept (floating types, floating types).
77
template <typename T1, typename T2>
78
using TypeFF = typename std::enable_if_t<IsWasmFloatV<T1> && IsWasmFloatV<T2>,
79
                                         Expect<void>>;
80
/// Accept (integer types, floating types).
81
template <typename T1, typename T2>
82
using TypeIF =
83
    typename std::enable_if_t<IsWasmIntV<T1> && IsWasmFloatV<T2>, Expect<void>>;
84
/// Accept (floating types, integer types).
85
template <typename T1, typename T2>
86
using TypeFI =
87
    typename std::enable_if_t<IsWasmFloatV<T1> && IsWasmIntV<T2>, Expect<void>>;
88
/// Accept (Wasm built-in num types, Wasm built-in num types).
89
template <typename T1, typename T2>
90
using TypeNN =
91
    typename std::enable_if_t<IsWasmNativeNumV<T1> && IsWasmNativeNumV<T2> &&
92
                                  sizeof(T1) == sizeof(T2),
93
                              Expect<void>>;
94
95
} // namespace
96
97
/// Helper class for handling the pre- and post-host functions.
98
class HostFuncHandler {
99
public:
100
0
  void setPreHost(void *HostData, std::function<void(void *)> HostFunc) {
101
0
    std::unique_lock Lock(Mutex);
102
0
    PreHostData = HostData;
103
0
    PreHostFunc = HostFunc;
104
0
  }
105
0
  void setPostHost(void *HostData, std::function<void(void *)> HostFunc) {
106
0
    std::unique_lock Lock(Mutex);
107
0
    PostHostData = HostData;
108
0
    PostHostFunc = HostFunc;
109
0
  }
110
0
  void invokePreHostFunc() {
111
0
    std::function<void(void *)> FuncSnapshot;
112
0
    void *DataSnapshot = nullptr;
113
0
    {
114
0
      std::shared_lock Lock(Mutex);
115
0
      FuncSnapshot = PreHostFunc;
116
0
      DataSnapshot = PreHostData;
117
0
    }
118
0
    if (FuncSnapshot.operator bool()) {
119
0
      FuncSnapshot(DataSnapshot);
120
0
    }
121
0
  }
122
0
  void invokePostHostFunc() {
123
0
    std::function<void(void *)> FuncSnapshot;
124
0
    void *DataSnapshot = nullptr;
125
0
    {
126
0
      std::shared_lock Lock(Mutex);
127
0
      FuncSnapshot = PostHostFunc;
128
0
      DataSnapshot = PostHostData;
129
0
    }
130
0
    if (FuncSnapshot.operator bool()) {
131
0
      FuncSnapshot(DataSnapshot);
132
0
    }
133
0
  }
134
135
private:
136
  void *PreHostData = nullptr;
137
  void *PostHostData = nullptr;
138
  std::function<void(void *)> PreHostFunc = {};
139
  std::function<void(void *)> PostHostFunc = {};
140
  mutable std::shared_mutex Mutex;
141
};
142
143
/// Executor flow control class.
144
class Executor {
145
public:
146
  Executor(const Configure &Conf, Statistics::Statistics *S = nullptr) noexcept
147
0
      : Conf(Conf) {
148
0
    if (Conf.getStatisticsConfigure().isInstructionCounting() ||
149
0
        Conf.getStatisticsConfigure().isCostMeasuring() ||
150
0
        Conf.getStatisticsConfigure().isTimeMeasuring()) {
151
0
      Stat = S;
152
0
    } else {
153
0
      Stat = nullptr;
154
0
    }
155
0
    if (Stat) {
156
0
      Stat->setCostLimit(Conf.getStatisticsConfigure().getCostLimit());
157
0
    }
158
0
  }
159
160
  /// Getter for configuration.
161
0
  const Configure &getConfigure() const { return Conf; }
162
163
  /// Instantiate a WASM Module as an anonymous module instance.
164
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
165
  instantiateModule(Runtime::StoreManager &StoreMgr, const AST::Module &Mod);
166
167
  /// Instantiate and register a WASM module as a named module instance.
168
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
169
  registerModule(Runtime::StoreManager &StoreMgr, const AST::Module &Mod,
170
                 std::string_view Name);
171
172
  /// Register an instantiated module as a named module instance.
173
  Expect<void> registerModule(Runtime::StoreManager &StoreMgr,
174
                              const Runtime::Instance::ModuleInstance &ModInst);
175
176
  /// Register an instantiated module under the given alias name.
177
  Expect<void> registerModule(Runtime::StoreManager &StoreMgr,
178
                              const Runtime::Instance::ModuleInstance &ModInst,
179
                              std::string_view Name);
180
181
  /// Instantiate a Component as an anonymous component instance.
182
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
183
  instantiateComponent(Runtime::StoreManager &StoreMgr,
184
                       const AST::Component::Component &Comp);
185
186
  /// Instantiate and register a Component as a named component instance.
187
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
188
  registerComponent(Runtime::StoreManager &StoreMgr,
189
                    const AST::Component::Component &Comp,
190
                    std::string_view Name);
191
192
  /// Register an instantiated component into a named component instance.
193
  Expect<void>
194
  registerComponent(Runtime::StoreManager &StoreMgr,
195
                    const Runtime::Instance::ComponentInstance &CompInst);
196
197
  /// Register a host function which will be invoked before calling a
198
  /// host function.
199
  Expect<void> registerPreHostFunction(void *HostData,
200
                                       std::function<void(void *)> HostFunc);
201
202
  /// Register a host function which will be invoked after calling a
203
  /// host function.
204
  Expect<void> registerPostHostFunction(void *HostData,
205
                                        std::function<void(void *)> HostFunc);
206
207
  /// Register a callback for lazy function compilation
208
  void registerLazyCompilationCallback(
209
      std::function<Expect<void>(const Runtime::Instance::FunctionInstance *)>
210
0
          Callback) {
211
0
    LazyCompilationHandler = std::move(Callback);
212
0
  }
213
214
  /// Invoke a WASM function by function instance.
215
  Expect<std::vector<std::pair<ValVariant, ValType>>>
216
  invoke(const Runtime::Instance::FunctionInstance *FuncInst,
217
         Span<const ValVariant> Params, Span<const ValType> ParamTypes);
218
219
  /// Invoke a Component function by function instance.
220
  Expect<std::vector<std::pair<ComponentValVariant, ComponentValType>>>
221
  invoke(const Runtime::Instance::Component::FunctionInstance *FuncInst,
222
         Span<const ComponentValVariant> Params,
223
         Span<const ComponentValType> ParamTypes);
224
225
  /// Module-qualified stack trace of the most recent failed invocation on this
226
  /// thread. Valid right after an invocation returns an error, else stale.
227
0
  static Span<const StackTraceEntry> getRecordedStackTrace() noexcept {
228
0
    return Span<const StackTraceEntry>{StackTrace}.first(StackTraceSize);
229
0
  }
230
231
  /// Asynchronous invoke a WASM function by function instance.
232
  Async<Expect<std::vector<std::pair<ValVariant, ValType>>>>
233
  asyncInvoke(const Runtime::Instance::FunctionInstance *FuncInst,
234
              Span<const ValVariant> Params, Span<const ValType> ParamTypes);
235
236
  /// Stop execution
237
0
  void stop() noexcept {
238
0
    StopToken.store(1, std::memory_order_relaxed);
239
0
    atomicNotifyAll();
240
0
  }
241
242
private:
243
  /// Run Wasm bytecode expression for initialization.
244
  Expect<void> runExpression(Runtime::StackManager &StackMgr,
245
                             AST::InstrView Instrs);
246
247
  /// Run Wasm function.
248
  Expect<void> runFunction(Runtime::StackManager &StackMgr,
249
                           const Runtime::Instance::FunctionInstance &Func,
250
                           Span<const ValVariant> Params);
251
252
  /// Execute instructions.
253
  Expect<void> execute(Runtime::StackManager &StackMgr,
254
                       const AST::InstrView::iterator Start,
255
                       const AST::InstrView::iterator End);
256
257
  /// \name Functions for instantiation.
258
  /// @{
259
  /// Instantiation of Module Instance.
260
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
261
  instantiate(Runtime::StoreManager &StoreMgr, const AST::Module &Mod,
262
              std::optional<std::string_view> Name = std::nullopt);
263
264
  /// Instantiation of Imports.
265
  Expect<void> instantiate(
266
      std::function<const Runtime::Instance::ModuleInstance *(std::string_view)>
267
          ModuleFinder,
268
      Runtime::Instance::ModuleInstance &ModInst,
269
      const AST::ImportSection &ImportSec);
270
271
  /// Instantiation of Function Instances.
272
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
273
                           const AST::FunctionSection &FuncSec,
274
                           const AST::CodeSection &CodeSec);
275
276
  /// Instantiation of Table Instances.
277
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
278
                           Runtime::Instance::ModuleInstance &ModInst,
279
                           const AST::TableSection &TabSec);
280
281
  /// Instantiation of Memory Instances.
282
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
283
                           const AST::MemorySection &MemSec);
284
285
  /// Instantiateion of Tag Instances.
286
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
287
                           const AST::TagSection &TagSec);
288
289
  /// Instantiation of Global Instances.
290
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
291
                           Runtime::Instance::ModuleInstance &ModInst,
292
                           const AST::GlobalSection &GlobSec);
293
294
  /// Instantiation of Element Instances.
295
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
296
                           Runtime::Instance::ModuleInstance &ModInst,
297
                           const AST::ElementSection &ElemSec);
298
299
  /// Initialize table with Element Instances.
300
  Expect<void> initTable(Runtime::StackManager &StackMgr,
301
                         const AST::ElementSection &ElemSec);
302
303
  /// Instantiation of Data Instances.
304
  Expect<void> instantiate(Runtime::StackManager &StackMgr,
305
                           Runtime::Instance::ModuleInstance &ModInst,
306
                           const AST::DataSection &DataSec);
307
308
  /// Initialize memory with Data Instances.
309
  Expect<void> initMemory(Runtime::StackManager &StackMgr,
310
                          const AST::DataSection &DataSec);
311
312
  /// Instantiation of Exports.
313
  Expect<void> instantiate(Runtime::Instance::ModuleInstance &ModInst,
314
                           const AST::ExportSection &ExportSec);
315
  /// @}
316
317
  /// \name Functions for instantiation of component model.
318
  /// @{
319
  /// Instantiation of Component Instance.
320
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
321
  instantiate(Runtime::StoreManager &StoreMgr,
322
              const AST::Component::Component &Comp,
323
              std::optional<std::string_view> Name = std::nullopt);
324
325
  /// Instantiation of Child Component Instance.
326
  Expect<std::unique_ptr<Runtime::Instance::ComponentInstance>>
327
  instantiate(Runtime::Instance::ComponentImportManager &ImportMgr,
328
              const AST::Component::Component &Comp);
329
330
  /// Instantiation of Child Core Module Instance.
331
  Expect<std::unique_ptr<Runtime::Instance::ModuleInstance>>
332
  instantiate(Runtime::Instance::ComponentImportManager &ImportMgr,
333
              const AST::Module &Mod);
334
335
  /// Instantiation of Core Module Section.
336
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
337
                           const AST::Component::CoreModuleSection &CoreModSec);
338
339
  /// Instantiation of Core Instance Section.
340
  Expect<void>
341
  instantiate(Runtime::Instance::ComponentInstance &CompInst,
342
              const AST::Component::CoreInstanceSection &CoreInstSec);
343
344
  /// Instantiation of Core Type Section.
345
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
346
                           const AST::Component::CoreTypeSection &CoreTypeSec);
347
348
  /// Instantiation of Component Section.
349
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
350
                           const AST::Component::ComponentSection &CompSec);
351
352
  /// Instantiation of Instance Section.
353
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
354
                           const AST::Component::InstanceSection &InstSec);
355
356
  /// Instantiation of Alias Section.
357
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
358
                           const AST::Component::AliasSection &AliasSec);
359
360
  /// Instantiation of Type Section.
361
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
362
                           const AST::Component::TypeSection &TypeSec);
363
364
  /// Instantiation of Canonical Section.
365
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
366
                           const AST::Component::CanonSection &CanonSec);
367
368
  /// Instantiation of Start Section.
369
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
370
                           const AST::Component::StartSection &StartSec);
371
372
  /// Instantiation of Import Section.
373
  Expect<void> instantiate(Runtime::StoreManager &StoreMgr,
374
                           Runtime::Instance::ComponentInstance &CompInst,
375
                           const AST::Component::ImportSection &ImportSec);
376
377
  /// Instantiation of Import Section in the child component instance.
378
  Expect<void> instantiate(Runtime::Instance::ComponentImportManager &ImportMgr,
379
                           Runtime::Instance::ComponentInstance &CompInst,
380
                           const AST::Component::ImportSection &ImportSec);
381
382
  /// Instantiation of Export Section.
383
  Expect<void> instantiate(Runtime::Instance::ComponentInstance &CompInst,
384
                           const AST::Component::ExportSection &ExportSec);
385
  /// @}
386
387
  /// \name Helper Functions for canonical ABI
388
  /// @{
389
  Expect<std::vector<ValVariant>>
390
  convValsToCoreWASM(Span<const ComponentValVariant> Vals,
391
                     Span<const ComponentValType> ValTypes,
392
                     Runtime::Instance::FunctionInstance *RFuncInst,
393
                     Runtime::Instance::MemoryInstance *MemInst,
394
                     const Runtime::Instance::ComponentInstance *CompInst,
395
                     StringEncoding Enc = StringEncoding::UTF8);
396
397
  Expect<std::vector<std::pair<ComponentValVariant, ComponentValType>>>
398
  convValsToComponent(Span<const std::pair<ValVariant, ValType>> CoreVals,
399
                      Span<const ComponentValType> ValTypes,
400
                      Runtime::Instance::MemoryInstance *MemInst,
401
                      const Runtime::Instance::ComponentInstance *CompInst,
402
                      StringEncoding Enc = StringEncoding::UTF8);
403
  /// @}
404
405
  /// \name Helper Functions for block controls.
406
  /// @{
407
  /// Helper function for calling functions. Return the continuation iterator.
408
  /// Set `IsNativeEntry` when entering from the native code.
409
  Expect<AST::InstrView::iterator> enterFunction(
410
      Runtime::StackManager &StackMgr,
411
      const Runtime::Instance::FunctionInstance &Func,
412
      const AST::InstrView::iterator RetIt, bool IsTailCall = false,
413
      bool IsNativeEntry = false,
414
      const Runtime::Instance::ModuleInstance *CallerModInst = nullptr);
415
416
  /// Live module instances whose compiled code may be on the native stack at
417
  /// fault time: the current frame-stack modules plus every instance in a store
418
  /// reachable through their linked-module back-links. Rebuilt on each fault.
419
  std::vector<const Runtime::Instance::ModuleInstance *>
420
  collectLiveModules(const Runtime::StackManager &StackMgr) const noexcept;
421
422
  /// Helper function for branching to label.
423
  Expect<void> branchToLabel(Runtime::StackManager &StackMgr,
424
                             const AST::Instruction::JumpDescriptor &JumpDesc,
425
                             AST::InstrView::iterator &PC) noexcept;
426
427
  /// Helper function for throwing an exception. Pass `ExnInst` on `throw_ref`
428
  /// to reuse the exception instance and preserve exnref identity.
429
  Expect<void> throwException(
430
      Runtime::StackManager &StackMgr, Runtime::Instance::TagInstance &TagInst,
431
      AST::InstrView::iterator &PC,
432
      const Runtime::Instance::ExceptionInstance *ExnInst = nullptr) noexcept;
433
  /// @}
434
435
  /// \name Helper Function for checking memory offset boundary.
436
  /// @{
437
  Expect<void>
438
  checkOffsetOverflow(const Runtime::Instance::MemoryInstance &MemInst,
439
                      const AST::Instruction &Instr, const uint64_t Val,
440
                      const uint64_t Size) const noexcept;
441
  /// @}
442
443
  /// \name Helper Functions for GC instructions.
444
  /// @{
445
  Expect<RefVariant> structNew(const Runtime::Instance::ModuleInstance *ModInst,
446
                               const uint32_t TypeIdx,
447
                               Span<const ValVariant> Args = {}) const noexcept;
448
  Expect<ValVariant> structGet(const Runtime::Instance::ModuleInstance *ModInst,
449
                               const RefVariant Ref, const uint32_t TypeIdx,
450
                               const uint32_t Off,
451
                               const bool IsSigned = false) const noexcept;
452
  Expect<void> structSet(const Runtime::Instance::ModuleInstance *ModInst,
453
                         const RefVariant Ref, const ValVariant Val,
454
                         const uint32_t TypeIdx,
455
                         const uint32_t Off) const noexcept;
456
  Expect<RefVariant> arrayNew(const Runtime::Instance::ModuleInstance *ModInst,
457
                              const uint32_t TypeIdx, const uint32_t Length,
458
                              Span<const ValVariant> Args = {}) const noexcept;
459
  Expect<RefVariant>
460
  arrayNewData(const Runtime::Instance::ModuleInstance *ModInst,
461
               const uint32_t TypeIdx, const uint32_t DataIdx,
462
               const uint32_t Start, const uint32_t Length) const noexcept;
463
  Expect<RefVariant>
464
  arrayNewElem(const Runtime::Instance::ModuleInstance *ModInst,
465
               const uint32_t TypeIdx, const uint32_t ElemIdx,
466
               const uint32_t Start, const uint32_t Length) const noexcept;
467
  Expect<ValVariant> arrayGet(const Runtime::Instance::ModuleInstance *ModInst,
468
                              const RefVariant &Ref, const uint32_t TypeIdx,
469
                              const uint32_t Idx,
470
                              const bool IsSigned = false) const noexcept;
471
  Expect<void> arraySet(const Runtime::Instance::ModuleInstance *ModInst,
472
                        const RefVariant &Ref, const ValVariant &Val,
473
                        const uint32_t TypeIdx,
474
                        const uint32_t Idx) const noexcept;
475
  Expect<void> arrayFill(const Runtime::Instance::ModuleInstance *ModInst,
476
                         const RefVariant &Ref, const ValVariant &Val,
477
                         const uint32_t TypeIdx, const uint32_t Idx,
478
                         const uint32_t Cnt) const noexcept;
479
  Expect<void> arrayInitData(const Runtime::Instance::ModuleInstance *ModInst,
480
                             const RefVariant &Ref, const uint32_t TypeIdx,
481
                             const uint32_t DataIdx, const uint32_t DstIdx,
482
                             const uint32_t SrcIdx,
483
                             const uint32_t Cnt) const noexcept;
484
  Expect<void> arrayInitElem(const Runtime::Instance::ModuleInstance *ModInst,
485
                             const RefVariant &Ref, const uint32_t TypeIdx,
486
                             const uint32_t ElemIdx, const uint32_t DstIdx,
487
                             const uint32_t SrcIdx,
488
                             const uint32_t Cnt) const noexcept;
489
  Expect<void> arrayCopy(const Runtime::Instance::ModuleInstance *ModInst,
490
                         const RefVariant &DstRef, const uint32_t DstTypeIdx,
491
                         const uint32_t DstIdx, const RefVariant &SrcRef,
492
                         const uint32_t SrcTypeIdx, const uint32_t SrcIdx,
493
                         const uint32_t Cnt) const noexcept;
494
  /// @}
495
496
  /// \name Helper Functions for atomic operations.
497
  /// @{
498
  template <typename T>
499
  Expect<uint64_t> atomicWait(Runtime::Instance::MemoryInstance &MemInst,
500
                              uint64_t Address, EndianValue<T> Expected,
501
                              int64_t Timeout) noexcept;
502
  Expect<uint64_t> atomicNotify(Runtime::Instance::MemoryInstance &MemInst,
503
                                uint64_t Address, uint64_t Count) noexcept;
504
  void atomicNotifyAll() noexcept;
505
  /// @}
506
507
  /// \name Helper Functions for getting instances or types.
508
  /// @{
509
  /// Helper function for getting defined type by index.
510
  const AST::SubType *
511
  getDefTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
512
                  const uint32_t Idx) const;
513
514
  /// Helper function for getting composite type by index. Assuming validated.
515
  const WasmEdge::AST::CompositeType &
516
  getCompositeTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
517
                        const uint32_t Idx) const noexcept;
518
519
  /// Helper function for getting struct storage type by index.
520
  const ValType &
521
  getStructStorageTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
522
                            const uint32_t Idx,
523
                            const uint32_t Off) const noexcept;
524
525
  /// Helper function for getting array storage type by index.
526
  const ValType &
527
  getArrayStorageTypeByIdx(const Runtime::Instance::ModuleInstance *ModInst,
528
                           const uint32_t Idx) const noexcept;
529
530
  /// Helper function for getting function instance by index.
531
  Runtime::Instance::FunctionInstance *
532
  getFuncInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
533
                   const uint32_t Idx) const;
534
535
  /// Helper function for getting table instance by index.
536
  Runtime::Instance::TableInstance *
537
  getTabInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
538
                  const uint32_t Idx) const;
539
540
  /// Helper function for getting memory instance by index.
541
  Runtime::Instance::MemoryInstance *
542
  getMemInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
543
                  const uint32_t Idx) const;
544
545
  /// Helper function for getting tag instance by index.
546
  Runtime::Instance::TagInstance *
547
  getTagInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
548
                  const uint32_t Idx) const;
549
550
  /// Helper function for getting global instance by index.
551
  Runtime::Instance::GlobalInstance *
552
  getGlobInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
553
                   const uint32_t Idx) const;
554
555
  /// Helper function for getting element instance by index.
556
  Runtime::Instance::ElementInstance *
557
  getElemInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
558
                   const uint32_t Idx) const;
559
560
  /// Helper function for getting data instance by index.
561
  Runtime::Instance::DataInstance *
562
  getDataInstByIdx(const Runtime::Instance::ModuleInstance *ModInst,
563
                   const uint32_t Idx) const;
564
565
  /// Helper function for converting into bottom abstract heap type.
566
  TypeCode toBottomType(const Runtime::Instance::ModuleInstance *ModInst,
567
                        const ValType &Type) const;
568
569
  /// Helper function for matching a reference against a reference type.
570
  bool matchRef(const Runtime::Instance::ModuleInstance *ModInst,
571
                const ValType &Exp, const RefVariant &Ref) const noexcept;
572
573
  /// Helper function for cleaning unused bits of numeric values in ValVariant.
574
  void cleanNumericVal(ValVariant &Val, const ValType &Type) const noexcept;
575
576
  /// Helper function for packing ValVariant for packed type.
577
  ValVariant packVal(const ValType &Type, const ValVariant &Val) const noexcept;
578
579
  /// Helper function for packing ValVariant vector for packed type.
580
  std::vector<ValVariant>
581
  packVals(const ValType &Type, std::vector<ValVariant> &&Vals) const noexcept;
582
583
  /// Helper function for unpacking ValVariant for packed type.
584
  ValVariant unpackVal(const ValType &Type, const ValVariant &Val,
585
                       bool IsSigned = false) const noexcept;
586
  /// @}
587
588
  /// \name Interpreter - Run instructions functions
589
  /// @{
590
  /// ======= Control instructions =======
591
  Expect<void> runIfElseOp(Runtime::StackManager &StackMgr,
592
                           const AST::Instruction &Instr,
593
                           AST::InstrView::iterator &PC) noexcept;
594
  Expect<void> runThrowOp(Runtime::StackManager &StackMgr,
595
                          const AST::Instruction &Instr,
596
                          AST::InstrView::iterator &PC) noexcept;
597
  Expect<void> runThrowRefOp(Runtime::StackManager &StackMgr,
598
                             const AST::Instruction &Instr,
599
                             AST::InstrView::iterator &PC) noexcept;
600
  Expect<void> runBrOp(Runtime::StackManager &StackMgr,
601
                       const AST::Instruction &Instr,
602
                       AST::InstrView::iterator &PC) noexcept;
603
  Expect<void> runBrIfOp(Runtime::StackManager &StackMgr,
604
                         const AST::Instruction &Instr,
605
                         AST::InstrView::iterator &PC) noexcept;
606
  Expect<void> runBrOnNullOp(Runtime::StackManager &StackMgr,
607
                             const AST::Instruction &Instr,
608
                             AST::InstrView::iterator &PC) noexcept;
609
  Expect<void> runBrOnNonNullOp(Runtime::StackManager &StackMgr,
610
                                const AST::Instruction &Instr,
611
                                AST::InstrView::iterator &PC) noexcept;
612
  Expect<void> runBrTableOp(Runtime::StackManager &StackMgr,
613
                            const AST::Instruction &Instr,
614
                            AST::InstrView::iterator &PC) noexcept;
615
  Expect<void> runBrOnCastOp(Runtime::StackManager &StackMgr,
616
                             const AST::Instruction &Instr,
617
                             AST::InstrView::iterator &PC,
618
                             bool IsReverse = false) noexcept;
619
  Expect<void> runReturnOp(Runtime::StackManager &StackMgr,
620
                           AST::InstrView::iterator &PC) noexcept;
621
  Expect<void> runCallOp(Runtime::StackManager &StackMgr,
622
                         const AST::Instruction &Instr,
623
                         AST::InstrView::iterator &PC,
624
                         bool IsTailCall = false) noexcept;
625
  Expect<void> runCallRefOp(Runtime::StackManager &StackMgr,
626
                            const AST::Instruction &Instr,
627
                            AST::InstrView::iterator &PC,
628
                            bool IsTailCall = false) noexcept;
629
  Expect<void> runCallIndirectOp(Runtime::StackManager &StackMgr,
630
                                 const AST::Instruction &Instr,
631
                                 AST::InstrView::iterator &PC,
632
                                 bool IsTailCall = false) noexcept;
633
  Expect<void> runTryTableOp(Runtime::StackManager &StackMgr,
634
                             const AST::Instruction &Instr,
635
                             AST::InstrView::iterator &PC) noexcept;
636
  /// ======= Variable instructions =======
637
  Expect<void> runLocalGetOp(Runtime::StackManager &StackMgr,
638
                             uint32_t StackOffset) const noexcept;
639
  Expect<void> runLocalSetOp(Runtime::StackManager &StackMgr,
640
                             uint32_t StackOffset) const noexcept;
641
  Expect<void> runLocalTeeOp(Runtime::StackManager &StackMgr,
642
                             uint32_t StackOffset) const noexcept;
643
  Expect<void> runGlobalGetOp(Runtime::StackManager &StackMgr,
644
                              uint32_t Idx) const noexcept;
645
  Expect<void> runGlobalSetOp(Runtime::StackManager &StackMgr,
646
                              uint32_t Idx) const noexcept;
647
  /// ======= Reference instructions =======
648
  Expect<void> runRefNullOp(Runtime::StackManager &StackMgr,
649
                            const ValType &Type) const noexcept;
650
  Expect<void> runRefIsNullOp(ValVariant &Val) const noexcept;
651
  Expect<void> runRefFuncOp(Runtime::StackManager &StackMgr,
652
                            uint32_t Idx) const noexcept;
653
  Expect<void> runRefEqOp(ValVariant &Val1,
654
                          const ValVariant &Val2) const noexcept;
655
  Expect<void> runRefAsNonNullOp(RefVariant &Val,
656
                                 const AST::Instruction &Instr) const noexcept;
657
  Expect<void> runStructNewOp(Runtime::StackManager &StackMgr,
658
                              const uint32_t TypeIdx,
659
                              const bool IsDefault = false) const noexcept;
660
  Expect<void> runStructGetOp(Runtime::StackManager &StackMgr,
661
                              const uint32_t TypeIdx, const uint32_t Off,
662
                              const AST::Instruction &Instr,
663
                              const bool IsSigned = false) const noexcept;
664
  Expect<void> runStructSetOp(Runtime::StackManager &StackMgr,
665
                              const ValVariant &Val, const uint32_t TypeIdx,
666
                              const uint32_t Off,
667
                              const AST::Instruction &Instr) const noexcept;
668
  Expect<void> runArrayNewOp(Runtime::StackManager &StackMgr,
669
                             const uint32_t TypeIdx, const uint32_t InitCnt,
670
                             uint32_t Length) const noexcept;
671
  Expect<void> runArrayNewDataOp(Runtime::StackManager &StackMgr,
672
                                 const uint32_t TypeIdx, const uint32_t DataIdx,
673
                                 const AST::Instruction &Instr) const noexcept;
674
  Expect<void> runArrayNewElemOp(Runtime::StackManager &StackMgr,
675
                                 const uint32_t TypeIdx, const uint32_t ElemIdx,
676
                                 const AST::Instruction &Instr) const noexcept;
677
  Expect<void> runArrayGetOp(Runtime::StackManager &StackMgr,
678
                             const uint32_t TypeIdx,
679
                             const AST::Instruction &Instr,
680
                             const bool IsSigned = false) const noexcept;
681
  Expect<void> runArraySetOp(Runtime::StackManager &StackMgr,
682
                             const ValVariant &Val, const uint32_t TypeIdx,
683
                             const AST::Instruction &Instr) const noexcept;
684
  Expect<void> runArrayLenOp(ValVariant &Val,
685
                             const AST::Instruction &Instr) const noexcept;
686
  Expect<void> runArrayFillOp(Runtime::StackManager &StackMgr,
687
                              const uint32_t Cnt, const ValVariant &Val,
688
                              const uint32_t TypeIdx,
689
                              const AST::Instruction &Instr) const noexcept;
690
  Expect<void> runArrayCopyOp(Runtime::StackManager &StackMgr,
691
                              const uint32_t Cnt, const uint32_t DstTypeIdx,
692
                              const uint32_t SrcTypeIdx,
693
                              const AST::Instruction &Instr) const noexcept;
694
  Expect<void> runArrayInitDataOp(Runtime::StackManager &StackMgr,
695
                                  const uint32_t Cnt, const uint32_t TypeIdx,
696
                                  const uint32_t DataIdx,
697
                                  const AST::Instruction &Instr) const noexcept;
698
  Expect<void> runArrayInitElemOp(Runtime::StackManager &StackMgr,
699
                                  const uint32_t Cnt, const uint32_t TypeIdx,
700
                                  const uint32_t ElemIdx,
701
                                  const AST::Instruction &Instr) const noexcept;
702
  Expect<void> runRefTestOp(const Runtime::Instance::ModuleInstance *ModInst,
703
                            ValVariant &Val, const AST::Instruction &Instr,
704
                            const bool IsCast = false) const noexcept;
705
  Expect<void> runRefConvOp(RefVariant &Val, TypeCode TCode) const noexcept;
706
  Expect<void> runRefI31Op(ValVariant &Val) const noexcept;
707
  Expect<void> runI31GetOp(ValVariant &Val, const AST::Instruction &Instr,
708
                           const bool IsSigned = false) const noexcept;
709
  /// ======= Table instructions =======
710
  Expect<void> runTableGetOp(Runtime::StackManager &StackMgr,
711
                             Runtime::Instance::TableInstance &TabInst,
712
                             const AST::Instruction &Instr);
713
  Expect<void> runTableSetOp(Runtime::StackManager &StackMgr,
714
                             Runtime::Instance::TableInstance &TabInst,
715
                             const AST::Instruction &Instr);
716
  Expect<void> runTableInitOp(Runtime::StackManager &StackMgr,
717
                              Runtime::Instance::TableInstance &TabInst,
718
                              Runtime::Instance::ElementInstance &ElemInst,
719
                              const AST::Instruction &Instr);
720
  Expect<void> runElemDropOp(Runtime::Instance::ElementInstance &ElemInst);
721
  Expect<void> runTableCopyOp(Runtime::StackManager &StackMgr,
722
                              Runtime::Instance::TableInstance &TabInstDst,
723
                              Runtime::Instance::TableInstance &TabInstSrc,
724
                              const AST::Instruction &Instr);
725
  Expect<void> runTableGrowOp(Runtime::StackManager &StackMgr,
726
                              Runtime::Instance::TableInstance &TabInst);
727
  Expect<void> runTableSizeOp(Runtime::StackManager &StackMgr,
728
                              Runtime::Instance::TableInstance &TabInst);
729
  Expect<void> runTableFillOp(Runtime::StackManager &StackMgr,
730
                              Runtime::Instance::TableInstance &TabInst,
731
                              const AST::Instruction &Instr);
732
  /// ======= Memory instructions =======
733
  template <typename T, uint32_t BitWidth = sizeof(T) * 8>
734
  TypeT<T> runLoadOp(Runtime::StackManager &StackMgr,
735
                     Runtime::Instance::MemoryInstance &MemInst,
736
                     const AST::Instruction &Instr);
737
  template <typename T, uint32_t BitWidth = sizeof(T) * 8>
738
  TypeN<T> runStoreOp(Runtime::StackManager &StackMgr,
739
                      Runtime::Instance::MemoryInstance &MemInst,
740
                      const AST::Instruction &Instr);
741
  Expect<void> runMemorySizeOp(Runtime::StackManager &StackMgr,
742
                               Runtime::Instance::MemoryInstance &MemInst);
743
  Expect<void> runMemoryGrowOp(Runtime::StackManager &StackMgr,
744
                               Runtime::Instance::MemoryInstance &MemInst);
745
  Expect<void> runMemoryInitOp(Runtime::StackManager &StackMgr,
746
                               Runtime::Instance::MemoryInstance &MemInst,
747
                               Runtime::Instance::DataInstance &DataInst,
748
                               const AST::Instruction &Instr);
749
  Expect<void> runDataDropOp(Runtime::Instance::DataInstance &DataInst);
750
  Expect<void> runMemoryCopyOp(Runtime::StackManager &StackMgr,
751
                               Runtime::Instance::MemoryInstance &MemInstDst,
752
                               Runtime::Instance::MemoryInstance &MemInstSrc,
753
                               const AST::Instruction &Instr);
754
  Expect<void> runMemoryFillOp(Runtime::StackManager &StackMgr,
755
                               Runtime::Instance::MemoryInstance &MemInst,
756
                               const AST::Instruction &Instr);
757
  /// ======= Test and Relation Numeric instructions =======
758
  template <typename T> TypeU<T> runEqzOp(ValVariant &Val) const;
759
  template <typename T>
760
  TypeT<T> runEqOp(ValVariant &Val1, const ValVariant &Val2) const;
761
  template <typename T>
762
  TypeT<T> runNeOp(ValVariant &Val1, const ValVariant &Val2) const;
763
  template <typename T>
764
  TypeT<T> runLtOp(ValVariant &Val1, const ValVariant &Val2) const;
765
  template <typename T>
766
  TypeT<T> runGtOp(ValVariant &Val1, const ValVariant &Val2) const;
767
  template <typename T>
768
  TypeT<T> runLeOp(ValVariant &Val1, const ValVariant &Val2) const;
769
  template <typename T>
770
  TypeT<T> runGeOp(ValVariant &Val1, const ValVariant &Val2) const;
771
  /// ======= Unary Numeric instructions =======
772
  template <typename T> TypeU<T> runClzOp(ValVariant &Val) const;
773
  template <typename T> TypeU<T> runCtzOp(ValVariant &Val) const;
774
  template <typename T> TypeU<T> runPopcntOp(ValVariant &Val) const;
775
  template <typename T> TypeF<T> runAbsOp(ValVariant &Val) const;
776
  template <typename T> TypeF<T> runNegOp(ValVariant &Val) const;
777
  template <typename T> TypeF<T> runCeilOp(ValVariant &Val) const;
778
  template <typename T> TypeF<T> runFloorOp(ValVariant &Val) const;
779
  template <typename T> TypeF<T> runTruncOp(ValVariant &Val) const;
780
  template <typename T> TypeF<T> runNearestOp(ValVariant &Val) const;
781
  template <typename T> TypeF<T> runSqrtOp(ValVariant &Val) const;
782
  /// ======= Binary Numeric instructions =======
783
  template <typename T>
784
  TypeN<T> runAddOp(ValVariant &Val1, const ValVariant &Val2) const;
785
  template <typename T>
786
  TypeN<T> runSubOp(ValVariant &Val1, const ValVariant &Val2) const;
787
  template <typename T>
788
  TypeN<T> runMulOp(ValVariant &Val1, const ValVariant &Val2) const;
789
  template <typename T>
790
  TypeT<T> runDivOp(const AST::Instruction &Instr, ValVariant &Val1,
791
                    const ValVariant &Val2) const;
792
  template <typename T>
793
  TypeI<T> runRemOp(const AST::Instruction &Instr, ValVariant &Val1,
794
                    const ValVariant &Val2) const;
795
  template <typename T>
796
  TypeU<T> runAndOp(ValVariant &Val1, const ValVariant &Val2) const;
797
  template <typename T>
798
  TypeU<T> runOrOp(ValVariant &Val1, const ValVariant &Val2) const;
799
  template <typename T>
800
  TypeU<T> runXorOp(ValVariant &Val1, const ValVariant &Val2) const;
801
  template <typename T>
802
  TypeU<T> runShlOp(ValVariant &Val1, const ValVariant &Val2) const;
803
  template <typename T>
804
  TypeI<T> runShrOp(ValVariant &Val1, const ValVariant &Val2) const;
805
  template <typename T>
806
  TypeU<T> runRotlOp(ValVariant &Val1, const ValVariant &Val2) const;
807
  template <typename T>
808
  TypeU<T> runRotrOp(ValVariant &Val1, const ValVariant &Val2) const;
809
  template <typename T>
810
  TypeF<T> runMinOp(ValVariant &Val1, const ValVariant &Val2) const;
811
  template <typename T>
812
  TypeF<T> runMaxOp(ValVariant &Val1, const ValVariant &Val2) const;
813
  template <typename T>
814
  TypeF<T> runCopysignOp(ValVariant &Val1, const ValVariant &Val2) const;
815
  inline Expect<void> runAdd128Op(ValVariant &Val1Lo, ValVariant &Val1Hi,
816
                                  const ValVariant &Val2Lo,
817
                                  const ValVariant &Val2Hi) const;
818
  inline Expect<void> runSub128Op(ValVariant &Val1Lo, ValVariant &Val1Hi,
819
                                  const ValVariant &Val2Lo,
820
                                  const ValVariant &Val2Hi) const;
821
  template <typename T>
822
  TypeI<T> runMulWideOp(ValVariant &Val1, ValVariant &Val2) const;
823
  /// ======= Cast Numeric instructions =======
824
  template <typename TIn, typename TOut>
825
  TypeUU<TIn, TOut> runWrapOp(ValVariant &Val) const;
826
  template <typename TIn, typename TOut>
827
  TypeFI<TIn, TOut> runTruncateOp(const AST::Instruction &Instr,
828
                                  ValVariant &Val) const;
829
  template <typename TIn, typename TOut>
830
  TypeFI<TIn, TOut> runTruncateSatOp(ValVariant &Val) const;
831
  template <typename TIn, typename TOut, size_t B = sizeof(TIn) * 8>
832
  TypeIU<TIn, TOut> runExtendOp(ValVariant &Val) const;
833
  template <typename TIn, typename TOut>
834
  TypeIF<TIn, TOut> runConvertOp(ValVariant &Val) const;
835
  template <typename TIn, typename TOut>
836
  TypeFF<TIn, TOut> runDemoteOp(ValVariant &Val) const;
837
  template <typename TIn, typename TOut>
838
  TypeFF<TIn, TOut> runPromoteOp(ValVariant &Val) const;
839
  template <typename TIn, typename TOut>
840
  TypeNN<TIn, TOut> runReinterpretOp(ValVariant &Val) const;
841
  /// ======= SIMD Memory instructions =======
842
  template <typename TIn, typename TOut>
843
  Expect<void> runLoadExpandOp(Runtime::StackManager &StackMgr,
844
                               Runtime::Instance::MemoryInstance &MemInst,
845
                               const AST::Instruction &Instr);
846
  template <typename T>
847
  Expect<void> runLoadSplatOp(Runtime::StackManager &StackMgr,
848
                              Runtime::Instance::MemoryInstance &MemInst,
849
                              const AST::Instruction &Instr);
850
  template <typename T>
851
  Expect<void> runLoadLaneOp(Runtime::StackManager &StackMgr,
852
                             Runtime::Instance::MemoryInstance &MemInst,
853
                             const AST::Instruction &Instr);
854
  template <typename T>
855
  Expect<void> runStoreLaneOp(Runtime::StackManager &StackMgr,
856
                              Runtime::Instance::MemoryInstance &MemInst,
857
                              const AST::Instruction &Instr);
858
  /// ======= SIMD Lane instructions =======
859
  template <typename TIn, typename TOut = TIn>
860
  Expect<void> runExtractLaneOp(ValVariant &Val, const uint8_t Index) const;
861
  template <typename TIn, typename TOut = TIn>
862
  Expect<void> runReplaceLaneOp(ValVariant &Val1, const ValVariant &Val2,
863
                                const uint8_t Index) const;
864
  /// ======= SIMD Numeric instructions =======
865
  template <typename TIn, typename TOut = TIn>
866
  Expect<void> runSplatOp(ValVariant &Val) const;
867
  template <typename T>
868
  Expect<void> runVectorEqOp(ValVariant &Val1, const ValVariant &Val2) const;
869
  template <typename T>
870
  Expect<void> runVectorNeOp(ValVariant &Val1, const ValVariant &Val2) const;
871
  template <typename T>
872
  Expect<void> runVectorLtOp(ValVariant &Val1, const ValVariant &Val2) const;
873
  template <typename T>
874
  Expect<void> runVectorGtOp(ValVariant &Val1, const ValVariant &Val2) const;
875
  template <typename T>
876
  Expect<void> runVectorLeOp(ValVariant &Val1, const ValVariant &Val2) const;
877
  template <typename T>
878
  Expect<void> runVectorGeOp(ValVariant &Val1, const ValVariant &Val2) const;
879
  template <typename T> Expect<void> runVectorAbsOp(ValVariant &Val) const;
880
  template <typename T> Expect<void> runVectorNegOp(ValVariant &Val) const;
881
  inline Expect<void> runVectorPopcntOp(ValVariant &Val) const;
882
  template <typename T> Expect<void> runVectorSqrtOp(ValVariant &Val) const;
883
  template <typename TIn, typename TOut>
884
  Expect<void> runVectorTruncSatOp(ValVariant &Val) const;
885
  template <typename TIn, typename TOut>
886
  Expect<void> runVectorConvertOp(ValVariant &Val) const;
887
  inline Expect<void> runVectorDemoteOp(ValVariant &Val) const;
888
  inline Expect<void> runVectorPromoteOp(ValVariant &Val) const;
889
  inline Expect<void> runVectorAnyTrueOp(ValVariant &Val) const;
890
  template <typename T> Expect<void> runVectorAllTrueOp(ValVariant &Val) const;
891
  template <typename T> Expect<void> runVectorBitMaskOp(ValVariant &Val) const;
892
  template <typename TIn, typename TOut>
893
  Expect<void> runVectorNarrowOp(ValVariant &Val1,
894
                                 const ValVariant &Val2) const;
895
  template <typename TIn, typename TOut>
896
  Expect<void> runVectorExtendLowOp(ValVariant &Val) const;
897
  template <typename TIn, typename TOut>
898
  Expect<void> runVectorExtendHighOp(ValVariant &Val) const;
899
  template <typename TIn, typename TOut>
900
  Expect<void> runVectorExtAddPairwiseOp(ValVariant &Val) const;
901
  template <typename TIn, typename TOut>
902
  Expect<void> runVectorExtMulLowOp(ValVariant &Val1,
903
                                    const ValVariant &Val2) const;
904
  template <typename TIn, typename TOut>
905
  Expect<void> runVectorExtMulHighOp(ValVariant &Val1,
906
                                     const ValVariant &Val2) const;
907
  inline Expect<void> runVectorQ15MulSatOp(ValVariant &Val1,
908
                                           const ValVariant &Val2) const;
909
  template <typename T>
910
  Expect<void> runVectorShlOp(ValVariant &Val1, const ValVariant &Val2) const;
911
  template <typename T>
912
  Expect<void> runVectorShrOp(ValVariant &Val1, const ValVariant &Val2) const;
913
  template <typename T>
914
  Expect<void> runVectorAddOp(ValVariant &Val1, const ValVariant &Val2) const;
915
  template <typename T>
916
  Expect<void> runVectorAddSatOp(ValVariant &Val1,
917
                                 const ValVariant &Val2) const;
918
  template <typename T>
919
  Expect<void> runVectorSubOp(ValVariant &Val1, const ValVariant &Val2) const;
920
  template <typename T>
921
  Expect<void> runVectorSubSatOp(ValVariant &Val1,
922
                                 const ValVariant &Val2) const;
923
  template <typename T>
924
  Expect<void> runVectorMulOp(ValVariant &Val1, const ValVariant &Val2) const;
925
  template <typename T>
926
  Expect<void> runVectorDivOp(ValVariant &Val1, const ValVariant &Val2) const;
927
  template <typename T>
928
  Expect<void> runVectorMinOp(ValVariant &Val1, const ValVariant &Val2) const;
929
  template <typename T>
930
  Expect<void> runVectorMaxOp(ValVariant &Val1, const ValVariant &Val2) const;
931
  template <typename T>
932
  Expect<void> runVectorFMinOp(ValVariant &Val1, const ValVariant &Val2) const;
933
  template <typename T>
934
  Expect<void> runVectorFMaxOp(ValVariant &Val1, const ValVariant &Val2) const;
935
  template <typename T>
936
  Expect<void> runVectorAvgrOp(ValVariant &Val1, const ValVariant &Val2) const;
937
  template <typename T> Expect<void> runVectorCeilOp(ValVariant &Val) const;
938
  template <typename T> Expect<void> runVectorFloorOp(ValVariant &Val) const;
939
  template <typename T> Expect<void> runVectorTruncOp(ValVariant &Val) const;
940
  template <typename T> Expect<void> runVectorNearestOp(ValVariant &Val) const;
941
  /// ======= Relaxed SIMD instructions =======
942
  template <typename T>
943
  Expect<void> runVectorRelaxedLaneselectOp(ValVariant &Val1,
944
                                            const ValVariant &Val2,
945
                                            const ValVariant &Mask) const;
946
  inline Expect<void>
947
  runVectorRelaxedIntegerDotProductOp(ValVariant &Val1,
948
                                      const ValVariant &Val2) const;
949
  inline Expect<void> runVectorRelaxedIntegerDotProductOpAdd(
950
      ValVariant &Val1, const ValVariant &Val2, const ValVariant &C) const;
951
  /// ======= Atomic instructions =======
952
  Expect<void> runAtomicNotifyOp(Runtime::StackManager &StackMgr,
953
                                 Runtime::Instance::MemoryInstance &MemInst,
954
                                 const AST::Instruction &Instr);
955
  Expect<void> runMemoryFenceOp();
956
  template <typename T>
957
  TypeT<T> runAtomicWaitOp(Runtime::StackManager &StackMgr,
958
                           Runtime::Instance::MemoryInstance &MemInst,
959
                           const AST::Instruction &Instr);
960
  template <typename T, typename I>
961
  TypeT<T> runAtomicLoadOp(Runtime::StackManager &StackMgr,
962
                           Runtime::Instance::MemoryInstance &MemInst,
963
                           const AST::Instruction &Instr);
964
  template <typename T, typename I>
965
  TypeT<T> runAtomicStoreOp(Runtime::StackManager &StackMgr,
966
                            Runtime::Instance::MemoryInstance &MemInst,
967
                            const AST::Instruction &Instr);
968
  template <typename T, typename I>
969
  TypeT<T> runAtomicAddOp(Runtime::StackManager &StackMgr,
970
                          Runtime::Instance::MemoryInstance &MemInst,
971
                          const AST::Instruction &Instr);
972
  template <typename T, typename I>
973
  TypeT<T> runAtomicSubOp(Runtime::StackManager &StackMgr,
974
                          Runtime::Instance::MemoryInstance &MemInst,
975
                          const AST::Instruction &Instr);
976
  template <typename T, typename I>
977
  TypeT<T> runAtomicOrOp(Runtime::StackManager &StackMgr,
978
                         Runtime::Instance::MemoryInstance &MemInst,
979
                         const AST::Instruction &Instr);
980
  template <typename T, typename I>
981
  TypeT<T> runAtomicAndOp(Runtime::StackManager &StackMgr,
982
                          Runtime::Instance::MemoryInstance &MemInst,
983
                          const AST::Instruction &Instr);
984
  template <typename T, typename I>
985
  TypeT<T> runAtomicXorOp(Runtime::StackManager &StackMgr,
986
                          Runtime::Instance::MemoryInstance &MemInst,
987
                          const AST::Instruction &Instr);
988
  template <typename T, typename I>
989
  TypeT<T> runAtomicExchangeOp(Runtime::StackManager &StackMgr,
990
                               Runtime::Instance::MemoryInstance &MemInst,
991
                               const AST::Instruction &Instr);
992
  template <typename T, typename I>
993
  TypeT<T>
994
  runAtomicCompareExchangeOp(Runtime::StackManager &StackMgr,
995
                             Runtime::Instance::MemoryInstance &MemInst,
996
                             const AST::Instruction &Instr);
997
  /// @}
998
999
public:
1000
  /// \name AOT/JIT - Run compiled functions
1001
  /// @{
1002
  Expect<void> proxyTrap(Runtime::StackManager &StackMgr,
1003
                         const uint32_t Code) noexcept;
1004
  Expect<void> proxyCall(Runtime::StackManager &StackMgr,
1005
                         const Runtime::Instance::ModuleInstance *ModInst,
1006
                         const uint32_t FuncIdx, const ValVariant *Args,
1007
                         ValVariant *Rets) noexcept;
1008
  Expect<void>
1009
  proxyCallIndirect(Runtime::StackManager &StackMgr,
1010
                    const Runtime::Instance::ModuleInstance *ModInst,
1011
                    const uint32_t TableIdx, const uint32_t FuncTypeIdx,
1012
                    const uint64_t FuncIdx, const ValVariant *Args,
1013
                    ValVariant *Rets) noexcept;
1014
  Expect<void> proxyCallRef(Runtime::StackManager &StackMgr,
1015
                            const Runtime::Instance::ModuleInstance *ModInst,
1016
                            const RefVariant Ref, const ValVariant *Args,
1017
                            ValVariant *Rets) noexcept;
1018
  Expect<RefVariant>
1019
  proxyRefFunc(Runtime::StackManager &StackMgr,
1020
               const Runtime::Instance::ModuleInstance *ModInst,
1021
               const uint32_t FuncIdx) noexcept;
1022
  Expect<RefVariant>
1023
  proxyStructNew(Runtime::StackManager &StackMgr,
1024
                 const Runtime::Instance::ModuleInstance *ModInst,
1025
                 const uint32_t TypeIdx, const ValVariant *Args,
1026
                 const uint32_t ArgSize) noexcept;
1027
  Expect<void> proxyStructGet(Runtime::StackManager &StackMgr,
1028
                              const Runtime::Instance::ModuleInstance *ModInst,
1029
                              const RefVariant Ref, const uint32_t TypeIdx,
1030
                              const uint32_t Off, const bool IsSigned,
1031
                              ValVariant *Ret) noexcept;
1032
  Expect<void> proxyStructSet(Runtime::StackManager &StackMgr,
1033
                              const Runtime::Instance::ModuleInstance *ModInst,
1034
                              const RefVariant Ref, const uint32_t TypeIdx,
1035
                              const uint32_t Off,
1036
                              const ValVariant *Val) noexcept;
1037
  Expect<RefVariant>
1038
  proxyArrayNew(Runtime::StackManager &StackMgr,
1039
                const Runtime::Instance::ModuleInstance *ModInst,
1040
                const uint32_t TypeIdx, const uint32_t Length,
1041
                const ValVariant *Args, const uint32_t ArgSize) noexcept;
1042
  Expect<RefVariant>
1043
  proxyArrayNewData(Runtime::StackManager &StackMgr,
1044
                    const Runtime::Instance::ModuleInstance *ModInst,
1045
                    const uint32_t TypeIdx, const uint32_t DataIdx,
1046
                    const uint32_t Start, const uint32_t Length) noexcept;
1047
  Expect<RefVariant>
1048
  proxyArrayNewElem(Runtime::StackManager &StackMgr,
1049
                    const Runtime::Instance::ModuleInstance *ModInst,
1050
                    const uint32_t TypeIdx, const uint32_t ElemIdx,
1051
                    const uint32_t Start, const uint32_t Length) noexcept;
1052
  Expect<void> proxyArrayGet(Runtime::StackManager &StackMgr,
1053
                             const Runtime::Instance::ModuleInstance *ModInst,
1054
                             const RefVariant Ref, const uint32_t TypeIdx,
1055
                             const uint32_t Idx, const bool IsSigned,
1056
                             ValVariant *Ret) noexcept;
1057
  Expect<void> proxyArraySet(Runtime::StackManager &StackMgr,
1058
                             const Runtime::Instance::ModuleInstance *ModInst,
1059
                             const RefVariant Ref, const uint32_t TypeIdx,
1060
                             const uint32_t Idx,
1061
                             const ValVariant *Val) noexcept;
1062
  Expect<uint32_t> proxyArrayLen(Runtime::StackManager &StackMgr,
1063
                                 const RefVariant Ref) noexcept;
1064
  Expect<void> proxyArrayFill(Runtime::StackManager &StackMgr,
1065
                              const Runtime::Instance::ModuleInstance *ModInst,
1066
                              const RefVariant Ref, const uint32_t TypeIdx,
1067
                              const uint32_t Idx, const uint32_t Cnt,
1068
                              const ValVariant *Val) noexcept;
1069
  Expect<void> proxyArrayCopy(Runtime::StackManager &StackMgr,
1070
                              const Runtime::Instance::ModuleInstance *ModInst,
1071
                              const RefVariant DstRef,
1072
                              const uint32_t DstTypeIdx, const uint32_t DstIdx,
1073
                              const RefVariant SrcRef,
1074
                              const uint32_t SrcTypeIdx, const uint32_t SrcIdx,
1075
                              const uint32_t Cnt) noexcept;
1076
  Expect<void>
1077
  proxyArrayInitData(Runtime::StackManager &StackMgr,
1078
                     const Runtime::Instance::ModuleInstance *ModInst,
1079
                     const RefVariant Ref, const uint32_t TypeIdx,
1080
                     const uint32_t DataIdx, const uint32_t DstIdx,
1081
                     const uint32_t SrcIdx, const uint32_t Cnt) noexcept;
1082
  Expect<void>
1083
  proxyArrayInitElem(Runtime::StackManager &StackMgr,
1084
                     const Runtime::Instance::ModuleInstance *ModInst,
1085
                     const RefVariant Ref, const uint32_t TypeIdx,
1086
                     const uint32_t ElemIdx, const uint32_t DstIdx,
1087
                     const uint32_t SrcIdx, const uint32_t Cnt) noexcept;
1088
  Expect<uint32_t>
1089
  proxyRefTest(Runtime::StackManager &StackMgr,
1090
               const Runtime::Instance::ModuleInstance *ModInst,
1091
               const RefVariant Ref, ValType VTTest) noexcept;
1092
  Expect<RefVariant>
1093
  proxyRefCast(Runtime::StackManager &StackMgr,
1094
               const Runtime::Instance::ModuleInstance *ModInst,
1095
               const RefVariant Ref, ValType VTCast) noexcept;
1096
  Expect<void> proxyTableInit(Runtime::StackManager &StackMgr,
1097
                              const Runtime::Instance::ModuleInstance *ModInst,
1098
                              const uint32_t TableIdx, const uint32_t ElemIdx,
1099
                              const uint64_t DstOff, const uint32_t SrcOff,
1100
                              const uint32_t Len) noexcept;
1101
  Expect<void> proxyElemDrop(Runtime::StackManager &StackMgr,
1102
                             const Runtime::Instance::ModuleInstance *ModInst,
1103
                             const uint32_t ElemIdx) noexcept;
1104
  Expect<void> proxyTableCopy(Runtime::StackManager &StackMgr,
1105
                              const Runtime::Instance::ModuleInstance *ModInst,
1106
                              const uint32_t TableIdxDst,
1107
                              const uint32_t TableIdxSrc, const uint64_t DstOff,
1108
                              const uint64_t SrcOff,
1109
                              const uint64_t Len) noexcept;
1110
  Expect<uint64_t>
1111
  proxyTableGrow(Runtime::StackManager &StackMgr,
1112
                 const Runtime::Instance::ModuleInstance *ModInst,
1113
                 const uint32_t TableIdx, const RefVariant Val,
1114
                 const uint64_t NewSize) noexcept;
1115
  Expect<void> proxyTableFill(Runtime::StackManager &StackMgr,
1116
                              const Runtime::Instance::ModuleInstance *ModInst,
1117
                              const uint32_t TableIdx, const uint64_t Off,
1118
                              const RefVariant Ref,
1119
                              const uint64_t Len) noexcept;
1120
  Expect<uint64_t>
1121
  proxyMemGrow(Runtime::StackManager &StackMgr,
1122
               const Runtime::Instance::ModuleInstance *ModInst,
1123
               const uint32_t MemIdx, const uint64_t NewSize) noexcept;
1124
  Expect<void> proxyMemInit(Runtime::StackManager &StackMgr,
1125
                            const Runtime::Instance::ModuleInstance *ModInst,
1126
                            const uint32_t MemIdx, const uint32_t DataIdx,
1127
                            const uint64_t DstOff, const uint32_t SrcOff,
1128
                            const uint32_t Len) noexcept;
1129
  Expect<void> proxyDataDrop(Runtime::StackManager &StackMgr,
1130
                             const Runtime::Instance::ModuleInstance *ModInst,
1131
                             const uint32_t DataIdx) noexcept;
1132
  Expect<void> proxyMemCopy(Runtime::StackManager &StackMgr,
1133
                            const Runtime::Instance::ModuleInstance *ModInst,
1134
                            const uint32_t DstMemIdx, const uint32_t SrcMemIdx,
1135
                            const uint64_t DstOff, const uint64_t SrcOff,
1136
                            const uint64_t Len) noexcept;
1137
  Expect<void> proxyMemFill(Runtime::StackManager &StackMgr,
1138
                            const Runtime::Instance::ModuleInstance *ModInst,
1139
                            const uint32_t MemIdx, const uint64_t Off,
1140
                            const uint8_t Val, const uint64_t Len) noexcept;
1141
  Expect<uint64_t>
1142
  proxyMemAtomicNotify(Runtime::StackManager &StackMgr,
1143
                       const Runtime::Instance::ModuleInstance *ModInst,
1144
                       const uint32_t MemIdx, const uint64_t Offset,
1145
                       const uint64_t Count) noexcept;
1146
  Expect<uint64_t>
1147
  proxyMemAtomicWait(Runtime::StackManager &StackMgr,
1148
                     const Runtime::Instance::ModuleInstance *ModInst,
1149
                     const uint32_t MemIdx, const uint64_t Offset,
1150
                     const uint64_t Expected, const int64_t Timeout,
1151
                     const uint32_t BitWidth) noexcept;
1152
  Expect<void *> proxyTableGetFuncSymbol(
1153
      Runtime::StackManager &StackMgr,
1154
      const Runtime::Instance::ModuleInstance *ModInst, const uint32_t TableIdx,
1155
      const uint32_t FuncTypeIdx, const uint64_t FuncIdx,
1156
      Runtime::Instance::ModuleInstance::ModuleContext **CalleeCtxOut) noexcept;
1157
  Expect<void *> proxyRefGetFuncSymbol(
1158
      Runtime::StackManager &StackMgr,
1159
      const Runtime::Instance::ModuleInstance *ModInst, const RefVariant Ref,
1160
      Runtime::Instance::ModuleInstance::ModuleContext **CalleeCtxOut) noexcept;
1161
  Expect<void *>
1162
  proxyFuncGetFuncSymbol(Runtime::StackManager &StackMgr,
1163
                         const Runtime::Instance::ModuleInstance *ModInst,
1164
                         const uint32_t FuncIdx) noexcept;
1165
  Expect<void> proxyThrow(Runtime::StackManager &StackMgr,
1166
                          const Runtime::Instance::ModuleInstance *ModInst,
1167
                          const uint32_t TagIdx, const ValVariant *Vals,
1168
                          const uint32_t Num) noexcept;
1169
  Expect<void> proxyThrowRef(Runtime::StackManager &StackMgr,
1170
                             const RefVariant Ref) noexcept;
1171
  Expect<void> proxyCatchPop(Runtime::StackManager &StackMgr,
1172
                             const Runtime::Instance::ModuleInstance *ModInst,
1173
                             ValVariant *Out, const uint32_t PopPayload,
1174
                             const uint32_t NeedRef) noexcept;
1175
  /// @}
1176
1177
  /// Callbacks for compiled modules
1178
  static const Executable::IntrinsicsTable Intrinsics;
1179
  /// Proxy helper template struct
1180
  template <typename FuncPtr> struct ProxyHelper;
1181
1182
private:
1183
  /// Execution context for compiled functions.
1184
  struct ExecutorContext {
1185
    std::atomic_uint64_t *InstrCount;
1186
    uint64_t *CostTable;
1187
    std::atomic_uint64_t *Gas;
1188
    uint64_t GasLimit;
1189
    std::atomic_uint32_t *StopToken;
1190
    void *const *PendingExnTagAddr;
1191
    uint8_t *StackLimit;
1192
  };
1193
1194
  /// Compiled code reads this struct by field index through the mirrored
1195
  /// ExecCtx type built in lib/llvm/compiler/context.cpp. Keep both in the same
1196
  /// order.
1197
  static_assert(offsetof(ExecutorContext, InstrCount) == 0);
1198
  static_assert(offsetof(ExecutorContext, InstrCount) <
1199
                offsetof(ExecutorContext, CostTable));
1200
  static_assert(offsetof(ExecutorContext, CostTable) <
1201
                offsetof(ExecutorContext, Gas));
1202
  static_assert(offsetof(ExecutorContext, Gas) <
1203
                offsetof(ExecutorContext, GasLimit));
1204
  static_assert(offsetof(ExecutorContext, GasLimit) <
1205
                offsetof(ExecutorContext, StopToken));
1206
  static_assert(offsetof(ExecutorContext, StopToken) <
1207
                offsetof(ExecutorContext, PendingExnTagAddr));
1208
  static_assert(offsetof(ExecutorContext, PendingExnTagAddr) <
1209
                offsetof(ExecutorContext, StackLimit));
1210
1211
  /// Restores thread local VM reference after overwriting it.
1212
  struct SavedThreadLocal {
1213
    SavedThreadLocal(Executor &Ex, Runtime::StackManager &StackMgr,
1214
                     [[maybe_unused]] const Runtime::Instance::FunctionInstance
1215
                         &Func) noexcept;
1216
1217
    SavedThreadLocal(const SavedThreadLocal &) = delete;
1218
    SavedThreadLocal(SavedThreadLocal &&) = delete;
1219
1220
    ~SavedThreadLocal() noexcept;
1221
1222
    Executor *SavedThis;
1223
    Runtime::StackManager *SavedCurrentStack;
1224
    ExecutorContext SavedExecutionContext;
1225
  };
1226
1227
  /// Pending exception passing across the compiled and the native-entered
1228
  /// frames. A non-null tag means pending; the exception instance keeps the
1229
  /// throw_ref identity.
1230
  struct PendingExnStruct {
1231
    /// Tag instance of the pending exception. Null when no exception is
1232
    /// pending.
1233
    Runtime::Instance::TagInstance *TagInst = nullptr;
1234
    /// Exception instance identity. Null for a fresh throw; set on rethrowing
1235
    /// to keep the exnref identity.
1236
    const Runtime::Instance::ExceptionInstance *Inst = nullptr;
1237
1238
    /// Getter and setter of the payload values.
1239
0
    const std::vector<ValVariant> &getPayload() const noexcept {
1240
0
      return Payload;
1241
0
    }
1242
0
    void setPayload(Span<const ValVariant> Vals) noexcept {
1243
0
      Payload.assign(Vals.begin(), Vals.end());
1244
0
    }
1245
1246
  private:
1247
    std::vector<ValVariant> Payload;
1248
  };
1249
1250
  /// Pointer to current object.
1251
  static thread_local Executor *This;
1252
  /// Stack passed into compiled functions
1253
  static thread_local Runtime::StackManager *CurrentStack;
1254
  /// Execution context for compiled functions
1255
  static thread_local ExecutorContext ExecutionContext;
1256
  /// Pending exception for compiled functions
1257
  static thread_local PendingExnStruct PendingExn;
1258
  /// Record stack trace on error
1259
  static thread_local std::array<StackTraceEntry, 256> StackTrace;
1260
  static thread_local size_t StackTraceSize;
1261
1262
  /// WasmEdge configuration
1263
  const Configure Conf;
1264
  /// Executor statistics
1265
  Statistics::Statistics *Stat;
1266
  /// Stop execution
1267
  std::atomic_uint32_t StopToken = 0;
1268
  /// Memory instance this Executor is currently waiting on (for stop()).
1269
  std::atomic<Runtime::Instance::MemoryInstance *> WaitingMemory = nullptr;
1270
  /// Executor Host Function Handler
1271
  HostFuncHandler HostFuncHelper = {};
1272
  /// Callback for lazy function compilation
1273
  std::function<Expect<void>(const Runtime::Instance::FunctionInstance *)>
1274
      LazyCompilationHandler;
1275
1276
  /// Helper function for triggering lazy compilation.
1277
  /// XXX: Calling checkLazyCompilation in one thread while another thread calls
1278
  /// unsafeUpgradeToCompiled on the same FuncInst could result in a race
1279
  /// condition if checking FuncInst->isCompiledFunction() directly here. As a
1280
  /// temporary workaround, checks for compilation state are deferred to the
1281
  /// LazyCompilationHandler, which must serialize them against compiled-state
1282
  /// upgrades (the lazy JIT engine does so under its internal lock).
1283
  Expect<void> checkLazyCompilation(
1284
0
      const Runtime::Instance::FunctionInstance *FuncInst) const noexcept {
1285
0
    if (unlikely(LazyCompilationHandler != nullptr)) {
1286
0
      return LazyCompilationHandler(FuncInst);
1287
0
    }
1288
0
    return {};
1289
0
  }
1290
};
1291
1292
} // namespace Executor
1293
} // namespace WasmEdge
1294
1295
#include "engine/atomic.ipp"
1296
#include "engine/binary_numeric.ipp"
1297
#include "engine/cast_numeric.ipp"
1298
#include "engine/memory.ipp"
1299
#include "engine/relation_numeric.ipp"
1300
#include "engine/unary_numeric.ipp"