/src/WasmEdge/include/executor/engine/atomic.ipp

Source
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors

#include "common/endian.h"
#include "common/types.h"
#include "executor/executor.h"
#include "runtime/instance/memory.h"
#include <experimental/scope.hpp>

#include <cstdint>

namespace WasmEdge {
namespace Executor {

template <typename T>
TypeT<T> Executor::runAtomicWaitOp(Runtime::StackManager &StackMgr,
                                   Runtime::Instance::MemoryInstance &MemInst,
                                   const AST::Instruction &Instr) {
  ValVariant RawTimeout = StackMgr.pop();
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(T) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  int64_t Timeout = RawTimeout.get<int64_t>();

  return atomicWait<T>(MemInst, Address, RawValue.get<T>(), Timeout)
      .map_error([&Instr](auto E) {
        spdlog::error(E);
        spdlog::error(
            ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
        return E;
      })
      .map([&](auto V) { RawAddress = emplaceAddr(V, AddrType); });
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicLoadOp(Runtime::StackManager &StackMgr,
                                   Runtime::Instance::MemoryInstance &MemInst,
                                   const AST::Instruction &Instr) {
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }

  EndianValue<I> Return = AtomicObj->load();
  RawAddress.emplace<T>(static_cast<T>(Return.le()));
  return {};
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicStoreOp(Runtime::StackManager &StackMgr,
                                    Runtime::Instance::MemoryInstance &MemInst,
                                    const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant RawAddress = StackMgr.pop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  EndianValue<I> Value = static_cast<I>(RawValue.get<T>());

  AtomicObj->store(Value.le());
  return {};
}

template <typename T, typename AtomicOp, typename BinaryOp>
T runAtomicOp(std::atomic<T> *AtomicObj, T Value, AtomicOp Op [[maybe_unused]],
              BinaryOp BinOp [[maybe_unused]]) {
  if constexpr (Endian::native == Endian::little) {
    return Op(AtomicObj, Value);
  } else {
    EndianValue<T> Return;
    EndianValue<T> Result;
    do {
      Return = AtomicObj->load();
      Result = BinOp(Return.le(), Value);
    } while (!AtomicObj->compare_exchange_weak(Return.raw(), Result.le()));
    return Return.le();
  }
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicAddOp(Runtime::StackManager &StackMgr,
                                  Runtime::Instance::MemoryInstance &MemInst,
                                  const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  I Value = static_cast<I>(RawValue.get<T>());

  I Return = runAtomicOp(
      AtomicObj, Value,
      [](std::atomic<I> *Obj, I Val) { return Obj->fetch_add(Val); },
      std::plus<I>{});
  RawAddress.emplace<T>(static_cast<T>(Return));
  return {};
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicSubOp(Runtime::StackManager &StackMgr,
                                  Runtime::Instance::MemoryInstance &MemInst,
                                  const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  I Value = static_cast<I>(RawValue.get<T>());

  I Return = runAtomicOp(
      AtomicObj, Value,
      [](std::atomic<I> *Obj, I Val) { return Obj->fetch_sub(Val); },
      std::minus<I>{});
  RawAddress.emplace<T>(static_cast<T>(Return));
  return {};
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicOrOp(Runtime::StackManager &StackMgr,
                                 Runtime::Instance::MemoryInstance &MemInst,
                                 const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  EndianValue<I> Value = static_cast<I>(RawValue.get<T>());

  EndianValue<I> Return = AtomicObj->fetch_or(Value.le());
  RawAddress.emplace<T>(static_cast<T>(Return.le()));
  return {};
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicAndOp(Runtime::StackManager &StackMgr,
                                  Runtime::Instance::MemoryInstance &MemInst,
                                  const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  EndianValue<I> Value = static_cast<I>(RawValue.get<T>());

  EndianValue<I> Return = AtomicObj->fetch_and(Value.le());
  RawAddress.emplace<T>(static_cast<T>(Return.le()));
  return {};
}

template <typename T, typename I>
TypeT<T> Executor::runAtomicXorOp(Runtime::StackManager &StackMgr,
                                  Runtime::Instance::MemoryInstance &MemInst,
                                  const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  EndianValue<I> Value = static_cast<I>(RawValue.get<T>());

  EndianValue<I> Return = AtomicObj->fetch_xor(Value.le());
  RawAddress.emplace<T>(static_cast<T>(Return.le()));
  return {};
}

template <typename T, typename I>
TypeT<T>
Executor::runAtomicExchangeOp(Runtime::StackManager &StackMgr,
                              Runtime::Instance::MemoryInstance &MemInst,
                              const AST::Instruction &Instr) {
  ValVariant RawValue = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  EndianValue<I> Value = static_cast<I>(RawValue.get<T>());

  EndianValue<I> Return = AtomicObj->exchange(Value.le());
  RawAddress.emplace<T>(static_cast<T>(Return.le()));
  return {};
}

template <typename T, typename I>
TypeT<T>
Executor::runAtomicCompareExchangeOp(Runtime::StackManager &StackMgr,
                                     Runtime::Instance::MemoryInstance &MemInst,
                                     const AST::Instruction &Instr) {
  ValVariant RawReplacement = StackMgr.pop();
  ValVariant RawExpected = StackMgr.pop();
  ValVariant &RawAddress = StackMgr.getTop();
  const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
  uint64_t Address = extractAddr(RawAddress, AddrType);
  EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
  Address += Instr.getMemoryOffset();

  if (Address % sizeof(I) != 0) {
    spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
  }

  // make sure the address no OOB with size I
  auto *AtomicObj = MemInst.getPointer<std::atomic<I> *>(Address);
  if (!AtomicObj) {
    spdlog::error(ErrCode::Value::MemoryOutOfBounds);
    spdlog::error(
        ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }
  EndianValue<I> Replacement = static_cast<I>(RawReplacement.get<T>());
  I Expected = EndianValue<I>(static_cast<I>(RawExpected.get<T>())).le();

  AtomicObj->compare_exchange_strong(Expected, Replacement.le());
  RawAddress.emplace<T>(static_cast<T>(Expected));
  return {};
}

template <typename T>
Expect<uint64_t>
Executor::atomicWait(Runtime::Instance::MemoryInstance &MemInst,
                     uint64_t Address, EndianValue<T> Expected,
                     int64_t Timeout) noexcept {
  // The error message should be handled by the caller, or the AOT mode will
  // produce the duplicated messages.
  if (!MemInst.isShared()) {
    return Unexpect(ErrCode::Value::ExpectSharedMemory);
  }

  if (auto *AtomicObj = MemInst.getPointer<std::atomic<T> *>(Address);
      !AtomicObj) {
    return Unexpect(ErrCode::Value::MemoryOutOfBounds);
  }

  std::optional<std::chrono::time_point<std::chrono::steady_clock>> Until;
  if (Timeout >= 0) {
    Until.emplace(std::chrono::steady_clock::now() +
                  std::chrono::nanoseconds(Timeout));
  }

  auto *AtomicObj = MemInst.getPointer<std::atomic<T> *>(Address);
  assuming(AtomicObj);

  if (AtomicObj->load() != Expected.le()) {
    return static_cast<uint64_t>(1); // NotEqual
  }

  auto &WaiterMapMtx = MemInst.getWaiterMapMutex();
  auto &WaiterMap = MemInst.getWaiterMap();

  std::remove_reference_t<decltype(WaiterMap)>::iterator WaiterIterator;
  {
    std::unique_lock<std::mutex> Locker(WaiterMapMtx);
    WaiterIterator = WaiterMap.emplace(std::piecewise_construct,
                                       std::forward_as_tuple(Address),
                                       std::forward_as_tuple());
  }

  WaitingMemory.store(&MemInst, std::memory_order_release);
  cxx20::scope_exit ScopeExitHolder([&]() noexcept {
    WaitingMemory.store(nullptr, std::memory_order_release);
    std::unique_lock<std::mutex> Locker(WaiterMapMtx);
    WaiterMap.erase(WaiterIterator);
  });

  while (true) {
    std::unique_lock<decltype(WaiterIterator->second.Mutex)> Locker(
        WaiterIterator->second.Mutex);
    // Check StopToken while holding the Waiter mutex, before entering wait.
    // This pairs with notifyAllWaiters() which acquires+releases this mutex
    // before calling notify_all(), ensuring we either see StopToken here or
    // get woken by the subsequent notify_all().
    if (unlikely(StopToken.load(std::memory_order_acquire) != 0)) {
      return Unexpect(ErrCode::Value::Interrupted);
    }
    std::cv_status WaitResult = std::cv_status::no_timeout;
    if (!Until) {
      WaiterIterator->second.Cond.wait(Locker);
    } else {
      WaitResult = WaiterIterator->second.Cond.wait_until(Locker, *Until);
    }
    if (unlikely(StopToken.load(std::memory_order_relaxed) != 0)) {
      return Unexpect(ErrCode::Value::Interrupted);
    }
    if (WaiterIterator->second.Notified) {
      // Woken by memory.atomic.notify — return "ok" per the spec,
      // regardless of the current memory value.
      return static_cast<uint64_t>(0); // ok
    }
    if (WaitResult == std::cv_status::timeout) {
      return static_cast<uint64_t>(2); // Timed-out
    }
  }
}

} // namespace Executor
} // namespace WasmEdge

Coverage Report

Created: 2026-06-30 06:10

Line	Count	Source
1		// SPDX-License-Identifier: Apache-2.0
2		// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4		#include "common/endian.h"
5		#include "common/types.h"
6		#include "executor/executor.h"
7		#include "runtime/instance/memory.h"
8		#include <experimental/scope.hpp>
9
10		#include <cstdint>
11
12		namespace WasmEdge {
13		namespace Executor {
14
15		template <typename T>
16		TypeT<T> Executor::runAtomicWaitOp(Runtime::StackManager &StackMgr,
17		Runtime::Instance::MemoryInstance &MemInst,
18	0	const AST::Instruction &Instr) {
19	0	ValVariant RawTimeout = StackMgr.pop();
20	0	ValVariant RawValue = StackMgr.pop();
21	0	ValVariant &RawAddress = StackMgr.getTop();
22	0	const auto AddrType = MemInst.getMemoryType().getLimit().getAddrType();
23	0	uint64_t Address = extractAddr(RawAddress, AddrType);
24	0	EXPECTED_TRY(checkOffsetOverflow(MemInst, Instr, Address, sizeof(T)));
25	0	Address += Instr.getMemoryOffset();
26
27	0	if (Address % sizeof(T) != 0) {
28	0	spdlog::error(ErrCode::Value::UnalignedAtomicAccess);
29	0	spdlog::error(
30	0	ErrInfo::InfoInstruction(Instr.getOpCode(), Instr.getOffset()));
31	0	return Unexpect(ErrCode::Value::UnalignedAtomicAccess);
32	0	}
33
34	0	int64_t Timeout = RawTimeout.get<int64_t>();
35
36	0	return atomicWait<T>(MemInst, Address, RawValue.get<T>(), Timeout)
37	0	.map_error([&Instr](auto E) {
38	0	spdlog::error(E);
39	0	spdlog::error(