/src/WasmEdge/lib/llvm/compiler/memoryInstr.cpp

Source
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The WasmEdge Authors

#include "compiler/function_compiler.h"

#include <limits>

namespace WasmEdge {

Expect<void>
FunctionCompiler::compileMemoryOp(const AST::Instruction &Instr) noexcept {
  switch (Instr.getOpCode()) {
  case OpCode::I32__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int32Ty);
    break;
  case OpCode::I64__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int64Ty);
    break;
  case OpCode::F32__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.FloatTy);
    break;
  case OpCode::F64__load:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.DoubleTy);
    break;
  case OpCode::I32__load8_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
                  true);
    break;
  case OpCode::I32__load8_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
                  false);
    break;
  case OpCode::I32__load16_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
                  true);
    break;
  case OpCode::I32__load16_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
                  false);
    break;
  case OpCode::I64__load8_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
                  true);
    break;
  case OpCode::I64__load8_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
                  false);
    break;
  case OpCode::I64__load16_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
                  true);
    break;
  case OpCode::I64__load16_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
                  false);
    break;
  case OpCode::I64__load32_s:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
                  true);
    break;
  case OpCode::I64__load32_u:
    compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                  Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
                  false);
    break;
  case OpCode::I32__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int32Ty);
    break;
  case OpCode::I64__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int64Ty);
    break;
  case OpCode::F32__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.FloatTy);
    break;
  case OpCode::F64__store:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.DoubleTy);
    break;
  case OpCode::I32__store8:
  case OpCode::I64__store8:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int8Ty, true);
    break;
  case OpCode::I32__store16:
  case OpCode::I64__store16:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int16Ty, true);
    break;
  case OpCode::I64__store32:
    compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
                   Instr.getMemoryAlign(), Context.Int32Ty, true);
    break;
  case OpCode::Memory__size:
    stackPush(Builder.createTrunc(
        Context.getMemorySize(Builder, ModCtx, Instr.getTargetIndex()),
        Context.MemoryAddrTypes[Instr.getTargetIndex()]));
    break;
  case OpCode::Memory__grow: {
    auto NewPageSize = Builder.createZExt(stackPop(), Context.Int64Ty);
    stackPush(Builder.createTrunc(
        Builder.createCall(
            Context.getIntrinsic(
                Builder, Executable::Intrinsics::kMemGrow,
                LLVM::Type::getFunctionType(
                    Context.Int64Ty,
                    {Context.Int8PtrTy, Context.Int32Ty, Context.Int64Ty},
                    false)),
            {Context.getModuleInst(Builder, ModCtx),
             LLContext.getInt32(Instr.getTargetIndex()), NewPageSize}),
        Context.MemoryAddrTypes[Instr.getTargetIndex()]));
    break;
  }
  case OpCode::Memory__init: {
    auto Len = stackPop();
    auto Src = stackPop();
    auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kMemInit,
            LLVM::Type::getFunctionType(Context.VoidTy,
                                        {Context.Int8PtrTy, Context.Int32Ty,
                                         Context.Int32Ty, Context.Int64Ty,
                                         Context.Int32Ty, Context.Int32Ty},
                                        false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex()),
         LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
    break;
  }
  case OpCode::Data__drop: {
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kDataDrop,
            LLVM::Type::getFunctionType(
                Context.VoidTy, {Context.Int8PtrTy, Context.Int32Ty}, false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex())});
    break;
  }
  case OpCode::Memory__copy: {
    auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
    auto Src = Builder.createZExt(stackPop(), Context.Int64Ty);
    auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kMemCopy,
            LLVM::Type::getFunctionType(Context.VoidTy,
                                        {Context.Int8PtrTy, Context.Int32Ty,
                                         Context.Int32Ty, Context.Int64Ty,
                                         Context.Int64Ty, Context.Int64Ty},
                                        false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex()),
         LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
    break;
  }
  case OpCode::Memory__fill: {
    auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
    auto Val = Builder.createTrunc(stackPop(), Context.Int8Ty);
    auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
    Builder.createCall(
        Context.getIntrinsic(
            Builder, Executable::Intrinsics::kMemFill,
            LLVM::Type::getFunctionType(Context.VoidTy,
                                        {Context.Int8PtrTy, Context.Int32Ty,
                                         Context.Int64Ty, Context.Int8Ty,
                                         Context.Int64Ty},
                                        false)),
        {Context.getModuleInst(Builder, ModCtx),
         LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});
    break;
  }

  // Const Numeric Instructions
  case OpCode::I32__const:
    stackPush(LLContext.getInt32(Instr.getNum().get<uint32_t>()));
    break;
  case OpCode::I64__const:
    stackPush(LLContext.getInt64(Instr.getNum().get<uint64_t>()));
    break;
  case OpCode::F32__const:
    stackPush(LLContext.getFloat(Instr.getNum().get<float>()));
    break;
  case OpCode::F64__const:
    stackPush(LLContext.getDouble(Instr.getNum().get<double>()));
    break;

  default:
    assumingUnreachable();
  }
  return {};
}

// Memory32 cannot escape the reservation's guard region, so it keeps the
// guard-page fast path. Memory64 addresses are guest-controlled 64-bit values.
void FunctionCompiler::boundsCheckMemory64(unsigned MemoryIndex,
                                           LLVM::Value Addr, uint64_t Offset,
                                           uint64_t AccessSize) noexcept {
  if (Context.MemoryAddrTypes[MemoryIndex].getIntegerBitWidth() != 64) {
    return;
  }
  if (Offset > std::numeric_limits<uint64_t>::max() - AccessSize) {
    // No address can satisfy the access.
    Builder.createBr(getTrapBB(ErrCode::Value::MemoryOutOfBounds));
    Builder.positionAtEnd(
        LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok"));
    return;
  }
  // Addr < usub_sat(SizeBytes, Offset + AccessSize - 1) is exactly
  // Addr + Offset + AccessSize <= SizeBytes with every term kept in 64 bits.
  const uint64_t OffsetAndSize = Offset + AccessSize;
  auto SizeBytes =
      Builder.createShl(Context.getMemorySize(Builder, ModCtx, MemoryIndex),
                        LLContext.getInt64(16));
  auto Limit = Builder.createIntrinsic(
      LLVM::Core::USubSat, {Context.Int64Ty},
      {SizeBytes, LLContext.getInt64(OffsetAndSize - 1)});
  auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok");
  Builder.createCondBr(Builder.createLikely(Builder.createICmpULT(Addr, Limit)),
                       OkBB, getTrapBB(ErrCode::Value::MemoryOutOfBounds));
  Builder.positionAtEnd(OkBB);
}

void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
                                     unsigned Alignment,
                                     LLVM::Type LoadTy) noexcept {
  if constexpr (LLVM::kForceUnalignment) {
    Alignment = 0;
  }
  auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
  boundsCheckMemory64(MemoryIndex, Off, Offset,
                      LoadTy.getPrimitiveSizeInBits() / 8);
  if (Offset != 0) {
    Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
  }

  auto VPtr = Builder.createInBoundsGEP1(
      Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
  auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
  auto LoadInst = Builder.createLoad(LoadTy, Ptr, true);
  LoadInst.setAlignment(1 << Alignment);
  stackPush(switchEndian(LoadInst));
}

void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
                                     unsigned Alignment, LLVM::Type LoadTy,
                                     LLVM::Type ExtendTy,
                                     bool Signed) noexcept {
  compileLoadOp(MemoryIndex, Offset, Alignment, LoadTy);
  if (Signed) {
    Stack.back() = Builder.createSExt(Stack.back(), ExtendTy);
  } else {
    Stack.back() = Builder.createZExt(Stack.back(), ExtendTy);
  }
}

void FunctionCompiler::compileStoreOp(uint32_t MemoryIndex, uint64_t Offset,
                                      uint32_t Alignment, LLVM::Type LoadTy,
                                      bool Trunc, bool BitCast) noexcept {
  if constexpr (LLVM::kForceUnalignment) {
    Alignment = 0;
  }
  auto V = stackPop();
  auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
  boundsCheckMemory64(MemoryIndex, Off, Offset,
                      LoadTy.getPrimitiveSizeInBits() / 8);
  if (Offset != 0) {
    Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
  }

  if (Trunc) {
    V = Builder.createTrunc(V, LoadTy);
  }
  if (BitCast) {
    V = Builder.createBitCast(V, LoadTy);
  }
  V = switchEndian(V);
  auto VPtr = Builder.createInBoundsGEP1(
      Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
  auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
  auto StoreInst = Builder.createStore(V, Ptr, true);
  StoreInst.setAlignment(1 << Alignment);
}

} // namespace WasmEdge

Coverage Report

Created: 2026-11-09 06:49

Line	Count	Source
1		// SPDX-License-Identifier: Apache-2.0
2		// SPDX-FileCopyrightText: Copyright The WasmEdge Authors
3
4		#include "compiler/function_compiler.h"
5
6		#include <limits>
7
8		namespace WasmEdge {
9
10		Expect<void>
11	599k	FunctionCompiler::compileMemoryOp(const AST::Instruction &Instr) noexcept {
12	599k	switch (Instr.getOpCode()) {
13	1.06k	case OpCode::I32__load:
14	1.06k	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
15	1.06k	Instr.getMemoryAlign(), Context.Int32Ty);
16	1.06k	break;
17	2.45k	case OpCode::I64__load:
18	2.45k	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
19	2.45k	Instr.getMemoryAlign(), Context.Int64Ty);
20	2.45k	break;
21	92	case OpCode::F32__load:
22	92	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
23	92	Instr.getMemoryAlign(), Context.FloatTy);
24	92	break;
25	224	case OpCode::F64__load:
26	224	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
27	224	Instr.getMemoryAlign(), Context.DoubleTy);
28	224	break;
29	514	case OpCode::I32__load8_s:
30	514	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
31	514	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
32	514	true);
33	514	break;
34	387	case OpCode::I32__load8_u:
35	387	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
36	387	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int32Ty,
37	387	false);
38	387	break;
39	300	case OpCode::I32__load16_s:
40	300	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
41	300	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
42	300	true);
43	300	break;
44	1.52k	case OpCode::I32__load16_u:
45	1.52k	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
46	1.52k	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int32Ty,
47	1.52k	false);
48	1.52k	break;
49	666	case OpCode::I64__load8_s:
50	666	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
51	666	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
52	666	true);
53	666	break;
54	417	case OpCode::I64__load8_u:
55	417	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
56	417	Instr.getMemoryAlign(), Context.Int8Ty, Context.Int64Ty,
57	417	false);
58	417	break;
59	308	case OpCode::I64__load16_s:
60	308	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
61	308	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
62	308	true);
63	308	break;
64	529	case OpCode::I64__load16_u:
65	529	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
66	529	Instr.getMemoryAlign(), Context.Int16Ty, Context.Int64Ty,
67	529	false);
68	529	break;
69	359	case OpCode::I64__load32_s:
70	359	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
71	359	Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
72	359	true);
73	359	break;
74	442	case OpCode::I64__load32_u:
75	442	compileLoadOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
76	442	Instr.getMemoryAlign(), Context.Int32Ty, Context.Int64Ty,
77	442	false);
78	442	break;
79	363	case OpCode::I32__store:
80	363	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
81	363	Instr.getMemoryAlign(), Context.Int32Ty);
82	363	break;
83	1.34k	case OpCode::I64__store:
84	1.34k	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
85	1.34k	Instr.getMemoryAlign(), Context.Int64Ty);
86	1.34k	break;
87	61	case OpCode::F32__store:
88	61	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
89	61	Instr.getMemoryAlign(), Context.FloatTy);
90	61	break;
91	43	case OpCode::F64__store:
92	43	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
93	43	Instr.getMemoryAlign(), Context.DoubleTy);
94	43	break;
95	279	case OpCode::I32__store8:
96	297	case OpCode::I64__store8:
97	297	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
98	297	Instr.getMemoryAlign(), Context.Int8Ty, true);
99	297	break;
100	232	case OpCode::I32__store16:
101	308	case OpCode::I64__store16:
102	308	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
103	308	Instr.getMemoryAlign(), Context.Int16Ty, true);
104	308	break;
105	30	case OpCode::I64__store32:
106	30	compileStoreOp(Instr.getTargetIndex(), Instr.getMemoryOffset(),
107	30	Instr.getMemoryAlign(), Context.Int32Ty, true);
108	30	break;
109	652	case OpCode::Memory__size:
110	652	stackPush(Builder.createTrunc(
111	652	Context.getMemorySize(Builder, ModCtx, Instr.getTargetIndex()),
112	652	Context.MemoryAddrTypes[Instr.getTargetIndex()]));
113	652	break;
114	774	case OpCode::Memory__grow: {
115	774	auto NewPageSize = Builder.createZExt(stackPop(), Context.Int64Ty);
116	774	stackPush(Builder.createTrunc(
117	774	Builder.createCall(
118	774	Context.getIntrinsic(
119	774	Builder, Executable::Intrinsics::kMemGrow,
120	774	LLVM::Type::getFunctionType(
121	774	Context.Int64Ty,
122	774	{Context.Int8PtrTy, Context.Int32Ty, Context.Int64Ty},
123	774	false)),
124	774	{Context.getModuleInst(Builder, ModCtx),
125	774	LLContext.getInt32(Instr.getTargetIndex()), NewPageSize}),
126	774	Context.MemoryAddrTypes[Instr.getTargetIndex()]));
127	774	break;
128	232	}
129	23	case OpCode::Memory__init: {
130	23	auto Len = stackPop();
131	23	auto Src = stackPop();
132	23	auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
133	23	Builder.createCall(
134	23	Context.getIntrinsic(
135	23	Builder, Executable::Intrinsics::kMemInit,
136	23	LLVM::Type::getFunctionType(Context.VoidTy,
137	23	{Context.Int8PtrTy, Context.Int32Ty,
138	23	Context.Int32Ty, Context.Int64Ty,
139	23	Context.Int32Ty, Context.Int32Ty},
140	23	false)),
141	23	{Context.getModuleInst(Builder, ModCtx),
142	23	LLContext.getInt32(Instr.getTargetIndex()),
143	23	LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
144	23	break;
145	232	}
146	22	case OpCode::Data__drop: {
147	22	Builder.createCall(
148	22	Context.getIntrinsic(
149	22	Builder, Executable::Intrinsics::kDataDrop,
150	22	LLVM::Type::getFunctionType(
151	22	Context.VoidTy, {Context.Int8PtrTy, Context.Int32Ty}, false)),
152	22	{Context.getModuleInst(Builder, ModCtx),
153	22	LLContext.getInt32(Instr.getTargetIndex())});
154	22	break;
155	232	}
156	348	case OpCode::Memory__copy: {
157	348	auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
158	348	auto Src = Builder.createZExt(stackPop(), Context.Int64Ty);
159	348	auto Dst = Builder.createZExt(stackPop(), Context.Int64Ty);
160	348	Builder.createCall(
161	348	Context.getIntrinsic(
162	348	Builder, Executable::Intrinsics::kMemCopy,
163	348	LLVM::Type::getFunctionType(Context.VoidTy,
164	348	{Context.Int8PtrTy, Context.Int32Ty,
165	348	Context.Int32Ty, Context.Int64Ty,
166	348	Context.Int64Ty, Context.Int64Ty},
167	348	false)),
168	348	{Context.getModuleInst(Builder, ModCtx),
169	348	LLContext.getInt32(Instr.getTargetIndex()),
170	348	LLContext.getInt32(Instr.getSourceIndex()), Dst, Src, Len});
171	348	break;
172	232	}
173	720	case OpCode::Memory__fill: {
174	720	auto Len = Builder.createZExt(stackPop(), Context.Int64Ty);
175	720	auto Val = Builder.createTrunc(stackPop(), Context.Int8Ty);
176	720	auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
177	720	Builder.createCall(
178	720	Context.getIntrinsic(
179	720	Builder, Executable::Intrinsics::kMemFill,
180	720	LLVM::Type::getFunctionType(Context.VoidTy,
181	720	{Context.Int8PtrTy, Context.Int32Ty,
182	720	Context.Int64Ty, Context.Int8Ty,
183	720	Context.Int64Ty},
184	720	false)),
185	720	{Context.getModuleInst(Builder, ModCtx),
186	720	LLContext.getInt32(Instr.getTargetIndex()), Off, Val, Len});
187	720	break;
188	232	}
189
190		// Const Numeric Instructions
191	500k	case OpCode::I32__const:
192	500k	stackPush(LLContext.getInt32(Instr.getNum().get<uint32_t>()));
193	500k	break;
194	67.8k	case OpCode::I64__const:
195	67.8k	stackPush(LLContext.getInt64(Instr.getNum().get<uint64_t>()));
196	67.8k	break;
197	11.8k	case OpCode::F32__const:
198	11.8k	stackPush(LLContext.getFloat(Instr.getNum().get<float>()));
199	11.8k	break;
200	5.51k	case OpCode::F64__const:
201	5.51k	stackPush(LLContext.getDouble(Instr.getNum().get<double>()));
202	5.51k	break;
203
204	0	default:
205	0	assumingUnreachable();
206	599k	}
207	599k	return {};
208	599k	}
209
210		// Memory32 cannot escape the reservation's guard region, so it keeps the
211		// guard-page fast path. Memory64 addresses are guest-controlled 64-bit values.
212		void FunctionCompiler::boundsCheckMemory64(unsigned MemoryIndex,
213		LLVM::Value Addr, uint64_t Offset,
214	18.9k	uint64_t AccessSize) noexcept {
215	18.9k	if (Context.MemoryAddrTypes[MemoryIndex].getIntegerBitWidth() != 64) {
216	18.9k	return;
217	18.9k	}
218	1	if (Offset > std::numeric_limits<uint64_t>::max() - AccessSize) {
219		// No address can satisfy the access.
220	0	Builder.createBr(getTrapBB(ErrCode::Value::MemoryOutOfBounds));
221	0	Builder.positionAtEnd(
222	0	LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok"));
223	0	return;
224	0	}
225		// Addr < usub_sat(SizeBytes, Offset + AccessSize - 1) is exactly
226		// Addr + Offset + AccessSize <= SizeBytes with every term kept in 64 bits.
227	1	const uint64_t OffsetAndSize = Offset + AccessSize;
228	1	auto SizeBytes =
229	1	Builder.createShl(Context.getMemorySize(Builder, ModCtx, MemoryIndex),
230	1	LLContext.getInt64(16));
231	1	auto Limit = Builder.createIntrinsic(
232	1	LLVM::Core::USubSat, {Context.Int64Ty},
233	1	{SizeBytes, LLContext.getInt64(OffsetAndSize - 1)});
234	1	auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "mem64.ok");
235	1	Builder.createCondBr(Builder.createLikely(Builder.createICmpULT(Addr, Limit)),
236	1	OkBB, getTrapBB(ErrCode::Value::MemoryOutOfBounds));
237	1	Builder.positionAtEnd(OkBB);
238	1	}
239
240		void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
241		unsigned Alignment,
242	15.8k	LLVM::Type LoadTy) noexcept {
243	15.8k	if constexpr (LLVM::kForceUnalignment) {
244	15.8k	Alignment = 0;
245	15.8k	}
246	15.8k	auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
247	15.8k	boundsCheckMemory64(MemoryIndex, Off, Offset,
248	15.8k	LoadTy.getPrimitiveSizeInBits() / 8);
249	15.8k	if (Offset != 0) {
250	10.0k	Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
251	10.0k	}
252
253	15.8k	auto VPtr = Builder.createInBoundsGEP1(
254	15.8k	Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
255	15.8k	auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
256	15.8k	auto LoadInst = Builder.createLoad(LoadTy, Ptr, true);
257	15.8k	LoadInst.setAlignment(1 << Alignment);
258	15.8k	stackPush(switchEndian(LoadInst));
259	15.8k	}
260
261		void FunctionCompiler::compileLoadOp(unsigned MemoryIndex, uint64_t Offset,
262		unsigned Alignment, LLVM::Type LoadTy,
263		LLVM::Type ExtendTy,
264	6.94k	bool Signed) noexcept {
265	6.94k	compileLoadOp(MemoryIndex, Offset, Alignment, LoadTy);
266	6.94k	if (Signed) {
267	2.76k	Stack.back() = Builder.createSExt(Stack.back(), ExtendTy);
268	4.18k	} else {
269	4.18k	Stack.back() = Builder.createZExt(Stack.back(), ExtendTy);
270	4.18k	}
271	6.94k	}
272
273		void FunctionCompiler::compileStoreOp(uint32_t MemoryIndex, uint64_t Offset,
274		uint32_t Alignment, LLVM::Type LoadTy,
275	3.07k	bool Trunc, bool BitCast) noexcept {
276	3.07k	if constexpr (LLVM::kForceUnalignment) {
277	3.07k	Alignment = 0;
278	3.07k	}
279	3.07k	auto V = stackPop();
280	3.07k	auto Off = Builder.createZExt(stackPop(), Context.Int64Ty);
281	3.07k	boundsCheckMemory64(MemoryIndex, Off, Offset,
282	3.07k	LoadTy.getPrimitiveSizeInBits() / 8);
283	3.07k	if (Offset != 0) {
284	2.26k	Off = Builder.createAdd(Off, LLContext.getInt64(Offset));
285	2.26k	}
286
287	3.07k	if (Trunc) {
288	635	V = Builder.createTrunc(V, LoadTy);
289	635	}
290	3.07k	if (BitCast) {
291	257	V = Builder.createBitCast(V, LoadTy);
292	257	}
293	3.07k	V = switchEndian(V);
294	3.07k	auto VPtr = Builder.createInBoundsGEP1(
295	3.07k	Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Off);
296	3.07k	auto Ptr = Builder.createBitCast(VPtr, LoadTy.getPointerTo());
297	3.07k	auto StoreInst = Builder.createStore(V, Ptr, true);
298	3.07k	StoreInst.setAlignment(1 << Alignment);
299	3.07k	}
300
301		} // namespace WasmEdge