/src/WasmEdge/lib/llvm/compiler/threadInstr.cpp
Line | Count | Source |
1 | | // SPDX-License-Identifier: Apache-2.0 |
2 | | // SPDX-FileCopyrightText: Copyright The WasmEdge Authors |
3 | | |
4 | | #include "compiler/function_compiler.h" |
5 | | |
6 | | namespace WasmEdge { |
7 | | |
8 | | Expect<void> |
9 | 258 | FunctionCompiler::compileAtomicOp(const AST::Instruction &Instr) noexcept { |
10 | 258 | switch (Instr.getOpCode()) { |
11 | 192 | case OpCode::Atomic__fence: |
12 | 192 | compileMemoryFence(); |
13 | 192 | break; |
14 | 60 | case OpCode::Memory__atomic__notify: |
15 | 60 | compileAtomicNotify(Instr.getTargetIndex(), Instr.getMemoryOffset()); |
16 | 60 | break; |
17 | 4 | case OpCode::Memory__atomic__wait32: |
18 | 4 | compileAtomicWait(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
19 | 4 | Context.Int32Ty, 32); |
20 | 4 | break; |
21 | 2 | case OpCode::Memory__atomic__wait64: |
22 | 2 | compileAtomicWait(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
23 | 2 | Context.Int64Ty, 64); |
24 | 2 | break; |
25 | 0 | case OpCode::I32__atomic__load: |
26 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
27 | 0 | Instr.getMemoryAlign(), Context.Int32Ty, Context.Int32Ty, |
28 | 0 | true); |
29 | 0 | break; |
30 | 0 | case OpCode::I64__atomic__load: |
31 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
32 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int64Ty, |
33 | 0 | true); |
34 | 0 | break; |
35 | 0 | case OpCode::I32__atomic__load8_u: |
36 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
37 | 0 | Instr.getMemoryAlign(), Context.Int32Ty, Context.Int8Ty); |
38 | 0 | break; |
39 | 0 | case OpCode::I32__atomic__load16_u: |
40 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
41 | 0 | Instr.getMemoryAlign(), Context.Int32Ty, Context.Int16Ty); |
42 | 0 | break; |
43 | 0 | case OpCode::I64__atomic__load8_u: |
44 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
45 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int8Ty); |
46 | 0 | break; |
47 | 0 | case OpCode::I64__atomic__load16_u: |
48 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
49 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int16Ty); |
50 | 0 | break; |
51 | 0 | case OpCode::I64__atomic__load32_u: |
52 | 0 | compileAtomicLoad(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
53 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int32Ty); |
54 | 0 | break; |
55 | 0 | case OpCode::I32__atomic__store: |
56 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
57 | 0 | Instr.getMemoryAlign(), Context.Int32Ty, Context.Int32Ty, |
58 | 0 | true); |
59 | 0 | break; |
60 | 0 | case OpCode::I64__atomic__store: |
61 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
62 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int64Ty, |
63 | 0 | true); |
64 | 0 | break; |
65 | 0 | case OpCode::I32__atomic__store8: |
66 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
67 | 0 | Instr.getMemoryAlign(), Context.Int32Ty, Context.Int8Ty, |
68 | 0 | true); |
69 | 0 | break; |
70 | 0 | case OpCode::I32__atomic__store16: |
71 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
72 | 0 | Instr.getMemoryAlign(), Context.Int32Ty, Context.Int16Ty, |
73 | 0 | true); |
74 | 0 | break; |
75 | 0 | case OpCode::I64__atomic__store8: |
76 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
77 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int8Ty, |
78 | 0 | true); |
79 | 0 | break; |
80 | 0 | case OpCode::I64__atomic__store16: |
81 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
82 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int16Ty, |
83 | 0 | true); |
84 | 0 | break; |
85 | 0 | case OpCode::I64__atomic__store32: |
86 | 0 | compileAtomicStore(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
87 | 0 | Instr.getMemoryAlign(), Context.Int64Ty, Context.Int32Ty, |
88 | 0 | true); |
89 | 0 | break; |
90 | 0 | case OpCode::I32__atomic__rmw__add: |
91 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
92 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
93 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
94 | 0 | break; |
95 | 0 | case OpCode::I64__atomic__rmw__add: |
96 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
97 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
98 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
99 | 0 | break; |
100 | 0 | case OpCode::I32__atomic__rmw8__add_u: |
101 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
102 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
103 | 0 | Context.Int32Ty, Context.Int8Ty); |
104 | 0 | break; |
105 | 0 | case OpCode::I32__atomic__rmw16__add_u: |
106 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
107 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
108 | 0 | Context.Int32Ty, Context.Int16Ty); |
109 | 0 | break; |
110 | 0 | case OpCode::I64__atomic__rmw8__add_u: |
111 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
112 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
113 | 0 | Context.Int64Ty, Context.Int8Ty); |
114 | 0 | break; |
115 | 0 | case OpCode::I64__atomic__rmw16__add_u: |
116 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
117 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
118 | 0 | Context.Int64Ty, Context.Int16Ty); |
119 | 0 | break; |
120 | 0 | case OpCode::I64__atomic__rmw32__add_u: |
121 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
122 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAdd, |
123 | 0 | Context.Int64Ty, Context.Int32Ty); |
124 | 0 | break; |
125 | 0 | case OpCode::I32__atomic__rmw__sub: |
126 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
127 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
128 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
129 | 0 | break; |
130 | 0 | case OpCode::I64__atomic__rmw__sub: |
131 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
132 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
133 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
134 | 0 | break; |
135 | 0 | case OpCode::I32__atomic__rmw8__sub_u: |
136 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
137 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
138 | 0 | Context.Int32Ty, Context.Int8Ty); |
139 | 0 | break; |
140 | 0 | case OpCode::I32__atomic__rmw16__sub_u: |
141 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
142 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
143 | 0 | Context.Int32Ty, Context.Int16Ty); |
144 | 0 | break; |
145 | 0 | case OpCode::I64__atomic__rmw8__sub_u: |
146 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
147 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
148 | 0 | Context.Int64Ty, Context.Int8Ty); |
149 | 0 | break; |
150 | 0 | case OpCode::I64__atomic__rmw16__sub_u: |
151 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
152 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
153 | 0 | Context.Int64Ty, Context.Int16Ty); |
154 | 0 | break; |
155 | 0 | case OpCode::I64__atomic__rmw32__sub_u: |
156 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
157 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpSub, |
158 | 0 | Context.Int64Ty, Context.Int32Ty); |
159 | 0 | break; |
160 | 0 | case OpCode::I32__atomic__rmw__and: |
161 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
162 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
163 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
164 | 0 | break; |
165 | 0 | case OpCode::I64__atomic__rmw__and: |
166 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
167 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
168 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
169 | 0 | break; |
170 | 0 | case OpCode::I32__atomic__rmw8__and_u: |
171 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
172 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
173 | 0 | Context.Int32Ty, Context.Int8Ty); |
174 | 0 | break; |
175 | 0 | case OpCode::I32__atomic__rmw16__and_u: |
176 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
177 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
178 | 0 | Context.Int32Ty, Context.Int16Ty); |
179 | 0 | break; |
180 | 0 | case OpCode::I64__atomic__rmw8__and_u: |
181 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
182 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
183 | 0 | Context.Int64Ty, Context.Int8Ty); |
184 | 0 | break; |
185 | 0 | case OpCode::I64__atomic__rmw16__and_u: |
186 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
187 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
188 | 0 | Context.Int64Ty, Context.Int16Ty); |
189 | 0 | break; |
190 | 0 | case OpCode::I64__atomic__rmw32__and_u: |
191 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
192 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpAnd, |
193 | 0 | Context.Int64Ty, Context.Int32Ty); |
194 | 0 | break; |
195 | 0 | case OpCode::I32__atomic__rmw__or: |
196 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
197 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
198 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
199 | 0 | break; |
200 | 0 | case OpCode::I64__atomic__rmw__or: |
201 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
202 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
203 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
204 | 0 | break; |
205 | 0 | case OpCode::I32__atomic__rmw8__or_u: |
206 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
207 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
208 | 0 | Context.Int32Ty, Context.Int8Ty); |
209 | 0 | break; |
210 | 0 | case OpCode::I32__atomic__rmw16__or_u: |
211 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
212 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
213 | 0 | Context.Int32Ty, Context.Int16Ty); |
214 | 0 | break; |
215 | 0 | case OpCode::I64__atomic__rmw8__or_u: |
216 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
217 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
218 | 0 | Context.Int64Ty, Context.Int8Ty); |
219 | 0 | break; |
220 | 0 | case OpCode::I64__atomic__rmw16__or_u: |
221 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
222 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
223 | 0 | Context.Int64Ty, Context.Int16Ty); |
224 | 0 | break; |
225 | 0 | case OpCode::I64__atomic__rmw32__or_u: |
226 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
227 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpOr, |
228 | 0 | Context.Int64Ty, Context.Int32Ty); |
229 | 0 | break; |
230 | 0 | case OpCode::I32__atomic__rmw__xor: |
231 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
232 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
233 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
234 | 0 | break; |
235 | 0 | case OpCode::I64__atomic__rmw__xor: |
236 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
237 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
238 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
239 | 0 | break; |
240 | 0 | case OpCode::I32__atomic__rmw8__xor_u: |
241 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
242 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
243 | 0 | Context.Int32Ty, Context.Int8Ty); |
244 | 0 | break; |
245 | 0 | case OpCode::I32__atomic__rmw16__xor_u: |
246 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
247 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
248 | 0 | Context.Int32Ty, Context.Int16Ty); |
249 | 0 | break; |
250 | 0 | case OpCode::I64__atomic__rmw8__xor_u: |
251 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
252 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
253 | 0 | Context.Int64Ty, Context.Int8Ty); |
254 | 0 | break; |
255 | 0 | case OpCode::I64__atomic__rmw16__xor_u: |
256 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
257 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
258 | 0 | Context.Int64Ty, Context.Int16Ty); |
259 | 0 | break; |
260 | 0 | case OpCode::I64__atomic__rmw32__xor_u: |
261 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
262 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXor, |
263 | 0 | Context.Int64Ty, Context.Int32Ty); |
264 | 0 | break; |
265 | 0 | case OpCode::I32__atomic__rmw__xchg: |
266 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
267 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
268 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
269 | 0 | break; |
270 | 0 | case OpCode::I64__atomic__rmw__xchg: |
271 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
272 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
273 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
274 | 0 | break; |
275 | 0 | case OpCode::I32__atomic__rmw8__xchg_u: |
276 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
277 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
278 | 0 | Context.Int32Ty, Context.Int8Ty); |
279 | 0 | break; |
280 | 0 | case OpCode::I32__atomic__rmw16__xchg_u: |
281 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
282 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
283 | 0 | Context.Int32Ty, Context.Int16Ty); |
284 | 0 | break; |
285 | 0 | case OpCode::I64__atomic__rmw8__xchg_u: |
286 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
287 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
288 | 0 | Context.Int64Ty, Context.Int8Ty); |
289 | 0 | break; |
290 | 0 | case OpCode::I64__atomic__rmw16__xchg_u: |
291 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
292 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
293 | 0 | Context.Int64Ty, Context.Int16Ty); |
294 | 0 | break; |
295 | 0 | case OpCode::I64__atomic__rmw32__xchg_u: |
296 | 0 | compileAtomicRMWOp(Instr.getTargetIndex(), Instr.getMemoryOffset(), |
297 | 0 | Instr.getMemoryAlign(), LLVMAtomicRMWBinOpXchg, |
298 | 0 | Context.Int64Ty, Context.Int32Ty); |
299 | 0 | break; |
300 | 0 | case OpCode::I32__atomic__rmw__cmpxchg: |
301 | 0 | compileAtomicCompareExchange( |
302 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
303 | 0 | Context.Int32Ty, Context.Int32Ty, true); |
304 | 0 | break; |
305 | 0 | case OpCode::I64__atomic__rmw__cmpxchg: |
306 | 0 | compileAtomicCompareExchange( |
307 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
308 | 0 | Context.Int64Ty, Context.Int64Ty, true); |
309 | 0 | break; |
310 | 0 | case OpCode::I32__atomic__rmw8__cmpxchg_u: |
311 | 0 | compileAtomicCompareExchange( |
312 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
313 | 0 | Context.Int32Ty, Context.Int8Ty); |
314 | 0 | break; |
315 | 0 | case OpCode::I32__atomic__rmw16__cmpxchg_u: |
316 | 0 | compileAtomicCompareExchange( |
317 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
318 | 0 | Context.Int32Ty, Context.Int16Ty); |
319 | 0 | break; |
320 | 0 | case OpCode::I64__atomic__rmw8__cmpxchg_u: |
321 | 0 | compileAtomicCompareExchange( |
322 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
323 | 0 | Context.Int64Ty, Context.Int8Ty); |
324 | 0 | break; |
325 | 0 | case OpCode::I64__atomic__rmw16__cmpxchg_u: |
326 | 0 | compileAtomicCompareExchange( |
327 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
328 | 0 | Context.Int64Ty, Context.Int16Ty); |
329 | 0 | break; |
330 | 0 | case OpCode::I64__atomic__rmw32__cmpxchg_u: |
331 | 0 | compileAtomicCompareExchange( |
332 | 0 | Instr.getTargetIndex(), Instr.getMemoryOffset(), Instr.getMemoryAlign(), |
333 | 0 | Context.Int64Ty, Context.Int32Ty); |
334 | 0 | break; |
335 | | |
336 | 0 | default: |
337 | 0 | assumingUnreachable(); |
338 | 258 | } |
339 | 258 | return {}; |
340 | 258 | } |
341 | | |
342 | | void FunctionCompiler::compileAtomicCheckOffsetAlignment( |
343 | 66 | LLVM::Value Offset, LLVM::Type IntType) noexcept { |
344 | 66 | const auto BitWidth = IntType.getIntegerBitWidth(); |
345 | 66 | auto BWMask = LLContext.getInt64((BitWidth >> 3) - 1); |
346 | 66 | auto Value = Builder.createAnd(Offset, BWMask); |
347 | 66 | auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "address_align_ok"); |
348 | 66 | auto IsAddressAligned = |
349 | 66 | Builder.createLikely(Builder.createICmpEQ(Value, LLContext.getInt64(0))); |
350 | 66 | Builder.createCondBr(IsAddressAligned, OkBB, |
351 | 66 | getTrapBB(ErrCode::Value::UnalignedAtomicAccess)); |
352 | | |
353 | 66 | Builder.positionAtEnd(OkBB); |
354 | 66 | } |
355 | | |
356 | 192 | void FunctionCompiler::compileMemoryFence() noexcept { |
357 | 192 | Builder.createFence(LLVMAtomicOrderingSequentiallyConsistent); |
358 | 192 | } |
359 | | |
360 | | void FunctionCompiler::compileAtomicNotify(unsigned MemoryIndex, |
361 | 60 | uint64_t MemoryOffset) noexcept { |
362 | 60 | auto Count = Builder.createZExt(stackPop(), Context.Int64Ty); |
363 | 60 | auto Addr = Builder.createZExt(stackPop(), Context.Int64Ty); |
364 | 60 | auto Offset = Addr; |
365 | 60 | if (MemoryOffset != 0) { |
366 | 53 | Offset = Builder.createAdd(Addr, LLContext.getInt64(MemoryOffset)); |
367 | 53 | } |
368 | 60 | compileAtomicCheckOffsetAlignment(Offset, Context.Int32Ty); |
369 | 60 | boundsCheckMemory64(MemoryIndex, Addr, MemoryOffset, sizeof(uint32_t)); |
370 | 60 | stackPush(Builder.createTrunc( |
371 | 60 | Builder.createCall( |
372 | 60 | Context.getIntrinsic( |
373 | 60 | Builder, Executable::Intrinsics::kMemAtomicNotify, |
374 | 60 | LLVM::Type::getFunctionType(Context.Int64Ty, |
375 | 60 | {Context.Int8PtrTy, Context.Int32Ty, |
376 | 60 | Context.Int64Ty, Context.Int64Ty}, |
377 | 60 | false)), |
378 | 60 | {Context.getModuleInst(Builder, ModCtx), |
379 | 60 | LLContext.getInt32(MemoryIndex), Offset, Count}), |
380 | 60 | Context.Int32Ty)); |
381 | 60 | } |
382 | | |
383 | | void FunctionCompiler::compileAtomicWait(unsigned MemoryIndex, |
384 | | uint64_t MemoryOffset, |
385 | | LLVM::Type TargetType, |
386 | 6 | uint32_t BitWidth) noexcept { |
387 | 6 | auto Timeout = stackPop(); |
388 | 6 | auto ExpectedValue = Builder.createZExtOrTrunc(stackPop(), Context.Int64Ty); |
389 | 6 | auto Addr = Builder.createZExt(stackPop(), Context.Int64Ty); |
390 | 6 | auto Offset = Addr; |
391 | 6 | if (MemoryOffset != 0) { |
392 | 3 | Offset = Builder.createAdd(Addr, LLContext.getInt64(MemoryOffset)); |
393 | 3 | } |
394 | 6 | compileAtomicCheckOffsetAlignment(Offset, TargetType); |
395 | 6 | boundsCheckMemory64(MemoryIndex, Addr, MemoryOffset, BitWidth / 8); |
396 | 6 | stackPush(Builder.createTrunc( |
397 | 6 | Builder.createCall( |
398 | 6 | Context.getIntrinsic( |
399 | 6 | Builder, Executable::Intrinsics::kMemAtomicWait, |
400 | 6 | LLVM::Type::getFunctionType(Context.Int64Ty, |
401 | 6 | {Context.Int8PtrTy, Context.Int32Ty, |
402 | 6 | Context.Int64Ty, Context.Int64Ty, |
403 | 6 | Context.Int64Ty, Context.Int32Ty}, |
404 | 6 | false)), |
405 | 6 | {Context.getModuleInst(Builder, ModCtx), |
406 | 6 | LLContext.getInt32(MemoryIndex), Offset, ExpectedValue, Timeout, |
407 | 6 | LLContext.getInt32(BitWidth)}), |
408 | 6 | Context.Int32Ty)); |
409 | 6 | } |
410 | | |
411 | | void FunctionCompiler::compileAtomicLoad(unsigned MemoryIndex, |
412 | | uint64_t MemoryOffset, |
413 | | unsigned Alignment, LLVM::Type IntType, |
414 | | LLVM::Type TargetType, |
415 | 0 | bool Signed) noexcept { |
416 | |
|
417 | 0 | auto Addr = Builder.createZExt(Stack.back(), Context.Int64Ty); |
418 | 0 | boundsCheckMemory64(MemoryIndex, Addr, MemoryOffset, |
419 | 0 | TargetType.getPrimitiveSizeInBits() / 8); |
420 | 0 | auto Offset = Addr; |
421 | 0 | if (MemoryOffset != 0) { |
422 | 0 | Offset = Builder.createAdd(Addr, LLContext.getInt64(MemoryOffset)); |
423 | 0 | } |
424 | 0 | compileAtomicCheckOffsetAlignment(Offset, TargetType); |
425 | 0 | auto VPtr = Builder.createInBoundsGEP1( |
426 | 0 | Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Offset); |
427 | |
|
428 | 0 | auto Ptr = Builder.createBitCast(VPtr, TargetType.getPointerTo()); |
429 | 0 | auto Load = switchEndian(Builder.createLoad(TargetType, Ptr, true)); |
430 | 0 | Load.setAlignment(1 << Alignment); |
431 | 0 | Load.setOrdering(LLVMAtomicOrderingSequentiallyConsistent); |
432 | |
|
433 | 0 | if (Signed) { |
434 | 0 | Stack.back() = Builder.createSExt(Load, IntType); |
435 | 0 | } else { |
436 | 0 | Stack.back() = Builder.createZExt(Load, IntType); |
437 | 0 | } |
438 | 0 | } |
439 | | |
440 | | void FunctionCompiler::compileAtomicStore(unsigned MemoryIndex, |
441 | | uint64_t MemoryOffset, |
442 | | unsigned Alignment, LLVM::Type, |
443 | | LLVM::Type TargetType, |
444 | 0 | bool Signed) noexcept { |
445 | 0 | auto V = stackPop(); |
446 | |
|
447 | 0 | if (Signed) { |
448 | 0 | V = Builder.createSExtOrTrunc(V, TargetType); |
449 | 0 | } else { |
450 | 0 | V = Builder.createZExtOrTrunc(V, TargetType); |
451 | 0 | } |
452 | 0 | V = switchEndian(V); |
453 | 0 | auto Addr = Builder.createZExt(Stack.back(), Context.Int64Ty); |
454 | 0 | boundsCheckMemory64(MemoryIndex, Addr, MemoryOffset, |
455 | 0 | TargetType.getPrimitiveSizeInBits() / 8); |
456 | 0 | auto Offset = Addr; |
457 | 0 | if (MemoryOffset != 0) { |
458 | 0 | Offset = Builder.createAdd(Addr, LLContext.getInt64(MemoryOffset)); |
459 | 0 | } |
460 | 0 | compileAtomicCheckOffsetAlignment(Offset, TargetType); |
461 | 0 | auto VPtr = Builder.createInBoundsGEP1( |
462 | 0 | Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Offset); |
463 | 0 | auto Ptr = Builder.createBitCast(VPtr, TargetType.getPointerTo()); |
464 | 0 | auto Store = Builder.createStore(V, Ptr, true); |
465 | 0 | Store.setAlignment(1 << Alignment); |
466 | 0 | Store.setOrdering(LLVMAtomicOrderingSequentiallyConsistent); |
467 | 0 | } |
468 | | |
469 | | void FunctionCompiler::compileAtomicRMWOp( |
470 | | unsigned MemoryIndex, uint64_t MemoryOffset, |
471 | | [[maybe_unused]] unsigned Alignment, LLVMAtomicRMWBinOp BinOp, |
472 | 0 | LLVM::Type IntType, LLVM::Type TargetType, bool Signed) noexcept { |
473 | 0 | auto Value = Builder.createSExtOrTrunc(stackPop(), TargetType); |
474 | 0 | auto Addr = Builder.createZExt(Stack.back(), Context.Int64Ty); |
475 | 0 | boundsCheckMemory64(MemoryIndex, Addr, MemoryOffset, |
476 | 0 | TargetType.getPrimitiveSizeInBits() / 8); |
477 | 0 | auto Offset = Addr; |
478 | 0 | if (MemoryOffset != 0) { |
479 | 0 | Offset = Builder.createAdd(Addr, LLContext.getInt64(MemoryOffset)); |
480 | 0 | } |
481 | 0 | compileAtomicCheckOffsetAlignment(Offset, TargetType); |
482 | 0 | auto VPtr = Builder.createInBoundsGEP1( |
483 | 0 | Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Offset); |
484 | 0 | auto Ptr = Builder.createBitCast(VPtr, TargetType.getPointerTo()); |
485 | |
|
486 | 0 | LLVM::Value Ret; |
487 | | if constexpr (Endian::native == Endian::big) { |
488 | | if (BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpAdd || |
489 | | BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpSub) { |
490 | | auto AtomicBB = LLVM::BasicBlock::create(LLContext, F.Fn, "atomic.rmw"); |
491 | | auto OkBB = LLVM::BasicBlock::create(LLContext, F.Fn, "atomic.rmw.ok"); |
492 | | Builder.createBr(AtomicBB); |
493 | | Builder.positionAtEnd(AtomicBB); |
494 | | |
495 | | auto Load = Builder.createLoad(TargetType, Ptr, true); |
496 | | Load.setOrdering(LLVMAtomicOrderingMonotonic); |
497 | | Load.setAlignment(1 << Alignment); |
498 | | |
499 | | LLVM::Value New; |
500 | | if (BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpAdd) |
501 | | New = Builder.createAdd(switchEndian(Load), Value); |
502 | | else if (BinOp == LLVMAtomicRMWBinOp::LLVMAtomicRMWBinOpSub) { |
503 | | New = Builder.createSub(switchEndian(Load), Value); |
504 | | } else { |
505 | | assumingUnreachable(); |
506 | | } |
507 | | New = switchEndian(New); |
508 | | |
509 | | auto Exchange = Builder.createAtomicCmpXchg( |
510 | | Ptr, Load, New, LLVMAtomicOrderingSequentiallyConsistent, |
511 | | LLVMAtomicOrderingSequentiallyConsistent); |
512 | | |
513 | | Ret = Builder.createExtractValue(Exchange, 0); |
514 | | auto Success = Builder.createExtractValue(Exchange, 1); |
515 | | Builder.createCondBr(Success, OkBB, AtomicBB); |
516 | | Builder.positionAtEnd(OkBB); |
517 | | } else { |
518 | | Ret = Builder.createAtomicRMW(BinOp, Ptr, switchEndian(Value), |
519 | | LLVMAtomicOrderingSequentiallyConsistent); |
520 | | } |
521 | 0 | } else { |
522 | 0 | Ret = Builder.createAtomicRMW(BinOp, Ptr, switchEndian(Value), |
523 | 0 | LLVMAtomicOrderingSequentiallyConsistent); |
524 | 0 | } |
525 | 0 | Ret = switchEndian(Ret); |
526 | | #if LLVM_VERSION_MAJOR >= 13 |
527 | | Ret.setAlignment(1 << Alignment); |
528 | | #endif |
529 | 0 | if (Signed) { |
530 | 0 | Stack.back() = Builder.createSExt(Ret, IntType); |
531 | 0 | } else { |
532 | 0 | Stack.back() = Builder.createZExt(Ret, IntType); |
533 | 0 | } |
534 | 0 | } |
535 | | |
536 | | void FunctionCompiler::compileAtomicCompareExchange( |
537 | | unsigned MemoryIndex, uint64_t MemoryOffset, |
538 | | [[maybe_unused]] unsigned Alignment, LLVM::Type IntType, |
539 | 0 | LLVM::Type TargetType, bool Signed) noexcept { |
540 | |
|
541 | 0 | auto Replacement = Builder.createSExtOrTrunc(stackPop(), TargetType); |
542 | 0 | auto Expected = Builder.createSExtOrTrunc(stackPop(), TargetType); |
543 | 0 | auto Addr = Builder.createZExt(Stack.back(), Context.Int64Ty); |
544 | 0 | boundsCheckMemory64(MemoryIndex, Addr, MemoryOffset, |
545 | 0 | TargetType.getPrimitiveSizeInBits() / 8); |
546 | 0 | auto Offset = Addr; |
547 | 0 | if (MemoryOffset != 0) { |
548 | 0 | Offset = Builder.createAdd(Addr, LLContext.getInt64(MemoryOffset)); |
549 | 0 | } |
550 | 0 | compileAtomicCheckOffsetAlignment(Offset, TargetType); |
551 | 0 | auto VPtr = Builder.createInBoundsGEP1( |
552 | 0 | Context.Int8Ty, Context.getMemory(Builder, ModCtx, MemoryIndex), Offset); |
553 | 0 | auto Ptr = Builder.createBitCast(VPtr, TargetType.getPointerTo()); |
554 | |
|
555 | 0 | auto Ret = Builder.createAtomicCmpXchg( |
556 | 0 | Ptr, switchEndian(Expected), switchEndian(Replacement), |
557 | 0 | LLVMAtomicOrderingSequentiallyConsistent, |
558 | 0 | LLVMAtomicOrderingSequentiallyConsistent); |
559 | | #if LLVM_VERSION_MAJOR >= 13 |
560 | | Ret.setAlignment(1 << Alignment); |
561 | | #endif |
562 | 0 | auto OldVal = Builder.createExtractValue(Ret, 0); |
563 | 0 | OldVal = switchEndian(OldVal); |
564 | 0 | if (Signed) { |
565 | 0 | Stack.back() = Builder.createSExt(OldVal, IntType); |
566 | 0 | } else { |
567 | 0 | Stack.back() = Builder.createZExt(OldVal, IntType); |
568 | 0 | } |
569 | 0 | } |
570 | | |
571 | | } // namespace WasmEdge |