/src/tarantool/third_party/luajit/src/lj_ccallback.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | ** FFI C callback handling. |
3 | | ** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h |
4 | | */ |
5 | | |
6 | | #include "lj_obj.h" |
7 | | |
8 | | #if LJ_HASFFI |
9 | | |
10 | | #include "lj_gc.h" |
11 | | #include "lj_err.h" |
12 | | #include "lj_tab.h" |
13 | | #include "lj_state.h" |
14 | | #include "lj_frame.h" |
15 | | #include "lj_ctype.h" |
16 | | #include "lj_cconv.h" |
17 | | #include "lj_ccall.h" |
18 | | #include "lj_ccallback.h" |
19 | | #include "lj_target.h" |
20 | | #include "lj_mcode.h" |
21 | | #include "lj_trace.h" |
22 | | #include "lj_vm.h" |
23 | | |
24 | | /* -- Target-specific handling of callback slots -------------------------- */ |
25 | | |
26 | 0 | #define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE) |
27 | | |
28 | | #if LJ_OS_NOJIT |
29 | | |
30 | | /* Callbacks disabled. */ |
31 | | #define CALLBACK_SLOT2OFS(slot) (0*(slot)) |
32 | | #define CALLBACK_OFS2SLOT(ofs) (0*(ofs)) |
33 | | #define CALLBACK_MAX_SLOT 0 |
34 | | |
35 | | #elif LJ_TARGET_X86ORX64 |
36 | | |
37 | 0 | #define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0) |
38 | 0 | #define CALLBACK_MCODE_GROUP (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5)) |
39 | | |
40 | | #define CALLBACK_SLOT2OFS(slot) \ |
41 | 0 | (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot)) |
42 | | |
43 | | static MSize CALLBACK_OFS2SLOT(MSize ofs) |
44 | 0 | { |
45 | 0 | MSize group; |
46 | 0 | ofs -= CALLBACK_MCODE_HEAD; |
47 | 0 | group = ofs / (32*4 + CALLBACK_MCODE_GROUP); |
48 | 0 | return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32; |
49 | 0 | } |
50 | | |
51 | | #define CALLBACK_MAX_SLOT \ |
52 | 0 | (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32) |
53 | | |
54 | | #elif LJ_TARGET_ARM |
55 | | |
56 | | #define CALLBACK_MCODE_HEAD 32 |
57 | | |
58 | | #elif LJ_TARGET_ARM64 |
59 | | |
60 | | #define CALLBACK_MCODE_HEAD 32 |
61 | | |
62 | | #elif LJ_TARGET_PPC |
63 | | |
64 | | #define CALLBACK_MCODE_HEAD 24 |
65 | | |
66 | | #elif LJ_TARGET_MIPS32 |
67 | | |
68 | | #define CALLBACK_MCODE_HEAD 20 |
69 | | |
70 | | #elif LJ_TARGET_MIPS64 |
71 | | |
72 | | #define CALLBACK_MCODE_HEAD 52 |
73 | | |
74 | | #else |
75 | | |
76 | | /* Missing support for this architecture. */ |
77 | | #define CALLBACK_SLOT2OFS(slot) (0*(slot)) |
78 | | #define CALLBACK_OFS2SLOT(ofs) (0*(ofs)) |
79 | | #define CALLBACK_MAX_SLOT 0 |
80 | | |
81 | | #endif |
82 | | |
83 | | #ifndef CALLBACK_SLOT2OFS |
84 | | #define CALLBACK_SLOT2OFS(slot) (CALLBACK_MCODE_HEAD + 8*(slot)) |
85 | | #define CALLBACK_OFS2SLOT(ofs) (((ofs)-CALLBACK_MCODE_HEAD)/8) |
86 | | #define CALLBACK_MAX_SLOT (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE)) |
87 | | #endif |
88 | | |
89 | | /* Convert callback slot number to callback function pointer. */ |
90 | | static void *callback_slot2ptr(CTState *cts, MSize slot) |
91 | 0 | { |
92 | 0 | return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot); |
93 | 0 | } |
94 | | |
95 | | /* Convert callback function pointer to slot number. */ |
96 | | MSize lj_ccallback_ptr2slot(CTState *cts, void *p) |
97 | 0 | { |
98 | 0 | uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode); |
99 | 0 | if (ofs < CALLBACK_MCODE_SIZE) { |
100 | 0 | MSize slot = CALLBACK_OFS2SLOT((MSize)ofs); |
101 | 0 | if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs) |
102 | 0 | return slot; |
103 | 0 | } |
104 | 0 | return ~0u; /* Not a known callback function pointer. */ |
105 | 0 | } |
106 | | |
107 | | /* Initialize machine code for callback function pointers. */ |
108 | | #if LJ_OS_NOJIT |
109 | | /* Disabled callback support. */ |
110 | | #define callback_mcode_init(g, p) (p) |
111 | | #elif LJ_TARGET_X86ORX64 |
112 | | static void *callback_mcode_init(global_State *g, uint8_t *page) |
113 | 0 | { |
114 | 0 | uint8_t *p = page; |
115 | 0 | uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback; |
116 | 0 | MSize slot; |
117 | 0 | #if LJ_64 |
118 | 0 | *(void **)p = target; p += 8; |
119 | 0 | #endif |
120 | 0 | for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { |
121 | | /* mov al, slot; jmp group */ |
122 | 0 | *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot; |
123 | 0 | if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) { |
124 | | /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */ |
125 | 0 | *p++ = XI_PUSH + RID_EBP; |
126 | 0 | *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8); |
127 | 0 | #if LJ_GC64 |
128 | 0 | *p++ = 0x48; *p++ = XI_MOVri | RID_EBP; |
129 | 0 | *(uint64_t *)p = (uint64_t)(g); p += 8; |
130 | | #else |
131 | | *p++ = XI_MOVri | RID_EBP; |
132 | | *(int32_t *)p = i32ptr(g); p += 4; |
133 | | #endif |
134 | 0 | #if LJ_64 |
135 | | /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */ |
136 | 0 | *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP; |
137 | 0 | *(int32_t *)p = (int32_t)(page-(p+4)); p += 4; |
138 | | #else |
139 | | /* jmp lj_vm_ffi_callback. */ |
140 | | *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4; |
141 | | #endif |
142 | 0 | } else { |
143 | 0 | *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2); |
144 | 0 | } |
145 | 0 | } |
146 | 0 | return p; |
147 | 0 | } |
148 | | #elif LJ_TARGET_ARM |
149 | | static void *callback_mcode_init(global_State *g, uint32_t *page) |
150 | | { |
151 | | uint32_t *p = page; |
152 | | void *target = (void *)lj_vm_ffi_callback; |
153 | | MSize slot; |
154 | | /* This must match with the saveregs macro in buildvm_arm.dasc. */ |
155 | | *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC); |
156 | | *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR); |
157 | | *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD; |
158 | | *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9); |
159 | | *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC); |
160 | | *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC); |
161 | | *p++ = u32ptr(g); |
162 | | *p++ = u32ptr(target); |
163 | | for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { |
164 | | *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC); |
165 | | *p = ARMI_B | ((page-p-2) & 0x00ffffffu); |
166 | | p++; |
167 | | } |
168 | | return p; |
169 | | } |
170 | | #elif LJ_TARGET_ARM64 |
171 | | static void *callback_mcode_init(global_State *g, uint32_t *page) |
172 | | { |
173 | | uint32_t *p = page; |
174 | | void *target = (void *)lj_vm_ffi_callback; |
175 | | MSize slot; |
176 | | *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4)); |
177 | | *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5)); |
178 | | *p++ = A64I_LE(A64I_BR | A64F_N(RID_X11)); |
179 | | *p++ = A64I_LE(A64I_NOP); |
180 | | ((void **)p)[0] = target; |
181 | | ((void **)p)[1] = g; |
182 | | p += 4; |
183 | | for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { |
184 | | *p++ = A64I_LE(A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot)); |
185 | | *p = A64I_LE(A64I_B | A64F_S26((page-p) & 0x03ffffffu)); |
186 | | p++; |
187 | | } |
188 | | return p; |
189 | | } |
190 | | #elif LJ_TARGET_PPC |
191 | | static void *callback_mcode_init(global_State *g, uint32_t *page) |
192 | | { |
193 | | uint32_t *p = page; |
194 | | void *target = (void *)lj_vm_ffi_callback; |
195 | | MSize slot; |
196 | | *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16); |
197 | | *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16); |
198 | | *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff); |
199 | | *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff); |
200 | | *p++ = PPCI_MTCTR | PPCF_T(RID_TMP); |
201 | | *p++ = PPCI_BCTR; |
202 | | for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { |
203 | | *p++ = PPCI_LI | PPCF_T(RID_R11) | slot; |
204 | | *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2); |
205 | | p++; |
206 | | } |
207 | | return p; |
208 | | } |
209 | | #elif LJ_TARGET_MIPS |
210 | | static void *callback_mcode_init(global_State *g, uint32_t *page) |
211 | | { |
212 | | uint32_t *p = page; |
213 | | uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback; |
214 | | uintptr_t ug = (uintptr_t)(void *)g; |
215 | | MSize slot; |
216 | | #if LJ_TARGET_MIPS32 |
217 | | *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16); |
218 | | *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16); |
219 | | #else |
220 | | *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 48); |
221 | | *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 48); |
222 | | *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff); |
223 | | *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff); |
224 | | *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16); |
225 | | *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16); |
226 | | *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff); |
227 | | *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff); |
228 | | *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16); |
229 | | *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16); |
230 | | #endif |
231 | | *p++ = MIPSI_ORI | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff); |
232 | | *p++ = MIPSI_JR | MIPSF_S(RID_R3); |
233 | | *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff); |
234 | | for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) { |
235 | | *p = MIPSI_B | ((page-p-1) & 0x0000ffffu); |
236 | | p++; |
237 | | *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot; |
238 | | } |
239 | | return p; |
240 | | } |
241 | | #else |
242 | | /* Missing support for this architecture. */ |
243 | | #define callback_mcode_init(g, p) (p) |
244 | | #endif |
245 | | |
246 | | /* -- Machine code management --------------------------------------------- */ |
247 | | |
248 | | #if LJ_TARGET_WINDOWS |
249 | | |
250 | | #define WIN32_LEAN_AND_MEAN |
251 | | #include <windows.h> |
252 | | |
253 | | #elif LJ_TARGET_POSIX |
254 | | |
255 | | #include <sys/mman.h> |
256 | | #ifndef MAP_ANONYMOUS |
257 | | #define MAP_ANONYMOUS MAP_ANON |
258 | | #endif |
259 | | |
260 | | #endif |
261 | | |
262 | | /* Allocate and initialize area for callback function pointers. */ |
263 | | static void callback_mcode_new(CTState *cts) |
264 | 0 | { |
265 | 0 | size_t sz = (size_t)CALLBACK_MCODE_SIZE; |
266 | 0 | void *p, *pe; |
267 | 0 | if (CALLBACK_MAX_SLOT == 0) |
268 | 0 | lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); |
269 | | #if LJ_TARGET_WINDOWS |
270 | | p = LJ_WIN_VALLOC(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); |
271 | | if (!p) |
272 | | lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); |
273 | | #elif LJ_TARGET_POSIX |
274 | 0 | p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS, |
275 | 0 | -1, 0); |
276 | 0 | if (p == MAP_FAILED) |
277 | 0 | lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); |
278 | | #else |
279 | | /* Fallback allocator. Fails if memory is not executable by default. */ |
280 | | p = lj_mem_new(cts->L, sz); |
281 | | #endif |
282 | 0 | cts->cb.mcode = p; |
283 | 0 | pe = callback_mcode_init(cts->g, p); |
284 | 0 | UNUSED(pe); |
285 | 0 | lj_assertCTS((size_t)((char *)pe - (char *)p) <= sz, |
286 | 0 | "miscalculated CALLBACK_MAX_SLOT"); |
287 | 0 | lj_mcode_sync(p, (char *)p + sz); |
288 | | #if LJ_TARGET_WINDOWS |
289 | | { |
290 | | DWORD oprot; |
291 | | LJ_WIN_VPROTECT(p, sz, PAGE_EXECUTE_READ, &oprot); |
292 | | } |
293 | | #elif LJ_TARGET_POSIX |
294 | 0 | mprotect(p, sz, (PROT_READ|PROT_EXEC)); |
295 | 0 | #endif |
296 | 0 | } |
297 | | |
298 | | /* Free area for callback function pointers. */ |
299 | | void lj_ccallback_mcode_free(CTState *cts) |
300 | 0 | { |
301 | 0 | size_t sz = (size_t)CALLBACK_MCODE_SIZE; |
302 | 0 | void *p = cts->cb.mcode; |
303 | 0 | if (p == NULL) return; |
304 | | #if LJ_TARGET_WINDOWS |
305 | | VirtualFree(p, 0, MEM_RELEASE); |
306 | | UNUSED(sz); |
307 | | #elif LJ_TARGET_POSIX |
308 | 0 | munmap(p, sz); |
309 | | #else |
310 | | lj_mem_free(cts->g, p, sz); |
311 | | #endif |
312 | 0 | } |
313 | | |
314 | | /* -- C callback entry ---------------------------------------------------- */ |
315 | | |
316 | | /* Target-specific handling of register arguments. Similar to lj_ccall.c. */ |
317 | | #if LJ_TARGET_X86 |
318 | | |
319 | | #define CALLBACK_HANDLE_REGARG \ |
320 | | if (!isfp) { /* Only non-FP values may be passed in registers. */ \ |
321 | | if (n > 1) { /* Anything > 32 bit is passed on the stack. */ \ |
322 | | if (!LJ_ABI_WIN) ngpr = maxgpr; /* Prevent reordering. */ \ |
323 | | } else if (ngpr + 1 <= maxgpr) { \ |
324 | | sp = &cts->cb.gpr[ngpr]; \ |
325 | | ngpr += n; \ |
326 | | goto done; \ |
327 | | } \ |
328 | | } |
329 | | |
330 | | #elif LJ_TARGET_X64 && LJ_ABI_WIN |
331 | | |
332 | | /* Windows/x64 argument registers are strictly positional (use ngpr). */ |
333 | | #define CALLBACK_HANDLE_REGARG \ |
334 | | if (isfp) { \ |
335 | | if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \ |
336 | | } else { \ |
337 | | if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \ |
338 | | } |
339 | | |
340 | | #elif LJ_TARGET_X64 |
341 | | |
342 | | #define CALLBACK_HANDLE_REGARG \ |
343 | 0 | if (isfp) { \ |
344 | 0 | if (nfpr + n <= CCALL_NARG_FPR) { \ |
345 | 0 | sp = &cts->cb.fpr[nfpr]; \ |
346 | 0 | nfpr += n; \ |
347 | 0 | goto done; \ |
348 | 0 | } \ |
349 | 0 | } else { \ |
350 | 0 | if (ngpr + n <= maxgpr) { \ |
351 | 0 | sp = &cts->cb.gpr[ngpr]; \ |
352 | 0 | ngpr += n; \ |
353 | 0 | goto done; \ |
354 | 0 | } \ |
355 | 0 | } |
356 | | |
357 | | #elif LJ_TARGET_ARM |
358 | | |
359 | | #if LJ_ABI_SOFTFP |
360 | | |
361 | | #define CALLBACK_HANDLE_REGARG_FP1 UNUSED(isfp); |
362 | | #define CALLBACK_HANDLE_REGARG_FP2 |
363 | | |
364 | | #else |
365 | | |
366 | | #define CALLBACK_HANDLE_REGARG_FP1 \ |
367 | | if (isfp) { \ |
368 | | if (n == 1) { \ |
369 | | if (fprodd) { \ |
370 | | sp = &cts->cb.fpr[fprodd-1]; \ |
371 | | fprodd = 0; \ |
372 | | goto done; \ |
373 | | } else if (nfpr + 1 <= CCALL_NARG_FPR) { \ |
374 | | sp = &cts->cb.fpr[nfpr++]; \ |
375 | | fprodd = nfpr; \ |
376 | | goto done; \ |
377 | | } \ |
378 | | } else { \ |
379 | | if (nfpr + 1 <= CCALL_NARG_FPR) { \ |
380 | | sp = &cts->cb.fpr[nfpr++]; \ |
381 | | goto done; \ |
382 | | } \ |
383 | | } \ |
384 | | fprodd = 0; /* No reordering after the first FP value is on stack. */ \ |
385 | | } else { |
386 | | |
387 | | #define CALLBACK_HANDLE_REGARG_FP2 } |
388 | | |
389 | | #endif |
390 | | |
391 | | #define CALLBACK_HANDLE_REGARG \ |
392 | | CALLBACK_HANDLE_REGARG_FP1 \ |
393 | | if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ |
394 | | if (ngpr + n <= maxgpr) { \ |
395 | | sp = &cts->cb.gpr[ngpr]; \ |
396 | | ngpr += n; \ |
397 | | goto done; \ |
398 | | } CALLBACK_HANDLE_REGARG_FP2 |
399 | | |
400 | | #elif LJ_TARGET_ARM64 |
401 | | |
402 | | #define CALLBACK_HANDLE_REGARG \ |
403 | | if (isfp) { \ |
404 | | if (nfpr + n <= CCALL_NARG_FPR) { \ |
405 | | sp = &cts->cb.fpr[nfpr]; \ |
406 | | nfpr += n; \ |
407 | | goto done; \ |
408 | | } else { \ |
409 | | nfpr = CCALL_NARG_FPR; /* Prevent reordering. */ \ |
410 | | } \ |
411 | | } else { \ |
412 | | if (!LJ_TARGET_OSX && n > 1) \ |
413 | | ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ |
414 | | if (ngpr + n <= maxgpr) { \ |
415 | | sp = &cts->cb.gpr[ngpr]; \ |
416 | | ngpr += n; \ |
417 | | goto done; \ |
418 | | } else { \ |
419 | | ngpr = CCALL_NARG_GPR; /* Prevent reordering. */ \ |
420 | | } \ |
421 | | } |
422 | | |
423 | | #elif LJ_TARGET_PPC |
424 | | |
425 | | #define CALLBACK_HANDLE_GPR \ |
426 | | if (n > 1) { \ |
427 | | lj_assertCTS(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) || /* double. */ \ |
428 | | ctype_isinteger(cta->info)) && n == 2, /* int64_t. */ \ |
429 | | "bad GPR type"); \ |
430 | | ngpr = (ngpr + 1u) & ~1u; /* Align int64_t to regpair. */ \ |
431 | | } \ |
432 | | if (ngpr + n <= maxgpr) { \ |
433 | | sp = &cts->cb.gpr[ngpr]; \ |
434 | | ngpr += n; \ |
435 | | goto done; \ |
436 | | } |
437 | | |
438 | | #if LJ_ABI_SOFTFP |
439 | | #define CALLBACK_HANDLE_REGARG \ |
440 | | CALLBACK_HANDLE_GPR \ |
441 | | UNUSED(isfp); |
442 | | #else |
443 | | #define CALLBACK_HANDLE_REGARG \ |
444 | | if (isfp) { \ |
445 | | if (nfpr + 1 <= CCALL_NARG_FPR) { \ |
446 | | sp = &cts->cb.fpr[nfpr++]; \ |
447 | | cta = ctype_get(cts, CTID_DOUBLE); /* FPRs always hold doubles. */ \ |
448 | | goto done; \ |
449 | | } \ |
450 | | } else { /* Try to pass argument in GPRs. */ \ |
451 | | CALLBACK_HANDLE_GPR \ |
452 | | } |
453 | | #endif |
454 | | |
455 | | #if !LJ_ABI_SOFTFP |
456 | | #define CALLBACK_HANDLE_RET \ |
457 | | if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ |
458 | | *(double *)dp = *(float *)dp; /* FPRs always hold doubles. */ |
459 | | #endif |
460 | | |
461 | | #elif LJ_TARGET_MIPS32 |
462 | | |
463 | | #define CALLBACK_HANDLE_GPR \ |
464 | | if (n > 1) ngpr = (ngpr + 1u) & ~1u; /* Align to regpair. */ \ |
465 | | if (ngpr + n <= maxgpr) { \ |
466 | | sp = &cts->cb.gpr[ngpr]; \ |
467 | | ngpr += n; \ |
468 | | goto done; \ |
469 | | } |
470 | | |
471 | | #if !LJ_ABI_SOFTFP /* MIPS32 hard-float */ |
472 | | #define CALLBACK_HANDLE_REGARG \ |
473 | | if (isfp && nfpr < CCALL_NARG_FPR) { /* Try to pass argument in FPRs. */ \ |
474 | | sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \ |
475 | | nfpr++; ngpr += n; \ |
476 | | goto done; \ |
477 | | } else { /* Try to pass argument in GPRs. */ \ |
478 | | nfpr = CCALL_NARG_FPR; \ |
479 | | CALLBACK_HANDLE_GPR \ |
480 | | } |
481 | | #else /* MIPS32 soft-float */ |
482 | | #define CALLBACK_HANDLE_REGARG \ |
483 | | CALLBACK_HANDLE_GPR \ |
484 | | UNUSED(isfp); |
485 | | #endif |
486 | | |
487 | | #define CALLBACK_HANDLE_RET \ |
488 | | if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ |
489 | | ((float *)dp)[1] = *(float *)dp; |
490 | | |
491 | | #elif LJ_TARGET_MIPS64 |
492 | | |
493 | | #if !LJ_ABI_SOFTFP /* MIPS64 hard-float */ |
494 | | #define CALLBACK_HANDLE_REGARG \ |
495 | | if (ngpr + n <= maxgpr) { \ |
496 | | sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \ |
497 | | ngpr += n; \ |
498 | | goto done; \ |
499 | | } |
500 | | #else /* MIPS64 soft-float */ |
501 | | #define CALLBACK_HANDLE_REGARG \ |
502 | | if (ngpr + n <= maxgpr) { \ |
503 | | UNUSED(isfp); \ |
504 | | sp = (void*) &cts->cb.gpr[ngpr]; \ |
505 | | ngpr += n; \ |
506 | | goto done; \ |
507 | | } |
508 | | #endif |
509 | | |
510 | | #define CALLBACK_HANDLE_RET \ |
511 | | if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \ |
512 | | ((float *)dp)[1] = *(float *)dp; |
513 | | |
514 | | #else |
515 | | #error "Missing calling convention definitions for this architecture" |
516 | | #endif |
517 | | |
518 | | /* Convert and push callback arguments to Lua stack. */ |
519 | | static void callback_conv_args(CTState *cts, lua_State *L) |
520 | 0 | { |
521 | 0 | TValue *o = L->top; |
522 | 0 | intptr_t *stack = cts->cb.stack; |
523 | 0 | MSize slot = cts->cb.slot; |
524 | 0 | CTypeID id = 0, rid, fid; |
525 | 0 | int gcsteps = 0; |
526 | 0 | CType *ct; |
527 | 0 | GCfunc *fn; |
528 | 0 | int fntp; |
529 | 0 | MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR; |
530 | 0 | #if CCALL_NARG_FPR |
531 | 0 | MSize nfpr = 0; |
532 | | #if LJ_TARGET_ARM |
533 | | MSize fprodd = 0; |
534 | | #endif |
535 | 0 | #endif |
536 | |
|
537 | 0 | if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) { |
538 | 0 | ct = ctype_get(cts, id); |
539 | 0 | rid = ctype_cid(ct->info); /* Return type. x86: +(spadj<<16). */ |
540 | 0 | fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot)); |
541 | 0 | fntp = LJ_TFUNC; |
542 | 0 | } else { /* Must set up frame first, before throwing the error. */ |
543 | 0 | ct = NULL; |
544 | 0 | rid = 0; |
545 | 0 | fn = (GCfunc *)L; |
546 | 0 | fntp = LJ_TTHREAD; |
547 | 0 | } |
548 | | /* Continuation returns from callback. */ |
549 | 0 | if (LJ_FR2) { |
550 | 0 | (o++)->u64 = LJ_CONT_FFI_CALLBACK; |
551 | 0 | (o++)->u64 = rid; |
552 | 0 | } else { |
553 | 0 | o->u32.lo = LJ_CONT_FFI_CALLBACK; |
554 | 0 | o->u32.hi = rid; |
555 | 0 | o++; |
556 | 0 | } |
557 | 0 | setframe_gc(o, obj2gco(fn), fntp); |
558 | 0 | if (LJ_FR2) o++; |
559 | 0 | setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT); |
560 | 0 | L->top = L->base = ++o; |
561 | 0 | if (!ct) |
562 | 0 | lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK); |
563 | 0 | if (isluafunc(fn)) |
564 | 0 | setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1); |
565 | 0 | lj_state_checkstack(L, LUA_MINSTACK); /* May throw. */ |
566 | 0 | o = L->base; /* Might have been reallocated. */ |
567 | |
|
568 | | #if LJ_TARGET_X86 |
569 | | /* x86 has several different calling conventions. */ |
570 | | switch (ctype_cconv(ct->info)) { |
571 | | case CTCC_FASTCALL: maxgpr = 2; break; |
572 | | case CTCC_THISCALL: maxgpr = 1; break; |
573 | | default: maxgpr = 0; break; |
574 | | } |
575 | | #endif |
576 | |
|
577 | 0 | fid = ct->sib; |
578 | 0 | while (fid) { |
579 | 0 | CType *ctf = ctype_get(cts, fid); |
580 | 0 | if (!ctype_isattrib(ctf->info)) { |
581 | 0 | CType *cta; |
582 | 0 | void *sp; |
583 | 0 | CTSize sz; |
584 | 0 | int isfp; |
585 | 0 | MSize n; |
586 | 0 | lj_assertCTS(ctype_isfield(ctf->info), "field expected"); |
587 | 0 | cta = ctype_rawchild(cts, ctf); |
588 | 0 | isfp = ctype_isfp(cta->info); |
589 | 0 | sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1); |
590 | 0 | n = sz / CTSIZE_PTR; /* Number of GPRs or stack slots needed. */ |
591 | |
|
592 | | CALLBACK_HANDLE_REGARG /* Handle register arguments. */ |
593 | 0 | |
594 | 0 | /* Otherwise pass argument on stack. */ |
595 | 0 | if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8) |
596 | 0 | nsp = (nsp + 1) & ~1u; /* Align 64 bit argument on stack. */ |
597 | 0 | sp = &stack[nsp]; |
598 | 0 | nsp += n; |
599 | |
|
600 | 0 | done: |
601 | 0 | if (LJ_BE && cta->size < CTSIZE_PTR |
602 | | #if LJ_TARGET_MIPS64 |
603 | | && !(isfp && nsp) |
604 | | #endif |
605 | 0 | ) |
606 | 0 | sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size); |
607 | 0 | gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp); |
608 | 0 | } |
609 | 0 | fid = ctf->sib; |
610 | 0 | } |
611 | 0 | L->top = o; |
612 | | #if LJ_TARGET_X86 |
613 | | /* Store stack adjustment for returns from non-cdecl callbacks. */ |
614 | | if (ctype_cconv(ct->info) != CTCC_CDECL) { |
615 | | #if LJ_FR2 |
616 | | (L->base-3)->u64 |= (nsp << (16+2)); |
617 | | #else |
618 | | (L->base-2)->u32.hi |= (nsp << (16+2)); |
619 | | #endif |
620 | | } |
621 | | #endif |
622 | 0 | while (gcsteps-- > 0) |
623 | 0 | lj_gc_check(L); |
624 | 0 | } |
625 | | |
626 | | /* Convert Lua object to callback result. */ |
627 | | static void callback_conv_result(CTState *cts, lua_State *L, TValue *o) |
628 | 0 | { |
629 | 0 | #if LJ_FR2 |
630 | 0 | CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64); |
631 | | #else |
632 | | CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi); |
633 | | #endif |
634 | | #if LJ_TARGET_X86 |
635 | | cts->cb.gpr[2] = 0; |
636 | | #endif |
637 | 0 | if (!ctype_isvoid(ctr->info)) { |
638 | 0 | uint8_t *dp = (uint8_t *)&cts->cb.gpr[0]; |
639 | 0 | #if CCALL_NUM_FPR |
640 | 0 | if (ctype_isfp(ctr->info)) |
641 | 0 | dp = (uint8_t *)&cts->cb.fpr[0]; |
642 | 0 | #endif |
643 | | #if LJ_TARGET_ARM64 && LJ_BE |
644 | | if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) |
645 | | dp = (uint8_t *)&cts->cb.fpr[0].f[1]; |
646 | | #endif |
647 | 0 | lj_cconv_ct_tv(cts, ctr, dp, o, 0); |
648 | | #ifdef CALLBACK_HANDLE_RET |
649 | | CALLBACK_HANDLE_RET |
650 | | #endif |
651 | | /* Extend returned integers to (at least) 32 bits. */ |
652 | 0 | if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) { |
653 | 0 | if (ctr->info & CTF_UNSIGNED) |
654 | 0 | *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp : |
655 | 0 | (uint32_t)*(uint16_t *)dp; |
656 | 0 | else |
657 | 0 | *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp : |
658 | 0 | (int32_t)*(int16_t *)dp; |
659 | 0 | } |
660 | | #if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE) |
661 | | /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */ |
662 | | if (ctr->size <= 4 && |
663 | | (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info))) |
664 | | *(int64_t *)dp = (int64_t)*(int32_t *)dp; |
665 | | #endif |
666 | | #if LJ_TARGET_X86 |
667 | | if (ctype_isfp(ctr->info)) |
668 | | cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2; |
669 | | #endif |
670 | 0 | } |
671 | 0 | } |
672 | | |
673 | | /* Enter callback. */ |
674 | | lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf) |
675 | 0 | { |
676 | 0 | lua_State *L = cts->L; |
677 | 0 | global_State *g = cts->g; |
678 | 0 | lj_assertG(L != NULL, "uninitialized cts->L in callback"); |
679 | 0 | if (tvref(g->jit_base)) { |
680 | 0 | setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK)); |
681 | 0 | if (g->panic) g->panic(L); |
682 | 0 | exit(EXIT_FAILURE); |
683 | 0 | } |
684 | 0 | lj_trace_abort(g); /* Never record across callback. */ |
685 | | /* Setup C frame. */ |
686 | 0 | cframe_prev(cf) = L->cframe; |
687 | 0 | setcframe_L(cf, L); |
688 | 0 | cframe_errfunc(cf) = -1; |
689 | 0 | cframe_nres(cf) = 0; |
690 | 0 | L->cframe = cf; |
691 | 0 | callback_conv_args(cts, L); |
692 | 0 | return L; /* Now call the function on this stack. */ |
693 | 0 | } |
694 | | |
695 | | /* Leave callback. */ |
696 | | void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o) |
697 | 0 | { |
698 | 0 | lua_State *L = cts->L; |
699 | 0 | GCfunc *fn; |
700 | 0 | TValue *obase = L->base; |
701 | 0 | L->base = L->top; /* Keep continuation frame for throwing errors. */ |
702 | 0 | if (o >= L->base) { |
703 | | /* PC of RET* is lost. Point to last line for result conv. errors. */ |
704 | 0 | fn = curr_func(L); |
705 | 0 | if (isluafunc(fn)) { |
706 | 0 | GCproto *pt = funcproto(fn); |
707 | 0 | setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1); |
708 | 0 | } |
709 | 0 | } |
710 | 0 | callback_conv_result(cts, L, o); |
711 | | /* Finally drop C frame and continuation frame. */ |
712 | 0 | L->top -= 2+2*LJ_FR2; |
713 | 0 | L->base = obase; |
714 | 0 | L->cframe = cframe_prev(L->cframe); |
715 | 0 | cts->cb.slot = 0; /* Blacklist C function that called the callback. */ |
716 | 0 | } |
717 | | |
718 | | /* -- C callback management ----------------------------------------------- */ |
719 | | |
720 | | /* Get an unused slot in the callback slot table. */ |
721 | | static MSize callback_slot_new(CTState *cts, CType *ct) |
722 | 0 | { |
723 | 0 | CTypeID id = ctype_typeid(cts, ct); |
724 | 0 | CTypeID1 *cbid = cts->cb.cbid; |
725 | 0 | MSize top; |
726 | 0 | for (top = cts->cb.topid; top < cts->cb.sizeid; top++) |
727 | 0 | if (LJ_LIKELY(cbid[top] == 0)) |
728 | 0 | goto found; |
729 | 0 | #if CALLBACK_MAX_SLOT |
730 | 0 | if (top >= CALLBACK_MAX_SLOT) |
731 | 0 | #endif |
732 | 0 | lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV); |
733 | 0 | if (!cts->cb.mcode) |
734 | 0 | callback_mcode_new(cts); |
735 | 0 | lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1); |
736 | 0 | cts->cb.cbid = cbid; |
737 | 0 | memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1)); |
738 | 0 | found: |
739 | 0 | cbid[top] = id; |
740 | 0 | cts->cb.topid = top+1; |
741 | 0 | return top; |
742 | 0 | } |
743 | | |
744 | | /* Check for function pointer and supported argument/result types. */ |
745 | | static CType *callback_checkfunc(CTState *cts, CType *ct) |
746 | 0 | { |
747 | 0 | int narg = 0; |
748 | 0 | if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR)) |
749 | 0 | return NULL; |
750 | 0 | ct = ctype_rawchild(cts, ct); |
751 | 0 | if (ctype_isfunc(ct->info)) { |
752 | 0 | CType *ctr = ctype_rawchild(cts, ct); |
753 | 0 | CTypeID fid = ct->sib; |
754 | 0 | if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) || |
755 | 0 | ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8))) |
756 | 0 | return NULL; |
757 | 0 | if ((ct->info & CTF_VARARG)) |
758 | 0 | return NULL; |
759 | 0 | while (fid) { |
760 | 0 | CType *ctf = ctype_get(cts, fid); |
761 | 0 | if (!ctype_isattrib(ctf->info)) { |
762 | 0 | CType *cta; |
763 | 0 | lj_assertCTS(ctype_isfield(ctf->info), "field expected"); |
764 | 0 | cta = ctype_rawchild(cts, ctf); |
765 | 0 | if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) || |
766 | 0 | (ctype_isnum(cta->info) && cta->size <= 8)) || |
767 | 0 | ++narg >= LUA_MINSTACK-3) |
768 | 0 | return NULL; |
769 | 0 | } |
770 | 0 | fid = ctf->sib; |
771 | 0 | } |
772 | 0 | return ct; |
773 | 0 | } |
774 | 0 | return NULL; |
775 | 0 | } |
776 | | |
777 | | /* Create a new callback and return the callback function pointer. */ |
778 | | void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn) |
779 | 0 | { |
780 | 0 | ct = callback_checkfunc(cts, ct); |
781 | 0 | if (ct) { |
782 | 0 | MSize slot = callback_slot_new(cts, ct); |
783 | 0 | GCtab *t = cts->miscmap; |
784 | 0 | setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn); |
785 | 0 | lj_gc_anybarriert(cts->L, t); |
786 | 0 | return callback_slot2ptr(cts, slot); |
787 | 0 | } |
788 | 0 | return NULL; /* Bad conversion. */ |
789 | 0 | } |
790 | | |
791 | | #endif |