Coverage Report

Created: 2024-04-24 06:23

/src/tarantool/third_party/luajit/src/lj_ccallback.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
** FFI C callback handling.
3
** Copyright (C) 2005-2017 Mike Pall. See Copyright Notice in luajit.h
4
*/
5
6
#include "lj_obj.h"
7
8
#if LJ_HASFFI
9
10
#include "lj_gc.h"
11
#include "lj_err.h"
12
#include "lj_tab.h"
13
#include "lj_state.h"
14
#include "lj_frame.h"
15
#include "lj_ctype.h"
16
#include "lj_cconv.h"
17
#include "lj_ccall.h"
18
#include "lj_ccallback.h"
19
#include "lj_target.h"
20
#include "lj_mcode.h"
21
#include "lj_trace.h"
22
#include "lj_vm.h"
23
24
/* -- Target-specific handling of callback slots -------------------------- */
25
26
0
#define CALLBACK_MCODE_SIZE (LJ_PAGESIZE * LJ_NUM_CBPAGE)
27
28
#if LJ_OS_NOJIT
29
30
/* Callbacks disabled. */
31
#define CALLBACK_SLOT2OFS(slot) (0*(slot))
32
#define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
33
#define CALLBACK_MAX_SLOT 0
34
35
#elif LJ_TARGET_X86ORX64
36
37
0
#define CALLBACK_MCODE_HEAD (LJ_64 ? 8 : 0)
38
0
#define CALLBACK_MCODE_GROUP  (-2+1+2+(LJ_GC64 ? 10 : 5)+(LJ_64 ? 6 : 5))
39
40
#define CALLBACK_SLOT2OFS(slot) \
41
0
  (CALLBACK_MCODE_HEAD + CALLBACK_MCODE_GROUP*((slot)/32) + 4*(slot))
42
43
static MSize CALLBACK_OFS2SLOT(MSize ofs)
44
0
{
45
0
  MSize group;
46
0
  ofs -= CALLBACK_MCODE_HEAD;
47
0
  group = ofs / (32*4 + CALLBACK_MCODE_GROUP);
48
0
  return (ofs % (32*4 + CALLBACK_MCODE_GROUP))/4 + group*32;
49
0
}
50
51
#define CALLBACK_MAX_SLOT \
52
0
  (((CALLBACK_MCODE_SIZE-CALLBACK_MCODE_HEAD)/(CALLBACK_MCODE_GROUP+4*32))*32)
53
54
#elif LJ_TARGET_ARM
55
56
#define CALLBACK_MCODE_HEAD   32
57
58
#elif LJ_TARGET_ARM64
59
60
#define CALLBACK_MCODE_HEAD   32
61
62
#elif LJ_TARGET_PPC
63
64
#define CALLBACK_MCODE_HEAD   24
65
66
#elif LJ_TARGET_MIPS32
67
68
#define CALLBACK_MCODE_HEAD   20
69
70
#elif LJ_TARGET_MIPS64
71
72
#define CALLBACK_MCODE_HEAD   52
73
74
#else
75
76
/* Missing support for this architecture. */
77
#define CALLBACK_SLOT2OFS(slot) (0*(slot))
78
#define CALLBACK_OFS2SLOT(ofs)  (0*(ofs))
79
#define CALLBACK_MAX_SLOT 0
80
81
#endif
82
83
#ifndef CALLBACK_SLOT2OFS
84
#define CALLBACK_SLOT2OFS(slot)   (CALLBACK_MCODE_HEAD + 8*(slot))
85
#define CALLBACK_OFS2SLOT(ofs)    (((ofs)-CALLBACK_MCODE_HEAD)/8)
86
#define CALLBACK_MAX_SLOT   (CALLBACK_OFS2SLOT(CALLBACK_MCODE_SIZE))
87
#endif
88
89
/* Convert callback slot number to callback function pointer. */
90
static void *callback_slot2ptr(CTState *cts, MSize slot)
91
0
{
92
0
  return (uint8_t *)cts->cb.mcode + CALLBACK_SLOT2OFS(slot);
93
0
}
94
95
/* Convert callback function pointer to slot number. */
96
MSize lj_ccallback_ptr2slot(CTState *cts, void *p)
97
0
{
98
0
  uintptr_t ofs = (uintptr_t)((uint8_t *)p -(uint8_t *)cts->cb.mcode);
99
0
  if (ofs < CALLBACK_MCODE_SIZE) {
100
0
    MSize slot = CALLBACK_OFS2SLOT((MSize)ofs);
101
0
    if (CALLBACK_SLOT2OFS(slot) == (MSize)ofs)
102
0
      return slot;
103
0
  }
104
0
  return ~0u;  /* Not a known callback function pointer. */
105
0
}
106
107
/* Initialize machine code for callback function pointers. */
108
#if LJ_OS_NOJIT
109
/* Disabled callback support. */
110
#define callback_mcode_init(g, p) (p)
111
#elif LJ_TARGET_X86ORX64
112
static void *callback_mcode_init(global_State *g, uint8_t *page)
113
0
{
114
0
  uint8_t *p = page;
115
0
  uint8_t *target = (uint8_t *)(void *)lj_vm_ffi_callback;
116
0
  MSize slot;
117
0
#if LJ_64
118
0
  *(void **)p = target; p += 8;
119
0
#endif
120
0
  for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
121
    /* mov al, slot; jmp group */
122
0
    *p++ = XI_MOVrib | RID_EAX; *p++ = (uint8_t)slot;
123
0
    if ((slot & 31) == 31 || slot == CALLBACK_MAX_SLOT-1) {
124
      /* push ebp/rbp; mov ah, slot>>8; mov ebp, &g. */
125
0
      *p++ = XI_PUSH + RID_EBP;
126
0
      *p++ = XI_MOVrib | (RID_EAX+4); *p++ = (uint8_t)(slot >> 8);
127
0
#if LJ_GC64
128
0
      *p++ = 0x48; *p++ = XI_MOVri | RID_EBP;
129
0
      *(uint64_t *)p = (uint64_t)(g); p += 8;
130
#else
131
      *p++ = XI_MOVri | RID_EBP;
132
      *(int32_t *)p = i32ptr(g); p += 4;
133
#endif
134
0
#if LJ_64
135
      /* jmp [rip-pageofs] where lj_vm_ffi_callback is stored. */
136
0
      *p++ = XI_GROUP5; *p++ = XM_OFS0 + (XOg_JMP<<3) + RID_EBP;
137
0
      *(int32_t *)p = (int32_t)(page-(p+4)); p += 4;
138
#else
139
      /* jmp lj_vm_ffi_callback. */
140
      *p++ = XI_JMP; *(int32_t *)p = target-(p+4); p += 4;
141
#endif
142
0
    } else {
143
0
      *p++ = XI_JMPs; *p++ = (uint8_t)((2+2)*(31-(slot&31)) - 2);
144
0
    }
145
0
  }
146
0
  return p;
147
0
}
148
#elif LJ_TARGET_ARM
149
static void *callback_mcode_init(global_State *g, uint32_t *page)
150
{
151
  uint32_t *p = page;
152
  void *target = (void *)lj_vm_ffi_callback;
153
  MSize slot;
154
  /* This must match with the saveregs macro in buildvm_arm.dasc. */
155
  *p++ = ARMI_SUB|ARMF_D(RID_R12)|ARMF_N(RID_R12)|ARMF_M(RID_PC);
156
  *p++ = ARMI_PUSH|ARMF_N(RID_SP)|RSET_RANGE(RID_R4,RID_R11+1)|RID2RSET(RID_LR);
157
  *p++ = ARMI_SUB|ARMI_K12|ARMF_D(RID_R12)|ARMF_N(RID_R12)|CALLBACK_MCODE_HEAD;
158
  *p++ = ARMI_STR|ARMI_LS_P|ARMI_LS_W|ARMF_D(RID_R12)|ARMF_N(RID_SP)|(CFRAME_SIZE-4*9);
159
  *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_R12)|ARMF_N(RID_PC);
160
  *p++ = ARMI_LDR|ARMI_LS_P|ARMI_LS_U|ARMF_D(RID_PC)|ARMF_N(RID_PC);
161
  *p++ = u32ptr(g);
162
  *p++ = u32ptr(target);
163
  for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
164
    *p++ = ARMI_MOV|ARMF_D(RID_R12)|ARMF_M(RID_PC);
165
    *p = ARMI_B | ((page-p-2) & 0x00ffffffu);
166
    p++;
167
  }
168
  return p;
169
}
170
#elif LJ_TARGET_ARM64
171
static void *callback_mcode_init(global_State *g, uint32_t *page)
172
{
173
  uint32_t *p = page;
174
  void *target = (void *)lj_vm_ffi_callback;
175
  MSize slot;
176
  *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X11) | A64F_S19(4));
177
  *p++ = A64I_LE(A64I_LDRLx | A64F_D(RID_X10) | A64F_S19(5));
178
  *p++ = A64I_LE(A64I_BR | A64F_N(RID_X11));
179
  *p++ = A64I_LE(A64I_NOP);
180
  ((void **)p)[0] = target;
181
  ((void **)p)[1] = g;
182
  p += 4;
183
  for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
184
    *p++ = A64I_LE(A64I_MOVZw | A64F_D(RID_X9) | A64F_U16(slot));
185
    *p = A64I_LE(A64I_B | A64F_S26((page-p) & 0x03ffffffu));
186
    p++;
187
  }
188
  return p;
189
}
190
#elif LJ_TARGET_PPC
191
static void *callback_mcode_init(global_State *g, uint32_t *page)
192
{
193
  uint32_t *p = page;
194
  void *target = (void *)lj_vm_ffi_callback;
195
  MSize slot;
196
  *p++ = PPCI_LIS | PPCF_T(RID_TMP) | (u32ptr(target) >> 16);
197
  *p++ = PPCI_LIS | PPCF_T(RID_R12) | (u32ptr(g) >> 16);
198
  *p++ = PPCI_ORI | PPCF_A(RID_TMP)|PPCF_T(RID_TMP) | (u32ptr(target) & 0xffff);
199
  *p++ = PPCI_ORI | PPCF_A(RID_R12)|PPCF_T(RID_R12) | (u32ptr(g) & 0xffff);
200
  *p++ = PPCI_MTCTR | PPCF_T(RID_TMP);
201
  *p++ = PPCI_BCTR;
202
  for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
203
    *p++ = PPCI_LI | PPCF_T(RID_R11) | slot;
204
    *p = PPCI_B | (((page-p) & 0x00ffffffu) << 2);
205
    p++;
206
  }
207
  return p;
208
}
209
#elif LJ_TARGET_MIPS
210
static void *callback_mcode_init(global_State *g, uint32_t *page)
211
{
212
  uint32_t *p = page;
213
  uintptr_t target = (uintptr_t)(void *)lj_vm_ffi_callback;
214
  uintptr_t ug = (uintptr_t)(void *)g;
215
  MSize slot;
216
#if LJ_TARGET_MIPS32
217
  *p++ = MIPSI_LUI | MIPSF_T(RID_R3) | (target >> 16);
218
  *p++ = MIPSI_LUI | MIPSF_T(RID_R2) | (ug >> 16);
219
#else
220
  *p++ = MIPSI_LUI  | MIPSF_T(RID_R3) | (target >> 48);
221
  *p++ = MIPSI_LUI  | MIPSF_T(RID_R2) | (ug >> 48);
222
  *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 32) & 0xffff);
223
  *p++ = MIPSI_ORI  | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 32) & 0xffff);
224
  *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
225
  *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
226
  *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | ((target >> 16) & 0xffff);
227
  *p++ = MIPSI_ORI  | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | ((ug >> 16) & 0xffff);
228
  *p++ = MIPSI_DSLL | MIPSF_D(RID_R3)|MIPSF_T(RID_R3) | MIPSF_A(16);
229
  *p++ = MIPSI_DSLL | MIPSF_D(RID_R2)|MIPSF_T(RID_R2) | MIPSF_A(16);
230
#endif
231
  *p++ = MIPSI_ORI  | MIPSF_T(RID_R3)|MIPSF_S(RID_R3) | (target & 0xffff);
232
  *p++ = MIPSI_JR | MIPSF_S(RID_R3);
233
  *p++ = MIPSI_ORI | MIPSF_T(RID_R2)|MIPSF_S(RID_R2) | (ug & 0xffff);
234
  for (slot = 0; slot < CALLBACK_MAX_SLOT; slot++) {
235
    *p = MIPSI_B | ((page-p-1) & 0x0000ffffu);
236
    p++;
237
    *p++ = MIPSI_LI | MIPSF_T(RID_R1) | slot;
238
  }
239
  return p;
240
}
241
#else
242
/* Missing support for this architecture. */
243
#define callback_mcode_init(g, p) (p)
244
#endif
245
246
/* -- Machine code management --------------------------------------------- */
247
248
#if LJ_TARGET_WINDOWS
249
250
#define WIN32_LEAN_AND_MEAN
251
#include <windows.h>
252
253
#elif LJ_TARGET_POSIX
254
255
#include <sys/mman.h>
256
#ifndef MAP_ANONYMOUS
257
#define MAP_ANONYMOUS   MAP_ANON
258
#endif
259
260
#endif
261
262
/* Allocate and initialize area for callback function pointers. */
263
static void callback_mcode_new(CTState *cts)
264
0
{
265
0
  size_t sz = (size_t)CALLBACK_MCODE_SIZE;
266
0
  void *p, *pe;
267
0
  if (CALLBACK_MAX_SLOT == 0)
268
0
    lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
269
#if LJ_TARGET_WINDOWS
270
  p = LJ_WIN_VALLOC(NULL, sz, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
271
  if (!p)
272
    lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
273
#elif LJ_TARGET_POSIX
274
0
  p = mmap(NULL, sz, (PROT_READ|PROT_WRITE), MAP_PRIVATE|MAP_ANONYMOUS,
275
0
     -1, 0);
276
0
  if (p == MAP_FAILED)
277
0
    lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
278
#else
279
  /* Fallback allocator. Fails if memory is not executable by default. */
280
  p = lj_mem_new(cts->L, sz);
281
#endif
282
0
  cts->cb.mcode = p;
283
0
  pe = callback_mcode_init(cts->g, p);
284
0
  UNUSED(pe);
285
0
  lj_assertCTS((size_t)((char *)pe - (char *)p) <= sz,
286
0
         "miscalculated CALLBACK_MAX_SLOT");
287
0
  lj_mcode_sync(p, (char *)p + sz);
288
#if LJ_TARGET_WINDOWS
289
  {
290
    DWORD oprot;
291
    LJ_WIN_VPROTECT(p, sz, PAGE_EXECUTE_READ, &oprot);
292
  }
293
#elif LJ_TARGET_POSIX
294
0
  mprotect(p, sz, (PROT_READ|PROT_EXEC));
295
0
#endif
296
0
}
297
298
/* Free area for callback function pointers. */
299
void lj_ccallback_mcode_free(CTState *cts)
300
0
{
301
0
  size_t sz = (size_t)CALLBACK_MCODE_SIZE;
302
0
  void *p = cts->cb.mcode;
303
0
  if (p == NULL) return;
304
#if LJ_TARGET_WINDOWS
305
  VirtualFree(p, 0, MEM_RELEASE);
306
  UNUSED(sz);
307
#elif LJ_TARGET_POSIX
308
0
  munmap(p, sz);
309
#else
310
  lj_mem_free(cts->g, p, sz);
311
#endif
312
0
}
313
314
/* -- C callback entry ---------------------------------------------------- */
315
316
/* Target-specific handling of register arguments. Similar to lj_ccall.c. */
317
#if LJ_TARGET_X86
318
319
#define CALLBACK_HANDLE_REGARG \
320
  if (!isfp) {  /* Only non-FP values may be passed in registers. */ \
321
    if (n > 1) {  /* Anything > 32 bit is passed on the stack. */ \
322
      if (!LJ_ABI_WIN) ngpr = maxgpr;  /* Prevent reordering. */ \
323
    } else if (ngpr + 1 <= maxgpr) { \
324
      sp = &cts->cb.gpr[ngpr]; \
325
      ngpr += n; \
326
      goto done; \
327
    } \
328
  }
329
330
#elif LJ_TARGET_X64 && LJ_ABI_WIN
331
332
/* Windows/x64 argument registers are strictly positional (use ngpr). */
333
#define CALLBACK_HANDLE_REGARG \
334
  if (isfp) { \
335
    if (ngpr < maxgpr) { sp = &cts->cb.fpr[ngpr++]; UNUSED(nfpr); goto done; } \
336
  } else { \
337
    if (ngpr < maxgpr) { sp = &cts->cb.gpr[ngpr++]; goto done; } \
338
  }
339
340
#elif LJ_TARGET_X64
341
342
#define CALLBACK_HANDLE_REGARG \
343
0
  if (isfp) { \
344
0
    if (nfpr + n <= CCALL_NARG_FPR) { \
345
0
      sp = &cts->cb.fpr[nfpr]; \
346
0
      nfpr += n; \
347
0
      goto done; \
348
0
    } \
349
0
  } else { \
350
0
    if (ngpr + n <= maxgpr) { \
351
0
      sp = &cts->cb.gpr[ngpr]; \
352
0
      ngpr += n; \
353
0
      goto done; \
354
0
    } \
355
0
  }
356
357
#elif LJ_TARGET_ARM
358
359
#if LJ_ABI_SOFTFP
360
361
#define CALLBACK_HANDLE_REGARG_FP1  UNUSED(isfp);
362
#define CALLBACK_HANDLE_REGARG_FP2
363
364
#else
365
366
#define CALLBACK_HANDLE_REGARG_FP1 \
367
  if (isfp) { \
368
    if (n == 1) { \
369
      if (fprodd) { \
370
  sp = &cts->cb.fpr[fprodd-1]; \
371
  fprodd = 0; \
372
  goto done; \
373
      } else if (nfpr + 1 <= CCALL_NARG_FPR) { \
374
  sp = &cts->cb.fpr[nfpr++]; \
375
  fprodd = nfpr; \
376
  goto done; \
377
      } \
378
    } else { \
379
      if (nfpr + 1 <= CCALL_NARG_FPR) { \
380
  sp = &cts->cb.fpr[nfpr++]; \
381
  goto done; \
382
      } \
383
    } \
384
    fprodd = 0;  /* No reordering after the first FP value is on stack. */ \
385
  } else {
386
387
#define CALLBACK_HANDLE_REGARG_FP2  }
388
389
#endif
390
391
#define CALLBACK_HANDLE_REGARG \
392
  CALLBACK_HANDLE_REGARG_FP1 \
393
  if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
394
  if (ngpr + n <= maxgpr) { \
395
    sp = &cts->cb.gpr[ngpr]; \
396
    ngpr += n; \
397
    goto done; \
398
  } CALLBACK_HANDLE_REGARG_FP2
399
400
#elif LJ_TARGET_ARM64
401
402
#define CALLBACK_HANDLE_REGARG \
403
  if (isfp) { \
404
    if (nfpr + n <= CCALL_NARG_FPR) { \
405
      sp = &cts->cb.fpr[nfpr]; \
406
      nfpr += n; \
407
      goto done; \
408
    } else { \
409
      nfpr = CCALL_NARG_FPR;  /* Prevent reordering. */ \
410
    } \
411
  } else { \
412
    if (!LJ_TARGET_OSX && n > 1) \
413
      ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
414
    if (ngpr + n <= maxgpr) { \
415
      sp = &cts->cb.gpr[ngpr]; \
416
      ngpr += n; \
417
      goto done; \
418
    } else { \
419
      ngpr = CCALL_NARG_GPR;  /* Prevent reordering. */ \
420
    } \
421
  }
422
423
#elif LJ_TARGET_PPC
424
425
#define CALLBACK_HANDLE_GPR \
426
  if (n > 1) { \
427
    lj_assertCTS(((LJ_ABI_SOFTFP && ctype_isnum(cta->info)) ||  /* double. */ \
428
     ctype_isinteger(cta->info)) && n == 2,  /* int64_t. */ \
429
     "bad GPR type"); \
430
    ngpr = (ngpr + 1u) & ~1u;  /* Align int64_t to regpair. */ \
431
  } \
432
  if (ngpr + n <= maxgpr) { \
433
    sp = &cts->cb.gpr[ngpr]; \
434
    ngpr += n; \
435
    goto done; \
436
  }
437
438
#if LJ_ABI_SOFTFP
439
#define CALLBACK_HANDLE_REGARG \
440
  CALLBACK_HANDLE_GPR \
441
  UNUSED(isfp);
442
#else
443
#define CALLBACK_HANDLE_REGARG \
444
  if (isfp) { \
445
    if (nfpr + 1 <= CCALL_NARG_FPR) { \
446
      sp = &cts->cb.fpr[nfpr++]; \
447
      cta = ctype_get(cts, CTID_DOUBLE);  /* FPRs always hold doubles. */ \
448
      goto done; \
449
    } \
450
  } else {  /* Try to pass argument in GPRs. */ \
451
    CALLBACK_HANDLE_GPR \
452
  }
453
#endif
454
455
#if !LJ_ABI_SOFTFP
456
#define CALLBACK_HANDLE_RET \
457
  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
458
    *(double *)dp = *(float *)dp;  /* FPRs always hold doubles. */
459
#endif
460
461
#elif LJ_TARGET_MIPS32
462
463
#define CALLBACK_HANDLE_GPR \
464
  if (n > 1) ngpr = (ngpr + 1u) & ~1u;  /* Align to regpair. */ \
465
  if (ngpr + n <= maxgpr) { \
466
    sp = &cts->cb.gpr[ngpr]; \
467
    ngpr += n; \
468
    goto done; \
469
  }
470
471
#if !LJ_ABI_SOFTFP  /* MIPS32 hard-float */
472
#define CALLBACK_HANDLE_REGARG \
473
  if (isfp && nfpr < CCALL_NARG_FPR) {  /* Try to pass argument in FPRs. */ \
474
    sp = (void *)((uint8_t *)&cts->cb.fpr[nfpr] + ((LJ_BE && n==1) ? 4 : 0)); \
475
    nfpr++; ngpr += n; \
476
    goto done; \
477
  } else {  /* Try to pass argument in GPRs. */ \
478
    nfpr = CCALL_NARG_FPR; \
479
    CALLBACK_HANDLE_GPR \
480
  }
481
#else     /* MIPS32 soft-float */
482
#define CALLBACK_HANDLE_REGARG \
483
  CALLBACK_HANDLE_GPR \
484
  UNUSED(isfp);
485
#endif
486
487
#define CALLBACK_HANDLE_RET \
488
  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
489
    ((float *)dp)[1] = *(float *)dp;
490
491
#elif LJ_TARGET_MIPS64
492
493
#if !LJ_ABI_SOFTFP  /* MIPS64 hard-float */
494
#define CALLBACK_HANDLE_REGARG \
495
  if (ngpr + n <= maxgpr) { \
496
    sp = isfp ? (void*) &cts->cb.fpr[ngpr] : (void*) &cts->cb.gpr[ngpr]; \
497
    ngpr += n; \
498
    goto done; \
499
  }
500
#else     /* MIPS64 soft-float */
501
#define CALLBACK_HANDLE_REGARG \
502
  if (ngpr + n <= maxgpr) { \
503
    UNUSED(isfp); \
504
    sp = (void*) &cts->cb.gpr[ngpr]; \
505
    ngpr += n; \
506
    goto done; \
507
  }
508
#endif
509
510
#define CALLBACK_HANDLE_RET \
511
  if (ctype_isfp(ctr->info) && ctr->size == sizeof(float)) \
512
    ((float *)dp)[1] = *(float *)dp;
513
514
#else
515
#error "Missing calling convention definitions for this architecture"
516
#endif
517
518
/* Convert and push callback arguments to Lua stack. */
519
static void callback_conv_args(CTState *cts, lua_State *L)
520
0
{
521
0
  TValue *o = L->top;
522
0
  intptr_t *stack = cts->cb.stack;
523
0
  MSize slot = cts->cb.slot;
524
0
  CTypeID id = 0, rid, fid;
525
0
  int gcsteps = 0;
526
0
  CType *ct;
527
0
  GCfunc *fn;
528
0
  int fntp;
529
0
  MSize ngpr = 0, nsp = 0, maxgpr = CCALL_NARG_GPR;
530
0
#if CCALL_NARG_FPR
531
0
  MSize nfpr = 0;
532
#if LJ_TARGET_ARM
533
  MSize fprodd = 0;
534
#endif
535
0
#endif
536
537
0
  if (slot < cts->cb.sizeid && (id = cts->cb.cbid[slot]) != 0) {
538
0
    ct = ctype_get(cts, id);
539
0
    rid = ctype_cid(ct->info);  /* Return type. x86: +(spadj<<16). */
540
0
    fn = funcV(lj_tab_getint(cts->miscmap, (int32_t)slot));
541
0
    fntp = LJ_TFUNC;
542
0
  } else {  /* Must set up frame first, before throwing the error. */
543
0
    ct = NULL;
544
0
    rid = 0;
545
0
    fn = (GCfunc *)L;
546
0
    fntp = LJ_TTHREAD;
547
0
  }
548
  /* Continuation returns from callback. */
549
0
  if (LJ_FR2) {
550
0
    (o++)->u64 = LJ_CONT_FFI_CALLBACK;
551
0
    (o++)->u64 = rid;
552
0
  } else {
553
0
    o->u32.lo = LJ_CONT_FFI_CALLBACK;
554
0
    o->u32.hi = rid;
555
0
    o++;
556
0
  }
557
0
  setframe_gc(o, obj2gco(fn), fntp);
558
0
  if (LJ_FR2) o++;
559
0
  setframe_ftsz(o, ((char *)(o+1) - (char *)L->base) + FRAME_CONT);
560
0
  L->top = L->base = ++o;
561
0
  if (!ct)
562
0
    lj_err_caller(cts->L, LJ_ERR_FFI_BADCBACK);
563
0
  if (isluafunc(fn))
564
0
    setcframe_pc(L->cframe, proto_bc(funcproto(fn))+1);
565
0
  lj_state_checkstack(L, LUA_MINSTACK);  /* May throw. */
566
0
  o = L->base;  /* Might have been reallocated. */
567
568
#if LJ_TARGET_X86
569
  /* x86 has several different calling conventions. */
570
  switch (ctype_cconv(ct->info)) {
571
  case CTCC_FASTCALL: maxgpr = 2; break;
572
  case CTCC_THISCALL: maxgpr = 1; break;
573
  default: maxgpr = 0; break;
574
  }
575
#endif
576
577
0
  fid = ct->sib;
578
0
  while (fid) {
579
0
    CType *ctf = ctype_get(cts, fid);
580
0
    if (!ctype_isattrib(ctf->info)) {
581
0
      CType *cta;
582
0
      void *sp;
583
0
      CTSize sz;
584
0
      int isfp;
585
0
      MSize n;
586
0
      lj_assertCTS(ctype_isfield(ctf->info), "field expected");
587
0
      cta = ctype_rawchild(cts, ctf);
588
0
      isfp = ctype_isfp(cta->info);
589
0
      sz = (cta->size + CTSIZE_PTR-1) & ~(CTSIZE_PTR-1);
590
0
      n = sz / CTSIZE_PTR;  /* Number of GPRs or stack slots needed. */
591
592
      CALLBACK_HANDLE_REGARG  /* Handle register arguments. */
593
0
594
0
      /* Otherwise pass argument on stack. */
595
0
      if (CCALL_ALIGN_STACKARG && LJ_32 && sz == 8)
596
0
  nsp = (nsp + 1) & ~1u;  /* Align 64 bit argument on stack. */
597
0
      sp = &stack[nsp];
598
0
      nsp += n;
599
600
0
    done:
601
0
      if (LJ_BE && cta->size < CTSIZE_PTR
602
#if LJ_TARGET_MIPS64
603
    && !(isfp && nsp)
604
#endif
605
0
   )
606
0
  sp = (void *)((uint8_t *)sp + CTSIZE_PTR-cta->size);
607
0
      gcsteps += lj_cconv_tv_ct(cts, cta, 0, o++, sp);
608
0
    }
609
0
    fid = ctf->sib;
610
0
  }
611
0
  L->top = o;
612
#if LJ_TARGET_X86
613
  /* Store stack adjustment for returns from non-cdecl callbacks. */
614
  if (ctype_cconv(ct->info) != CTCC_CDECL) {
615
#if LJ_FR2
616
    (L->base-3)->u64 |= (nsp << (16+2));
617
#else
618
    (L->base-2)->u32.hi |= (nsp << (16+2));
619
#endif
620
  }
621
#endif
622
0
  while (gcsteps-- > 0)
623
0
    lj_gc_check(L);
624
0
}
625
626
/* Convert Lua object to callback result. */
627
static void callback_conv_result(CTState *cts, lua_State *L, TValue *o)
628
0
{
629
0
#if LJ_FR2
630
0
  CType *ctr = ctype_raw(cts, (uint16_t)(L->base-3)->u64);
631
#else
632
  CType *ctr = ctype_raw(cts, (uint16_t)(L->base-2)->u32.hi);
633
#endif
634
#if LJ_TARGET_X86
635
  cts->cb.gpr[2] = 0;
636
#endif
637
0
  if (!ctype_isvoid(ctr->info)) {
638
0
    uint8_t *dp = (uint8_t *)&cts->cb.gpr[0];
639
0
#if CCALL_NUM_FPR
640
0
    if (ctype_isfp(ctr->info))
641
0
      dp = (uint8_t *)&cts->cb.fpr[0];
642
0
#endif
643
#if LJ_TARGET_ARM64 && LJ_BE
644
    if (ctype_isfp(ctr->info) && ctr->size == sizeof(float))
645
      dp = (uint8_t *)&cts->cb.fpr[0].f[1];
646
#endif
647
0
    lj_cconv_ct_tv(cts, ctr, dp, o, 0);
648
#ifdef CALLBACK_HANDLE_RET
649
    CALLBACK_HANDLE_RET
650
#endif
651
    /* Extend returned integers to (at least) 32 bits. */
652
0
    if (ctype_isinteger_or_bool(ctr->info) && ctr->size < 4) {
653
0
      if (ctr->info & CTF_UNSIGNED)
654
0
  *(uint32_t *)dp = ctr->size == 1 ? (uint32_t)*(uint8_t *)dp :
655
0
             (uint32_t)*(uint16_t *)dp;
656
0
      else
657
0
  *(int32_t *)dp = ctr->size == 1 ? (int32_t)*(int8_t *)dp :
658
0
            (int32_t)*(int16_t *)dp;
659
0
    }
660
#if LJ_TARGET_MIPS64 || (LJ_TARGET_ARM64 && LJ_BE)
661
    /* Always sign-extend results to 64 bits. Even a soft-fp 'float'. */
662
    if (ctr->size <= 4 &&
663
  (LJ_ABI_SOFTFP || ctype_isinteger_or_bool(ctr->info)))
664
      *(int64_t *)dp = (int64_t)*(int32_t *)dp;
665
#endif
666
#if LJ_TARGET_X86
667
    if (ctype_isfp(ctr->info))
668
      cts->cb.gpr[2] = ctr->size == sizeof(float) ? 1 : 2;
669
#endif
670
0
  }
671
0
}
672
673
/* Enter callback. */
674
lua_State * LJ_FASTCALL lj_ccallback_enter(CTState *cts, void *cf)
675
0
{
676
0
  lua_State *L = cts->L;
677
0
  global_State *g = cts->g;
678
0
  lj_assertG(L != NULL, "uninitialized cts->L in callback");
679
0
  if (tvref(g->jit_base)) {
680
0
    setstrV(L, L->top++, lj_err_str(L, LJ_ERR_FFI_BADCBACK));
681
0
    if (g->panic) g->panic(L);
682
0
    exit(EXIT_FAILURE);
683
0
  }
684
0
  lj_trace_abort(g);  /* Never record across callback. */
685
  /* Setup C frame. */
686
0
  cframe_prev(cf) = L->cframe;
687
0
  setcframe_L(cf, L);
688
0
  cframe_errfunc(cf) = -1;
689
0
  cframe_nres(cf) = 0;
690
0
  L->cframe = cf;
691
0
  callback_conv_args(cts, L);
692
0
  return L;  /* Now call the function on this stack. */
693
0
}
694
695
/* Leave callback. */
696
void LJ_FASTCALL lj_ccallback_leave(CTState *cts, TValue *o)
697
0
{
698
0
  lua_State *L = cts->L;
699
0
  GCfunc *fn;
700
0
  TValue *obase = L->base;
701
0
  L->base = L->top;  /* Keep continuation frame for throwing errors. */
702
0
  if (o >= L->base) {
703
    /* PC of RET* is lost. Point to last line for result conv. errors. */
704
0
    fn = curr_func(L);
705
0
    if (isluafunc(fn)) {
706
0
      GCproto *pt = funcproto(fn);
707
0
      setcframe_pc(L->cframe, proto_bc(pt)+pt->sizebc+1);
708
0
    }
709
0
  }
710
0
  callback_conv_result(cts, L, o);
711
  /* Finally drop C frame and continuation frame. */
712
0
  L->top -= 2+2*LJ_FR2;
713
0
  L->base = obase;
714
0
  L->cframe = cframe_prev(L->cframe);
715
0
  cts->cb.slot = 0;  /* Blacklist C function that called the callback. */
716
0
}
717
718
/* -- C callback management ----------------------------------------------- */
719
720
/* Get an unused slot in the callback slot table. */
721
static MSize callback_slot_new(CTState *cts, CType *ct)
722
0
{
723
0
  CTypeID id = ctype_typeid(cts, ct);
724
0
  CTypeID1 *cbid = cts->cb.cbid;
725
0
  MSize top;
726
0
  for (top = cts->cb.topid; top < cts->cb.sizeid; top++)
727
0
    if (LJ_LIKELY(cbid[top] == 0))
728
0
      goto found;
729
0
#if CALLBACK_MAX_SLOT
730
0
  if (top >= CALLBACK_MAX_SLOT)
731
0
#endif
732
0
    lj_err_caller(cts->L, LJ_ERR_FFI_CBACKOV);
733
0
  if (!cts->cb.mcode)
734
0
    callback_mcode_new(cts);
735
0
  lj_mem_growvec(cts->L, cbid, cts->cb.sizeid, CALLBACK_MAX_SLOT, CTypeID1);
736
0
  cts->cb.cbid = cbid;
737
0
  memset(cbid+top, 0, (cts->cb.sizeid-top)*sizeof(CTypeID1));
738
0
found:
739
0
  cbid[top] = id;
740
0
  cts->cb.topid = top+1;
741
0
  return top;
742
0
}
743
744
/* Check for function pointer and supported argument/result types. */
745
static CType *callback_checkfunc(CTState *cts, CType *ct)
746
0
{
747
0
  int narg = 0;
748
0
  if (!ctype_isptr(ct->info) || (LJ_64 && ct->size != CTSIZE_PTR))
749
0
    return NULL;
750
0
  ct = ctype_rawchild(cts, ct);
751
0
  if (ctype_isfunc(ct->info)) {
752
0
    CType *ctr = ctype_rawchild(cts, ct);
753
0
    CTypeID fid = ct->sib;
754
0
    if (!(ctype_isvoid(ctr->info) || ctype_isenum(ctr->info) ||
755
0
    ctype_isptr(ctr->info) || (ctype_isnum(ctr->info) && ctr->size <= 8)))
756
0
      return NULL;
757
0
    if ((ct->info & CTF_VARARG))
758
0
      return NULL;
759
0
    while (fid) {
760
0
      CType *ctf = ctype_get(cts, fid);
761
0
      if (!ctype_isattrib(ctf->info)) {
762
0
  CType *cta;
763
0
  lj_assertCTS(ctype_isfield(ctf->info), "field expected");
764
0
  cta = ctype_rawchild(cts, ctf);
765
0
  if (!(ctype_isenum(cta->info) || ctype_isptr(cta->info) ||
766
0
        (ctype_isnum(cta->info) && cta->size <= 8)) ||
767
0
      ++narg >= LUA_MINSTACK-3)
768
0
    return NULL;
769
0
      }
770
0
      fid = ctf->sib;
771
0
    }
772
0
    return ct;
773
0
  }
774
0
  return NULL;
775
0
}
776
777
/* Create a new callback and return the callback function pointer. */
778
void *lj_ccallback_new(CTState *cts, CType *ct, GCfunc *fn)
779
0
{
780
0
  ct = callback_checkfunc(cts, ct);
781
0
  if (ct) {
782
0
    MSize slot = callback_slot_new(cts, ct);
783
0
    GCtab *t = cts->miscmap;
784
0
    setfuncV(cts->L, lj_tab_setint(cts->L, t, (int32_t)slot), fn);
785
0
    lj_gc_anybarriert(cts->L, t);
786
0
    return callback_slot2ptr(cts, slot);
787
0
  }
788
0
  return NULL;  /* Bad conversion. */
789
0
}
790
791
#endif