Coverage Report

Created: 2026-10-31 08:12

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/tcg/tcg.c
Line
Count
Source
1
/*
2
 * Tiny Code Generator for QEMU
3
 *
4
 * Copyright (c) 2008 Fabrice Bellard
5
 *
6
 * Permission is hereby granted, free of charge, to any person obtaining a copy
7
 * of this software and associated documentation files (the "Software"), to deal
8
 * in the Software without restriction, including without limitation the rights
9
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
 * copies of the Software, and to permit persons to whom the Software is
11
 * furnished to do so, subject to the following conditions:
12
 *
13
 * The above copyright notice and this permission notice shall be included in
14
 * all copies or substantial portions of the Software.
15
 *
16
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22
 * THE SOFTWARE.
23
 */
24
25
#include "qemu/osdep.h"
26
27
/* Define to jump the ELF file used to communicate with GDB.  */
28
#undef DEBUG_JIT
29
30
#include "qemu/error-report.h"
31
#include "qemu/cutils.h"
32
#include "qemu/host-utils.h"
33
#include "qemu/qemu-print.h"
34
#include "qemu/cacheflush.h"
35
#include "qemu/cacheinfo.h"
36
#include "qemu/timer.h"
37
#include "exec/target_page.h"
38
#include "exec/translation-block.h"
39
#include "exec/tlb-common.h"
40
#include "tcg/startup.h"
41
#include "tcg/tcg-op-common.h"
42
43
#if UINTPTR_MAX == UINT32_MAX
44
# define ELF_CLASS  ELFCLASS32
45
#else
46
0
# define ELF_CLASS  ELFCLASS64
47
#endif
48
#if HOST_BIG_ENDIAN
49
# define ELF_DATA   ELFDATA2MSB
50
#else
51
0
# define ELF_DATA   ELFDATA2LSB
52
#endif
53
54
#include "elf.h"
55
#include "exec/log.h"
56
#include "tcg/tcg-ldst.h"
57
#include "tcg/tcg-temp-internal.h"
58
#include "tcg-internal.h"
59
#include "tcg/perf.h"
60
#include "tcg-has.h"
61
#ifdef CONFIG_USER_ONLY
62
#include "user/guest-base.h"
63
#endif
64
65
/* Forward declarations for functions declared in tcg-target.c.inc and
66
   used here. */
67
static void tcg_target_init(TCGContext *s);
68
static void tcg_target_qemu_prologue(TCGContext *s);
69
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
70
                        intptr_t value, intptr_t addend);
71
static void tcg_out_nop_fill(tcg_insn_unit *p, int count);
72
73
typedef struct TCGLabelQemuLdst TCGLabelQemuLdst;
74
static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
75
static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
76
77
/* The CIE and FDE header definitions will be common to all hosts.  */
78
typedef struct {
79
    uint32_t len __attribute__((aligned((sizeof(void *)))));
80
    uint32_t id;
81
    uint8_t version;
82
    char augmentation[1];
83
    uint8_t code_align;
84
    uint8_t data_align;
85
    uint8_t return_column;
86
} DebugFrameCIE;
87
88
typedef struct QEMU_PACKED {
89
    uint32_t len __attribute__((aligned((sizeof(void *)))));
90
    uint32_t cie_offset;
91
    uintptr_t func_start;
92
    uintptr_t func_len;
93
} DebugFrameFDEHeader;
94
95
typedef struct QEMU_PACKED {
96
    DebugFrameCIE cie;
97
    DebugFrameFDEHeader fde;
98
} DebugFrameHeader;
99
100
struct TCGLabelQemuLdst {
101
    bool is_ld;             /* qemu_ld: true, qemu_st: false */
102
    MemOpIdx oi;
103
    TCGType type;           /* result type of a load */
104
    TCGReg addr_reg;        /* reg index for guest virtual addr */
105
    TCGReg datalo_reg;      /* reg index for low word to be loaded or stored */
106
    TCGReg datahi_reg;      /* reg index for high word to be loaded or stored */
107
    const tcg_insn_unit *raddr;   /* addr of the next IR of qemu_ld/st IR */
108
    tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
109
    QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
110
};
111
112
static void tcg_register_jit_int(const void *buf, size_t size,
113
                                 const void *debug_frame,
114
                                 size_t debug_frame_size)
115
    __attribute__((unused));
116
117
/* Forward declarations for functions declared and used in tcg-target.c.inc. */
118
static void tcg_out_tb_start(TCGContext *s);
119
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
120
                       intptr_t arg2);
121
static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
122
static void tcg_out_movi(TCGContext *s, TCGType type,
123
                         TCGReg ret, tcg_target_long arg);
124
static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
125
static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
126
static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
127
static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
128
static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
129
static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
130
static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
131
static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
132
static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
133
static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
134
static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
135
static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
136
static void tcg_out_goto_tb(TCGContext *s, int which);
137
static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest);
138
static void tcg_out_mb(TCGContext *s, unsigned bar);
139
static void tcg_out_br(TCGContext *s, TCGLabel *l);
140
static void tcg_out_set_carry(TCGContext *s);
141
static void tcg_out_set_borrow(TCGContext *s);
142
#if TCG_TARGET_MAYBE_vec
143
static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
144
                            TCGReg dst, TCGReg src);
145
static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
146
                             TCGReg dst, TCGReg base, intptr_t offset);
147
static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
148
                             TCGReg dst, int64_t arg);
149
static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
150
                           unsigned vecl, unsigned vece,
151
                           const TCGArg args[TCG_MAX_OP_ARGS],
152
                           const int const_args[TCG_MAX_OP_ARGS]);
153
#else
154
static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
155
                                   TCGReg dst, TCGReg src)
156
{
157
    g_assert_not_reached();
158
}
159
static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
160
                                    TCGReg dst, TCGReg base, intptr_t offset)
161
{
162
    g_assert_not_reached();
163
}
164
static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
165
                                    TCGReg dst, int64_t arg)
166
{
167
    g_assert_not_reached();
168
}
169
static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
170
                                  unsigned vecl, unsigned vece,
171
                                  const TCGArg args[TCG_MAX_OP_ARGS],
172
                                  const int const_args[TCG_MAX_OP_ARGS])
173
{
174
    g_assert_not_reached();
175
}
176
int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
177
{
178
    return 0;
179
}
180
#endif
181
static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
182
                       intptr_t arg2);
183
static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
184
                        TCGReg base, intptr_t ofs);
185
static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
186
                         const TCGHelperInfo *info);
187
static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
188
static bool tcg_target_const_match(int64_t val, int ct,
189
                                   TCGType type, TCGCond cond, int vece);
190
191
#ifndef CONFIG_USER_ONLY
192
0
#define guest_base  ({ qemu_build_not_reached(); (uintptr_t)0; })
193
#endif
194
195
typedef struct TCGLdstHelperParam {
196
    TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
197
    unsigned ntmp;
198
    int tmp[3];
199
} TCGLdstHelperParam;
200
201
static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
202
                                   const TCGLdstHelperParam *p)
203
    __attribute__((unused));
204
static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l,
205
                                  bool load_sign, const TCGLdstHelperParam *p)
206
    __attribute__((unused));
207
static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
208
                                   const TCGLdstHelperParam *p)
209
    __attribute__((unused));
210
211
static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
212
    [MO_UB] = helper_ldub_mmu,
213
    [MO_SB] = helper_ldsb_mmu,
214
    [MO_UW] = helper_lduw_mmu,
215
    [MO_SW] = helper_ldsw_mmu,
216
    [MO_UL] = helper_ldul_mmu,
217
    [MO_UQ] = helper_ldq_mmu,
218
    [MO_SL] = helper_ldsl_mmu,
219
    [MO_128] = helper_ld16_mmu,
220
};
221
222
static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
223
    [MO_8]  = helper_stb_mmu,
224
    [MO_16] = helper_stw_mmu,
225
    [MO_32] = helper_stl_mmu,
226
    [MO_64] = helper_stq_mmu,
227
    [MO_128] = helper_st16_mmu,
228
};
229
230
typedef struct {
231
    MemOp atom;   /* lg2 bits of atomicity required */
232
    MemOp align;  /* lg2 bits of alignment to use */
233
} TCGAtomAlign;
234
235
static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
236
                                           MemOp host_atom, bool allow_two_ops)
237
    __attribute__((unused));
238
239
TCGContext tcg_init_ctx;
240
__thread TCGContext *tcg_ctx;
241
242
TCGContext **tcg_ctxs;
243
unsigned int tcg_cur_ctxs;
244
unsigned int tcg_max_ctxs;
245
TCGv_env tcg_env;
246
const void *tcg_code_gen_epilogue;
247
ptrdiff_t tcg_splitwx_diff;
248
249
#ifndef CONFIG_TCG_INTERPRETER
250
tcg_prologue_fn *tcg_qemu_tb_exec;
251
#endif
252
253
static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
254
static TCGRegSet tcg_target_call_clobber_regs;
255
256
#if TCG_TARGET_INSN_UNIT_SIZE == 1
257
static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
258
0
{
259
0
    *s->code_ptr++ = v;
260
0
}
261
262
static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
263
                                                      uint8_t v)
264
0
{
265
0
    *p = v;
266
0
}
267
#endif
268
269
#if TCG_TARGET_INSN_UNIT_SIZE <= 2
270
static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
271
0
{
272
0
    if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
273
0
        *s->code_ptr++ = v;
274
0
    } else {
275
0
        tcg_insn_unit *p = s->code_ptr;
276
0
        memcpy(p, &v, sizeof(v));
277
0
        s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
278
0
    }
279
0
}
280
281
static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
282
                                                       uint16_t v)
283
0
{
284
0
    if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
285
0
        *p = v;
286
0
    } else {
287
0
        memcpy(p, &v, sizeof(v));
288
0
    }
289
0
}
290
#endif
291
292
#if TCG_TARGET_INSN_UNIT_SIZE <= 4
293
static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
294
0
{
295
0
    if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
296
0
        *s->code_ptr++ = v;
297
0
    } else {
298
0
        tcg_insn_unit *p = s->code_ptr;
299
0
        memcpy(p, &v, sizeof(v));
300
0
        s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
301
0
    }
302
0
}
303
304
static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
305
                                                       uint32_t v)
306
0
{
307
0
    if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
308
0
        *p = v;
309
0
    } else {
310
0
        memcpy(p, &v, sizeof(v));
311
0
    }
312
0
}
313
#endif
314
315
#if TCG_TARGET_INSN_UNIT_SIZE <= 8
316
static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
317
0
{
318
0
    if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
319
0
        *s->code_ptr++ = v;
320
0
    } else {
321
0
        tcg_insn_unit *p = s->code_ptr;
322
0
        memcpy(p, &v, sizeof(v));
323
0
        s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
324
0
    }
325
0
}
326
327
static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
328
                                                       uint64_t v)
329
0
{
330
0
    if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
331
0
        *p = v;
332
0
    } else {
333
0
        memcpy(p, &v, sizeof(v));
334
0
    }
335
0
}
336
#endif
337
338
/* label relocation processing */
339
340
static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
341
                          TCGLabel *l, intptr_t addend)
342
0
{
343
0
    TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
344
345
0
    r->type = type;
346
0
    r->ptr = code_ptr;
347
0
    r->addend = addend;
348
0
    QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
349
0
}
350
351
static void tcg_out_label(TCGContext *s, TCGLabel *l)
352
0
{
353
0
    tcg_debug_assert(!l->has_value);
354
0
    l->has_value = 1;
355
0
    l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
356
0
}
357
358
TCGLabel *gen_new_label(void)
359
0
{
360
0
    TCGContext *s = tcg_ctx;
361
0
    TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
362
363
0
    memset(l, 0, sizeof(TCGLabel));
364
0
    l->id = s->nb_labels++;
365
0
    QSIMPLEQ_INIT(&l->branches);
366
0
    QSIMPLEQ_INIT(&l->relocs);
367
368
0
    QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
369
370
0
    return l;
371
0
}
372
373
static bool tcg_resolve_relocs(TCGContext *s)
374
0
{
375
0
    TCGLabel *l;
376
377
0
    QSIMPLEQ_FOREACH(l, &s->labels, next) {
378
0
        TCGRelocation *r;
379
0
        uintptr_t value = l->u.value;
380
381
0
        QSIMPLEQ_FOREACH(r, &l->relocs, next) {
382
0
            if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
383
0
                return false;
384
0
            }
385
0
        }
386
0
    }
387
0
    return true;
388
0
}
389
390
static void set_jmp_reset_offset(TCGContext *s, int which)
391
0
{
392
    /*
393
     * We will check for overflow at the end of the opcode loop in
394
     * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
395
     */
396
0
    s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
397
0
}
398
399
static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
400
0
{
401
    /*
402
     * We will check for overflow at the end of the opcode loop in
403
     * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
404
     */
405
0
    s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
406
0
}
407
408
static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
409
0
{
410
0
    /*
411
0
     * Return the read-execute version of the pointer, for the benefit
412
0
     * of any pc-relative addressing mode.
413
0
     */
414
0
    return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
415
0
}
416
417
static int __attribute__((unused))
418
tlb_mask_table_ofs(TCGContext *s, int which)
419
0
{
420
0
    int fi = mmuidx_to_fast_index(which);
421
0
    return (offsetof(CPUNegativeOffsetState, tlb.f[fi]) -
422
0
            sizeof(CPUNegativeOffsetState));
423
0
}
424
425
/* Signal overflow, starting over with fewer guest insns. */
426
static G_NORETURN
427
void tcg_raise_tb_overflow(TCGContext *s)
428
0
{
429
0
    siglongjmp(s->jmp_trans, -2);
430
0
}
431
432
/*
433
 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
434
 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
435
 *
436
 * However, tcg_out_helper_load_slots reuses this field to hold an
437
 * argument slot number (which may designate a argument register or an
438
 * argument stack slot), converting to TCGReg once all arguments that
439
 * are destined for the stack are processed.
440
 */
441
typedef struct TCGMovExtend {
442
    unsigned dst;
443
    TCGReg src;
444
    TCGType dst_type;
445
    TCGType src_type;
446
    MemOp src_ext;
447
} TCGMovExtend;
448
449
/**
450
 * tcg_out_movext -- move and extend
451
 * @s: tcg context
452
 * @dst_type: integral type for destination
453
 * @dst: destination register
454
 * @src_type: integral type for source
455
 * @src_ext: extension to apply to source
456
 * @src: source register
457
 *
458
 * Move or extend @src into @dst, depending on @src_ext and the types.
459
 */
460
static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
461
                           TCGType src_type, MemOp src_ext, TCGReg src)
462
0
{
463
0
    switch (src_ext) {
464
0
    case MO_UB:
465
0
        tcg_out_ext8u(s, dst, src);
466
0
        break;
467
0
    case MO_SB:
468
0
        tcg_out_ext8s(s, dst_type, dst, src);
469
0
        break;
470
0
    case MO_UW:
471
0
        tcg_out_ext16u(s, dst, src);
472
0
        break;
473
0
    case MO_SW:
474
0
        tcg_out_ext16s(s, dst_type, dst, src);
475
0
        break;
476
0
    case MO_UL:
477
0
    case MO_SL:
478
0
        if (dst_type == TCG_TYPE_I32) {
479
0
            if (src_type == TCG_TYPE_I32) {
480
0
                tcg_out_mov(s, TCG_TYPE_I32, dst, src);
481
0
            } else {
482
0
                tcg_out_extrl_i64_i32(s, dst, src);
483
0
            }
484
0
        } else if (src_type == TCG_TYPE_I32) {
485
0
            if (src_ext & MO_SIGN) {
486
0
                tcg_out_exts_i32_i64(s, dst, src);
487
0
            } else {
488
0
                tcg_out_extu_i32_i64(s, dst, src);
489
0
            }
490
0
        } else {
491
0
            if (src_ext & MO_SIGN) {
492
0
                tcg_out_ext32s(s, dst, src);
493
0
            } else {
494
0
                tcg_out_ext32u(s, dst, src);
495
0
            }
496
0
        }
497
0
        break;
498
0
    case MO_UQ:
499
0
        if (dst_type == TCG_TYPE_I32) {
500
0
            tcg_out_extrl_i64_i32(s, dst, src);
501
0
        } else {
502
0
            tcg_out_mov(s, TCG_TYPE_I64, dst, src);
503
0
        }
504
0
        break;
505
0
    default:
506
0
        g_assert_not_reached();
507
0
    }
508
0
}
509
510
/* Minor variations on a theme, using a structure. */
511
static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
512
                                    TCGReg src)
513
0
{
514
0
    tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
515
0
}
516
517
static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
518
0
{
519
0
    tcg_out_movext1_new_src(s, i, i->src);
520
0
}
521
522
/**
523
 * tcg_out_movext2 -- move and extend two pair
524
 * @s: tcg context
525
 * @i1: first move description
526
 * @i2: second move description
527
 * @scratch: temporary register, or -1 for none
528
 *
529
 * As tcg_out_movext, for both @i1 and @i2, caring for overlap
530
 * between the sources and destinations.
531
 */
532
533
static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
534
                            const TCGMovExtend *i2, int scratch)
535
0
{
536
0
    TCGReg src1 = i1->src;
537
0
    TCGReg src2 = i2->src;
538
539
0
    if (i1->dst != src2) {
540
0
        tcg_out_movext1(s, i1);
541
0
        tcg_out_movext1(s, i2);
542
0
        return;
543
0
    }
544
0
    if (i2->dst == src1) {
545
0
        TCGType src1_type = i1->src_type;
546
0
        TCGType src2_type = i2->src_type;
547
548
0
        if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
549
            /* The data is now in the correct registers, now extend. */
550
0
            src1 = i2->src;
551
0
            src2 = i1->src;
552
0
        } else {
553
0
            tcg_debug_assert(scratch >= 0);
554
0
            tcg_out_mov(s, src1_type, scratch, src1);
555
0
            src1 = scratch;
556
0
        }
557
0
    }
558
0
    tcg_out_movext1_new_src(s, i2, src2);
559
0
    tcg_out_movext1_new_src(s, i1, src1);
560
0
}
561
562
/**
563
 * tcg_out_movext3 -- move and extend three pair
564
 * @s: tcg context
565
 * @i1: first move description
566
 * @i2: second move description
567
 * @i3: third move description
568
 * @scratch: temporary register, or -1 for none
569
 *
570
 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
571
 * between the sources and destinations.
572
 */
573
574
static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1,
575
                            const TCGMovExtend *i2, const TCGMovExtend *i3,
576
                            int scratch)
577
0
{
578
0
    TCGReg src1 = i1->src;
579
0
    TCGReg src2 = i2->src;
580
0
    TCGReg src3 = i3->src;
581
582
0
    if (i1->dst != src2 && i1->dst != src3) {
583
0
        tcg_out_movext1(s, i1);
584
0
        tcg_out_movext2(s, i2, i3, scratch);
585
0
        return;
586
0
    }
587
0
    if (i2->dst != src1 && i2->dst != src3) {
588
0
        tcg_out_movext1(s, i2);
589
0
        tcg_out_movext2(s, i1, i3, scratch);
590
0
        return;
591
0
    }
592
0
    if (i3->dst != src1 && i3->dst != src2) {
593
0
        tcg_out_movext1(s, i3);
594
0
        tcg_out_movext2(s, i1, i2, scratch);
595
0
        return;
596
0
    }
597
598
    /*
599
     * There is a cycle.  Since there are only 3 nodes, the cycle is
600
     * either "clockwise" or "anti-clockwise", and can be solved with
601
     * a single scratch or two xchg.
602
     */
603
0
    if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
604
        /* "Clockwise" */
605
0
        if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
606
0
            tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
607
            /* The data is now in the correct registers, now extend. */
608
0
            tcg_out_movext1_new_src(s, i1, i1->dst);
609
0
            tcg_out_movext1_new_src(s, i2, i2->dst);
610
0
            tcg_out_movext1_new_src(s, i3, i3->dst);
611
0
        } else {
612
0
            tcg_debug_assert(scratch >= 0);
613
0
            tcg_out_mov(s, i1->src_type, scratch, src1);
614
0
            tcg_out_movext1(s, i3);
615
0
            tcg_out_movext1(s, i2);
616
0
            tcg_out_movext1_new_src(s, i1, scratch);
617
0
        }
618
0
    } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
619
        /* "Anti-clockwise" */
620
0
        if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
621
0
            tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
622
            /* The data is now in the correct registers, now extend. */
623
0
            tcg_out_movext1_new_src(s, i1, i1->dst);
624
0
            tcg_out_movext1_new_src(s, i2, i2->dst);
625
0
            tcg_out_movext1_new_src(s, i3, i3->dst);
626
0
        } else {
627
0
            tcg_debug_assert(scratch >= 0);
628
0
            tcg_out_mov(s, i1->src_type, scratch, src1);
629
0
            tcg_out_movext1(s, i2);
630
0
            tcg_out_movext1(s, i3);
631
0
            tcg_out_movext1_new_src(s, i1, scratch);
632
0
        }
633
0
    } else {
634
0
        g_assert_not_reached();
635
0
    }
636
0
}
637
638
/*
639
 * Allocate a new TCGLabelQemuLdst entry.
640
 */
641
642
__attribute__((unused))
643
static TCGLabelQemuLdst *new_ldst_label(TCGContext *s)
644
0
{
645
0
    TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l));
646
647
0
    memset(l, 0, sizeof(*l));
648
0
    QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next);
649
650
0
    return l;
651
0
}
652
653
/*
654
 * Allocate new constant pool entries.
655
 */
656
657
typedef struct TCGLabelPoolData {
658
    struct TCGLabelPoolData *next;
659
    tcg_insn_unit *label;
660
    intptr_t addend;
661
    int rtype;
662
    unsigned nlong;
663
    tcg_target_ulong data[];
664
} TCGLabelPoolData;
665
666
static TCGLabelPoolData *new_pool_alloc(TCGContext *s, int nlong, int rtype,
667
                                        tcg_insn_unit *label, intptr_t addend)
668
0
{
669
0
    TCGLabelPoolData *n = tcg_malloc(sizeof(TCGLabelPoolData)
670
0
                                     + sizeof(tcg_target_ulong) * nlong);
671
672
0
    n->label = label;
673
0
    n->addend = addend;
674
0
    n->rtype = rtype;
675
0
    n->nlong = nlong;
676
0
    return n;
677
0
}
678
679
static void new_pool_insert(TCGContext *s, TCGLabelPoolData *n)
680
0
{
681
0
    TCGLabelPoolData *i, **pp;
682
0
    int nlong = n->nlong;
683
684
    /* Insertion sort on the pool.  */
685
0
    for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) {
686
0
        if (nlong > i->nlong) {
687
0
            break;
688
0
        }
689
0
        if (nlong < i->nlong) {
690
0
            continue;
691
0
        }
692
0
        if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) {
693
0
            break;
694
0
        }
695
0
    }
696
0
    n->next = *pp;
697
0
    *pp = n;
698
0
}
699
700
/* The "usual" for generic integer code.  */
701
__attribute__((unused))
702
static void new_pool_label(TCGContext *s, tcg_target_ulong d, int rtype,
703
                           tcg_insn_unit *label, intptr_t addend)
704
0
{
705
0
    TCGLabelPoolData *n = new_pool_alloc(s, 1, rtype, label, addend);
706
0
    n->data[0] = d;
707
0
    new_pool_insert(s, n);
708
0
}
709
710
/* For v64 or v128, depending on the host.  */
711
__attribute__((unused))
712
static void new_pool_l2(TCGContext *s, int rtype, tcg_insn_unit *label,
713
                        intptr_t addend, tcg_target_ulong d0,
714
                        tcg_target_ulong d1)
715
0
{
716
0
    TCGLabelPoolData *n = new_pool_alloc(s, 2, rtype, label, addend);
717
0
    n->data[0] = d0;
718
0
    n->data[1] = d1;
719
0
    new_pool_insert(s, n);
720
0
}
721
722
/* For v128 or v256, depending on the host.  */
723
__attribute__((unused))
724
static void new_pool_l4(TCGContext *s, int rtype, tcg_insn_unit *label,
725
                        intptr_t addend, tcg_target_ulong d0,
726
                        tcg_target_ulong d1, tcg_target_ulong d2,
727
                        tcg_target_ulong d3)
728
0
{
729
0
    TCGLabelPoolData *n = new_pool_alloc(s, 4, rtype, label, addend);
730
0
    n->data[0] = d0;
731
0
    n->data[1] = d1;
732
0
    n->data[2] = d2;
733
0
    n->data[3] = d3;
734
0
    new_pool_insert(s, n);
735
0
}
736
737
/* For v256, for 32-bit host.  */
738
__attribute__((unused))
739
static void new_pool_l8(TCGContext *s, int rtype, tcg_insn_unit *label,
740
                        intptr_t addend, tcg_target_ulong d0,
741
                        tcg_target_ulong d1, tcg_target_ulong d2,
742
                        tcg_target_ulong d3, tcg_target_ulong d4,
743
                        tcg_target_ulong d5, tcg_target_ulong d6,
744
                        tcg_target_ulong d7)
745
0
{
746
0
    TCGLabelPoolData *n = new_pool_alloc(s, 8, rtype, label, addend);
747
0
    n->data[0] = d0;
748
0
    n->data[1] = d1;
749
0
    n->data[2] = d2;
750
0
    n->data[3] = d3;
751
0
    n->data[4] = d4;
752
0
    n->data[5] = d5;
753
0
    n->data[6] = d6;
754
0
    n->data[7] = d7;
755
0
    new_pool_insert(s, n);
756
0
}
757
758
/*
759
 * Generate TB finalization at the end of block
760
 */
761
762
static int tcg_out_ldst_finalize(TCGContext *s)
763
0
{
764
0
    TCGLabelQemuLdst *lb;
765
766
    /* qemu_ld/st slow paths */
767
0
    QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) {
768
0
        if (lb->is_ld
769
0
            ? !tcg_out_qemu_ld_slow_path(s, lb)
770
0
            : !tcg_out_qemu_st_slow_path(s, lb)) {
771
0
            return -2;
772
0
        }
773
774
        /*
775
         * Test for (pending) buffer overflow.  The assumption is that any
776
         * one operation beginning below the high water mark cannot overrun
777
         * the buffer completely.  Thus we can test for overflow after
778
         * generating code without having to check during generation.
779
         */
780
0
        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
781
0
            return -1;
782
0
        }
783
0
    }
784
0
    return 0;
785
0
}
786
787
static int tcg_out_pool_finalize(TCGContext *s)
788
0
{
789
0
    TCGLabelPoolData *p = s->pool_labels;
790
0
    TCGLabelPoolData *l = NULL;
791
0
    void *a;
792
793
0
    if (p == NULL) {
794
0
        return 0;
795
0
    }
796
797
    /*
798
     * ??? Round up to qemu_icache_linesize, but then do not round
799
     * again when allocating the next TranslationBlock structure.
800
     */
801
0
    a = (void *)ROUND_UP((uintptr_t)s->code_ptr,
802
0
                         sizeof(tcg_target_ulong) * p->nlong);
803
0
    tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr);
804
0
    s->data_gen_ptr = a;
805
806
0
    for (; p != NULL; p = p->next) {
807
0
        size_t size = sizeof(tcg_target_ulong) * p->nlong;
808
0
        uintptr_t value;
809
810
0
        if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) {
811
0
            if (unlikely(a > s->code_gen_highwater)) {
812
0
                return -1;
813
0
            }
814
0
            memcpy(a, p->data, size);
815
0
            a += size;
816
0
            l = p;
817
0
        }
818
819
0
        value = (uintptr_t)tcg_splitwx_to_rx(a) - size;
820
0
        if (!patch_reloc(p->label, p->rtype, value, p->addend)) {
821
0
            return -2;
822
0
        }
823
0
    }
824
825
0
    s->code_ptr = a;
826
0
    return 0;
827
0
}
828
829
#define C_PFX1(P, A)                    P##A
830
0
#define C_PFX2(P, A, B)                 P##A##_##B
831
0
#define C_PFX3(P, A, B, C)              P##A##_##B##_##C
832
0
#define C_PFX4(P, A, B, C, D)           P##A##_##B##_##C##_##D
833
0
#define C_PFX5(P, A, B, C, D, E)        P##A##_##B##_##C##_##D##_##E
834
#define C_PFX6(P, A, B, C, D, E, F)     P##A##_##B##_##C##_##D##_##E##_##F
835
836
/* Define an enumeration for the various combinations. */
837
838
#define C_O0_I1(I1)                     C_PFX1(c_o0_i1_, I1),
839
#define C_O0_I2(I1, I2)                 C_PFX2(c_o0_i2_, I1, I2),
840
#define C_O0_I3(I1, I2, I3)             C_PFX3(c_o0_i3_, I1, I2, I3),
841
#define C_O0_I4(I1, I2, I3, I4)         C_PFX4(c_o0_i4_, I1, I2, I3, I4),
842
843
#define C_O1_I1(O1, I1)                 C_PFX2(c_o1_i1_, O1, I1),
844
#define C_O1_I2(O1, I1, I2)             C_PFX3(c_o1_i2_, O1, I1, I2),
845
#define C_O1_I3(O1, I1, I2, I3)         C_PFX4(c_o1_i3_, O1, I1, I2, I3),
846
#define C_O1_I4(O1, I1, I2, I3, I4)     C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
847
848
#define C_N1_I2(O1, I1, I2)             C_PFX3(c_n1_i2_, O1, I1, I2),
849
#define C_N1O1_I1(O1, O2, I1)           C_PFX3(c_n1o1_i1_, O1, O2, I1),
850
#define C_N2_I1(O1, O2, I1)             C_PFX3(c_n2_i1_, O1, O2, I1),
851
852
#define C_O2_I1(O1, O2, I1)             C_PFX3(c_o2_i1_, O1, O2, I1),
853
#define C_O2_I2(O1, O2, I1, I2)         C_PFX4(c_o2_i2_, O1, O2, I1, I2),
854
#define C_O2_I3(O1, O2, I1, I2, I3)     C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
855
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
856
#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
857
858
typedef enum {
859
    C_Dynamic = -2,
860
    C_NotImplemented = -1,
861
#include "tcg-target-con-set.h"
862
} TCGConstraintSetIndex;
863
864
static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode, TCGType, unsigned);
865
866
#undef C_O0_I1
867
#undef C_O0_I2
868
#undef C_O0_I3
869
#undef C_O0_I4
870
#undef C_O1_I1
871
#undef C_O1_I2
872
#undef C_O1_I3
873
#undef C_O1_I4
874
#undef C_N1_I2
875
#undef C_N1O1_I1
876
#undef C_N2_I1
877
#undef C_O2_I1
878
#undef C_O2_I2
879
#undef C_O2_I3
880
#undef C_O2_I4
881
#undef C_N1_O1_I4
882
883
/* Put all of the constraint sets into an array, indexed by the enum. */
884
885
typedef struct TCGConstraintSet {
886
    uint8_t nb_oargs, nb_iargs;
887
    const char *args_ct_str[TCG_MAX_OP_ARGS];
888
} TCGConstraintSet;
889
890
#define C_O0_I1(I1)                     { 0, 1, { #I1 } },
891
#define C_O0_I2(I1, I2)                 { 0, 2, { #I1, #I2 } },
892
#define C_O0_I3(I1, I2, I3)             { 0, 3, { #I1, #I2, #I3 } },
893
#define C_O0_I4(I1, I2, I3, I4)         { 0, 4, { #I1, #I2, #I3, #I4 } },
894
895
#define C_O1_I1(O1, I1)                 { 1, 1, { #O1, #I1 } },
896
#define C_O1_I2(O1, I1, I2)             { 1, 2, { #O1, #I1, #I2 } },
897
#define C_O1_I3(O1, I1, I2, I3)         { 1, 3, { #O1, #I1, #I2, #I3 } },
898
#define C_O1_I4(O1, I1, I2, I3, I4)     { 1, 4, { #O1, #I1, #I2, #I3, #I4 } },
899
900
#define C_N1_I2(O1, I1, I2)             { 1, 2, { "&" #O1, #I1, #I2 } },
901
#define C_N1O1_I1(O1, O2, I1)           { 2, 1, { "&" #O1, #O2, #I1 } },
902
#define C_N2_I1(O1, O2, I1)             { 2, 1, { "&" #O1, "&" #O2, #I1 } },
903
904
#define C_O2_I1(O1, O2, I1)             { 2, 1, { #O1, #O2, #I1 } },
905
#define C_O2_I2(O1, O2, I1, I2)         { 2, 2, { #O1, #O2, #I1, #I2 } },
906
#define C_O2_I3(O1, O2, I1, I2, I3)     { 2, 3, { #O1, #O2, #I1, #I2, #I3 } },
907
#define C_O2_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { #O1, #O2, #I1, #I2, #I3, #I4 } },
908
#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
909
910
static const TCGConstraintSet constraint_sets[] = {
911
#include "tcg-target-con-set.h"
912
};
913
914
#undef C_O0_I1
915
#undef C_O0_I2
916
#undef C_O0_I3
917
#undef C_O0_I4
918
#undef C_O1_I1
919
#undef C_O1_I2
920
#undef C_O1_I3
921
#undef C_O1_I4
922
#undef C_N1_I2
923
#undef C_N1O1_I1
924
#undef C_N2_I1
925
#undef C_O2_I1
926
#undef C_O2_I2
927
#undef C_O2_I3
928
#undef C_O2_I4
929
#undef C_N1_O1_I4
930
931
/* Expand the enumerator to be returned from tcg_target_op_def(). */
932
933
#define C_O0_I1(I1)                     C_PFX1(c_o0_i1_, I1)
934
0
#define C_O0_I2(I1, I2)                 C_PFX2(c_o0_i2_, I1, I2)
935
#define C_O0_I3(I1, I2, I3)             C_PFX3(c_o0_i3_, I1, I2, I3)
936
#define C_O0_I4(I1, I2, I3, I4)         C_PFX4(c_o0_i4_, I1, I2, I3, I4)
937
938
0
#define C_O1_I1(O1, I1)                 C_PFX2(c_o1_i1_, O1, I1)
939
0
#define C_O1_I2(O1, I1, I2)             C_PFX3(c_o1_i2_, O1, I1, I2)
940
0
#define C_O1_I3(O1, I1, I2, I3)         C_PFX4(c_o1_i3_, O1, I1, I2, I3)
941
0
#define C_O1_I4(O1, I1, I2, I3, I4)     C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
942
943
0
#define C_N1_I2(O1, I1, I2)             C_PFX3(c_n1_i2_, O1, I1, I2)
944
#define C_N1O1_I1(O1, O2, I1)           C_PFX3(c_n1o1_i1_, O1, O2, I1)
945
#define C_N2_I1(O1, O2, I1)             C_PFX3(c_n2_i1_, O1, O2, I1)
946
947
#define C_O2_I1(O1, O2, I1)             C_PFX3(c_o2_i1_, O1, O2, I1)
948
0
#define C_O2_I2(O1, O2, I1, I2)         C_PFX4(c_o2_i2_, O1, O2, I1, I2)
949
#define C_O2_I3(O1, O2, I1, I2, I3)     C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
950
#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
951
#define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
952
953
/*
954
 * TCGOutOp is the base class for a set of structures that describe how
955
 * to generate code for a given TCGOpcode.
956
 *
957
 * @static_constraint:
958
 *   C_NotImplemented: The TCGOpcode is not supported by the backend.
959
 *   C_Dynamic:        Use @dynamic_constraint to select a constraint set
960
 *                     based on any of @type, @flags, or host isa.
961
 *   Otherwise:        The register allocation constrains for the TCGOpcode.
962
 *
963
 * Subclasses of TCGOutOp will define a set of output routines that may
964
 * be used.  Such routines will often be selected by the set of registers
965
 * and constants that come out of register allocation.  The set of
966
 * routines that are provided will guide the set of constraints that are
967
 * legal.  In particular, assume that tcg_optimize() has done its job in
968
 * swapping commutative operands and folding operations for which all
969
 * operands are constant.
970
 */
971
typedef struct TCGOutOp {
972
    TCGConstraintSetIndex static_constraint;
973
    TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags);
974
} TCGOutOp;
975
976
typedef struct TCGOutOpAddSubCarry {
977
    TCGOutOp base;
978
    void (*out_rrr)(TCGContext *s, TCGType type,
979
                    TCGReg a0, TCGReg a1, TCGReg a2);
980
    void (*out_rri)(TCGContext *s, TCGType type,
981
                    TCGReg a0, TCGReg a1, tcg_target_long a2);
982
    void (*out_rir)(TCGContext *s, TCGType type,
983
                    TCGReg a0, tcg_target_long a1, TCGReg a2);
984
    void (*out_rii)(TCGContext *s, TCGType type,
985
                    TCGReg a0, tcg_target_long a1, tcg_target_long a2);
986
} TCGOutOpAddSubCarry;
987
988
typedef struct TCGOutOpBinary {
989
    TCGOutOp base;
990
    void (*out_rrr)(TCGContext *s, TCGType type,
991
                    TCGReg a0, TCGReg a1, TCGReg a2);
992
    void (*out_rri)(TCGContext *s, TCGType type,
993
                    TCGReg a0, TCGReg a1, tcg_target_long a2);
994
} TCGOutOpBinary;
995
996
typedef struct TCGOutOpBrcond {
997
    TCGOutOp base;
998
    void (*out_rr)(TCGContext *s, TCGType type, TCGCond cond,
999
                   TCGReg a1, TCGReg a2, TCGLabel *label);
1000
    void (*out_ri)(TCGContext *s, TCGType type, TCGCond cond,
1001
                   TCGReg a1, tcg_target_long a2, TCGLabel *label);
1002
} TCGOutOpBrcond;
1003
1004
typedef struct TCGOutOpBswap {
1005
    TCGOutOp base;
1006
    void (*out_rr)(TCGContext *s, TCGType type,
1007
                   TCGReg a0, TCGReg a1, unsigned flags);
1008
} TCGOutOpBswap;
1009
1010
typedef struct TCGOutOpDeposit {
1011
    TCGOutOp base;
1012
    void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1013
                    TCGReg a2, unsigned ofs, unsigned len);
1014
    void (*out_rri)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1015
                    tcg_target_long a2, unsigned ofs, unsigned len);
1016
    void (*out_rzr)(TCGContext *s, TCGType type, TCGReg a0,
1017
                    TCGReg a2, unsigned ofs, unsigned len);
1018
} TCGOutOpDeposit;
1019
1020
typedef struct TCGOutOpDivRem {
1021
    TCGOutOp base;
1022
    void (*out_rr01r)(TCGContext *s, TCGType type,
1023
                      TCGReg a0, TCGReg a1, TCGReg a4);
1024
} TCGOutOpDivRem;
1025
1026
typedef struct TCGOutOpExtract {
1027
    TCGOutOp base;
1028
    void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1029
                   unsigned ofs, unsigned len);
1030
} TCGOutOpExtract;
1031
1032
typedef struct TCGOutOpExtract2 {
1033
    TCGOutOp base;
1034
    void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1035
                    TCGReg a2, unsigned shr);
1036
} TCGOutOpExtract2;
1037
1038
typedef struct TCGOutOpLoad {
1039
    TCGOutOp base;
1040
    void (*out)(TCGContext *s, TCGType type, TCGReg dest,
1041
                TCGReg base, intptr_t offset);
1042
} TCGOutOpLoad;
1043
1044
typedef struct TCGOutOpMovcond {
1045
    TCGOutOp base;
1046
    void (*out)(TCGContext *s, TCGType type, TCGCond cond,
1047
                TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2,
1048
                TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf);
1049
} TCGOutOpMovcond;
1050
1051
typedef struct TCGOutOpMul2 {
1052
    TCGOutOp base;
1053
    void (*out_rrrr)(TCGContext *s, TCGType type,
1054
                     TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3);
1055
} TCGOutOpMul2;
1056
1057
typedef struct TCGOutOpQemuLdSt {
1058
    TCGOutOp base;
1059
    void (*out)(TCGContext *s, TCGType type, TCGReg dest,
1060
                TCGReg addr, MemOpIdx oi);
1061
} TCGOutOpQemuLdSt;
1062
1063
typedef struct TCGOutOpQemuLdSt2 {
1064
    TCGOutOp base;
1065
    void (*out)(TCGContext *s, TCGType type, TCGReg dlo, TCGReg dhi,
1066
                TCGReg addr, MemOpIdx oi);
1067
} TCGOutOpQemuLdSt2;
1068
1069
typedef struct TCGOutOpUnary {
1070
    TCGOutOp base;
1071
    void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1);
1072
} TCGOutOpUnary;
1073
1074
typedef struct TCGOutOpSetcond {
1075
    TCGOutOp base;
1076
    void (*out_rrr)(TCGContext *s, TCGType type, TCGCond cond,
1077
                    TCGReg ret, TCGReg a1, TCGReg a2);
1078
    void (*out_rri)(TCGContext *s, TCGType type, TCGCond cond,
1079
                    TCGReg ret, TCGReg a1, tcg_target_long a2);
1080
} TCGOutOpSetcond;
1081
1082
typedef struct TCGOutOpStore {
1083
    TCGOutOp base;
1084
    void (*out_r)(TCGContext *s, TCGType type, TCGReg data,
1085
                  TCGReg base, intptr_t offset);
1086
    void (*out_i)(TCGContext *s, TCGType type, tcg_target_long data,
1087
                  TCGReg base, intptr_t offset);
1088
} TCGOutOpStore;
1089
1090
typedef struct TCGOutOpSubtract {
1091
    TCGOutOp base;
1092
    void (*out_rrr)(TCGContext *s, TCGType type,
1093
                    TCGReg a0, TCGReg a1, TCGReg a2);
1094
    void (*out_rir)(TCGContext *s, TCGType type,
1095
                    TCGReg a0, tcg_target_long a1, TCGReg a2);
1096
} TCGOutOpSubtract;
1097
1098
typedef struct TCGOutOpLea {
1099
    TCGOutOp base;
1100
    void (*out)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1101
                TCGReg a2, unsigned sh, tcg_target_long imm);
1102
} TCGOutOpLea;
1103
1104
#include "tcg-target.c.inc"
1105
1106
#ifndef CONFIG_TCG_INTERPRETER
1107
/* Validate CPUTLBDescFast placement. */
1108
QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
1109
                        sizeof(CPUNegativeOffsetState))
1110
                  < MIN_TLB_MASK_TABLE_OFS);
1111
#endif
1112
1113
/*
1114
 * We require these functions for slow-path function calls.
1115
 * Adapt them generically for opcode output.
1116
 */
1117
1118
static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1119
0
{
1120
0
    tcg_out_exts_i32_i64(s, a0, a1);
1121
0
}
1122
1123
static const TCGOutOpUnary outop_exts_i32_i64 = {
1124
    .base.static_constraint = C_O1_I1(r, r),
1125
    .out_rr = tgen_exts_i32_i64,
1126
};
1127
1128
static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1129
0
{
1130
0
    tcg_out_extu_i32_i64(s, a0, a1);
1131
0
}
1132
1133
static const TCGOutOpUnary outop_extu_i32_i64 = {
1134
    .base.static_constraint = C_O1_I1(r, r),
1135
    .out_rr = tgen_extu_i32_i64,
1136
};
1137
1138
static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1139
0
{
1140
0
    tcg_out_extrl_i64_i32(s, a0, a1);
1141
0
}
1142
1143
static const TCGOutOpUnary outop_extrl_i64_i32 = {
1144
    .base.static_constraint = C_O1_I1(r, r),
1145
    .out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL,
1146
};
1147
1148
static const TCGOutOp outop_goto_ptr = {
1149
    .static_constraint = C_O0_I1(r),
1150
};
1151
1152
static const TCGOutOpLoad outop_ld = {
1153
    .base.static_constraint = C_O1_I1(r, r),
1154
    .out = tcg_out_ld,
1155
};
1156
1157
/*
1158
 * Register V as the TCGOutOp for O.
1159
 * This verifies that V is of type T, otherwise give a nice compiler error.
1160
 * This prevents trivial mistakes within each arch/tcg-target.c.inc.
1161
 */
1162
#define OUTOP(O, T, V)  [O] = _Generic(V, T: &V.base)
1163
1164
/* Register allocation descriptions for every TCGOpcode. */
1165
static const TCGOutOp * const all_outop[NB_OPS] = {
1166
    OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add),
1167
    /* addc1o is implemented with set_carry + addcio */
1168
    OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio),
1169
    OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci),
1170
    OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio),
1171
    OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco),
1172
    OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and),
1173
    OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc),
1174
    OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond),
1175
    OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16),
1176
    OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32),
1177
    OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64),
1178
    OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz),
1179
    OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop),
1180
    OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz),
1181
    OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit),
1182
    OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs),
1183
    OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2),
1184
    OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu),
1185
    OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2),
1186
    OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv),
1187
    OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract),
1188
    OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2),
1189
    OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64),
1190
    OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32),
1191
    OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32),
1192
    OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64),
1193
    OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld),
1194
    OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s),
1195
    OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u),
1196
    OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s),
1197
    OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u),
1198
    OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s),
1199
    OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u),
1200
    OUTOP(INDEX_op_lea, TCGOutOpLea, outop_lea),
1201
    OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond),
1202
    OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul),
1203
    OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2),
1204
    OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh),
1205
    OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2),
1206
    OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh),
1207
    OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand),
1208
    OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg),
1209
    OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond),
1210
    OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor),
1211
    OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not),
1212
    OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or),
1213
    OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc),
1214
    OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld),
1215
    OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2),
1216
    OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st),
1217
    OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2),
1218
    OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems),
1219
    OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu),
1220
    OUTOP(INDEX_op_revbit8, TCGOutOpUnary, outop_revbit8),
1221
    OUTOP(INDEX_op_revbit32, TCGOutOpBswap, outop_revbit32),
1222
    OUTOP(INDEX_op_revbit64, TCGOutOpUnary, outop_revbit64),
1223
    OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl),
1224
    OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr),
1225
    OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar),
1226
    OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond),
1227
    OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract),
1228
    OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl),
1229
    OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr),
1230
    OUTOP(INDEX_op_smax, TCGOutOpBinary, outop_smax),
1231
    OUTOP(INDEX_op_smin, TCGOutOpBinary, outop_smin),
1232
    OUTOP(INDEX_op_st, TCGOutOpStore, outop_st),
1233
    OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8),
1234
    OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16),
1235
    OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st),
1236
    OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub),
1237
    /* subb1o is implemented with set_borrow + subbio */
1238
    OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio),
1239
    OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi),
1240
    OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio),
1241
    OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo),
1242
    OUTOP(INDEX_op_umax, TCGOutOpBinary, outop_umax),
1243
    OUTOP(INDEX_op_umin, TCGOutOpBinary, outop_umin),
1244
    OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor),
1245
1246
    [INDEX_op_goto_ptr] = &outop_goto_ptr,
1247
};
1248
1249
#undef OUTOP
1250
1251
/*
1252
 * All TCG threads except the parent (i.e. the one that called tcg_context_init
1253
 * and registered the target's TCG globals) must register with this function
1254
 * before initiating translation.
1255
 *
1256
 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
1257
 * of tcg_region_init() for the reasoning behind this.
1258
 *
1259
 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
1260
 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
1261
 * is not used anymore for translation once this function is called.
1262
 *
1263
 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
1264
 * iterates over the array (e.g. tcg_code_size() the same for both system/user
1265
 * modes.
1266
 */
1267
#ifdef CONFIG_USER_ONLY
1268
void tcg_register_thread(void)
1269
{
1270
    tcg_ctx = &tcg_init_ctx;
1271
}
1272
#else
1273
void tcg_register_thread(void)
1274
0
{
1275
0
    TCGContext *s = g_malloc(sizeof(*s));
1276
0
    unsigned int i, n;
1277
1278
0
    *s = tcg_init_ctx;
1279
1280
    /* Relink mem_base.  */
1281
0
    for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
1282
0
        if (tcg_init_ctx.temps[i].mem_base) {
1283
0
            ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
1284
0
            tcg_debug_assert(b >= 0 && b < n);
1285
0
            s->temps[i].mem_base = &s->temps[b];
1286
0
        }
1287
0
    }
1288
1289
    /* Claim an entry in tcg_ctxs */
1290
0
    n = qatomic_fetch_inc(&tcg_cur_ctxs);
1291
0
    g_assert(n < tcg_max_ctxs);
1292
0
    qatomic_set(&tcg_ctxs[n], s);
1293
1294
0
    if (n > 0) {
1295
0
        tcg_region_thread_initial_alloc(s);
1296
0
    }
1297
1298
0
    tcg_ctx = s;
1299
0
}
1300
#endif /* !CONFIG_USER_ONLY */
1301
1302
/* pool based memory allocation */
1303
void *tcg_malloc_internal(TCGContext *s, int size)
1304
0
{
1305
0
    TCGPool *p;
1306
0
    int pool_size;
1307
1308
0
    if (size > TCG_POOL_CHUNK_SIZE) {
1309
        /* big malloc: insert a new pool (XXX: could optimize) */
1310
0
        p = g_malloc(sizeof(TCGPool) + size);
1311
0
        p->size = size;
1312
0
        p->next = s->pool_first_large;
1313
0
        s->pool_first_large = p;
1314
0
        return p->data;
1315
0
    } else {
1316
0
        p = s->pool_current;
1317
0
        if (!p) {
1318
0
            p = s->pool_first;
1319
0
            if (!p) {
1320
0
                goto new_pool;
1321
0
            }
1322
0
        } else {
1323
0
            if (!p->next) {
1324
0
            new_pool:
1325
0
                pool_size = TCG_POOL_CHUNK_SIZE;
1326
0
                p = g_malloc(sizeof(TCGPool) + pool_size);
1327
0
                p->size = pool_size;
1328
0
                p->next = NULL;
1329
0
                if (s->pool_current) {
1330
0
                    s->pool_current->next = p;
1331
0
                } else {
1332
0
                    s->pool_first = p;
1333
0
                }
1334
0
            } else {
1335
0
                p = p->next;
1336
0
            }
1337
0
        }
1338
0
    }
1339
0
    s->pool_current = p;
1340
0
    s->pool_cur = (uintptr_t)p->data + size;
1341
0
    s->pool_end = (uintptr_t)p->data + p->size;
1342
0
    return p->data;
1343
0
}
1344
1345
void tcg_pool_reset(TCGContext *s)
1346
0
{
1347
0
    TCGPool *p, *t;
1348
0
    for (p = s->pool_first_large; p; p = t) {
1349
0
        t = p->next;
1350
0
        g_free(p);
1351
0
    }
1352
0
    s->pool_first_large = NULL;
1353
0
    s->pool_cur = s->pool_end = 0;
1354
0
    s->pool_current = NULL;
1355
0
}
1356
1357
/*
1358
 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
1359
 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
1360
 * We only use these for layout in tcg_out_ld_helper_ret and
1361
 * tcg_out_st_helper_args, and share them between several of
1362
 * the helpers, with the end result that it's easier to build manually.
1363
 */
1364
1365
#define dh_typecode_ttl  dh_typecode_i64
1366
1367
static TCGHelperInfo info_helper_ld32_mmu = {
1368
    .flags = TCG_CALL_NO_WG,
1369
    .typemask = dh_typemask(ttl, 0)  /* return tcg_target_ulong */
1370
              | dh_typemask(env, 1)
1371
              | dh_typemask(i64, 2)  /* uint64_t addr */
1372
              | dh_typemask(i32, 3)  /* unsigned oi */
1373
              | dh_typemask(ptr, 4)  /* uintptr_t ra */
1374
};
1375
1376
static TCGHelperInfo info_helper_ld64_mmu = {
1377
    .flags = TCG_CALL_NO_WG,
1378
    .typemask = dh_typemask(i64, 0)  /* return uint64_t */
1379
              | dh_typemask(env, 1)
1380
              | dh_typemask(i64, 2)  /* uint64_t addr */
1381
              | dh_typemask(i32, 3)  /* unsigned oi */
1382
              | dh_typemask(ptr, 4)  /* uintptr_t ra */
1383
};
1384
1385
static TCGHelperInfo info_helper_ld128_mmu = {
1386
    .flags = TCG_CALL_NO_WG,
1387
    .typemask = dh_typemask(i128, 0) /* return Int128 */
1388
              | dh_typemask(env, 1)
1389
              | dh_typemask(i64, 2)  /* uint64_t addr */
1390
              | dh_typemask(i32, 3)  /* unsigned oi */
1391
              | dh_typemask(ptr, 4)  /* uintptr_t ra */
1392
};
1393
1394
static TCGHelperInfo info_helper_st32_mmu = {
1395
    .flags = TCG_CALL_NO_WG,
1396
    .typemask = dh_typemask(void, 0)
1397
              | dh_typemask(env, 1)
1398
              | dh_typemask(i64, 2)  /* uint64_t addr */
1399
              | dh_typemask(i32, 3)  /* uint32_t data */
1400
              | dh_typemask(i32, 4)  /* unsigned oi */
1401
              | dh_typemask(ptr, 5)  /* uintptr_t ra */
1402
};
1403
1404
static TCGHelperInfo info_helper_st64_mmu = {
1405
    .flags = TCG_CALL_NO_WG,
1406
    .typemask = dh_typemask(void, 0)
1407
              | dh_typemask(env, 1)
1408
              | dh_typemask(i64, 2)  /* uint64_t addr */
1409
              | dh_typemask(i64, 3)  /* uint64_t data */
1410
              | dh_typemask(i32, 4)  /* unsigned oi */
1411
              | dh_typemask(ptr, 5)  /* uintptr_t ra */
1412
};
1413
1414
static TCGHelperInfo info_helper_st128_mmu = {
1415
    .flags = TCG_CALL_NO_WG,
1416
    .typemask = dh_typemask(void, 0)
1417
              | dh_typemask(env, 1)
1418
              | dh_typemask(i64, 2)  /* uint64_t addr */
1419
              | dh_typemask(i128, 3) /* Int128 data */
1420
              | dh_typemask(i32, 4)  /* unsigned oi */
1421
              | dh_typemask(ptr, 5)  /* uintptr_t ra */
1422
};
1423
1424
#ifdef CONFIG_TCG_INTERPRETER
1425
static ffi_type *typecode_to_ffi(int argmask)
1426
{
1427
    /*
1428
     * libffi does not support __int128_t, so we have forced Int128
1429
     * to use the structure definition instead of the builtin type.
1430
     */
1431
    static ffi_type *ffi_type_i128_elements[3] = {
1432
        &ffi_type_uint64,
1433
        &ffi_type_uint64,
1434
        NULL
1435
    };
1436
    static ffi_type ffi_type_i128 = {
1437
        .size = 16,
1438
        .alignment = __alignof__(Int128),
1439
        .type = FFI_TYPE_STRUCT,
1440
        .elements = ffi_type_i128_elements,
1441
    };
1442
1443
    switch (argmask) {
1444
    case dh_typecode_void:
1445
        return &ffi_type_void;
1446
    case dh_typecode_i32:
1447
        return &ffi_type_uint32;
1448
    case dh_typecode_s32:
1449
        return &ffi_type_sint32;
1450
    case dh_typecode_i64:
1451
        return &ffi_type_uint64;
1452
    case dh_typecode_s64:
1453
        return &ffi_type_sint64;
1454
    case dh_typecode_ptr:
1455
        return &ffi_type_pointer;
1456
    case dh_typecode_i128:
1457
        return &ffi_type_i128;
1458
    }
1459
    g_assert_not_reached();
1460
}
1461
1462
static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
1463
{
1464
    unsigned typemask = info->typemask;
1465
    struct {
1466
        ffi_cif cif;
1467
        ffi_type *args[];
1468
    } *ca;
1469
    ffi_status status;
1470
    int nargs;
1471
1472
    /* Ignoring the return type, find the last non-zero field. */
1473
    nargs = 32 - clz32(typemask >> 3);
1474
    nargs = DIV_ROUND_UP(nargs, 3);
1475
    assert(nargs <= MAX_CALL_IARGS);
1476
1477
    ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
1478
    ca->cif.rtype = typecode_to_ffi(typemask & 7);
1479
    ca->cif.nargs = nargs;
1480
1481
    if (nargs != 0) {
1482
        ca->cif.arg_types = ca->args;
1483
        for (int j = 0; j < nargs; ++j) {
1484
            int typecode = extract32(typemask, (j + 1) * 3, 3);
1485
            ca->args[j] = typecode_to_ffi(typecode);
1486
        }
1487
    }
1488
1489
    status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
1490
                          ca->cif.rtype, ca->cif.arg_types);
1491
    assert(status == FFI_OK);
1492
1493
    return &ca->cif;
1494
}
1495
1496
#define HELPER_INFO_INIT(I)      (&(I)->cif)
1497
#define HELPER_INFO_INIT_VAL(I)  init_ffi_layout(I)
1498
#else
1499
#define HELPER_INFO_INIT(I)      (&(I)->init)
1500
#define HELPER_INFO_INIT_VAL(I)  1
1501
#endif /* CONFIG_TCG_INTERPRETER */
1502
1503
static inline bool arg_slot_reg_p(unsigned arg_slot)
1504
0
{
1505
    /*
1506
     * Split the sizeof away from the comparison to avoid Werror from
1507
     * "unsigned < 0 is always false", when iarg_regs is empty.
1508
     */
1509
0
    unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
1510
0
    return arg_slot < nreg;
1511
0
}
1512
1513
static inline int arg_slot_stk_ofs(unsigned arg_slot)
1514
0
{
1515
0
    unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1516
0
    unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1517
1518
0
    tcg_debug_assert(stk_slot < max);
1519
0
    return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
1520
0
}
1521
1522
typedef struct TCGCumulativeArgs {
1523
    int arg_idx;                /* tcg_gen_callN args[] */
1524
    int info_in_idx;            /* TCGHelperInfo in[] */
1525
    int arg_slot;               /* regs+stack slot */
1526
    int ref_slot;               /* stack slots for references */
1527
} TCGCumulativeArgs;
1528
1529
static void layout_arg_even(TCGCumulativeArgs *cum)
1530
0
{
1531
0
    cum->arg_slot += cum->arg_slot & 1;
1532
0
}
1533
1534
static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
1535
                         TCGCallArgumentKind kind)
1536
0
{
1537
0
    TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1538
1539
0
    *loc = (TCGCallArgumentLoc){
1540
0
        .kind = kind,
1541
0
        .arg_idx = cum->arg_idx,
1542
0
        .arg_slot = cum->arg_slot,
1543
0
    };
1544
0
    cum->info_in_idx++;
1545
0
    cum->arg_slot++;
1546
0
}
1547
1548
static void layout_arg_normal_n(TCGCumulativeArgs *cum,
1549
                                TCGHelperInfo *info, int n)
1550
0
{
1551
0
    TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1552
1553
0
    for (int i = 0; i < n; ++i) {
1554
        /* Layout all using the same arg_idx, adjusting the subindex. */
1555
0
        loc[i] = (TCGCallArgumentLoc){
1556
0
            .kind = TCG_CALL_ARG_NORMAL,
1557
0
            .arg_idx = cum->arg_idx,
1558
0
            .tmp_subindex = i,
1559
0
            .arg_slot = cum->arg_slot + i,
1560
0
        };
1561
0
    }
1562
0
    cum->info_in_idx += n;
1563
0
    cum->arg_slot += n;
1564
0
}
1565
1566
static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
1567
0
{
1568
0
    TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1569
0
    int n = 128 / TCG_TARGET_REG_BITS;
1570
0
1571
0
    /* The first subindex carries the pointer. */
1572
0
    layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
1573
0
1574
0
    /*
1575
0
     * The callee is allowed to clobber memory associated with
1576
0
     * structure pass by-reference.  Therefore we must make copies.
1577
0
     * Allocate space from "ref_slot", which will be adjusted to
1578
0
     * follow the parameters on the stack.
1579
0
     */
1580
0
    loc[0].ref_slot = cum->ref_slot;
1581
0
1582
0
    /*
1583
0
     * Subsequent words also go into the reference slot, but
1584
0
     * do not accumulate into the regular arguments.
1585
0
     */
1586
0
    for (int i = 1; i < n; ++i) {
1587
0
        loc[i] = (TCGCallArgumentLoc){
1588
0
            .kind = TCG_CALL_ARG_BY_REF_N,
1589
0
            .arg_idx = cum->arg_idx,
1590
0
            .tmp_subindex = i,
1591
0
            .ref_slot = cum->ref_slot + i,
1592
0
        };
1593
0
    }
1594
0
    cum->info_in_idx += n - 1;  /* i=0 accounted for in layout_arg_1 */
1595
0
    cum->ref_slot += n;
1596
0
}
1597
1598
static void init_call_layout(TCGHelperInfo *info)
1599
0
{
1600
0
    int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
1601
0
    int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1602
0
    unsigned typemask = info->typemask;
1603
0
    unsigned typecode;
1604
0
    TCGCumulativeArgs cum = { };
1605
1606
    /*
1607
     * Parse and place any function return value.
1608
     */
1609
0
    typecode = typemask & 7;
1610
0
    switch (typecode) {
1611
0
    case dh_typecode_void:
1612
0
        info->nr_out = 0;
1613
0
        break;
1614
0
    case dh_typecode_i32:
1615
0
    case dh_typecode_s32:
1616
0
    case dh_typecode_i64:
1617
0
    case dh_typecode_s64:
1618
0
    case dh_typecode_ptr:
1619
0
        info->nr_out = 1;
1620
0
        info->out_kind = TCG_CALL_RET_NORMAL;
1621
0
        break;
1622
0
    case dh_typecode_i128:
1623
0
        info->nr_out = 128 / TCG_TARGET_REG_BITS;
1624
0
        info->out_kind = TCG_TARGET_CALL_RET_I128;
1625
0
        switch (TCG_TARGET_CALL_RET_I128) {
1626
0
        case TCG_CALL_RET_NORMAL:
1627
            /* Query the last register now to trigger any assert early. */
1628
0
            tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1629
0
            break;
1630
0
        case TCG_CALL_RET_BY_VEC:
1631
            /* Query the single register now to trigger any assert early. */
1632
0
            tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
1633
0
            break;
1634
0
        case TCG_CALL_RET_BY_REF:
1635
            /*
1636
             * Allocate the first argument to the output.
1637
             * We don't need to store this anywhere, just make it
1638
             * unavailable for use in the input loop below.
1639
             */
1640
0
            cum.arg_slot = 1;
1641
0
            break;
1642
0
        default:
1643
0
            qemu_build_not_reached();
1644
0
        }
1645
0
        break;
1646
0
    default:
1647
0
        g_assert_not_reached();
1648
0
    }
1649
1650
    /*
1651
     * Parse and place function arguments.
1652
     */
1653
0
    for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
1654
0
        TCGCallArgumentKind kind;
1655
0
        TCGType type;
1656
1657
0
        typecode = typemask & 7;
1658
0
        switch (typecode) {
1659
0
        case dh_typecode_i32:
1660
0
        case dh_typecode_s32:
1661
0
            type = TCG_TYPE_I32;
1662
0
            break;
1663
0
        case dh_typecode_i64:
1664
0
        case dh_typecode_s64:
1665
0
            type = TCG_TYPE_I64;
1666
0
            break;
1667
0
        case dh_typecode_ptr:
1668
0
            type = TCG_TYPE_PTR;
1669
0
            break;
1670
0
        case dh_typecode_i128:
1671
0
            type = TCG_TYPE_I128;
1672
0
            break;
1673
0
        default:
1674
0
            g_assert_not_reached();
1675
0
        }
1676
1677
0
        switch (type) {
1678
0
        case TCG_TYPE_I32:
1679
0
            switch (TCG_TARGET_CALL_ARG_I32) {
1680
0
            case TCG_CALL_ARG_EVEN:
1681
0
                layout_arg_even(&cum);
1682
                /* fall through */
1683
0
            case TCG_CALL_ARG_NORMAL:
1684
0
                layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1685
0
                break;
1686
0
            case TCG_CALL_ARG_EXTEND:
1687
0
                kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
1688
0
                layout_arg_1(&cum, info, kind);
1689
0
                break;
1690
0
            default:
1691
0
                qemu_build_not_reached();
1692
0
            }
1693
0
            break;
1694
1695
0
        case TCG_TYPE_I64:
1696
0
            switch (TCG_TARGET_CALL_ARG_I64) {
1697
0
            case TCG_CALL_ARG_EVEN:
1698
0
                layout_arg_even(&cum);
1699
                /* fall through */
1700
0
            case TCG_CALL_ARG_NORMAL:
1701
0
                layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1702
0
                break;
1703
0
            default:
1704
0
                qemu_build_not_reached();
1705
0
            }
1706
0
            break;
1707
1708
0
        case TCG_TYPE_I128:
1709
0
            switch (TCG_TARGET_CALL_ARG_I128) {
1710
0
            case TCG_CALL_ARG_EVEN:
1711
0
                layout_arg_even(&cum);
1712
                /* fall through */
1713
0
            case TCG_CALL_ARG_NORMAL:
1714
0
                layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1715
0
                break;
1716
0
            case TCG_CALL_ARG_BY_REF:
1717
0
                layout_arg_by_ref(&cum, info);
1718
0
                break;
1719
0
            default:
1720
0
                qemu_build_not_reached();
1721
0
            }
1722
0
            break;
1723
1724
0
        default:
1725
0
            g_assert_not_reached();
1726
0
        }
1727
0
    }
1728
0
    info->nr_in = cum.info_in_idx;
1729
1730
    /* Validate that we didn't overrun the input array. */
1731
0
    assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1732
    /* Validate the backend has enough argument space. */
1733
0
    assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
1734
1735
    /*
1736
     * Relocate the "ref_slot" area to the end of the parameters.
1737
     * Minimizing this stack offset helps code size for x86,
1738
     * which has a signed 8-bit offset encoding.
1739
     */
1740
0
    if (cum.ref_slot != 0) {
1741
0
        int ref_base = 0;
1742
1743
0
        if (cum.arg_slot > max_reg_slots) {
1744
0
            int align = __alignof(Int128) / sizeof(tcg_target_long);
1745
1746
0
            ref_base = cum.arg_slot - max_reg_slots;
1747
0
            if (align > 1) {
1748
0
                ref_base = ROUND_UP(ref_base, align);
1749
0
            }
1750
0
        }
1751
0
        assert(ref_base + cum.ref_slot <= max_stk_slots);
1752
0
        ref_base += max_reg_slots;
1753
1754
0
        if (ref_base != 0) {
1755
0
            for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1756
0
                TCGCallArgumentLoc *loc = &info->in[i];
1757
0
                switch (loc->kind) {
1758
0
                case TCG_CALL_ARG_BY_REF:
1759
0
                case TCG_CALL_ARG_BY_REF_N:
1760
0
                    loc->ref_slot += ref_base;
1761
0
                    break;
1762
0
                default:
1763
0
                    break;
1764
0
                }
1765
0
            }
1766
0
        }
1767
0
    }
1768
0
}
1769
1770
static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
1771
static void process_constraint_sets(void);
1772
static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1773
                                            TCGReg reg, const char *name);
1774
1775
static void tcg_context_init(unsigned max_threads)
1776
0
{
1777
0
    TCGContext *s = &tcg_init_ctx;
1778
0
    int n, i;
1779
0
    TCGTemp *ts;
1780
1781
0
    memset(s, 0, sizeof(*s));
1782
0
    s->nb_globals = 0;
1783
1784
0
    init_call_layout(&info_helper_ld32_mmu);
1785
0
    init_call_layout(&info_helper_ld64_mmu);
1786
0
    init_call_layout(&info_helper_ld128_mmu);
1787
0
    init_call_layout(&info_helper_st32_mmu);
1788
0
    init_call_layout(&info_helper_st64_mmu);
1789
0
    init_call_layout(&info_helper_st128_mmu);
1790
1791
0
    tcg_target_init(s);
1792
0
    process_constraint_sets();
1793
1794
    /* Reverse the order of the saved registers, assuming they're all at
1795
       the start of tcg_target_reg_alloc_order.  */
1796
0
    for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1797
0
        int r = tcg_target_reg_alloc_order[n];
1798
0
        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1799
0
            break;
1800
0
        }
1801
0
    }
1802
0
    for (i = 0; i < n; ++i) {
1803
0
        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1804
0
    }
1805
0
    for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1806
0
        indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1807
0
    }
1808
1809
0
    tcg_ctx = s;
1810
    /*
1811
     * In user-mode we simply share the init context among threads, since we
1812
     * use a single region. See the documentation tcg_region_init() for the
1813
     * reasoning behind this.
1814
     * In system-mode we will have at most max_threads TCG threads.
1815
     */
1816
#ifdef CONFIG_USER_ONLY
1817
    tcg_ctxs = &tcg_ctx;
1818
    tcg_cur_ctxs = 1;
1819
    tcg_max_ctxs = 1;
1820
#else
1821
0
    tcg_max_ctxs = max_threads;
1822
0
    tcg_ctxs = g_new0(TCGContext *, max_threads);
1823
0
#endif
1824
1825
0
    tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1826
0
    ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1827
0
    tcg_env = temp_tcgv_ptr(ts);
1828
0
}
1829
1830
void tcg_init(size_t tb_size, int splitwx, unsigned max_threads)
1831
0
{
1832
0
    tcg_context_init(max_threads);
1833
0
    tcg_region_init(tb_size, splitwx, max_threads);
1834
0
}
1835
1836
/*
1837
 * Allocate TBs right before their corresponding translated code, making
1838
 * sure that TBs and code are on different cache lines.
1839
 */
1840
TranslationBlock *tcg_tb_alloc(TCGContext *s)
1841
0
{
1842
0
    uintptr_t align = qemu_icache_linesize;
1843
0
    TranslationBlock *tb;
1844
0
    void *next;
1845
1846
0
    while (1) {
1847
0
        tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1848
1849
        /*
1850
         * Note that code_gen_ptr can be NULL after vCPU hotplug.
1851
         * See tcg_region_thread_initial_alloc.
1852
         */
1853
0
        if (tb) {
1854
0
            next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1855
0
            if (next <= s->code_gen_highwater) {
1856
0
                qatomic_set(&s->code_gen_ptr, next);
1857
0
                return tb;
1858
0
            }
1859
0
        }
1860
0
        if (!tcg_region_alloc(s)) {
1861
0
            return NULL;
1862
0
        }
1863
0
    }
1864
0
}
1865
1866
void tcg_prologue_init(void)
1867
0
{
1868
0
    TCGContext *s = tcg_ctx;
1869
0
    size_t prologue_size;
1870
1871
0
    s->code_ptr = s->code_gen_ptr;
1872
0
    s->code_buf = s->code_gen_ptr;
1873
0
    s->data_gen_ptr = NULL;
1874
1875
0
#ifndef CONFIG_TCG_INTERPRETER
1876
0
    tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
1877
0
#endif
1878
1879
0
    s->pool_labels = NULL;
1880
1881
0
    qemu_thread_jit_write();
1882
    /* Generate the prologue.  */
1883
0
    tcg_target_qemu_prologue(s);
1884
1885
    /* Allow the prologue to put e.g. guest_base into a pool entry.  */
1886
0
    {
1887
0
        int result = tcg_out_pool_finalize(s);
1888
0
        tcg_debug_assert(result == 0);
1889
0
    }
1890
1891
0
    prologue_size = tcg_current_code_size(s);
1892
0
    perf_report_prologue(s->code_gen_ptr, prologue_size);
1893
1894
0
#ifndef CONFIG_TCG_INTERPRETER
1895
0
    flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1896
0
                        (uintptr_t)s->code_buf, prologue_size);
1897
0
#endif
1898
1899
0
    if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1900
0
        FILE *logfile = qemu_log_trylock();
1901
0
        if (logfile) {
1902
0
            fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1903
0
            if (s->data_gen_ptr) {
1904
0
                size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1905
0
                size_t data_size = prologue_size - code_size;
1906
0
                size_t i;
1907
1908
0
                disas(logfile, s->code_gen_ptr, code_size);
1909
1910
0
                for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1911
0
                    if (sizeof(tcg_target_ulong) == 8) {
1912
0
                        fprintf(logfile,
1913
0
                                "0x%08" PRIxPTR ":  .quad  0x%016" PRIx64 "\n",
1914
0
                                (uintptr_t)s->data_gen_ptr + i,
1915
0
                                *(uint64_t *)(s->data_gen_ptr + i));
1916
0
                    } else {
1917
0
                        fprintf(logfile,
1918
0
                                "0x%08" PRIxPTR ":  .long  0x%08x\n",
1919
0
                                (uintptr_t)s->data_gen_ptr + i,
1920
0
                                *(uint32_t *)(s->data_gen_ptr + i));
1921
0
                    }
1922
0
                }
1923
0
            } else {
1924
0
                disas(logfile, s->code_gen_ptr, prologue_size);
1925
0
            }
1926
0
            fprintf(logfile, "\n");
1927
0
            qemu_log_unlock(logfile);
1928
0
        }
1929
0
    }
1930
1931
0
#ifndef CONFIG_TCG_INTERPRETER
1932
    /*
1933
     * Assert that goto_ptr is implemented completely, setting an epilogue.
1934
     * For tci, we use NULL as the signal to return from the interpreter,
1935
     * so skip this check.
1936
     */
1937
0
    tcg_debug_assert(tcg_code_gen_epilogue != NULL);
1938
0
#endif
1939
1940
0
    tcg_region_prologue_set(s);
1941
0
}
1942
1943
void tcg_func_start(TCGContext *s)
1944
0
{
1945
0
    tcg_pool_reset(s);
1946
0
    s->nb_temps = s->nb_globals;
1947
1948
    /* No temps have been previously allocated for size or locality.  */
1949
0
    tcg_temp_ebb_reset_freed(s);
1950
1951
    /* No constant temps have been previously allocated. */
1952
0
    for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1953
0
        if (s->const_table[i]) {
1954
0
            g_hash_table_remove_all(s->const_table[i]);
1955
0
        }
1956
0
    }
1957
1958
0
    s->nb_ops = 0;
1959
0
    s->nb_labels = 0;
1960
0
    s->current_frame_offset = s->frame_start;
1961
1962
#ifdef CONFIG_DEBUG_TCG
1963
    s->goto_tb_issue_mask = 0;
1964
#endif
1965
1966
0
    QTAILQ_INIT(&s->ops);
1967
0
    QTAILQ_INIT(&s->free_ops);
1968
0
    s->emit_before_op = NULL;
1969
0
    QSIMPLEQ_INIT(&s->labels);
1970
1971
0
    tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
1972
0
}
1973
1974
static TCGTemp *tcg_temp_alloc(TCGContext *s)
1975
0
{
1976
0
    int n = s->nb_temps++;
1977
1978
0
    if (n >= TCG_MAX_TEMPS) {
1979
0
        tcg_raise_tb_overflow(s);
1980
0
    }
1981
0
    return memset(&s->temps[n], 0, sizeof(TCGTemp));
1982
0
}
1983
1984
static TCGTemp *tcg_global_alloc(TCGContext *s)
1985
0
{
1986
0
    TCGTemp *ts;
1987
1988
0
    tcg_debug_assert(s->nb_globals == s->nb_temps);
1989
0
    tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
1990
0
    s->nb_globals++;
1991
0
    ts = tcg_temp_alloc(s);
1992
0
    ts->kind = TEMP_GLOBAL;
1993
1994
0
    return ts;
1995
0
}
1996
1997
static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1998
                                            TCGReg reg, const char *name)
1999
0
{
2000
0
    TCGTemp *ts = tcg_global_alloc(s);
2001
2002
0
    ts->base_type = type;
2003
0
    ts->type = type;
2004
0
    ts->kind = TEMP_FIXED;
2005
0
    ts->reg = reg;
2006
0
    ts->name = name;
2007
0
    tcg_regset_set_reg(s->reserved_regs, reg);
2008
2009
0
    return ts;
2010
0
}
2011
2012
void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
2013
0
{
2014
0
    s->frame_start = start;
2015
0
    s->frame_end = start + size;
2016
0
    s->frame_temp
2017
0
        = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
2018
0
}
2019
2020
static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
2021
                                            const char *name, TCGType type)
2022
0
{
2023
0
    TCGContext *s = tcg_ctx;
2024
0
    TCGTemp *base_ts = tcgv_ptr_temp(base);
2025
0
    TCGTemp *ts = tcg_global_alloc(s);
2026
0
    int indirect_reg = 0;
2027
2028
0
    switch (base_ts->kind) {
2029
0
    case TEMP_FIXED:
2030
0
        break;
2031
0
    case TEMP_GLOBAL:
2032
        /* We do not support double-indirect registers.  */
2033
0
        tcg_debug_assert(!base_ts->indirect_reg);
2034
0
        base_ts->indirect_base = 1;
2035
0
        s->nb_indirects += 1;
2036
0
        indirect_reg = 1;
2037
0
        break;
2038
0
    default:
2039
0
        g_assert_not_reached();
2040
0
    }
2041
2042
0
    ts->base_type = type;
2043
0
    ts->type = type;
2044
0
    ts->indirect_reg = indirect_reg;
2045
0
    ts->mem_allocated = 1;
2046
0
    ts->mem_base = base_ts;
2047
0
    ts->mem_offset = offset;
2048
0
    ts->name = name;
2049
0
    return ts;
2050
0
}
2051
2052
TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
2053
0
{
2054
0
    TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
2055
0
    return temp_tcgv_i32(ts);
2056
0
}
2057
2058
TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
2059
0
{
2060
0
    TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
2061
0
    return temp_tcgv_i64(ts);
2062
0
}
2063
2064
TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
2065
0
{
2066
0
    TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
2067
0
    return temp_tcgv_ptr(ts);
2068
0
}
2069
2070
TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
2071
0
{
2072
0
    TCGContext *s = tcg_ctx;
2073
0
    TCGTemp *ts;
2074
0
    int n;
2075
2076
0
    if (kind == TEMP_EBB) {
2077
0
        int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
2078
2079
0
        if (idx < TCG_MAX_TEMPS) {
2080
            /* There is already an available temp with the right type.  */
2081
0
            clear_bit(idx, s->free_temps[type].l);
2082
2083
0
            ts = &s->temps[idx];
2084
0
            ts->temp_allocated = 1;
2085
0
            tcg_debug_assert(ts->base_type == type);
2086
0
            tcg_debug_assert(ts->kind == kind);
2087
0
            return ts;
2088
0
        }
2089
0
    } else {
2090
0
        tcg_debug_assert(kind == TEMP_TB);
2091
0
    }
2092
2093
0
    switch (type) {
2094
0
    case TCG_TYPE_I32:
2095
0
    case TCG_TYPE_I64:
2096
0
    case TCG_TYPE_V64:
2097
0
    case TCG_TYPE_V128:
2098
0
    case TCG_TYPE_V256:
2099
0
        n = 1;
2100
0
        break;
2101
0
    case TCG_TYPE_I128:
2102
0
        n = 128 / TCG_TARGET_REG_BITS;
2103
0
        break;
2104
0
    default:
2105
0
        g_assert_not_reached();
2106
0
    }
2107
2108
0
    ts = tcg_temp_alloc(s);
2109
0
    ts->base_type = type;
2110
0
    ts->temp_allocated = 1;
2111
0
    ts->kind = kind;
2112
2113
0
    if (n == 1) {
2114
0
        ts->type = type;
2115
0
    } else {
2116
0
        ts->type = TCG_TYPE_REG;
2117
2118
0
        for (int i = 1; i < n; ++i) {
2119
0
            TCGTemp *ts2 = tcg_temp_alloc(s);
2120
2121
0
            tcg_debug_assert(ts2 == ts + i);
2122
0
            ts2->base_type = type;
2123
0
            ts2->type = TCG_TYPE_REG;
2124
0
            ts2->temp_allocated = 1;
2125
0
            ts2->temp_subindex = i;
2126
0
            ts2->kind = kind;
2127
0
        }
2128
0
    }
2129
0
    return ts;
2130
0
}
2131
2132
TCGv_i32 tcg_temp_new_i32(void)
2133
0
{
2134
0
    return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
2135
0
}
2136
2137
TCGv_i32 tcg_temp_ebb_new_i32(void)
2138
0
{
2139
0
    return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
2140
0
}
2141
2142
TCGv_i64 tcg_temp_new_i64(void)
2143
0
{
2144
0
    return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
2145
0
}
2146
2147
TCGv_i64 tcg_temp_ebb_new_i64(void)
2148
0
{
2149
0
    return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
2150
0
}
2151
2152
TCGv_ptr tcg_temp_new_ptr(void)
2153
0
{
2154
0
    return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
2155
0
}
2156
2157
TCGv_ptr tcg_temp_ebb_new_ptr(void)
2158
0
{
2159
0
    return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
2160
0
}
2161
2162
TCGv_i128 tcg_temp_new_i128(void)
2163
0
{
2164
0
    return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
2165
0
}
2166
2167
TCGv_i128 tcg_temp_ebb_new_i128(void)
2168
0
{
2169
0
    return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
2170
0
}
2171
2172
TCGv_vec tcg_temp_new_vec(TCGType type)
2173
0
{
2174
0
    TCGTemp *t;
2175
2176
#ifdef CONFIG_DEBUG_TCG
2177
    switch (type) {
2178
    case TCG_TYPE_V64:
2179
        assert(TCG_TARGET_HAS_v64);
2180
        break;
2181
    case TCG_TYPE_V128:
2182
        assert(TCG_TARGET_HAS_v128);
2183
        break;
2184
    case TCG_TYPE_V256:
2185
        assert(TCG_TARGET_HAS_v256);
2186
        break;
2187
    default:
2188
        g_assert_not_reached();
2189
    }
2190
#endif
2191
2192
0
    t = tcg_temp_new_internal(type, TEMP_EBB);
2193
0
    return temp_tcgv_vec(t);
2194
0
}
2195
2196
/* Create a new temp of the same type as an existing temp.  */
2197
TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
2198
0
{
2199
0
    TCGTemp *t = tcgv_vec_temp(match);
2200
2201
0
    tcg_debug_assert(t->temp_allocated != 0);
2202
2203
0
    t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
2204
0
    return temp_tcgv_vec(t);
2205
0
}
2206
2207
void tcg_temp_free_internal(TCGTemp *ts)
2208
0
{
2209
0
    TCGContext *s = tcg_ctx;
2210
2211
0
    if (ts == NULL) {
2212
0
        return;
2213
0
    }
2214
2215
0
    switch (ts->kind) {
2216
0
    case TEMP_CONST:
2217
0
    case TEMP_TB:
2218
        /* Silently ignore free. */
2219
0
        break;
2220
0
    case TEMP_EBB:
2221
0
        tcg_debug_assert(ts->temp_allocated != 0);
2222
0
        ts->temp_allocated = 0;
2223
0
        set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
2224
0
        break;
2225
0
    default:
2226
        /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
2227
0
        g_assert_not_reached();
2228
0
    }
2229
0
}
2230
2231
void tcg_temp_free_i32(TCGv_i32 arg)
2232
0
{
2233
0
    tcg_temp_free_internal(tcgv_i32_temp(arg));
2234
0
}
2235
2236
void tcg_temp_free_i64(TCGv_i64 arg)
2237
0
{
2238
0
    tcg_temp_free_internal(tcgv_i64_temp(arg));
2239
0
}
2240
2241
void tcg_temp_free_i128(TCGv_i128 arg)
2242
0
{
2243
0
    tcg_temp_free_internal(tcgv_i128_temp(arg));
2244
0
}
2245
2246
void tcg_temp_free_ptr(TCGv_ptr arg)
2247
0
{
2248
0
    tcg_temp_free_internal(tcgv_ptr_temp(arg));
2249
0
}
2250
2251
void tcg_temp_free_vec(TCGv_vec arg)
2252
0
{
2253
0
    tcg_temp_free_internal(tcgv_vec_temp(arg));
2254
0
}
2255
2256
TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
2257
0
{
2258
0
    TCGContext *s = tcg_ctx;
2259
0
    GHashTable *h = s->const_table[type];
2260
0
    TCGTemp *ts;
2261
2262
0
    if (h == NULL) {
2263
0
        h = g_hash_table_new(g_int64_hash, g_int64_equal);
2264
0
        s->const_table[type] = h;
2265
0
    }
2266
2267
0
    if (type == TCG_TYPE_I32) {
2268
0
        val = (int32_t)val;
2269
0
    }
2270
2271
0
    ts = g_hash_table_lookup(h, &val);
2272
0
    if (ts == NULL) {
2273
0
        ts = tcg_temp_alloc(s);
2274
0
        ts->base_type = type;
2275
0
        ts->type = type;
2276
0
        ts->kind = TEMP_CONST;
2277
0
        ts->temp_allocated = 1;
2278
0
        ts->val = val;
2279
0
        g_hash_table_insert(h, &ts->val, ts);
2280
0
    }
2281
2282
0
    return ts;
2283
0
}
2284
2285
TCGv_i32 tcg_constant_i32(int32_t val)
2286
0
{
2287
0
    return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
2288
0
}
2289
2290
TCGv_i64 tcg_constant_i64(int64_t val)
2291
0
{
2292
0
    return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
2293
0
}
2294
2295
TCGv_vaddr tcg_constant_vaddr(uintptr_t val)
2296
0
{
2297
0
    return temp_tcgv_vaddr(tcg_constant_internal(TCG_TYPE_PTR, val));
2298
0
}
2299
2300
TCGv_ptr tcg_constant_ptr_int(intptr_t val)
2301
0
{
2302
0
    return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
2303
0
}
2304
2305
TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
2306
0
{
2307
0
    val = dup_const(vece, val);
2308
0
    return temp_tcgv_vec(tcg_constant_internal(type, val));
2309
0
}
2310
2311
TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
2312
0
{
2313
0
    TCGTemp *t = tcgv_vec_temp(match);
2314
2315
0
    tcg_debug_assert(t->temp_allocated != 0);
2316
0
    return tcg_constant_vec(t->base_type, vece, val);
2317
0
}
2318
2319
#ifdef CONFIG_DEBUG_TCG
2320
size_t temp_idx(TCGTemp *ts)
2321
{
2322
    ptrdiff_t n = ts - tcg_ctx->temps;
2323
    assert(n >= 0 && n < tcg_ctx->nb_temps);
2324
    return n;
2325
}
2326
2327
TCGTemp *tcgv_i32_temp(TCGv_i32 v)
2328
{
2329
    uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
2330
2331
    assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
2332
    assert(o % sizeof(TCGTemp) == 0);
2333
2334
    return (void *)tcg_ctx + (uintptr_t)v;
2335
}
2336
#endif /* CONFIG_DEBUG_TCG */
2337
2338
/*
2339
 * Return true if OP may appear in the opcode stream with TYPE.
2340
 * Test the runtime variable that controls each opcode.
2341
 */
2342
bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags)
2343
0
{
2344
0
    bool has_type;
2345
2346
0
    switch (type) {
2347
0
    case TCG_TYPE_I32:
2348
0
    case TCG_TYPE_I64:
2349
0
        has_type = true;
2350
0
        break;
2351
0
    case TCG_TYPE_V64:
2352
0
        has_type = TCG_TARGET_HAS_v64;
2353
0
        break;
2354
0
    case TCG_TYPE_V128:
2355
0
        has_type = TCG_TARGET_HAS_v128;
2356
0
        break;
2357
0
    case TCG_TYPE_V256:
2358
0
        has_type = TCG_TARGET_HAS_v256;
2359
0
        break;
2360
0
    default:
2361
0
        has_type = false;
2362
0
        break;
2363
0
    }
2364
2365
0
    switch (op) {
2366
0
    case INDEX_op_discard:
2367
0
    case INDEX_op_set_label:
2368
0
    case INDEX_op_call:
2369
0
    case INDEX_op_br:
2370
0
    case INDEX_op_mb:
2371
0
    case INDEX_op_insn_start:
2372
0
    case INDEX_op_exit_tb:
2373
0
    case INDEX_op_goto_tb:
2374
0
    case INDEX_op_goto_ptr:
2375
0
        return true;
2376
2377
0
    case INDEX_op_qemu_ld:
2378
0
    case INDEX_op_qemu_st:
2379
0
        tcg_debug_assert(type <= TCG_TYPE_REG);
2380
0
        return true;
2381
2382
0
    case INDEX_op_qemu_ld2:
2383
0
    case INDEX_op_qemu_st2:
2384
0
        tcg_debug_assert(type == TCG_TYPE_I128);
2385
0
        goto do_lookup;
2386
2387
0
    case INDEX_op_add:
2388
0
    case INDEX_op_and:
2389
0
    case INDEX_op_brcond:
2390
0
    case INDEX_op_deposit:
2391
0
    case INDEX_op_extract:
2392
0
    case INDEX_op_ld8u:
2393
0
    case INDEX_op_ld8s:
2394
0
    case INDEX_op_ld16u:
2395
0
    case INDEX_op_ld16s:
2396
0
    case INDEX_op_ld:
2397
0
    case INDEX_op_mov:
2398
0
    case INDEX_op_movcond:
2399
0
    case INDEX_op_negsetcond:
2400
0
    case INDEX_op_or:
2401
0
    case INDEX_op_setcond:
2402
0
    case INDEX_op_sextract:
2403
0
    case INDEX_op_st8:
2404
0
    case INDEX_op_st16:
2405
0
    case INDEX_op_st:
2406
0
    case INDEX_op_xor:
2407
0
        return has_type;
2408
2409
0
    case INDEX_op_ld32u:
2410
0
    case INDEX_op_ld32s:
2411
0
    case INDEX_op_st32:
2412
0
    case INDEX_op_ext_i32_i64:
2413
0
    case INDEX_op_extu_i32_i64:
2414
0
    case INDEX_op_extrl_i64_i32:
2415
0
    case INDEX_op_extrh_i64_i32:
2416
0
        return true;
2417
2418
0
    case INDEX_op_mov_vec:
2419
0
    case INDEX_op_dup_vec:
2420
0
    case INDEX_op_dupm_vec:
2421
0
    case INDEX_op_ld_vec:
2422
0
    case INDEX_op_st_vec:
2423
0
    case INDEX_op_add_vec:
2424
0
    case INDEX_op_sub_vec:
2425
0
    case INDEX_op_and_vec:
2426
0
    case INDEX_op_or_vec:
2427
0
    case INDEX_op_xor_vec:
2428
0
    case INDEX_op_cmp_vec:
2429
0
        return has_type;
2430
0
    case INDEX_op_not_vec:
2431
0
        return has_type && TCG_TARGET_HAS_not_vec;
2432
0
    case INDEX_op_neg_vec:
2433
0
        return has_type && TCG_TARGET_HAS_neg_vec;
2434
0
    case INDEX_op_abs_vec:
2435
0
        return has_type && TCG_TARGET_HAS_abs_vec;
2436
0
    case INDEX_op_andc_vec:
2437
0
        return has_type && TCG_TARGET_HAS_andc_vec;
2438
0
    case INDEX_op_orc_vec:
2439
0
        return has_type && TCG_TARGET_HAS_orc_vec;
2440
0
    case INDEX_op_nand_vec:
2441
0
        return has_type && TCG_TARGET_HAS_nand_vec;
2442
0
    case INDEX_op_nor_vec:
2443
0
        return has_type && TCG_TARGET_HAS_nor_vec;
2444
0
    case INDEX_op_eqv_vec:
2445
0
        return has_type && TCG_TARGET_HAS_eqv_vec;
2446
0
    case INDEX_op_mul_vec:
2447
0
        return has_type && TCG_TARGET_HAS_mul_vec;
2448
0
    case INDEX_op_shli_vec:
2449
0
    case INDEX_op_shri_vec:
2450
0
    case INDEX_op_sari_vec:
2451
0
        return has_type && TCG_TARGET_HAS_shi_vec;
2452
0
    case INDEX_op_shls_vec:
2453
0
    case INDEX_op_shrs_vec:
2454
0
    case INDEX_op_sars_vec:
2455
0
        return has_type && TCG_TARGET_HAS_shs_vec;
2456
0
    case INDEX_op_shlv_vec:
2457
0
    case INDEX_op_shrv_vec:
2458
0
    case INDEX_op_sarv_vec:
2459
0
        return has_type && TCG_TARGET_HAS_shv_vec;
2460
0
    case INDEX_op_rotli_vec:
2461
0
        return has_type && TCG_TARGET_HAS_roti_vec;
2462
0
    case INDEX_op_rotls_vec:
2463
0
        return has_type && TCG_TARGET_HAS_rots_vec;
2464
0
    case INDEX_op_rotlv_vec:
2465
0
    case INDEX_op_rotrv_vec:
2466
0
        return has_type && TCG_TARGET_HAS_rotv_vec;
2467
0
    case INDEX_op_ssadd_vec:
2468
0
    case INDEX_op_usadd_vec:
2469
0
    case INDEX_op_sssub_vec:
2470
0
    case INDEX_op_ussub_vec:
2471
0
        return has_type && TCG_TARGET_HAS_sat_vec;
2472
0
    case INDEX_op_smin_vec:
2473
0
    case INDEX_op_umin_vec:
2474
0
    case INDEX_op_smax_vec:
2475
0
    case INDEX_op_umax_vec:
2476
0
        return has_type && TCG_TARGET_HAS_minmax_vec;
2477
0
    case INDEX_op_bitsel_vec:
2478
0
        return has_type && TCG_TARGET_HAS_bitsel_vec;
2479
0
    case INDEX_op_cmpsel_vec:
2480
0
        return has_type && TCG_TARGET_HAS_cmpsel_vec;
2481
2482
0
    default:
2483
0
        if (op < INDEX_op_last_generic) {
2484
0
            const TCGOutOp *outop;
2485
0
            TCGConstraintSetIndex con_set;
2486
2487
0
            if (!has_type) {
2488
0
                return false;
2489
0
            }
2490
2491
0
    do_lookup:
2492
0
            outop = all_outop[op];
2493
0
            tcg_debug_assert(outop != NULL);
2494
2495
0
            con_set = outop->static_constraint;
2496
0
            if (con_set == C_Dynamic) {
2497
0
                con_set = outop->dynamic_constraint(type, flags);
2498
0
            }
2499
0
            if (con_set >= 0) {
2500
0
                return true;
2501
0
            }
2502
0
            tcg_debug_assert(con_set == C_NotImplemented);
2503
0
            return false;
2504
0
        }
2505
0
        tcg_debug_assert(op < NB_OPS);
2506
0
        return true;
2507
2508
0
    case INDEX_op_last_generic:
2509
0
        g_assert_not_reached();
2510
0
    }
2511
0
}
2512
2513
bool tcg_op_deposit_valid(TCGType type, unsigned ofs, unsigned len)
2514
0
{
2515
0
    unsigned width;
2516
2517
0
    tcg_debug_assert(type == TCG_TYPE_I32 || type == TCG_TYPE_I64);
2518
0
    width = (type == TCG_TYPE_I32 ? 32 : 64);
2519
2520
0
    tcg_debug_assert(ofs < width);
2521
0
    tcg_debug_assert(len > 0);
2522
0
    tcg_debug_assert(len <= width - ofs);
2523
2524
0
    return TCG_TARGET_deposit_valid(type, ofs, len);
2525
0
}
2526
2527
static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
2528
2529
static void tcg_gen_callN(void *func, TCGHelperInfo *info,
2530
                          TCGTemp *ret, TCGTemp **args)
2531
0
{
2532
0
    TCGv_i64 extend_free[MAX_CALL_IARGS];
2533
0
    int n_extend = 0;
2534
0
    TCGOp *op;
2535
0
    int i, n, pi = 0, total_args;
2536
2537
0
    if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
2538
0
        init_call_layout(info);
2539
0
        g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
2540
0
    }
2541
2542
0
    total_args = info->nr_out + info->nr_in + 2;
2543
0
    op = tcg_op_alloc(INDEX_op_call, total_args);
2544
2545
0
#ifdef CONFIG_PLUGIN
2546
    /* Flag helpers that may affect guest state */
2547
0
    if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
2548
0
        tcg_ctx->plugin_insn->calls_helpers = true;
2549
0
    }
2550
0
#endif
2551
2552
0
    TCGOP_CALLO(op) = n = info->nr_out;
2553
0
    switch (n) {
2554
0
    case 0:
2555
0
        tcg_debug_assert(ret == NULL);
2556
0
        break;
2557
0
    case 1:
2558
0
        tcg_debug_assert(ret != NULL);
2559
0
        op->args[pi++] = temp_arg(ret);
2560
0
        break;
2561
0
    case 2:
2562
0
    case 4:
2563
0
        tcg_debug_assert(ret != NULL);
2564
0
        tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
2565
0
        tcg_debug_assert(ret->temp_subindex == 0);
2566
0
        for (i = 0; i < n; ++i) {
2567
0
            op->args[pi++] = temp_arg(ret + i);
2568
0
        }
2569
0
        break;
2570
0
    default:
2571
0
        g_assert_not_reached();
2572
0
    }
2573
2574
0
    TCGOP_CALLI(op) = n = info->nr_in;
2575
0
    for (i = 0; i < n; i++) {
2576
0
        const TCGCallArgumentLoc *loc = &info->in[i];
2577
0
        TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
2578
2579
0
        switch (loc->kind) {
2580
0
        case TCG_CALL_ARG_NORMAL:
2581
0
        case TCG_CALL_ARG_BY_REF:
2582
0
        case TCG_CALL_ARG_BY_REF_N:
2583
0
            op->args[pi++] = temp_arg(ts);
2584
0
            break;
2585
2586
0
        case TCG_CALL_ARG_EXTEND_U:
2587
0
        case TCG_CALL_ARG_EXTEND_S:
2588
0
            {
2589
0
                TCGv_i64 temp = tcg_temp_ebb_new_i64();
2590
0
                TCGv_i32 orig = temp_tcgv_i32(ts);
2591
2592
0
                if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
2593
0
                    tcg_gen_ext_i32_i64(temp, orig);
2594
0
                } else {
2595
0
                    tcg_gen_extu_i32_i64(temp, orig);
2596
0
                }
2597
0
                op->args[pi++] = tcgv_i64_arg(temp);
2598
0
                extend_free[n_extend++] = temp;
2599
0
            }
2600
0
            break;
2601
2602
0
        default:
2603
0
            g_assert_not_reached();
2604
0
        }
2605
0
    }
2606
0
    op->args[pi++] = (uintptr_t)func;
2607
0
    op->args[pi++] = (uintptr_t)info;
2608
0
    tcg_debug_assert(pi == total_args);
2609
2610
0
    if (tcg_ctx->emit_before_op) {
2611
0
        QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
2612
0
    } else {
2613
0
        QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2614
0
    }
2615
2616
0
    tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
2617
0
    for (i = 0; i < n_extend; ++i) {
2618
0
        tcg_temp_free_i64(extend_free[i]);
2619
0
    }
2620
0
}
2621
2622
void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret)
2623
0
{
2624
0
    tcg_gen_callN(func, info, ret, NULL);
2625
0
}
2626
2627
void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
2628
0
{
2629
0
    tcg_gen_callN(func, info, ret, &t1);
2630
0
}
2631
2632
void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret,
2633
                   TCGTemp *t1, TCGTemp *t2)
2634
0
{
2635
0
    TCGTemp *args[2] = { t1, t2 };
2636
0
    tcg_gen_callN(func, info, ret, args);
2637
0
}
2638
2639
void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret,
2640
                   TCGTemp *t1, TCGTemp *t2, TCGTemp *t3)
2641
0
{
2642
0
    TCGTemp *args[3] = { t1, t2, t3 };
2643
0
    tcg_gen_callN(func, info, ret, args);
2644
0
}
2645
2646
void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret,
2647
                   TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
2648
0
{
2649
0
    TCGTemp *args[4] = { t1, t2, t3, t4 };
2650
0
    tcg_gen_callN(func, info, ret, args);
2651
0
}
2652
2653
void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2654
                   TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
2655
0
{
2656
0
    TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
2657
0
    tcg_gen_callN(func, info, ret, args);
2658
0
}
2659
2660
void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret,
2661
                   TCGTemp *t1, TCGTemp *t2, TCGTemp *t3,
2662
                   TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
2663
0
{
2664
0
    TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
2665
0
    tcg_gen_callN(func, info, ret, args);
2666
0
}
2667
2668
void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2669
                   TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
2670
                   TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
2671
0
{
2672
0
    TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
2673
0
    tcg_gen_callN(func, info, ret, args);
2674
0
}
2675
2676
static void tcg_reg_alloc_start(TCGContext *s)
2677
0
{
2678
0
    int i, n;
2679
2680
0
    for (i = 0, n = s->nb_temps; i < n; i++) {
2681
0
        TCGTemp *ts = &s->temps[i];
2682
0
        TCGTempVal val = TEMP_VAL_MEM;
2683
2684
0
        switch (ts->kind) {
2685
0
        case TEMP_CONST:
2686
0
            val = TEMP_VAL_CONST;
2687
0
            break;
2688
0
        case TEMP_FIXED:
2689
0
            val = TEMP_VAL_REG;
2690
0
            break;
2691
0
        case TEMP_GLOBAL:
2692
0
            break;
2693
0
        case TEMP_EBB:
2694
0
            val = TEMP_VAL_DEAD;
2695
            /* fall through */
2696
0
        case TEMP_TB:
2697
0
            ts->mem_allocated = 0;
2698
0
            break;
2699
0
        default:
2700
0
            g_assert_not_reached();
2701
0
        }
2702
0
        ts->val_type = val;
2703
0
    }
2704
2705
0
    memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
2706
0
}
2707
2708
static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2709
                                 TCGTemp *ts)
2710
0
{
2711
0
    int idx = temp_idx(ts);
2712
2713
0
    switch (ts->kind) {
2714
0
    case TEMP_FIXED:
2715
0
    case TEMP_GLOBAL:
2716
0
        pstrcpy(buf, buf_size, ts->name);
2717
0
        break;
2718
0
    case TEMP_TB:
2719
0
        snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
2720
0
        break;
2721
0
    case TEMP_EBB:
2722
0
        snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
2723
0
        break;
2724
0
    case TEMP_CONST:
2725
0
        switch (ts->type) {
2726
0
        case TCG_TYPE_I32:
2727
0
            snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2728
0
            break;
2729
0
        case TCG_TYPE_I64:
2730
0
            snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2731
0
            break;
2732
0
        case TCG_TYPE_V64:
2733
0
        case TCG_TYPE_V128:
2734
0
        case TCG_TYPE_V256:
2735
0
            snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2736
0
                     64 << (ts->type - TCG_TYPE_V64), ts->val);
2737
0
            break;
2738
0
        default:
2739
0
            g_assert_not_reached();
2740
0
        }
2741
0
        break;
2742
0
    }
2743
0
    return buf;
2744
0
}
2745
2746
static char *tcg_get_arg_str(TCGContext *s, char *buf,
2747
                             int buf_size, TCGArg arg)
2748
0
{
2749
0
    return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
2750
0
}
2751
2752
static const char * const cond_name[] =
2753
{
2754
    [TCG_COND_NEVER] = "never",
2755
    [TCG_COND_ALWAYS] = "always",
2756
    [TCG_COND_EQ] = "eq",
2757
    [TCG_COND_NE] = "ne",
2758
    [TCG_COND_LT] = "lt",
2759
    [TCG_COND_GE] = "ge",
2760
    [TCG_COND_LE] = "le",
2761
    [TCG_COND_GT] = "gt",
2762
    [TCG_COND_LTU] = "ltu",
2763
    [TCG_COND_GEU] = "geu",
2764
    [TCG_COND_LEU] = "leu",
2765
    [TCG_COND_GTU] = "gtu",
2766
    [TCG_COND_TSTEQ] = "tsteq",
2767
    [TCG_COND_TSTNE] = "tstne",
2768
};
2769
2770
static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
2771
{
2772
    [MO_UB]   = "ub",
2773
    [MO_SB]   = "sb",
2774
    [MO_LEUW] = "leuw",
2775
    [MO_LESW] = "lesw",
2776
    [MO_LEUL] = "leul",
2777
    [MO_LESL] = "lesl",
2778
    [MO_LEUQ] = "leq",
2779
    [MO_BEUW] = "beuw",
2780
    [MO_BESW] = "besw",
2781
    [MO_BEUL] = "beul",
2782
    [MO_BESL] = "besl",
2783
    [MO_BEUQ] = "beq",
2784
    [MO_128 + MO_BE] = "beo",
2785
    [MO_128 + MO_LE] = "leo",
2786
};
2787
2788
static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
2789
    [MO_UNALN >> MO_ASHIFT]    = "un+",
2790
    [MO_ALIGN >> MO_ASHIFT]    = "al+",
2791
    [MO_ALIGN_2 >> MO_ASHIFT]  = "al2+",
2792
    [MO_ALIGN_4 >> MO_ASHIFT]  = "al4+",
2793
    [MO_ALIGN_8 >> MO_ASHIFT]  = "al8+",
2794
    [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2795
    [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2796
    [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2797
};
2798
2799
static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
2800
    [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
2801
    [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
2802
    [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
2803
    [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
2804
    [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
2805
    [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
2806
};
2807
2808
static const char bswap_flag_name[][6] = {
2809
    [TCG_BSWAP_IZ] = "iz",
2810
    [TCG_BSWAP_OZ] = "oz",
2811
    [TCG_BSWAP_OS] = "os",
2812
    [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2813
    [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2814
};
2815
2816
#ifdef CONFIG_PLUGIN
2817
static const char * const plugin_from_name[] = {
2818
    "from-tb",
2819
    "from-insn",
2820
    "after-insn",
2821
    "after-tb",
2822
};
2823
#endif
2824
2825
static inline bool tcg_regset_single(TCGRegSet d)
2826
0
{
2827
0
    return (d & (d - 1)) == 0;
2828
0
}
2829
2830
static inline TCGReg tcg_regset_first(TCGRegSet d)
2831
0
{
2832
0
    if (TCG_TARGET_NB_REGS <= 32) {
2833
0
        return ctz32(d);
2834
0
    } else {
2835
0
        return ctz64(d);
2836
0
    }
2837
0
}
2838
2839
/* Return only the number of characters output -- no error return. */
2840
#define ne_fprintf(...) \
2841
0
    ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2842
2843
void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
2844
0
{
2845
0
    char buf[128];
2846
0
    TCGOp *op;
2847
2848
0
    QTAILQ_FOREACH(op, &s->ops, link) {
2849
0
        int i, k, nb_oargs, nb_iargs, nb_cargs;
2850
0
        const TCGOpDef *def;
2851
0
        TCGOpcode c;
2852
0
        int col = 0;
2853
2854
0
        c = op->opc;
2855
0
        def = &tcg_op_defs[c];
2856
2857
0
        if (c == INDEX_op_insn_start) {
2858
0
            nb_oargs = 0;
2859
0
            col += ne_fprintf(f, "\n ----");
2860
2861
0
            for (i = 0, k = INSN_START_WORDS; i < k; ++i) {
2862
0
                col += ne_fprintf(f, " %016" PRIx64,
2863
0
                                  tcg_get_insn_start_param(op, i));
2864
0
            }
2865
0
        } else if (c == INDEX_op_call) {
2866
0
            const TCGHelperInfo *info = tcg_call_info(op);
2867
0
            void *func = tcg_call_func(op);
2868
2869
            /* variable number of arguments */
2870
0
            nb_oargs = TCGOP_CALLO(op);
2871
0
            nb_iargs = TCGOP_CALLI(op);
2872
0
            nb_cargs = def->nb_cargs;
2873
2874
0
            col += ne_fprintf(f, " %s ", def->name);
2875
2876
            /*
2877
             * Print the function name from TCGHelperInfo, if available.
2878
             * Note that plugins have a template function for the info,
2879
             * but the actual function pointer comes from the plugin.
2880
             */
2881
0
            if (func == info->func) {
2882
0
                col += ne_fprintf(f, "%s", info->name);
2883
0
            } else {
2884
0
                col += ne_fprintf(f, "plugin(%p)", func);
2885
0
            }
2886
2887
0
            col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
2888
0
            for (i = 0; i < nb_oargs; i++) {
2889
0
                col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2890
0
                                                            op->args[i]));
2891
0
            }
2892
0
            for (i = 0; i < nb_iargs; i++) {
2893
0
                TCGArg arg = op->args[nb_oargs + i];
2894
0
                const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2895
0
                col += ne_fprintf(f, ",%s", t);
2896
0
            }
2897
0
        } else {
2898
0
            if (def->flags & TCG_OPF_INT) {
2899
0
                col += ne_fprintf(f, " %s_i%d ",
2900
0
                                  def->name,
2901
0
                                  tcg_type_bits(TCGOP_TYPE(op)));
2902
0
            } else if (def->flags & TCG_OPF_VECTOR) {
2903
0
                col += ne_fprintf(f, "%s v%d,e%d,",
2904
0
                                  def->name,
2905
0
                                  tcg_type_bits(TCGOP_TYPE(op)),
2906
0
                                  8 << TCGOP_VECE(op));
2907
0
            } else {
2908
0
                col += ne_fprintf(f, " %s ", def->name);
2909
0
            }
2910
2911
0
            nb_oargs = def->nb_oargs;
2912
0
            nb_iargs = def->nb_iargs;
2913
0
            nb_cargs = def->nb_cargs;
2914
2915
0
            k = 0;
2916
0
            for (i = 0; i < nb_oargs; i++) {
2917
0
                const char *sep =  k ? "," : "";
2918
0
                col += ne_fprintf(f, "%s%s", sep,
2919
0
                                  tcg_get_arg_str(s, buf, sizeof(buf),
2920
0
                                                  op->args[k++]));
2921
0
            }
2922
0
            for (i = 0; i < nb_iargs; i++) {
2923
0
                const char *sep =  k ? "," : "";
2924
0
                col += ne_fprintf(f, "%s%s", sep,
2925
0
                                  tcg_get_arg_str(s, buf, sizeof(buf),
2926
0
                                                  op->args[k++]));
2927
0
            }
2928
0
            switch (c) {
2929
0
            case INDEX_op_brcond:
2930
0
            case INDEX_op_setcond:
2931
0
            case INDEX_op_negsetcond:
2932
0
            case INDEX_op_movcond:
2933
0
            case INDEX_op_cmp_vec:
2934
0
            case INDEX_op_cmpsel_vec:
2935
0
                if (op->args[k] < ARRAY_SIZE(cond_name)
2936
0
                    && cond_name[op->args[k]]) {
2937
0
                    col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
2938
0
                } else {
2939
0
                    col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
2940
0
                }
2941
0
                i = 1;
2942
0
                break;
2943
0
            case INDEX_op_qemu_ld:
2944
0
            case INDEX_op_qemu_st:
2945
0
            case INDEX_op_qemu_ld2:
2946
0
            case INDEX_op_qemu_st2:
2947
0
                {
2948
0
                    const char *s_al, *s_tlb, *s_op, *s_at;
2949
0
                    MemOpIdx oi = op->args[k++];
2950
0
                    MemOp mop = get_memop(oi);
2951
0
                    unsigned ix = get_mmuidx(oi);
2952
2953
0
                    s_tlb = mop & MO_ALIGN_TLB_ONLY ? "tlb+" : "";
2954
0
                    s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
2955
0
                    s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
2956
0
                    s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
2957
0
                    mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE |
2958
0
                             MO_ATOM_MASK | MO_ALIGN_TLB_ONLY);
2959
2960
                    /* If all fields are accounted for, print symbolically. */
2961
0
                    if (!mop && s_al && s_op && s_at) {
2962
0
                        col += ne_fprintf(f, ",%s%s%s%s,%u",
2963
0
                                          s_at, s_al, s_tlb, s_op, ix);
2964
0
                    } else {
2965
0
                        mop = get_memop(oi);
2966
0
                        col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
2967
0
                    }
2968
0
                    i = 1;
2969
0
                }
2970
0
                break;
2971
0
            case INDEX_op_bswap16:
2972
0
            case INDEX_op_bswap32:
2973
0
            case INDEX_op_bswap64:
2974
0
            case INDEX_op_revbit32:
2975
0
                {
2976
0
                    TCGArg flags = op->args[k];
2977
0
                    const char *name = NULL;
2978
2979
0
                    if (flags < ARRAY_SIZE(bswap_flag_name)) {
2980
0
                        name = bswap_flag_name[flags];
2981
0
                    }
2982
0
                    if (name && name[0]) {
2983
0
                        col += ne_fprintf(f, ",%s", name);
2984
0
                    } else {
2985
0
                        col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
2986
0
                    }
2987
0
                    i = k = 1;
2988
0
                }
2989
0
                break;
2990
0
#ifdef CONFIG_PLUGIN
2991
0
            case INDEX_op_plugin_cb:
2992
0
                {
2993
0
                    TCGArg from = op->args[k++];
2994
0
                    const char *name = NULL;
2995
2996
0
                    if (from < ARRAY_SIZE(plugin_from_name)) {
2997
0
                        name = plugin_from_name[from];
2998
0
                    }
2999
0
                    if (name) {
3000
0
                        col += ne_fprintf(f, "%s", name);
3001
0
                    } else {
3002
0
                        col += ne_fprintf(f, "$0x%" TCG_PRIlx, from);
3003
0
                    }
3004
0
                    i = 1;
3005
0
                }
3006
0
                break;
3007
0
#endif
3008
0
            default:
3009
0
                i = 0;
3010
0
                break;
3011
0
            }
3012
0
            switch (c) {
3013
0
            case INDEX_op_set_label:
3014
0
            case INDEX_op_br:
3015
0
            case INDEX_op_brcond:
3016
0
                col += ne_fprintf(f, "%s$L%d", k ? "," : "",
3017
0
                                  arg_label(op->args[k])->id);
3018
0
                i++, k++;
3019
0
                break;
3020
0
            case INDEX_op_mb:
3021
0
                {
3022
0
                    TCGBar membar = op->args[k];
3023
0
                    const char *b_op, *m_op;
3024
3025
0
                    switch (membar & TCG_BAR_SC) {
3026
0
                    case 0:
3027
0
                        b_op = "none";
3028
0
                        break;
3029
0
                    case TCG_BAR_LDAQ:
3030
0
                        b_op = "acq";
3031
0
                        break;
3032
0
                    case TCG_BAR_STRL:
3033
0
                        b_op = "rel";
3034
0
                        break;
3035
0
                    case TCG_BAR_SC:
3036
0
                        b_op = "seq";
3037
0
                        break;
3038
0
                    default:
3039
0
                        g_assert_not_reached();
3040
0
                    }
3041
3042
0
                    switch (membar & TCG_MO_ALL) {
3043
0
                    case 0:
3044
0
                        m_op = "none";
3045
0
                        break;
3046
0
                    case TCG_MO_LD_LD:
3047
0
                        m_op = "rr";
3048
0
                        break;
3049
0
                    case TCG_MO_LD_ST:
3050
0
                        m_op = "rw";
3051
0
                        break;
3052
0
                    case TCG_MO_ST_LD:
3053
0
                        m_op = "wr";
3054
0
                        break;
3055
0
                    case TCG_MO_ST_ST:
3056
0
                        m_op = "ww";
3057
0
                        break;
3058
0
                    case TCG_MO_LD_LD | TCG_MO_LD_ST:
3059
0
                        m_op = "rr+rw";
3060
0
                        break;
3061
0
                    case TCG_MO_LD_LD | TCG_MO_ST_LD:
3062
0
                        m_op = "rr+wr";
3063
0
                        break;
3064
0
                    case TCG_MO_LD_LD | TCG_MO_ST_ST:
3065
0
                        m_op = "rr+ww";
3066
0
                        break;
3067
0
                    case TCG_MO_LD_ST | TCG_MO_ST_LD:
3068
0
                        m_op = "rw+wr";
3069
0
                        break;
3070
0
                    case TCG_MO_LD_ST | TCG_MO_ST_ST:
3071
0
                        m_op = "rw+ww";
3072
0
                        break;
3073
0
                    case TCG_MO_ST_LD | TCG_MO_ST_ST:
3074
0
                        m_op = "wr+ww";
3075
0
                        break;
3076
0
                    case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
3077
0
                        m_op = "rr+rw+wr";
3078
0
                        break;
3079
0
                    case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
3080
0
                        m_op = "rr+rw+ww";
3081
0
                        break;
3082
0
                    case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
3083
0
                        m_op = "rr+wr+ww";
3084
0
                        break;
3085
0
                    case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
3086
0
                        m_op = "rw+wr+ww";
3087
0
                        break;
3088
0
                    case TCG_MO_ALL:
3089
0
                        m_op = "all";
3090
0
                        break;
3091
0
                    default:
3092
0
                        g_assert_not_reached();
3093
0
                    }
3094
3095
0
                    col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
3096
0
                    i++, k++;
3097
0
                }
3098
0
                break;
3099
0
            default:
3100
0
                break;
3101
0
            }
3102
0
            for (; i < nb_cargs; i++, k++) {
3103
0
                col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
3104
0
                                  op->args[k]);
3105
0
            }
3106
0
        }
3107
3108
0
        if (have_prefs || op->life) {
3109
0
            for (; col < 40; ++col) {
3110
0
                putc(' ', f);
3111
0
            }
3112
0
        }
3113
3114
0
        if (op->life) {
3115
0
            unsigned life = op->life;
3116
3117
0
            if (life & (SYNC_ARG * 3)) {
3118
0
                ne_fprintf(f, "  sync:");
3119
0
                for (i = 0; i < 2; ++i) {
3120
0
                    if (life & (SYNC_ARG << i)) {
3121
0
                        ne_fprintf(f, " %d", i);
3122
0
                    }
3123
0
                }
3124
0
            }
3125
0
            life /= DEAD_ARG;
3126
0
            if (life) {
3127
0
                ne_fprintf(f, "  dead:");
3128
0
                for (i = 0; life; ++i, life >>= 1) {
3129
0
                    if (life & 1) {
3130
0
                        ne_fprintf(f, " %d", i);
3131
0
                    }
3132
0
                }
3133
0
            }
3134
0
        }
3135
3136
0
        if (have_prefs) {
3137
0
            for (i = 0; i < nb_oargs; ++i) {
3138
0
                TCGRegSet set = output_pref(op, i);
3139
3140
0
                if (i == 0) {
3141
0
                    ne_fprintf(f, "  pref=");
3142
0
                } else {
3143
0
                    ne_fprintf(f, ",");
3144
0
                }
3145
0
                if (set == 0) {
3146
0
                    ne_fprintf(f, "none");
3147
0
                } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
3148
0
                    ne_fprintf(f, "all");
3149
#ifdef CONFIG_DEBUG_TCG
3150
                } else if (tcg_regset_single(set)) {
3151
                    TCGReg reg = tcg_regset_first(set);
3152
                    ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
3153
#endif
3154
0
                } else if (TCG_TARGET_NB_REGS <= 32) {
3155
0
                    ne_fprintf(f, "0x%x", (uint32_t)set);
3156
0
                } else {
3157
0
                    ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
3158
0
                }
3159
0
            }
3160
0
        }
3161
3162
0
        putc('\n', f);
3163
0
    }
3164
0
}
3165
3166
/* we give more priority to constraints with less registers */
3167
static int get_constraint_priority(const TCGArgConstraint *arg_ct, int k)
3168
0
{
3169
0
    int n;
3170
3171
0
    arg_ct += k;
3172
0
    n = ctpop64(arg_ct->regs);
3173
3174
    /*
3175
     * Sort constraints of a single register first, which includes output
3176
     * aliases (which must exactly match the input already allocated).
3177
     */
3178
0
    if (n == 1 || arg_ct->oalias) {
3179
0
        return INT_MAX;
3180
0
    }
3181
3182
    /*
3183
     * Sort register pairs next, first then second immediately after.
3184
     * Arbitrarily sort multiple pairs by the index of the first reg;
3185
     * there shouldn't be many pairs.
3186
     */
3187
0
    switch (arg_ct->pair) {
3188
0
    case 1:
3189
0
    case 3:
3190
0
        return (k + 1) * 2;
3191
0
    case 2:
3192
0
        return (arg_ct->pair_index + 1) * 2 - 1;
3193
0
    }
3194
3195
    /* Finally, sort by decreasing register count. */
3196
0
    assert(n > 1);
3197
0
    return -n;
3198
0
}
3199
3200
/* sort from highest priority to lowest */
3201
static void sort_constraints(TCGArgConstraint *a, int start, int n)
3202
0
{
3203
0
    int i, j;
3204
3205
0
    for (i = 0; i < n; i++) {
3206
0
        a[start + i].sort_index = start + i;
3207
0
    }
3208
0
    if (n <= 1) {
3209
0
        return;
3210
0
    }
3211
0
    for (i = 0; i < n - 1; i++) {
3212
0
        for (j = i + 1; j < n; j++) {
3213
0
            int p1 = get_constraint_priority(a, a[start + i].sort_index);
3214
0
            int p2 = get_constraint_priority(a, a[start + j].sort_index);
3215
0
            if (p1 < p2) {
3216
0
                int tmp = a[start + i].sort_index;
3217
0
                a[start + i].sort_index = a[start + j].sort_index;
3218
0
                a[start + j].sort_index = tmp;
3219
0
            }
3220
0
        }
3221
0
    }
3222
0
}
3223
3224
static const TCGArgConstraint empty_cts[TCG_MAX_OP_ARGS];
3225
static TCGArgConstraint all_cts[ARRAY_SIZE(constraint_sets)][TCG_MAX_OP_ARGS];
3226
3227
static void process_constraint_sets(void)
3228
0
{
3229
0
    for (size_t c = 0; c < ARRAY_SIZE(constraint_sets); ++c) {
3230
0
        const TCGConstraintSet *tdefs = &constraint_sets[c];
3231
0
        TCGArgConstraint *args_ct = all_cts[c];
3232
0
        int nb_oargs = tdefs->nb_oargs;
3233
0
        int nb_iargs = tdefs->nb_iargs;
3234
0
        int nb_args = nb_oargs + nb_iargs;
3235
0
        bool saw_alias_pair = false;
3236
3237
0
        for (int i = 0; i < nb_args; i++) {
3238
0
            const char *ct_str = tdefs->args_ct_str[i];
3239
0
            bool input_p = i >= nb_oargs;
3240
0
            int o;
3241
3242
0
            switch (*ct_str) {
3243
0
            case '0' ... '9':
3244
0
                o = *ct_str - '0';
3245
0
                tcg_debug_assert(input_p);
3246
0
                tcg_debug_assert(o < nb_oargs);
3247
0
                tcg_debug_assert(args_ct[o].regs != 0);
3248
0
                tcg_debug_assert(!args_ct[o].oalias);
3249
0
                args_ct[i] = args_ct[o];
3250
                /* The output sets oalias.  */
3251
0
                args_ct[o].oalias = 1;
3252
0
                args_ct[o].alias_index = i;
3253
                /* The input sets ialias. */
3254
0
                args_ct[i].ialias = 1;
3255
0
                args_ct[i].alias_index = o;
3256
0
                if (args_ct[i].pair) {
3257
0
                    saw_alias_pair = true;
3258
0
                }
3259
0
                tcg_debug_assert(ct_str[1] == '\0');
3260
0
                continue;
3261
3262
0
            case '&':
3263
0
                tcg_debug_assert(!input_p);
3264
0
                args_ct[i].newreg = true;
3265
0
                ct_str++;
3266
0
                break;
3267
3268
0
            case 'p': /* plus */
3269
                /* Allocate to the register after the previous. */
3270
0
                tcg_debug_assert(i > (input_p ? nb_oargs : 0));
3271
0
                o = i - 1;
3272
0
                tcg_debug_assert(!args_ct[o].pair);
3273
0
                tcg_debug_assert(!args_ct[o].ct);
3274
0
                args_ct[i] = (TCGArgConstraint){
3275
0
                    .pair = 2,
3276
0
                    .pair_index = o,
3277
0
                    .regs = args_ct[o].regs << 1,
3278
0
                    .newreg = args_ct[o].newreg,
3279
0
                };
3280
0
                args_ct[o].pair = 1;
3281
0
                args_ct[o].pair_index = i;
3282
0
                tcg_debug_assert(ct_str[1] == '\0');
3283
0
                continue;
3284
3285
0
            case 'm': /* minus */
3286
                /* Allocate to the register before the previous. */
3287
0
                tcg_debug_assert(i > (input_p ? nb_oargs : 0));
3288
0
                o = i - 1;
3289
0
                tcg_debug_assert(!args_ct[o].pair);
3290
0
                tcg_debug_assert(!args_ct[o].ct);
3291
0
                args_ct[i] = (TCGArgConstraint){
3292
0
                    .pair = 1,
3293
0
                    .pair_index = o,
3294
0
                    .regs = args_ct[o].regs >> 1,
3295
0
                    .newreg = args_ct[o].newreg,
3296
0
                };
3297
0
                args_ct[o].pair = 2;
3298
0
                args_ct[o].pair_index = i;
3299
0
                tcg_debug_assert(ct_str[1] == '\0');
3300
0
                continue;
3301
0
            }
3302
3303
0
            do {
3304
0
                switch (*ct_str) {
3305
0
                case 'i':
3306
0
                    args_ct[i].ct |= TCG_CT_CONST;
3307
0
                    break;
3308
0
#ifdef TCG_REG_ZERO
3309
0
                case 'z':
3310
0
                    args_ct[i].ct |= TCG_CT_REG_ZERO;
3311
0
                    break;
3312
0
#endif
3313
3314
                /* Include all of the target-specific constraints. */
3315
3316
0
#undef CONST
3317
0
#define CONST(CASE, MASK) \
3318
0
    case CASE: args_ct[i].ct |= MASK; break;
3319
0
#define REGS(CASE, MASK) \
3320
0
    case CASE: args_ct[i].regs |= MASK; break;
3321
3322
0
#include "tcg-target-con-str.h"
3323
3324
0
#undef REGS
3325
0
#undef CONST
3326
0
                default:
3327
0
                case '0' ... '9':
3328
0
                case '&':
3329
0
                case 'p':
3330
0
                case 'm':
3331
                    /* Typo in TCGConstraintSet constraint. */
3332
0
                    g_assert_not_reached();
3333
0
                }
3334
0
            } while (*++ct_str != '\0');
3335
0
        }
3336
3337
        /*
3338
         * Fix up output pairs that are aliased with inputs.
3339
         * When we created the alias, we copied pair from the output.
3340
         * There are three cases:
3341
         *    (1a) Pairs of inputs alias pairs of outputs.
3342
         *    (1b) One input aliases the first of a pair of outputs.
3343
         *    (2)  One input aliases the second of a pair of outputs.
3344
         *
3345
         * Case 1a is handled by making sure that the pair_index'es are
3346
         * properly updated so that they appear the same as a pair of inputs.
3347
         *
3348
         * Case 1b is handled by setting the pair_index of the input to
3349
         * itself, simply so it doesn't point to an unrelated argument.
3350
         * Since we don't encounter the "second" during the input allocation
3351
         * phase, nothing happens with the second half of the input pair.
3352
         *
3353
         * Case 2 is handled by setting the second input to pair=3, the
3354
         * first output to pair=3, and the pair_index'es to match.
3355
         */
3356
0
        if (saw_alias_pair) {
3357
0
            for (int i = nb_oargs; i < nb_args; i++) {
3358
0
                int o, o2, i2;
3359
3360
                /*
3361
                 * Since [0-9pm] must be alone in the constraint string,
3362
                 * the only way they can both be set is if the pair comes
3363
                 * from the output alias.
3364
                 */
3365
0
                if (!args_ct[i].ialias) {
3366
0
                    continue;
3367
0
                }
3368
0
                switch (args_ct[i].pair) {
3369
0
                case 0:
3370
0
                    break;
3371
0
                case 1:
3372
0
                    o = args_ct[i].alias_index;
3373
0
                    o2 = args_ct[o].pair_index;
3374
0
                    tcg_debug_assert(args_ct[o].pair == 1);
3375
0
                    tcg_debug_assert(args_ct[o2].pair == 2);
3376
0
                    if (args_ct[o2].oalias) {
3377
                        /* Case 1a */
3378
0
                        i2 = args_ct[o2].alias_index;
3379
0
                        tcg_debug_assert(args_ct[i2].pair == 2);
3380
0
                        args_ct[i2].pair_index = i;
3381
0
                        args_ct[i].pair_index = i2;
3382
0
                    } else {
3383
                        /* Case 1b */
3384
0
                        args_ct[i].pair_index = i;
3385
0
                    }
3386
0
                    break;
3387
0
                case 2:
3388
0
                    o = args_ct[i].alias_index;
3389
0
                    o2 = args_ct[o].pair_index;
3390
0
                    tcg_debug_assert(args_ct[o].pair == 2);
3391
0
                    tcg_debug_assert(args_ct[o2].pair == 1);
3392
0
                    if (args_ct[o2].oalias) {
3393
                        /* Case 1a */
3394
0
                        i2 = args_ct[o2].alias_index;
3395
0
                        tcg_debug_assert(args_ct[i2].pair == 1);
3396
0
                        args_ct[i2].pair_index = i;
3397
0
                        args_ct[i].pair_index = i2;
3398
0
                    } else {
3399
                        /* Case 2 */
3400
0
                        args_ct[i].pair = 3;
3401
0
                        args_ct[o2].pair = 3;
3402
0
                        args_ct[i].pair_index = o2;
3403
0
                        args_ct[o2].pair_index = i;
3404
0
                    }
3405
0
                    break;
3406
0
                default:
3407
0
                    g_assert_not_reached();
3408
0
                }
3409
0
            }
3410
0
        }
3411
3412
        /* sort the constraints (XXX: this is just an heuristic) */
3413
0
        sort_constraints(args_ct, 0, nb_oargs);
3414
0
        sort_constraints(args_ct, nb_oargs, nb_iargs);
3415
0
    }
3416
0
}
3417
3418
static const TCGArgConstraint *op_args_ct(TCGOpcode opc, TCGType type,
3419
                                          unsigned flags)
3420
0
{
3421
0
    const TCGOpDef *def = &tcg_op_defs[opc];
3422
0
    const TCGOutOp *outop = all_outop[opc];
3423
0
    TCGConstraintSetIndex con_set;
3424
3425
0
    if (def->flags & TCG_OPF_NOT_PRESENT) {
3426
0
        return empty_cts;
3427
0
    }
3428
3429
0
    if (outop) {
3430
0
        con_set = outop->static_constraint;
3431
0
        if (con_set == C_Dynamic) {
3432
0
            con_set = outop->dynamic_constraint(type, flags);
3433
0
        }
3434
0
    } else {
3435
0
        con_set = tcg_target_op_def(opc, type, flags);
3436
0
    }
3437
0
    tcg_debug_assert(con_set >= 0);
3438
0
    tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
3439
3440
    /* The constraint arguments must match TCGOpcode arguments. */
3441
0
    tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs);
3442
0
    tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs);
3443
3444
0
    return all_cts[con_set];
3445
0
}
3446
3447
static const TCGArgConstraint *opcode_args_ct(const TCGOp *op)
3448
0
{
3449
0
    return op_args_ct(op->opc, TCGOP_TYPE(op), TCGOP_FLAGS(op));
3450
0
}
3451
3452
bool tcg_op_imm_match(TCGOpcode opc, TCGType type, tcg_target_ulong imm)
3453
0
{
3454
0
    const TCGArgConstraint *args_ct = op_args_ct(opc, type, 0);
3455
0
    const TCGOpDef *def = &tcg_op_defs[opc];
3456
3457
0
    tcg_debug_assert(def->nb_oargs == 1);
3458
0
    tcg_debug_assert(def->nb_iargs == 2);
3459
0
    return tcg_target_const_match(imm, args_ct[2].ct, type, 0, 0);
3460
0
}
3461
3462
static void remove_label_use(TCGOp *op, int idx)
3463
0
{
3464
0
    TCGLabel *label = arg_label(op->args[idx]);
3465
0
    TCGLabelUse *use;
3466
3467
0
    QSIMPLEQ_FOREACH(use, &label->branches, next) {
3468
0
        if (use->op == op) {
3469
0
            QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3470
0
            return;
3471
0
        }
3472
0
    }
3473
0
    g_assert_not_reached();
3474
0
}
3475
3476
void tcg_op_remove(TCGContext *s, TCGOp *op)
3477
0
{
3478
0
    switch (op->opc) {
3479
0
    case INDEX_op_br:
3480
0
        remove_label_use(op, 0);
3481
0
        break;
3482
0
    case INDEX_op_brcond:
3483
0
        remove_label_use(op, 3);
3484
0
        break;
3485
0
    default:
3486
0
        break;
3487
0
    }
3488
3489
0
    QTAILQ_REMOVE(&s->ops, op, link);
3490
0
    QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
3491
0
    s->nb_ops--;
3492
0
}
3493
3494
void tcg_remove_ops_after(TCGOp *op)
3495
0
{
3496
0
    TCGContext *s = tcg_ctx;
3497
3498
0
    while (true) {
3499
0
        TCGOp *last = tcg_last_op();
3500
0
        if (last == op) {
3501
0
            return;
3502
0
        }
3503
0
        tcg_op_remove(s, last);
3504
0
    }
3505
0
}
3506
3507
static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
3508
0
{
3509
0
    TCGContext *s = tcg_ctx;
3510
0
    TCGOp *op = NULL;
3511
3512
0
    if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3513
0
        QTAILQ_FOREACH(op, &s->free_ops, link) {
3514
0
            if (nargs <= op->nargs) {
3515
0
                QTAILQ_REMOVE(&s->free_ops, op, link);
3516
0
                nargs = op->nargs;
3517
0
                goto found;
3518
0
            }
3519
0
        }
3520
0
    }
3521
3522
    /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
3523
0
    nargs = MAX(4, nargs);
3524
0
    op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
3525
3526
0
 found:
3527
0
    memset(op, 0, offsetof(TCGOp, link));
3528
0
    op->opc = opc;
3529
0
    op->nargs = nargs;
3530
3531
    /* Check for bitfield overflow. */
3532
0
    tcg_debug_assert(op->nargs == nargs);
3533
3534
0
    s->nb_ops++;
3535
0
    return op;
3536
0
}
3537
3538
TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
3539
0
{
3540
0
    TCGOp *op = tcg_op_alloc(opc, nargs);
3541
3542
0
    if (tcg_ctx->emit_before_op) {
3543
0
        QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
3544
0
    } else {
3545
0
        QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
3546
0
    }
3547
0
    return op;
3548
0
}
3549
3550
TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
3551
                            TCGOpcode opc, TCGType type, unsigned nargs)
3552
0
{
3553
0
    TCGOp *new_op = tcg_op_alloc(opc, nargs);
3554
3555
0
    TCGOP_TYPE(new_op) = type;
3556
0
    QTAILQ_INSERT_BEFORE(old_op, new_op, link);
3557
0
    return new_op;
3558
0
}
3559
3560
TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
3561
                           TCGOpcode opc, TCGType type, unsigned nargs)
3562
0
{
3563
0
    TCGOp *new_op = tcg_op_alloc(opc, nargs);
3564
3565
0
    TCGOP_TYPE(new_op) = type;
3566
0
    QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
3567
0
    return new_op;
3568
0
}
3569
3570
static void move_label_uses(TCGLabel *to, TCGLabel *from)
3571
0
{
3572
0
    TCGLabelUse *u;
3573
3574
0
    QSIMPLEQ_FOREACH(u, &from->branches, next) {
3575
0
        TCGOp *op = u->op;
3576
0
        switch (op->opc) {
3577
0
        case INDEX_op_br:
3578
0
            op->args[0] = label_arg(to);
3579
0
            break;
3580
0
        case INDEX_op_brcond:
3581
0
            op->args[3] = label_arg(to);
3582
0
            break;
3583
0
        default:
3584
0
            g_assert_not_reached();
3585
0
        }
3586
0
    }
3587
3588
0
    QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3589
0
}
3590
3591
/* Reachable analysis : remove unreachable code.  */
3592
static void __attribute__((noinline))
3593
reachable_code_pass(TCGContext *s)
3594
0
{
3595
0
    TCGOp *op, *op_next, *op_prev;
3596
0
    bool dead = false;
3597
3598
0
    QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3599
0
        bool remove = dead;
3600
0
        TCGLabel *label;
3601
3602
0
        switch (op->opc) {
3603
0
        case INDEX_op_set_label:
3604
0
            label = arg_label(op->args[0]);
3605
3606
            /*
3607
             * Note that the first op in the TB is always a load,
3608
             * so there is always something before a label.
3609
             */
3610
0
            op_prev = QTAILQ_PREV(op, link);
3611
3612
            /*
3613
             * If we find two sequential labels, move all branches to
3614
             * reference the second label and remove the first label.
3615
             * Do this before branch to next optimization, so that the
3616
             * middle label is out of the way.
3617
             */
3618
0
            if (op_prev->opc == INDEX_op_set_label) {
3619
0
                move_label_uses(label, arg_label(op_prev->args[0]));
3620
0
                tcg_op_remove(s, op_prev);
3621
0
                op_prev = QTAILQ_PREV(op, link);
3622
0
            }
3623
3624
            /*
3625
             * Optimization can fold conditional branches to unconditional.
3626
             * If we find a label which is preceded by an unconditional
3627
             * branch to next, remove the branch.  We couldn't do this when
3628
             * processing the branch because any dead code between the branch
3629
             * and label had not yet been removed.
3630
             */
3631
0
            if (op_prev->opc == INDEX_op_br &&
3632
0
                label == arg_label(op_prev->args[0])) {
3633
0
                tcg_op_remove(s, op_prev);
3634
                /* Fall through means insns become live again.  */
3635
0
                dead = false;
3636
0
            }
3637
3638
0
            if (QSIMPLEQ_EMPTY(&label->branches)) {
3639
                /*
3640
                 * While there is an occasional backward branch, virtually
3641
                 * all branches generated by the translators are forward.
3642
                 * Which means that generally we will have already removed
3643
                 * all references to the label that will be, and there is
3644
                 * little to be gained by iterating.
3645
                 */
3646
0
                remove = true;
3647
0
            } else {
3648
                /* Once we see a label, insns become live again.  */
3649
0
                dead = false;
3650
0
                remove = false;
3651
0
            }
3652
0
            break;
3653
3654
0
        case INDEX_op_br:
3655
0
        case INDEX_op_exit_tb:
3656
0
        case INDEX_op_goto_ptr:
3657
            /* Unconditional branches; everything following is dead.  */
3658
0
            dead = true;
3659
0
            break;
3660
3661
0
        case INDEX_op_call:
3662
            /* Notice noreturn helper calls, raising exceptions.  */
3663
0
            if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
3664
0
                dead = true;
3665
0
            }
3666
0
            break;
3667
3668
0
        case INDEX_op_insn_start:
3669
            /* Never remove -- we need to keep these for unwind.  */
3670
0
            remove = false;
3671
0
            break;
3672
3673
0
        default:
3674
0
            break;
3675
0
        }
3676
3677
0
        if (remove) {
3678
0
            tcg_op_remove(s, op);
3679
0
        }
3680
0
    }
3681
0
}
3682
3683
0
#define TS_DEAD  1
3684
0
#define TS_MEM   2
3685
3686
0
#define IS_DEAD_ARG(n)   (arg_life & (DEAD_ARG << (n)))
3687
0
#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
3688
3689
/* For liveness_pass_1, the register preferences for a given temp.  */
3690
static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
3691
0
{
3692
0
    return ts->state_ptr;
3693
0
}
3694
3695
/* For liveness_pass_1, reset the preferences for a given temp to the
3696
 * maximal regset for its type.
3697
 */
3698
static inline void la_reset_pref(TCGTemp *ts)
3699
0
{
3700
0
    *la_temp_pref(ts)
3701
0
        = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
3702
0
}
3703
3704
/* liveness analysis: end of function: all temps are dead, and globals
3705
   should be in memory. */
3706
static void la_func_end(TCGContext *s, int ng, int nt)
3707
0
{
3708
0
    int i;
3709
3710
0
    for (i = 0; i < ng; ++i) {
3711
0
        s->temps[i].state = TS_DEAD | TS_MEM;
3712
0
        la_reset_pref(&s->temps[i]);
3713
0
    }
3714
0
    for (i = ng; i < nt; ++i) {
3715
0
        s->temps[i].state = TS_DEAD;
3716
0
        la_reset_pref(&s->temps[i]);
3717
0
    }
3718
0
}
3719
3720
/* liveness analysis: end of basic block: all temps are dead, globals
3721
   and local temps should be in memory. */
3722
static void la_bb_end(TCGContext *s, int ng, int nt)
3723
0
{
3724
0
    int i;
3725
3726
0
    for (i = 0; i < nt; ++i) {
3727
0
        TCGTemp *ts = &s->temps[i];
3728
0
        int state;
3729
3730
0
        switch (ts->kind) {
3731
0
        case TEMP_FIXED:
3732
0
        case TEMP_GLOBAL:
3733
0
        case TEMP_TB:
3734
0
            state = TS_DEAD | TS_MEM;
3735
0
            break;
3736
0
        case TEMP_EBB:
3737
0
        case TEMP_CONST:
3738
0
            state = TS_DEAD;
3739
0
            break;
3740
0
        default:
3741
0
            g_assert_not_reached();
3742
0
        }
3743
0
        ts->state = state;
3744
0
        la_reset_pref(ts);
3745
0
    }
3746
0
}
3747
3748
/* liveness analysis: sync globals back to memory.  */
3749
static void la_global_sync(TCGContext *s, int ng)
3750
0
{
3751
0
    int i;
3752
3753
0
    for (i = 0; i < ng; ++i) {
3754
0
        int state = s->temps[i].state;
3755
0
        s->temps[i].state = state | TS_MEM;
3756
0
        if (state == TS_DEAD) {
3757
            /* If the global was previously dead, reset prefs.  */
3758
0
            la_reset_pref(&s->temps[i]);
3759
0
        }
3760
0
    }
3761
0
}
3762
3763
/*
3764
 * liveness analysis: conditional branch: all temps are dead unless
3765
 * explicitly live-across-conditional-branch, globals and local temps
3766
 * should be synced.
3767
 */
3768
static void la_bb_sync(TCGContext *s, int ng, int nt)
3769
0
{
3770
0
    la_global_sync(s, ng);
3771
3772
0
    for (int i = ng; i < nt; ++i) {
3773
0
        TCGTemp *ts = &s->temps[i];
3774
0
        int state;
3775
3776
0
        switch (ts->kind) {
3777
0
        case TEMP_TB:
3778
0
            state = ts->state;
3779
0
            ts->state = state | TS_MEM;
3780
0
            if (state != TS_DEAD) {
3781
0
                continue;
3782
0
            }
3783
0
            break;
3784
0
        case TEMP_EBB:
3785
0
        case TEMP_CONST:
3786
0
            continue;
3787
0
        default:
3788
0
            g_assert_not_reached();
3789
0
        }
3790
0
        la_reset_pref(&s->temps[i]);
3791
0
    }
3792
0
}
3793
3794
/* liveness analysis: sync globals back to memory and kill.  */
3795
static void la_global_kill(TCGContext *s, int ng)
3796
0
{
3797
0
    int i;
3798
3799
0
    for (i = 0; i < ng; i++) {
3800
0
        s->temps[i].state = TS_DEAD | TS_MEM;
3801
0
        la_reset_pref(&s->temps[i]);
3802
0
    }
3803
0
}
3804
3805
/* liveness analysis: note live globals crossing calls.  */
3806
static void la_cross_call(TCGContext *s, int nt)
3807
0
{
3808
0
    TCGRegSet mask = ~tcg_target_call_clobber_regs;
3809
0
    int i;
3810
3811
0
    for (i = 0; i < nt; i++) {
3812
0
        TCGTemp *ts = &s->temps[i];
3813
0
        if (!(ts->state & TS_DEAD)) {
3814
0
            TCGRegSet *pset = la_temp_pref(ts);
3815
0
            TCGRegSet set = *pset;
3816
3817
0
            set &= mask;
3818
            /* If the combination is not possible, restart.  */
3819
0
            if (set == 0) {
3820
0
                set = tcg_target_available_regs[ts->type] & mask;
3821
0
            }
3822
0
            *pset = set;
3823
0
        }
3824
0
    }
3825
0
}
3826
3827
/*
3828
 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3829
 * to TEMP_EBB, if possible.
3830
 */
3831
static void __attribute__((noinline))
3832
liveness_pass_0(TCGContext *s)
3833
0
{
3834
0
    void * const multiple_ebb = (void *)(uintptr_t)-1;
3835
0
    int nb_temps = s->nb_temps;
3836
0
    TCGOp *op, *ebb;
3837
3838
0
    for (int i = s->nb_globals; i < nb_temps; ++i) {
3839
0
        s->temps[i].state_ptr = NULL;
3840
0
    }
3841
3842
    /*
3843
     * Represent each EBB by the op at which it begins.  In the case of
3844
     * the first EBB, this is the first op, otherwise it is a label.
3845
     * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3846
     * within a single EBB, else MULTIPLE_EBB.
3847
     */
3848
0
    ebb = QTAILQ_FIRST(&s->ops);
3849
0
    QTAILQ_FOREACH(op, &s->ops, link) {
3850
0
        const TCGOpDef *def;
3851
0
        int nb_oargs, nb_iargs;
3852
3853
0
        switch (op->opc) {
3854
0
        case INDEX_op_set_label:
3855
0
            ebb = op;
3856
0
            continue;
3857
0
        case INDEX_op_discard:
3858
0
            continue;
3859
0
        case INDEX_op_call:
3860
0
            nb_oargs = TCGOP_CALLO(op);
3861
0
            nb_iargs = TCGOP_CALLI(op);
3862
0
            break;
3863
0
        default:
3864
0
            def = &tcg_op_defs[op->opc];
3865
0
            nb_oargs = def->nb_oargs;
3866
0
            nb_iargs = def->nb_iargs;
3867
0
            break;
3868
0
        }
3869
3870
0
        for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3871
0
            TCGTemp *ts = arg_temp(op->args[i]);
3872
3873
0
            if (ts->kind != TEMP_TB) {
3874
0
                continue;
3875
0
            }
3876
0
            if (ts->state_ptr == NULL) {
3877
0
                ts->state_ptr = ebb;
3878
0
            } else if (ts->state_ptr != ebb) {
3879
0
                ts->state_ptr = multiple_ebb;
3880
0
            }
3881
0
        }
3882
0
    }
3883
3884
    /*
3885
     * For TEMP_TB that turned out not to be used beyond one EBB,
3886
     * reduce the liveness to TEMP_EBB.
3887
     */
3888
0
    for (int i = s->nb_globals; i < nb_temps; ++i) {
3889
0
        TCGTemp *ts = &s->temps[i];
3890
0
        if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3891
0
            ts->kind = TEMP_EBB;
3892
0
        }
3893
0
    }
3894
0
}
3895
3896
static void assert_carry_dead(TCGContext *s)
3897
0
{
3898
    /*
3899
     * Carry operations can be separated by a few insns like mov,
3900
     * load or store, but they should always be "close", and
3901
     * carry-out operations should always be paired with carry-in.
3902
     * At various boundaries, carry must have been consumed.
3903
     */
3904
0
    tcg_debug_assert(!s->carry_live);
3905
0
}
3906
3907
/* Liveness analysis : update the opc_arg_life array to tell if a
3908
   given input arguments is dead. Instructions updating dead
3909
   temporaries are removed. */
3910
static void __attribute__((noinline))
3911
liveness_pass_1(TCGContext *s)
3912
0
{
3913
0
    int nb_globals = s->nb_globals;
3914
0
    int nb_temps = s->nb_temps;
3915
0
    TCGOp *op, *op_prev;
3916
0
    TCGRegSet *prefs;
3917
3918
0
    prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
3919
0
    for (int i = 0; i < nb_temps; ++i) {
3920
0
        s->temps[i].state_ptr = prefs + i;
3921
0
    }
3922
3923
    /* ??? Should be redundant with the exit_tb that ends the TB.  */
3924
0
    la_func_end(s, nb_globals, nb_temps);
3925
3926
0
    s->carry_live = false;
3927
0
    QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
3928
0
        int nb_iargs, nb_oargs;
3929
0
        TCGOpcode opc_new, opc_new2;
3930
0
        TCGLifeData arg_life = 0;
3931
0
        TCGTemp *ts;
3932
0
        TCGOpcode opc = op->opc;
3933
0
        const TCGOpDef *def;
3934
0
        const TCGArgConstraint *args_ct;
3935
3936
0
        switch (opc) {
3937
0
        case INDEX_op_call:
3938
0
            assert_carry_dead(s);
3939
0
            {
3940
0
                const TCGHelperInfo *info = tcg_call_info(op);
3941
0
                int call_flags = tcg_call_flags(op);
3942
3943
0
                nb_oargs = TCGOP_CALLO(op);
3944
0
                nb_iargs = TCGOP_CALLI(op);
3945
3946
                /* pure functions can be removed if their result is unused */
3947
0
                if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
3948
0
                    for (int i = 0; i < nb_oargs; i++) {
3949
0
                        ts = arg_temp(op->args[i]);
3950
0
                        if (ts->state != TS_DEAD) {
3951
0
                            goto do_not_remove_call;
3952
0
                        }
3953
0
                    }
3954
0
                    goto do_remove;
3955
0
                }
3956
0
            do_not_remove_call:
3957
3958
                /* Output args are dead.  */
3959
0
                for (int i = 0; i < nb_oargs; i++) {
3960
0
                    ts = arg_temp(op->args[i]);
3961
0
                    if (ts->state & TS_DEAD) {
3962
0
                        arg_life |= DEAD_ARG << i;
3963
0
                    }
3964
0
                    if (ts->state & TS_MEM) {
3965
0
                        arg_life |= SYNC_ARG << i;
3966
0
                    }
3967
0
                    ts->state = TS_DEAD;
3968
0
                    la_reset_pref(ts);
3969
0
                }
3970
3971
                /* Not used -- it will be tcg_target_call_oarg_reg().  */
3972
0
                memset(op->output_pref, 0, sizeof(op->output_pref));
3973
3974
0
                if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
3975
0
                                    TCG_CALL_NO_READ_GLOBALS))) {
3976
0
                    la_global_kill(s, nb_globals);
3977
0
                } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
3978
0
                    la_global_sync(s, nb_globals);
3979
0
                }
3980
3981
                /* Record arguments that die in this helper.  */
3982
0
                for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
3983
0
                    ts = arg_temp(op->args[i]);
3984
0
                    if (ts->state & TS_DEAD) {
3985
0
                        arg_life |= DEAD_ARG << i;
3986
0
                    }
3987
0
                }
3988
3989
                /* For all live registers, remove call-clobbered prefs.  */
3990
0
                la_cross_call(s, nb_temps);
3991
3992
                /*
3993
                 * Input arguments are live for preceding opcodes.
3994
                 *
3995
                 * For those arguments that die, and will be allocated in
3996
                 * registers, clear the register set for that arg, to be
3997
                 * filled in below.  For args that will be on the stack,
3998
                 * reset to any available reg.  Process arguments in reverse
3999
                 * order so that if a temp is used more than once, the stack
4000
                 * reset to max happens before the register reset to 0.
4001
                 */
4002
0
                for (int i = nb_iargs - 1; i >= 0; i--) {
4003
0
                    const TCGCallArgumentLoc *loc = &info->in[i];
4004
0
                    ts = arg_temp(op->args[nb_oargs + i]);
4005
4006
0
                    if (ts->state & TS_DEAD) {
4007
0
                        switch (loc->kind) {
4008
0
                        case TCG_CALL_ARG_NORMAL:
4009
0
                        case TCG_CALL_ARG_EXTEND_U:
4010
0
                        case TCG_CALL_ARG_EXTEND_S:
4011
0
                            if (arg_slot_reg_p(loc->arg_slot)) {
4012
0
                                *la_temp_pref(ts) = 0;
4013
0
                                break;
4014
0
                            }
4015
                            /* fall through */
4016
0
                        default:
4017
0
                            *la_temp_pref(ts) =
4018
0
                                tcg_target_available_regs[ts->type];
4019
0
                            break;
4020
0
                        }
4021
0
                        ts->state &= ~TS_DEAD;
4022
0
                    }
4023
0
                }
4024
4025
                /*
4026
                 * For each input argument, add its input register to prefs.
4027
                 * If a temp is used once, this produces a single set bit;
4028
                 * if a temp is used multiple times, this produces a set.
4029
                 */
4030
0
                for (int i = 0; i < nb_iargs; i++) {
4031
0
                    const TCGCallArgumentLoc *loc = &info->in[i];
4032
0
                    ts = arg_temp(op->args[nb_oargs + i]);
4033
4034
0
                    switch (loc->kind) {
4035
0
                    case TCG_CALL_ARG_NORMAL:
4036
0
                    case TCG_CALL_ARG_EXTEND_U:
4037
0
                    case TCG_CALL_ARG_EXTEND_S:
4038
0
                        if (arg_slot_reg_p(loc->arg_slot)) {
4039
0
                            tcg_regset_set_reg(*la_temp_pref(ts),
4040
0
                                tcg_target_call_iarg_regs[loc->arg_slot]);
4041
0
                        }
4042
0
                        break;
4043
0
                    default:
4044
0
                        break;
4045
0
                    }
4046
0
                }
4047
0
            }
4048
0
            break;
4049
0
        case INDEX_op_insn_start:
4050
0
            assert_carry_dead(s);
4051
0
            break;
4052
0
        case INDEX_op_discard:
4053
            /* mark the temporary as dead */
4054
0
            ts = arg_temp(op->args[0]);
4055
0
            ts->state = TS_DEAD;
4056
0
            la_reset_pref(ts);
4057
0
            break;
4058
4059
0
        case INDEX_op_muls2:
4060
0
            opc_new = INDEX_op_mul;
4061
0
            opc_new2 = INDEX_op_mulsh;
4062
0
            goto do_mul2;
4063
0
        case INDEX_op_mulu2:
4064
0
            opc_new = INDEX_op_mul;
4065
0
            opc_new2 = INDEX_op_muluh;
4066
0
        do_mul2:
4067
0
            assert_carry_dead(s);
4068
0
            if (arg_temp(op->args[1])->state == TS_DEAD) {
4069
0
                if (arg_temp(op->args[0])->state == TS_DEAD) {
4070
                    /* Both parts of the operation are dead.  */
4071
0
                    goto do_remove;
4072
0
                }
4073
                /* The high part of the operation is dead; generate the low. */
4074
0
                op->opc = opc = opc_new;
4075
0
                op->args[1] = op->args[2];
4076
0
                op->args[2] = op->args[3];
4077
0
            } else if (arg_temp(op->args[0])->state == TS_DEAD &&
4078
0
                       tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) {
4079
                /* The low part of the operation is dead; generate the high. */
4080
0
                op->opc = opc = opc_new2;
4081
0
                op->args[0] = op->args[1];
4082
0
                op->args[1] = op->args[2];
4083
0
                op->args[2] = op->args[3];
4084
0
            } else {
4085
0
                goto do_not_remove;
4086
0
            }
4087
            /* Mark the single-word operation live.  */
4088
0
            goto do_not_remove;
4089
4090
0
        case INDEX_op_addco:
4091
0
            if (s->carry_live) {
4092
0
                goto do_not_remove;
4093
0
            }
4094
0
            op->opc = opc = INDEX_op_add;
4095
0
            goto do_default;
4096
4097
0
        case INDEX_op_addcio:
4098
0
            if (s->carry_live) {
4099
0
                goto do_not_remove;
4100
0
            }
4101
0
            op->opc = opc = INDEX_op_addci;
4102
0
            goto do_default;
4103
4104
0
        case INDEX_op_subbo:
4105
0
            if (s->carry_live) {
4106
0
                goto do_not_remove;
4107
0
            }
4108
            /* Lower to sub, but this may also require canonicalization. */
4109
0
            op->opc = opc = INDEX_op_sub;
4110
0
            ts = arg_temp(op->args[2]);
4111
0
            if (ts->kind == TEMP_CONST) {
4112
0
                ts = tcg_constant_internal(ts->type, -ts->val);
4113
0
                if (ts->state_ptr == NULL) {
4114
0
                    tcg_debug_assert(temp_idx(ts) == nb_temps);
4115
0
                    nb_temps++;
4116
0
                    ts->state_ptr = tcg_malloc(sizeof(TCGRegSet));
4117
0
                    ts->state = TS_DEAD;
4118
0
                    la_reset_pref(ts);
4119
0
                }
4120
0
                op->args[2] = temp_arg(ts);
4121
0
                op->opc = opc = INDEX_op_add;
4122
0
            }
4123
0
            goto do_default;
4124
4125
0
        case INDEX_op_subbio:
4126
0
            if (s->carry_live) {
4127
0
                goto do_not_remove;
4128
0
            }
4129
0
            op->opc = opc = INDEX_op_subbi;
4130
0
            goto do_default;
4131
4132
0
        case INDEX_op_addc1o:
4133
0
            if (s->carry_live) {
4134
0
                goto do_not_remove;
4135
0
            }
4136
            /* Lower to add, add +1. */
4137
0
            op_prev = tcg_op_insert_before(s, op, INDEX_op_add,
4138
0
                                           TCGOP_TYPE(op), 3);
4139
0
            op_prev->args[0] = op->args[0];
4140
0
            op_prev->args[1] = op->args[1];
4141
0
            op_prev->args[2] = op->args[2];
4142
0
            op->opc = opc = INDEX_op_add;
4143
0
            op->args[1] = op->args[0];
4144
0
            ts = arg_temp(op->args[0]);
4145
0
            ts = tcg_constant_internal(ts->type, 1);
4146
0
            op->args[2] = temp_arg(ts);
4147
0
            goto do_default;
4148
4149
0
        case INDEX_op_subb1o:
4150
0
            if (s->carry_live) {
4151
0
                goto do_not_remove;
4152
0
            }
4153
            /* Lower to sub, add -1. */
4154
0
            op_prev = tcg_op_insert_before(s, op, INDEX_op_sub,
4155
0
                                           TCGOP_TYPE(op), 3);
4156
0
            op_prev->args[0] = op->args[0];
4157
0
            op_prev->args[1] = op->args[1];
4158
0
            op_prev->args[2] = op->args[2];
4159
0
            op->opc = opc = INDEX_op_add;
4160
0
            op->args[1] = op->args[0];
4161
0
            ts = arg_temp(op->args[0]);
4162
0
            ts = tcg_constant_internal(ts->type, -1);
4163
0
            op->args[2] = temp_arg(ts);
4164
0
            goto do_default;
4165
4166
0
        default:
4167
0
        do_default:
4168
            /*
4169
             * Test if the operation can be removed because all
4170
             * its outputs are dead. We assume that nb_oargs == 0
4171
             * implies side effects.
4172
             */
4173
0
            def = &tcg_op_defs[opc];
4174
0
            if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) {
4175
0
                for (int i = def->nb_oargs - 1; i >= 0; i--) {
4176
0
                    if (arg_temp(op->args[i])->state != TS_DEAD) {
4177
0
                        goto do_not_remove;
4178
0
                    }
4179
0
                }
4180
0
                goto do_remove;
4181
0
            }
4182
0
            goto do_not_remove;
4183
4184
0
        do_remove:
4185
0
            tcg_op_remove(s, op);
4186
0
            break;
4187
4188
0
        do_not_remove:
4189
0
            def = &tcg_op_defs[opc];
4190
0
            nb_iargs = def->nb_iargs;
4191
0
            nb_oargs = def->nb_oargs;
4192
4193
0
            for (int i = 0; i < nb_oargs; i++) {
4194
0
                ts = arg_temp(op->args[i]);
4195
4196
                /* Remember the preference of the uses that followed.  */
4197
0
                if (i < ARRAY_SIZE(op->output_pref)) {
4198
0
                    op->output_pref[i] = *la_temp_pref(ts);
4199
0
                }
4200
4201
                /* Output args are dead.  */
4202
0
                if (ts->state & TS_DEAD) {
4203
0
                    arg_life |= DEAD_ARG << i;
4204
0
                }
4205
0
                if (ts->state & TS_MEM) {
4206
0
                    arg_life |= SYNC_ARG << i;
4207
0
                }
4208
0
                ts->state = TS_DEAD;
4209
0
                la_reset_pref(ts);
4210
0
            }
4211
4212
            /* If end of basic block, update.  */
4213
0
            if (def->flags & TCG_OPF_BB_EXIT) {
4214
0
                assert_carry_dead(s);
4215
0
                la_func_end(s, nb_globals, nb_temps);
4216
0
            } else if (def->flags & TCG_OPF_COND_BRANCH) {
4217
0
                assert_carry_dead(s);
4218
0
                la_bb_sync(s, nb_globals, nb_temps);
4219
0
            } else if (def->flags & TCG_OPF_BB_END) {
4220
0
                assert_carry_dead(s);
4221
0
                la_bb_end(s, nb_globals, nb_temps);
4222
0
            } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4223
0
                assert_carry_dead(s);
4224
0
                la_global_sync(s, nb_globals);
4225
0
                if (def->flags & TCG_OPF_CALL_CLOBBER) {
4226
0
                    la_cross_call(s, nb_temps);
4227
0
                }
4228
0
            }
4229
4230
            /* Record arguments that die in this opcode.  */
4231
0
            for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4232
0
                ts = arg_temp(op->args[i]);
4233
0
                if (ts->state & TS_DEAD) {
4234
0
                    arg_life |= DEAD_ARG << i;
4235
0
                }
4236
0
            }
4237
0
            if (def->flags & TCG_OPF_CARRY_OUT) {
4238
0
                s->carry_live = false;
4239
0
            }
4240
4241
            /* Input arguments are live for preceding opcodes.  */
4242
0
            for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4243
0
                ts = arg_temp(op->args[i]);
4244
0
                if (ts->state & TS_DEAD) {
4245
                    /* For operands that were dead, initially allow
4246
                       all regs for the type.  */
4247
0
                    *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
4248
0
                    ts->state &= ~TS_DEAD;
4249
0
                }
4250
0
            }
4251
0
            if (def->flags & TCG_OPF_CARRY_IN) {
4252
0
                s->carry_live = true;
4253
0
            }
4254
4255
            /* Incorporate constraints for this operand.  */
4256
0
            switch (opc) {
4257
0
            case INDEX_op_mov:
4258
                /* Note that these are TCG_OPF_NOT_PRESENT and do not
4259
                   have proper constraints.  That said, special case
4260
                   moves to propagate preferences backward.  */
4261
0
                if (IS_DEAD_ARG(1)) {
4262
0
                    *la_temp_pref(arg_temp(op->args[0]))
4263
0
                        = *la_temp_pref(arg_temp(op->args[1]));
4264
0
                }
4265
0
                break;
4266
4267
0
            default:
4268
0
                args_ct = opcode_args_ct(op);
4269
0
                for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4270
0
                    const TCGArgConstraint *ct = &args_ct[i];
4271
0
                    TCGRegSet set, *pset;
4272
4273
0
                    ts = arg_temp(op->args[i]);
4274
0
                    pset = la_temp_pref(ts);
4275
0
                    set = *pset;
4276
4277
0
                    set &= ct->regs;
4278
0
                    if (ct->ialias) {
4279
0
                        set &= output_pref(op, ct->alias_index);
4280
0
                    }
4281
                    /* If the combination is not possible, restart.  */
4282
0
                    if (set == 0) {
4283
0
                        set = ct->regs;
4284
0
                    }
4285
0
                    *pset = set;
4286
0
                }
4287
0
                break;
4288
0
            }
4289
0
            break;
4290
0
        }
4291
0
        op->life = arg_life;
4292
0
    }
4293
0
    assert_carry_dead(s);
4294
0
}
4295
4296
/* Liveness analysis: Convert indirect regs to direct temporaries.  */
4297
static bool __attribute__((noinline))
4298
liveness_pass_2(TCGContext *s)
4299
0
{
4300
0
    int nb_globals = s->nb_globals;
4301
0
    int nb_temps, i;
4302
0
    bool changes = false;
4303
0
    TCGOp *op, *op_next;
4304
4305
    /* Create a temporary for each indirect global.  */
4306
0
    for (i = 0; i < nb_globals; ++i) {
4307
0
        TCGTemp *its = &s->temps[i];
4308
0
        if (its->indirect_reg) {
4309
0
            TCGTemp *dts = tcg_temp_alloc(s);
4310
0
            dts->type = its->type;
4311
0
            dts->base_type = its->base_type;
4312
0
            dts->temp_subindex = its->temp_subindex;
4313
0
            dts->kind = TEMP_EBB;
4314
0
            its->state_ptr = dts;
4315
0
        } else {
4316
0
            its->state_ptr = NULL;
4317
0
        }
4318
        /* All globals begin dead.  */
4319
0
        its->state = TS_DEAD;
4320
0
    }
4321
0
    for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
4322
0
        TCGTemp *its = &s->temps[i];
4323
0
        its->state_ptr = NULL;
4324
0
        its->state = TS_DEAD;
4325
0
    }
4326
4327
0
    QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
4328
0
        TCGOpcode opc = op->opc;
4329
0
        const TCGOpDef *def = &tcg_op_defs[opc];
4330
0
        TCGLifeData arg_life = op->life;
4331
0
        int nb_iargs, nb_oargs, call_flags;
4332
0
        TCGTemp *arg_ts, *dir_ts;
4333
4334
0
        if (opc == INDEX_op_call) {
4335
0
            nb_oargs = TCGOP_CALLO(op);
4336
0
            nb_iargs = TCGOP_CALLI(op);
4337
0
            call_flags = tcg_call_flags(op);
4338
0
        } else {
4339
0
            nb_iargs = def->nb_iargs;
4340
0
            nb_oargs = def->nb_oargs;
4341
4342
            /* Set flags similar to how calls require.  */
4343
0
            if (def->flags & TCG_OPF_COND_BRANCH) {
4344
                /* Like reading globals: sync_globals */
4345
0
                call_flags = TCG_CALL_NO_WRITE_GLOBALS;
4346
0
            } else if (def->flags & TCG_OPF_BB_END) {
4347
                /* Like writing globals: save_globals */
4348
0
                call_flags = 0;
4349
0
            } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4350
                /* Like reading globals: sync_globals */
4351
0
                call_flags = TCG_CALL_NO_WRITE_GLOBALS;
4352
0
            } else {
4353
                /* No effect on globals.  */
4354
0
                call_flags = (TCG_CALL_NO_READ_GLOBALS |
4355
0
                              TCG_CALL_NO_WRITE_GLOBALS);
4356
0
            }
4357
0
        }
4358
4359
        /* Make sure that input arguments are available.  */
4360
0
        for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4361
0
            arg_ts = arg_temp(op->args[i]);
4362
0
            dir_ts = arg_ts->state_ptr;
4363
0
            if (dir_ts && arg_ts->state == TS_DEAD) {
4364
0
                TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld,
4365
0
                                                  arg_ts->type, 3);
4366
4367
0
                lop->args[0] = temp_arg(dir_ts);
4368
0
                lop->args[1] = temp_arg(arg_ts->mem_base);
4369
0
                lop->args[2] = arg_ts->mem_offset;
4370
4371
                /* Loaded, but synced with memory.  */
4372
0
                arg_ts->state = TS_MEM;
4373
0
            }
4374
0
        }
4375
4376
        /* Perform input replacement, and mark inputs that became dead.
4377
           No action is required except keeping temp_state up to date
4378
           so that we reload when needed.  */
4379
0
        for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4380
0
            arg_ts = arg_temp(op->args[i]);
4381
0
            dir_ts = arg_ts->state_ptr;
4382
0
            if (dir_ts) {
4383
0
                op->args[i] = temp_arg(dir_ts);
4384
0
                changes = true;
4385
0
                if (IS_DEAD_ARG(i)) {
4386
0
                    arg_ts->state = TS_DEAD;
4387
0
                }
4388
0
            }
4389
0
        }
4390
4391
        /* Liveness analysis should ensure that the following are
4392
           all correct, for call sites and basic block end points.  */
4393
0
        if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
4394
            /* Nothing to do */
4395
0
        } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
4396
0
            for (i = 0; i < nb_globals; ++i) {
4397
                /* Liveness should see that globals are synced back,
4398
                   that is, either TS_DEAD or TS_MEM.  */
4399
0
                arg_ts = &s->temps[i];
4400
0
                tcg_debug_assert(arg_ts->state_ptr == 0
4401
0
                                 || arg_ts->state != 0);
4402
0
            }
4403
0
        } else {
4404
0
            for (i = 0; i < nb_globals; ++i) {
4405
                /* Liveness should see that globals are saved back,
4406
                   that is, TS_DEAD, waiting to be reloaded.  */
4407
0
                arg_ts = &s->temps[i];
4408
0
                tcg_debug_assert(arg_ts->state_ptr == 0
4409
0
                                 || arg_ts->state == TS_DEAD);
4410
0
            }
4411
0
        }
4412
4413
        /* Outputs become available.  */
4414
0
        if (opc == INDEX_op_mov) {
4415
0
            arg_ts = arg_temp(op->args[0]);
4416
0
            dir_ts = arg_ts->state_ptr;
4417
0
            if (dir_ts) {
4418
0
                op->args[0] = temp_arg(dir_ts);
4419
0
                changes = true;
4420
4421
                /* The output is now live and modified.  */
4422
0
                arg_ts->state = 0;
4423
4424
0
                if (NEED_SYNC_ARG(0)) {
4425
0
                    TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st,
4426
0
                                                     arg_ts->type, 3);
4427
0
                    TCGTemp *out_ts = dir_ts;
4428
4429
0
                    if (IS_DEAD_ARG(0)) {
4430
0
                        out_ts = arg_temp(op->args[1]);
4431
0
                        arg_ts->state = TS_DEAD;
4432
0
                        tcg_op_remove(s, op);
4433
0
                    } else {
4434
0
                        arg_ts->state = TS_MEM;
4435
0
                    }
4436
4437
0
                    sop->args[0] = temp_arg(out_ts);
4438
0
                    sop->args[1] = temp_arg(arg_ts->mem_base);
4439
0
                    sop->args[2] = arg_ts->mem_offset;
4440
0
                } else {
4441
0
                    tcg_debug_assert(!IS_DEAD_ARG(0));
4442
0
                }
4443
0
            }
4444
0
        } else {
4445
0
            for (i = 0; i < nb_oargs; i++) {
4446
0
                arg_ts = arg_temp(op->args[i]);
4447
0
                dir_ts = arg_ts->state_ptr;
4448
0
                if (!dir_ts) {
4449
0
                    continue;
4450
0
                }
4451
0
                op->args[i] = temp_arg(dir_ts);
4452
0
                changes = true;
4453
4454
                /* The output is now live and modified.  */
4455
0
                arg_ts->state = 0;
4456
4457
                /* Sync outputs upon their last write.  */
4458
0
                if (NEED_SYNC_ARG(i)) {
4459
0
                    TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st,
4460
0
                                                     arg_ts->type, 3);
4461
4462
0
                    sop->args[0] = temp_arg(dir_ts);
4463
0
                    sop->args[1] = temp_arg(arg_ts->mem_base);
4464
0
                    sop->args[2] = arg_ts->mem_offset;
4465
4466
0
                    arg_ts->state = TS_MEM;
4467
0
                }
4468
                /* Drop outputs that are dead.  */
4469
0
                if (IS_DEAD_ARG(i)) {
4470
0
                    arg_ts->state = TS_DEAD;
4471
0
                }
4472
0
            }
4473
0
        }
4474
0
    }
4475
4476
0
    return changes;
4477
0
}
4478
4479
static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
4480
0
{
4481
0
    intptr_t off;
4482
0
    int size, align;
4483
4484
    /* When allocating an object, look at the full type. */
4485
0
    size = tcg_type_size(ts->base_type);
4486
0
    switch (ts->base_type) {
4487
0
    case TCG_TYPE_I32:
4488
0
        align = 4;
4489
0
        break;
4490
0
    case TCG_TYPE_I64:
4491
0
    case TCG_TYPE_V64:
4492
0
        align = 8;
4493
0
        break;
4494
0
    case TCG_TYPE_I128:
4495
0
    case TCG_TYPE_V128:
4496
0
    case TCG_TYPE_V256:
4497
        /*
4498
         * Note that we do not require aligned storage for V256,
4499
         * and that we provide alignment for I128 to match V128,
4500
         * even if that's above what the host ABI requires.
4501
         */
4502
0
        align = 16;
4503
0
        break;
4504
0
    default:
4505
0
        g_assert_not_reached();
4506
0
    }
4507
4508
    /*
4509
     * Assume the stack is sufficiently aligned.
4510
     * This affects e.g. ARM NEON, where we have 8 byte stack alignment
4511
     * and do not require 16 byte vector alignment.  This seems slightly
4512
     * easier than fully parameterizing the above switch statement.
4513
     */
4514
0
    align = MIN(TCG_TARGET_STACK_ALIGN, align);
4515
0
    off = ROUND_UP(s->current_frame_offset, align);
4516
4517
    /* If we've exhausted the stack frame, restart with a smaller TB. */
4518
0
    if (off + size > s->frame_end) {
4519
0
        tcg_raise_tb_overflow(s);
4520
0
    }
4521
0
    s->current_frame_offset = off + size;
4522
#if defined(__sparc__)
4523
    off += TCG_TARGET_STACK_BIAS;
4524
#endif
4525
4526
    /* If the object was subdivided, assign memory to all the parts. */
4527
0
    if (ts->base_type != ts->type) {
4528
0
        int part_size = tcg_type_size(ts->type);
4529
0
        int part_count = size / part_size;
4530
4531
        /*
4532
         * Each part is allocated sequentially in tcg_temp_new_internal.
4533
         * Jump back to the first part by subtracting the current index.
4534
         */
4535
0
        ts -= ts->temp_subindex;
4536
0
        for (int i = 0; i < part_count; ++i) {
4537
0
            ts[i].mem_offset = off + i * part_size;
4538
0
            ts[i].mem_base = s->frame_temp;
4539
0
            ts[i].mem_allocated = 1;
4540
0
        }
4541
0
    } else {
4542
0
        ts->mem_offset = off;
4543
0
        ts->mem_base = s->frame_temp;
4544
0
        ts->mem_allocated = 1;
4545
0
    }
4546
0
}
4547
4548
/* Assign @reg to @ts, and update reg_to_temp[]. */
4549
static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
4550
0
{
4551
0
    if (ts->val_type == TEMP_VAL_REG) {
4552
0
        TCGReg old = ts->reg;
4553
0
        tcg_debug_assert(s->reg_to_temp[old] == ts);
4554
0
        if (old == reg) {
4555
0
            return;
4556
0
        }
4557
0
        s->reg_to_temp[old] = NULL;
4558
0
    }
4559
0
    tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4560
0
    s->reg_to_temp[reg] = ts;
4561
0
    ts->val_type = TEMP_VAL_REG;
4562
0
    ts->reg = reg;
4563
0
}
4564
4565
/* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4566
static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
4567
0
{
4568
0
    tcg_debug_assert(type != TEMP_VAL_REG);
4569
0
    if (ts->val_type == TEMP_VAL_REG) {
4570
0
        TCGReg reg = ts->reg;
4571
0
        tcg_debug_assert(s->reg_to_temp[reg] == ts);
4572
0
        s->reg_to_temp[reg] = NULL;
4573
0
    }
4574
0
    ts->val_type = type;
4575
0
}
4576
4577
static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
4578
4579
/* Mark a temporary as free or dead.  If 'free_or_dead' is negative,
4580
   mark it free; otherwise mark it dead.  */
4581
static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
4582
0
{
4583
0
    TCGTempVal new_type;
4584
4585
0
    switch (ts->kind) {
4586
0
    case TEMP_FIXED:
4587
0
        return;
4588
0
    case TEMP_GLOBAL:
4589
0
    case TEMP_TB:
4590
0
        new_type = TEMP_VAL_MEM;
4591
0
        break;
4592
0
    case TEMP_EBB:
4593
0
        new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
4594
0
        break;
4595
0
    case TEMP_CONST:
4596
0
        new_type = TEMP_VAL_CONST;
4597
0
        break;
4598
0
    default:
4599
0
        g_assert_not_reached();
4600
0
    }
4601
0
    set_temp_val_nonreg(s, ts, new_type);
4602
0
}
4603
4604
/* Mark a temporary as dead.  */
4605
static inline void temp_dead(TCGContext *s, TCGTemp *ts)
4606
0
{
4607
0
    temp_free_or_dead(s, ts, 1);
4608
0
}
4609
4610
/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
4611
   registers needs to be allocated to store a constant.  If 'free_or_dead'
4612
   is non-zero, subsequently release the temporary; if it is positive, the
4613
   temp is dead; if it is negative, the temp is free.  */
4614
static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
4615
                      TCGRegSet preferred_regs, int free_or_dead)
4616
0
{
4617
0
    if (!temp_readonly(ts) && !ts->mem_coherent) {
4618
0
        if (!ts->mem_allocated) {
4619
0
            temp_allocate_frame(s, ts);
4620
0
        }
4621
0
        switch (ts->val_type) {
4622
0
        case TEMP_VAL_CONST:
4623
            /* If we're going to free the temp immediately, then we won't
4624
               require it later in a register, so attempt to store the
4625
               constant to memory directly.  */
4626
0
            if (free_or_dead
4627
0
                && tcg_out_sti(s, ts->type, ts->val,
4628
0
                               ts->mem_base->reg, ts->mem_offset)) {
4629
0
                break;
4630
0
            }
4631
0
            temp_load(s, ts, tcg_target_available_regs[ts->type],
4632
0
                      allocated_regs, preferred_regs);
4633
            /* fallthrough */
4634
4635
0
        case TEMP_VAL_REG:
4636
0
            tcg_out_st(s, ts->type, ts->reg,
4637
0
                       ts->mem_base->reg, ts->mem_offset);
4638
0
            break;
4639
4640
0
        case TEMP_VAL_MEM:
4641
0
            break;
4642
4643
0
        case TEMP_VAL_DEAD:
4644
0
        default:
4645
0
            g_assert_not_reached();
4646
0
        }
4647
0
        ts->mem_coherent = 1;
4648
0
    }
4649
0
    if (free_or_dead) {
4650
0
        temp_free_or_dead(s, ts, free_or_dead);
4651
0
    }
4652
0
}
4653
4654
/* free register 'reg' by spilling the corresponding temporary if necessary */
4655
static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
4656
0
{
4657
0
    TCGTemp *ts = s->reg_to_temp[reg];
4658
0
    if (ts != NULL) {
4659
0
        temp_sync(s, ts, allocated_regs, 0, -1);
4660
0
    }
4661
0
}
4662
4663
/**
4664
 * tcg_reg_alloc:
4665
 * @required_regs: Set of registers in which we must allocate.
4666
 * @allocated_regs: Set of registers which must be avoided.
4667
 * @preferred_regs: Set of registers we should prefer.
4668
 * @rev: True if we search the registers in "indirect" order.
4669
 *
4670
 * The allocated register must be in @required_regs & ~@allocated_regs,
4671
 * but if we can put it in @preferred_regs we may save a move later.
4672
 */
4673
static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
4674
                            TCGRegSet allocated_regs,
4675
                            TCGRegSet preferred_regs, bool rev)
4676
0
{
4677
0
    int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4678
0
    TCGRegSet reg_ct[2];
4679
0
    const int *order;
4680
4681
0
    reg_ct[1] = required_regs & ~allocated_regs;
4682
0
    tcg_debug_assert(reg_ct[1] != 0);
4683
0
    reg_ct[0] = reg_ct[1] & preferred_regs;
4684
4685
    /* Skip the preferred_regs option if it cannot be satisfied,
4686
       or if the preference made no difference.  */
4687
0
    f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4688
4689
0
    order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4690
4691
    /* Try free registers, preferences first.  */
4692
0
    for (j = f; j < 2; j++) {
4693
0
        TCGRegSet set = reg_ct[j];
4694
4695
0
        if (tcg_regset_single(set)) {
4696
            /* One register in the set.  */
4697
0
            TCGReg reg = tcg_regset_first(set);
4698
0
            if (s->reg_to_temp[reg] == NULL) {
4699
0
                return reg;
4700
0
            }
4701
0
        } else {
4702
0
            for (i = 0; i < n; i++) {
4703
0
                TCGReg reg = order[i];
4704
0
                if (s->reg_to_temp[reg] == NULL &&
4705
0
                    tcg_regset_test_reg(set, reg)) {
4706
0
                    return reg;
4707
0
                }
4708
0
            }
4709
0
        }
4710
0
    }
4711
4712
    /* We must spill something.  */
4713
0
    for (j = f; j < 2; j++) {
4714
0
        TCGRegSet set = reg_ct[j];
4715
4716
0
        if (tcg_regset_single(set)) {
4717
            /* One register in the set.  */
4718
0
            TCGReg reg = tcg_regset_first(set);
4719
0
            tcg_reg_free(s, reg, allocated_regs);
4720
0
            return reg;
4721
0
        } else {
4722
0
            for (i = 0; i < n; i++) {
4723
0
                TCGReg reg = order[i];
4724
0
                if (tcg_regset_test_reg(set, reg)) {
4725
0
                    tcg_reg_free(s, reg, allocated_regs);
4726
0
                    return reg;
4727
0
                }
4728
0
            }
4729
0
        }
4730
0
    }
4731
4732
0
    g_assert_not_reached();
4733
0
}
4734
4735
static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
4736
                                 TCGRegSet allocated_regs,
4737
                                 TCGRegSet preferred_regs, bool rev)
4738
0
{
4739
0
    int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4740
0
    TCGRegSet reg_ct[2];
4741
0
    const int *order;
4742
4743
    /* Ensure that if I is not in allocated_regs, I+1 is not either. */
4744
0
    reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
4745
0
    tcg_debug_assert(reg_ct[1] != 0);
4746
0
    reg_ct[0] = reg_ct[1] & preferred_regs;
4747
4748
0
    order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4749
4750
    /*
4751
     * Skip the preferred_regs option if it cannot be satisfied,
4752
     * or if the preference made no difference.
4753
     */
4754
0
    k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4755
4756
    /*
4757
     * Minimize the number of flushes by looking for 2 free registers first,
4758
     * then a single flush, then two flushes.
4759
     */
4760
0
    for (fmin = 2; fmin >= 0; fmin--) {
4761
0
        for (j = k; j < 2; j++) {
4762
0
            TCGRegSet set = reg_ct[j];
4763
4764
0
            for (i = 0; i < n; i++) {
4765
0
                TCGReg reg = order[i];
4766
4767
0
                if (tcg_regset_test_reg(set, reg)) {
4768
0
                    int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
4769
0
                    if (f >= fmin) {
4770
0
                        tcg_reg_free(s, reg, allocated_regs);
4771
0
                        tcg_reg_free(s, reg + 1, allocated_regs);
4772
0
                        return reg;
4773
0
                    }
4774
0
                }
4775
0
            }
4776
0
        }
4777
0
    }
4778
0
    g_assert_not_reached();
4779
0
}
4780
4781
/* Make sure the temporary is in a register.  If needed, allocate the register
4782
   from DESIRED while avoiding ALLOCATED.  */
4783
static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
4784
                      TCGRegSet allocated_regs, TCGRegSet preferred_regs)
4785
0
{
4786
0
    TCGReg reg;
4787
4788
0
    switch (ts->val_type) {
4789
0
    case TEMP_VAL_REG:
4790
0
        return;
4791
0
    case TEMP_VAL_CONST:
4792
0
        reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4793
0
                            preferred_regs, ts->indirect_base);
4794
0
        if (ts->type <= TCG_TYPE_I64) {
4795
0
            tcg_out_movi(s, ts->type, reg, ts->val);
4796
0
        } else {
4797
0
            uint64_t val = ts->val;
4798
0
            MemOp vece = MO_64;
4799
4800
            /*
4801
             * Find the minimal vector element that matches the constant.
4802
             * The targets will, in general, have to do this search anyway,
4803
             * do this generically.
4804
             */
4805
0
            if (val == dup_const(MO_8, val)) {
4806
0
                vece = MO_8;
4807
0
            } else if (val == dup_const(MO_16, val)) {
4808
0
                vece = MO_16;
4809
0
            } else if (val == dup_const(MO_32, val)) {
4810
0
                vece = MO_32;
4811
0
            }
4812
4813
0
            tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
4814
0
        }
4815
0
        ts->mem_coherent = 0;
4816
0
        break;
4817
0
    case TEMP_VAL_MEM:
4818
0
        if (!ts->mem_allocated) {
4819
0
            temp_allocate_frame(s, ts);
4820
0
        }
4821
0
        reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4822
0
                            preferred_regs, ts->indirect_base);
4823
0
        tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4824
0
        ts->mem_coherent = 1;
4825
0
        break;
4826
0
    case TEMP_VAL_DEAD:
4827
0
    default:
4828
0
        g_assert_not_reached();
4829
0
    }
4830
0
    set_temp_val_reg(s, ts, reg);
4831
0
}
4832
4833
/* Save a temporary to memory. 'allocated_regs' is used in case a
4834
   temporary registers needs to be allocated to store a constant.  */
4835
static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
4836
0
{
4837
    /* The liveness analysis already ensures that globals are back
4838
       in memory. Keep an tcg_debug_assert for safety. */
4839
0
    tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
4840
0
}
4841
4842
/* save globals to their canonical location and assume they can be
4843
   modified be the following code. 'allocated_regs' is used in case a
4844
   temporary registers needs to be allocated to store a constant. */
4845
static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
4846
0
{
4847
0
    int i, n;
4848
4849
0
    for (i = 0, n = s->nb_globals; i < n; i++) {
4850
0
        temp_save(s, &s->temps[i], allocated_regs);
4851
0
    }
4852
0
}
4853
4854
/* sync globals to their canonical location and assume they can be
4855
   read by the following code. 'allocated_regs' is used in case a
4856
   temporary registers needs to be allocated to store a constant. */
4857
static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
4858
0
{
4859
0
    int i, n;
4860
4861
0
    for (i = 0, n = s->nb_globals; i < n; i++) {
4862
0
        TCGTemp *ts = &s->temps[i];
4863
0
        tcg_debug_assert(ts->val_type != TEMP_VAL_REG
4864
0
                         || ts->kind == TEMP_FIXED
4865
0
                         || ts->mem_coherent);
4866
0
    }
4867
0
}
4868
4869
/* at the end of a basic block, we assume all temporaries are dead and
4870
   all globals are stored at their canonical location. */
4871
static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
4872
0
{
4873
0
    assert_carry_dead(s);
4874
0
    for (int i = s->nb_globals; i < s->nb_temps; i++) {
4875
0
        TCGTemp *ts = &s->temps[i];
4876
4877
0
        switch (ts->kind) {
4878
0
        case TEMP_TB:
4879
0
            temp_save(s, ts, allocated_regs);
4880
0
            break;
4881
0
        case TEMP_EBB:
4882
            /* The liveness analysis already ensures that temps are dead.
4883
               Keep an tcg_debug_assert for safety. */
4884
0
            tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
4885
0
            break;
4886
0
        case TEMP_CONST:
4887
            /* Similarly, we should have freed any allocated register. */
4888
0
            tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4889
0
            break;
4890
0
        default:
4891
0
            g_assert_not_reached();
4892
0
        }
4893
0
    }
4894
4895
0
    save_globals(s, allocated_regs);
4896
0
}
4897
4898
/*
4899
 * At a conditional branch, we assume all temporaries are dead unless
4900
 * explicitly live-across-conditional-branch; all globals and local
4901
 * temps are synced to their location.
4902
 */
4903
static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4904
0
{
4905
0
    assert_carry_dead(s);
4906
0
    sync_globals(s, allocated_regs);
4907
4908
0
    for (int i = s->nb_globals; i < s->nb_temps; i++) {
4909
0
        TCGTemp *ts = &s->temps[i];
4910
        /*
4911
         * The liveness analysis already ensures that temps are dead.
4912
         * Keep tcg_debug_asserts for safety.
4913
         */
4914
0
        switch (ts->kind) {
4915
0
        case TEMP_TB:
4916
0
            tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
4917
0
            break;
4918
0
        case TEMP_EBB:
4919
0
        case TEMP_CONST:
4920
0
            break;
4921
0
        default:
4922
0
            g_assert_not_reached();
4923
0
        }
4924
0
    }
4925
0
}
4926
4927
/*
4928
 * Specialized code generation for INDEX_op_mov_* with a constant.
4929
 */
4930
static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
4931
                                  tcg_target_ulong val, TCGLifeData arg_life,
4932
                                  TCGRegSet preferred_regs)
4933
0
{
4934
    /* ENV should not be modified.  */
4935
0
    tcg_debug_assert(!temp_readonly(ots));
4936
4937
    /* The movi is not explicitly generated here.  */
4938
0
    set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
4939
0
    ots->val = val;
4940
0
    ots->mem_coherent = 0;
4941
0
    if (NEED_SYNC_ARG(0)) {
4942
0
        temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
4943
0
    } else if (IS_DEAD_ARG(0)) {
4944
0
        temp_dead(s, ots);
4945
0
    }
4946
0
}
4947
4948
/*
4949
 * Specialized code generation for INDEX_op_mov_*.
4950
 */
4951
static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
4952
0
{
4953
0
    const TCGLifeData arg_life = op->life;
4954
0
    TCGRegSet allocated_regs, preferred_regs;
4955
0
    TCGTemp *ts, *ots;
4956
0
    TCGType otype, itype;
4957
0
    TCGReg oreg, ireg;
4958
4959
0
    allocated_regs = s->reserved_regs;
4960
0
    preferred_regs = output_pref(op, 0);
4961
0
    ots = arg_temp(op->args[0]);
4962
0
    ts = arg_temp(op->args[1]);
4963
4964
    /* ENV should not be modified.  */
4965
0
    tcg_debug_assert(!temp_readonly(ots));
4966
4967
    /* Note that otype != itype for no-op truncation.  */
4968
0
    otype = ots->type;
4969
0
    itype = ts->type;
4970
4971
0
    if (ts->val_type == TEMP_VAL_CONST) {
4972
        /* propagate constant or generate sti */
4973
0
        tcg_target_ulong val = ts->val;
4974
0
        if (IS_DEAD_ARG(1)) {
4975
0
            temp_dead(s, ts);
4976
0
        }
4977
0
        tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
4978
0
        return;
4979
0
    }
4980
4981
    /* If the source value is in memory we're going to be forced
4982
       to have it in a register in order to perform the copy.  Copy
4983
       the SOURCE value into its own register first, that way we
4984
       don't have to reload SOURCE the next time it is used. */
4985
0
    if (ts->val_type == TEMP_VAL_MEM) {
4986
0
        temp_load(s, ts, tcg_target_available_regs[itype],
4987
0
                  allocated_regs, preferred_regs);
4988
0
    }
4989
0
    tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
4990
0
    ireg = ts->reg;
4991
4992
0
    if (IS_DEAD_ARG(0)) {
4993
        /* mov to a non-saved dead register makes no sense (even with
4994
           liveness analysis disabled). */
4995
0
        tcg_debug_assert(NEED_SYNC_ARG(0));
4996
0
        if (!ots->mem_allocated) {
4997
0
            temp_allocate_frame(s, ots);
4998
0
        }
4999
0
        tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
5000
0
        if (IS_DEAD_ARG(1)) {
5001
0
            temp_dead(s, ts);
5002
0
        }
5003
0
        temp_dead(s, ots);
5004
0
        return;
5005
0
    }
5006
5007
0
    if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
5008
        /*
5009
         * The mov can be suppressed.  Kill input first, so that it
5010
         * is unlinked from reg_to_temp, then set the output to the
5011
         * reg that we saved from the input.
5012
         */
5013
0
        temp_dead(s, ts);
5014
0
        oreg = ireg;
5015
0
    } else {
5016
0
        if (ots->val_type == TEMP_VAL_REG) {
5017
0
            oreg = ots->reg;
5018
0
        } else {
5019
            /* Make sure to not spill the input register during allocation. */
5020
0
            oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
5021
0
                                 allocated_regs | ((TCGRegSet)1 << ireg),
5022
0
                                 preferred_regs, ots->indirect_base);
5023
0
        }
5024
0
        if (!tcg_out_mov(s, otype, oreg, ireg)) {
5025
            /*
5026
             * Cross register class move not supported.
5027
             * Store the source register into the destination slot
5028
             * and leave the destination temp as TEMP_VAL_MEM.
5029
             */
5030
0
            assert(!temp_readonly(ots));
5031
0
            if (!ts->mem_allocated) {
5032
0
                temp_allocate_frame(s, ots);
5033
0
            }
5034
0
            tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
5035
0
            set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
5036
0
            ots->mem_coherent = 1;
5037
0
            return;
5038
0
        }
5039
0
    }
5040
0
    set_temp_val_reg(s, ots, oreg);
5041
0
    ots->mem_coherent = 0;
5042
5043
0
    if (NEED_SYNC_ARG(0)) {
5044
0
        temp_sync(s, ots, allocated_regs, 0, 0);
5045
0
    }
5046
0
}
5047
5048
/*
5049
 * Specialized code generation for INDEX_op_dup_vec.
5050
 */
5051
static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
5052
0
{
5053
0
    const TCGLifeData arg_life = op->life;
5054
0
    TCGRegSet dup_out_regs, dup_in_regs;
5055
0
    const TCGArgConstraint *dup_args_ct;
5056
0
    TCGTemp *its, *ots;
5057
0
    TCGType itype, vtype;
5058
0
    unsigned vece;
5059
0
    int lowpart_ofs;
5060
0
    bool ok;
5061
5062
0
    ots = arg_temp(op->args[0]);
5063
0
    its = arg_temp(op->args[1]);
5064
5065
    /* ENV should not be modified.  */
5066
0
    tcg_debug_assert(!temp_readonly(ots));
5067
5068
0
    itype = its->type;
5069
0
    vece = TCGOP_VECE(op);
5070
0
    vtype = TCGOP_TYPE(op);
5071
5072
0
    if (its->val_type == TEMP_VAL_CONST) {
5073
        /* Propagate constant via movi -> dupi.  */
5074
0
        tcg_target_ulong val = dup_const(vece, its->val);
5075
0
        if (IS_DEAD_ARG(1)) {
5076
0
            temp_dead(s, its);
5077
0
        }
5078
0
        tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
5079
0
        return;
5080
0
    }
5081
5082
0
    dup_args_ct = opcode_args_ct(op);
5083
0
    dup_out_regs = dup_args_ct[0].regs;
5084
0
    dup_in_regs = dup_args_ct[1].regs;
5085
5086
    /* Allocate the output register now.  */
5087
0
    if (ots->val_type != TEMP_VAL_REG) {
5088
0
        TCGRegSet allocated_regs = s->reserved_regs;
5089
0
        TCGReg oreg;
5090
5091
0
        if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
5092
            /* Make sure to not spill the input register. */
5093
0
            tcg_regset_set_reg(allocated_regs, its->reg);
5094
0
        }
5095
0
        oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
5096
0
                             output_pref(op, 0), ots->indirect_base);
5097
0
        set_temp_val_reg(s, ots, oreg);
5098
0
    }
5099
5100
0
    switch (its->val_type) {
5101
0
    case TEMP_VAL_REG:
5102
        /*
5103
         * The dup constriaints must be broad, covering all possible VECE.
5104
         * However, tcg_op_dup_vec() gets to see the VECE and we allow it
5105
         * to fail, indicating that extra moves are required for that case.
5106
         */
5107
0
        if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
5108
0
            if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
5109
0
                goto done;
5110
0
            }
5111
            /* Try again from memory or a vector input register.  */
5112
0
        }
5113
0
        if (!its->mem_coherent) {
5114
            /*
5115
             * The input register is not synced, and so an extra store
5116
             * would be required to use memory.  Attempt an integer-vector
5117
             * register move first.  We do not have a TCGRegSet for this.
5118
             */
5119
0
            if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
5120
0
                break;
5121
0
            }
5122
            /* Sync the temp back to its slot and load from there.  */
5123
0
            temp_sync(s, its, s->reserved_regs, 0, 0);
5124
0
        }
5125
        /* fall through */
5126
5127
0
    case TEMP_VAL_MEM:
5128
0
        lowpart_ofs = 0;
5129
0
        if (HOST_BIG_ENDIAN) {
5130
0
            lowpart_ofs = tcg_type_size(itype) - (1 << vece);
5131
0
        }
5132
0
        if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
5133
0
                             its->mem_offset + lowpart_ofs)) {
5134
0
            goto done;
5135
0
        }
5136
        /* Load the input into the destination vector register. */
5137
0
        tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
5138
0
        break;
5139
5140
0
    default:
5141
0
        g_assert_not_reached();
5142
0
    }
5143
5144
    /* We now have a vector input register, so dup must succeed. */
5145
0
    ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
5146
0
    tcg_debug_assert(ok);
5147
5148
0
 done:
5149
0
    ots->mem_coherent = 0;
5150
0
    if (IS_DEAD_ARG(1)) {
5151
0
        temp_dead(s, its);
5152
0
    }
5153
0
    if (NEED_SYNC_ARG(0)) {
5154
0
        temp_sync(s, ots, s->reserved_regs, 0, 0);
5155
0
    }
5156
0
    if (IS_DEAD_ARG(0)) {
5157
0
        temp_dead(s, ots);
5158
0
    }
5159
0
}
5160
5161
static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
5162
0
{
5163
0
    const TCGLifeData arg_life = op->life;
5164
0
    const TCGOpDef * const def = &tcg_op_defs[op->opc];
5165
0
    TCGRegSet i_allocated_regs;
5166
0
    TCGRegSet o_allocated_regs;
5167
0
    int i, k, nb_iargs, nb_oargs;
5168
0
    TCGReg reg;
5169
0
    TCGArg arg;
5170
0
    const TCGArgConstraint *args_ct;
5171
0
    const TCGArgConstraint *arg_ct;
5172
0
    TCGTemp *ts;
5173
0
    TCGArg new_args[TCG_MAX_OP_ARGS];
5174
0
    int const_args[TCG_MAX_OP_ARGS];
5175
0
    TCGCond op_cond;
5176
5177
0
    if (def->flags & TCG_OPF_CARRY_IN) {
5178
0
        tcg_debug_assert(s->carry_live);
5179
0
    }
5180
5181
0
    nb_oargs = def->nb_oargs;
5182
0
    nb_iargs = def->nb_iargs;
5183
5184
    /* copy constants */
5185
0
    memcpy(new_args + nb_oargs + nb_iargs,
5186
0
           op->args + nb_oargs + nb_iargs,
5187
0
           sizeof(TCGArg) * def->nb_cargs);
5188
5189
0
    i_allocated_regs = s->reserved_regs;
5190
0
    o_allocated_regs = s->reserved_regs;
5191
5192
0
    switch (op->opc) {
5193
0
    case INDEX_op_brcond:
5194
0
        op_cond = op->args[2];
5195
0
        break;
5196
0
    case INDEX_op_setcond:
5197
0
    case INDEX_op_negsetcond:
5198
0
    case INDEX_op_cmp_vec:
5199
0
        op_cond = op->args[3];
5200
0
        break;
5201
0
    case INDEX_op_movcond:
5202
0
    case INDEX_op_cmpsel_vec:
5203
0
        op_cond = op->args[5];
5204
0
        break;
5205
0
    default:
5206
        /* No condition within opcode. */
5207
0
        op_cond = TCG_COND_ALWAYS;
5208
0
        break;
5209
0
    }
5210
5211
0
    args_ct = opcode_args_ct(op);
5212
5213
    /* satisfy input constraints */
5214
0
    for (k = 0; k < nb_iargs; k++) {
5215
0
        TCGRegSet i_preferred_regs, i_required_regs;
5216
0
        bool allocate_new_reg, copyto_new_reg;
5217
0
        TCGTemp *ts2;
5218
0
        int i1, i2;
5219
5220
0
        i = args_ct[nb_oargs + k].sort_index;
5221
0
        arg = op->args[i];
5222
0
        arg_ct = &args_ct[i];
5223
0
        ts = arg_temp(arg);
5224
5225
0
        if (ts->val_type == TEMP_VAL_CONST) {
5226
0
#ifdef TCG_REG_ZERO
5227
0
            if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) {
5228
                /* Hardware zero register: indicate register via non-const. */
5229
0
                const_args[i] = 0;
5230
0
                new_args[i] = TCG_REG_ZERO;
5231
0
                continue;
5232
0
            }
5233
0
#endif
5234
5235
0
            if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type,
5236
0
                                       op_cond, TCGOP_VECE(op))) {
5237
                /* constant is OK for instruction */
5238
0
                const_args[i] = 1;
5239
0
                new_args[i] = ts->val;
5240
0
                continue;
5241
0
            }
5242
0
        }
5243
5244
0
        reg = ts->reg;
5245
0
        i_preferred_regs = 0;
5246
0
        i_required_regs = arg_ct->regs;
5247
0
        allocate_new_reg = false;
5248
0
        copyto_new_reg = false;
5249
5250
0
        switch (arg_ct->pair) {
5251
0
        case 0: /* not paired */
5252
0
            if (arg_ct->ialias) {
5253
0
                i_preferred_regs = output_pref(op, arg_ct->alias_index);
5254
5255
                /*
5256
                 * If the input is readonly, then it cannot also be an
5257
                 * output and aliased to itself.  If the input is not
5258
                 * dead after the instruction, we must allocate a new
5259
                 * register and move it.
5260
                 */
5261
0
                if (temp_readonly(ts) || !IS_DEAD_ARG(i)
5262
0
                    || args_ct[arg_ct->alias_index].newreg) {
5263
0
                    allocate_new_reg = true;
5264
0
                } else if (ts->val_type == TEMP_VAL_REG) {
5265
                    /*
5266
                     * Check if the current register has already been
5267
                     * allocated for another input.
5268
                     */
5269
0
                    allocate_new_reg =
5270
0
                        tcg_regset_test_reg(i_allocated_regs, reg);
5271
0
                }
5272
0
            }
5273
0
            if (!allocate_new_reg) {
5274
0
                temp_load(s, ts, i_required_regs, i_allocated_regs,
5275
0
                          i_preferred_regs);
5276
0
                reg = ts->reg;
5277
0
                allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
5278
0
            }
5279
0
            if (allocate_new_reg) {
5280
                /*
5281
                 * Allocate a new register matching the constraint
5282
                 * and move the temporary register into it.
5283
                 */
5284
0
                temp_load(s, ts, tcg_target_available_regs[ts->type],
5285
0
                          i_allocated_regs, 0);
5286
0
                reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
5287
0
                                    i_preferred_regs, ts->indirect_base);
5288
0
                copyto_new_reg = true;
5289
0
            }
5290
0
            break;
5291
5292
0
        case 1:
5293
            /* First of an input pair; if i1 == i2, the second is an output. */
5294
0
            i1 = i;
5295
0
            i2 = arg_ct->pair_index;
5296
0
            ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
5297
5298
            /*
5299
             * It is easier to default to allocating a new pair
5300
             * and to identify a few cases where it's not required.
5301
             */
5302
0
            if (arg_ct->ialias) {
5303
0
                i_preferred_regs = output_pref(op, arg_ct->alias_index);
5304
0
                if (IS_DEAD_ARG(i1) &&
5305
0
                    IS_DEAD_ARG(i2) &&
5306
0
                    !temp_readonly(ts) &&
5307
0
                    ts->val_type == TEMP_VAL_REG &&
5308
0
                    ts->reg < TCG_TARGET_NB_REGS - 1 &&
5309
0
                    tcg_regset_test_reg(i_required_regs, reg) &&
5310
0
                    !tcg_regset_test_reg(i_allocated_regs, reg) &&
5311
0
                    !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
5312
0
                    (ts2
5313
0
                     ? ts2->val_type == TEMP_VAL_REG &&
5314
0
                       ts2->reg == reg + 1 &&
5315
0
                       !temp_readonly(ts2)
5316
0
                     : s->reg_to_temp[reg + 1] == NULL)) {
5317
0
                    break;
5318
0
                }
5319
0
            } else {
5320
                /* Without aliasing, the pair must also be an input. */
5321
0
                tcg_debug_assert(ts2);
5322
0
                if (ts->val_type == TEMP_VAL_REG &&
5323
0
                    ts2->val_type == TEMP_VAL_REG &&
5324
0
                    ts2->reg == reg + 1 &&
5325
0
                    tcg_regset_test_reg(i_required_regs, reg)) {
5326
0
                    break;
5327
0
                }
5328
0
            }
5329
0
            reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
5330
0
                                     0, ts->indirect_base);
5331
0
            goto do_pair;
5332
5333
0
        case 2: /* pair second */
5334
0
            reg = new_args[arg_ct->pair_index] + 1;
5335
0
            goto do_pair;
5336
5337
0
        case 3: /* ialias with second output, no first input */
5338
0
            tcg_debug_assert(arg_ct->ialias);
5339
0
            i_preferred_regs = output_pref(op, arg_ct->alias_index);
5340
5341
0
            if (IS_DEAD_ARG(i) &&
5342
0
                !temp_readonly(ts) &&
5343
0
                ts->val_type == TEMP_VAL_REG &&
5344
0
                reg > 0 &&
5345
0
                s->reg_to_temp[reg - 1] == NULL &&
5346
0
                tcg_regset_test_reg(i_required_regs, reg) &&
5347
0
                !tcg_regset_test_reg(i_allocated_regs, reg) &&
5348
0
                !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
5349
0
                tcg_regset_set_reg(i_allocated_regs, reg - 1);
5350
0
                break;
5351
0
            }
5352
0
            reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
5353
0
                                     i_allocated_regs, 0,
5354
0
                                     ts->indirect_base);
5355
0
            tcg_regset_set_reg(i_allocated_regs, reg);
5356
0
            reg += 1;
5357
0
            goto do_pair;
5358
5359
0
        do_pair:
5360
            /*
5361
             * If an aliased input is not dead after the instruction,
5362
             * we must allocate a new register and move it.
5363
             */
5364
0
            if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
5365
0
                TCGRegSet t_allocated_regs = i_allocated_regs;
5366
5367
                /*
5368
                 * Because of the alias, and the continued life, make sure
5369
                 * that the temp is somewhere *other* than the reg pair,
5370
                 * and we get a copy in reg.
5371
                 */
5372
0
                tcg_regset_set_reg(t_allocated_regs, reg);
5373
0
                tcg_regset_set_reg(t_allocated_regs, reg + 1);
5374
0
                if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
5375
                    /* If ts was already in reg, copy it somewhere else. */
5376
0
                    TCGReg nr;
5377
0
                    bool ok;
5378
5379
0
                    tcg_debug_assert(ts->kind != TEMP_FIXED);
5380
0
                    nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
5381
0
                                       t_allocated_regs, 0, ts->indirect_base);
5382
0
                    ok = tcg_out_mov(s, ts->type, nr, reg);
5383
0
                    tcg_debug_assert(ok);
5384
5385
0
                    set_temp_val_reg(s, ts, nr);
5386
0
                } else {
5387
0
                    temp_load(s, ts, tcg_target_available_regs[ts->type],
5388
0
                              t_allocated_regs, 0);
5389
0
                    copyto_new_reg = true;
5390
0
                }
5391
0
            } else {
5392
                /* Preferably allocate to reg, otherwise copy. */
5393
0
                i_required_regs = (TCGRegSet)1 << reg;
5394
0
                temp_load(s, ts, i_required_regs, i_allocated_regs,
5395
0
                          i_preferred_regs);
5396
0
                copyto_new_reg = ts->reg != reg;
5397
0
            }
5398
0
            break;
5399
5400
0
        default:
5401
0
            g_assert_not_reached();
5402
0
        }
5403
5404
0
        if (copyto_new_reg) {
5405
0
            if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5406
                /*
5407
                 * Cross register class move not supported.  Sync the
5408
                 * temp back to its slot and load from there.
5409
                 */
5410
0
                temp_sync(s, ts, i_allocated_regs, 0, 0);
5411
0
                tcg_out_ld(s, ts->type, reg,
5412
0
                           ts->mem_base->reg, ts->mem_offset);
5413
0
            }
5414
0
        }
5415
0
        new_args[i] = reg;
5416
0
        const_args[i] = 0;
5417
0
        tcg_regset_set_reg(i_allocated_regs, reg);
5418
0
    }
5419
5420
    /* mark dead temporaries and free the associated registers */
5421
0
    for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
5422
0
        if (IS_DEAD_ARG(i)) {
5423
0
            temp_dead(s, arg_temp(op->args[i]));
5424
0
        }
5425
0
    }
5426
5427
0
    if (def->flags & TCG_OPF_COND_BRANCH) {
5428
0
        tcg_reg_alloc_cbranch(s, i_allocated_regs);
5429
0
    } else if (def->flags & TCG_OPF_BB_END) {
5430
0
        tcg_reg_alloc_bb_end(s, i_allocated_regs);
5431
0
    } else {
5432
0
        if (def->flags & TCG_OPF_CALL_CLOBBER) {
5433
0
            assert_carry_dead(s);
5434
            /* XXX: permit generic clobber register list ? */
5435
0
            for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5436
0
                if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
5437
0
                    tcg_reg_free(s, i, i_allocated_regs);
5438
0
                }
5439
0
            }
5440
0
        }
5441
0
        if (def->flags & TCG_OPF_SIDE_EFFECTS) {
5442
            /* sync globals if the op has side effects and might trigger
5443
               an exception. */
5444
0
            sync_globals(s, i_allocated_regs);
5445
0
        }
5446
5447
        /* satisfy the output constraints */
5448
0
        for (k = 0; k < nb_oargs; k++) {
5449
0
            i = args_ct[k].sort_index;
5450
0
            arg = op->args[i];
5451
0
            arg_ct = &args_ct[i];
5452
0
            ts = arg_temp(arg);
5453
5454
            /* ENV should not be modified.  */
5455
0
            tcg_debug_assert(!temp_readonly(ts));
5456
5457
0
            switch (arg_ct->pair) {
5458
0
            case 0: /* not paired */
5459
0
                if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
5460
0
                    reg = new_args[arg_ct->alias_index];
5461
0
                } else if (arg_ct->newreg) {
5462
0
                    reg = tcg_reg_alloc(s, arg_ct->regs,
5463
0
                                        i_allocated_regs | o_allocated_regs,
5464
0
                                        output_pref(op, k), ts->indirect_base);
5465
0
                } else {
5466
0
                    reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
5467
0
                                        output_pref(op, k), ts->indirect_base);
5468
0
                }
5469
0
                break;
5470
5471
0
            case 1: /* first of pair */
5472
0
                if (arg_ct->oalias) {
5473
0
                    reg = new_args[arg_ct->alias_index];
5474
0
                } else if (arg_ct->newreg) {
5475
0
                    reg = tcg_reg_alloc_pair(s, arg_ct->regs,
5476
0
                                             i_allocated_regs | o_allocated_regs,
5477
0
                                             output_pref(op, k),
5478
0
                                             ts->indirect_base);
5479
0
                } else {
5480
0
                    reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
5481
0
                                             output_pref(op, k),
5482
0
                                             ts->indirect_base);
5483
0
                }
5484
0
                break;
5485
5486
0
            case 2: /* second of pair */
5487
0
                if (arg_ct->oalias) {
5488
0
                    reg = new_args[arg_ct->alias_index];
5489
0
                } else {
5490
0
                    reg = new_args[arg_ct->pair_index] + 1;
5491
0
                }
5492
0
                break;
5493
5494
0
            case 3: /* first of pair, aliasing with a second input */
5495
0
                tcg_debug_assert(!arg_ct->newreg);
5496
0
                reg = new_args[arg_ct->pair_index] - 1;
5497
0
                break;
5498
5499
0
            default:
5500
0
                g_assert_not_reached();
5501
0
            }
5502
0
            tcg_regset_set_reg(o_allocated_regs, reg);
5503
0
            set_temp_val_reg(s, ts, reg);
5504
0
            ts->mem_coherent = 0;
5505
0
            new_args[i] = reg;
5506
0
        }
5507
0
    }
5508
5509
    /* emit instruction */
5510
0
    TCGType type = TCGOP_TYPE(op);
5511
0
    switch (op->opc) {
5512
0
    case INDEX_op_addc1o:
5513
0
        tcg_out_set_carry(s);
5514
        /* fall through */
5515
0
    case INDEX_op_add:
5516
0
    case INDEX_op_addcio:
5517
0
    case INDEX_op_addco:
5518
0
    case INDEX_op_and:
5519
0
    case INDEX_op_andc:
5520
0
    case INDEX_op_clz:
5521
0
    case INDEX_op_ctz:
5522
0
    case INDEX_op_divs:
5523
0
    case INDEX_op_divu:
5524
0
    case INDEX_op_eqv:
5525
0
    case INDEX_op_mul:
5526
0
    case INDEX_op_mulsh:
5527
0
    case INDEX_op_muluh:
5528
0
    case INDEX_op_nand:
5529
0
    case INDEX_op_nor:
5530
0
    case INDEX_op_or:
5531
0
    case INDEX_op_orc:
5532
0
    case INDEX_op_rems:
5533
0
    case INDEX_op_remu:
5534
0
    case INDEX_op_rotl:
5535
0
    case INDEX_op_rotr:
5536
0
    case INDEX_op_sar:
5537
0
    case INDEX_op_shl:
5538
0
    case INDEX_op_shr:
5539
0
    case INDEX_op_smax:
5540
0
    case INDEX_op_smin:
5541
0
    case INDEX_op_umax:
5542
0
    case INDEX_op_umin:
5543
0
    case INDEX_op_xor:
5544
0
        {
5545
0
            const TCGOutOpBinary *out =
5546
0
                container_of(all_outop[op->opc], TCGOutOpBinary, base);
5547
5548
            /* Constants should never appear in the first source operand. */
5549
0
            tcg_debug_assert(!const_args[1]);
5550
0
            if (const_args[2]) {
5551
0
                out->out_rri(s, type, new_args[0], new_args[1], new_args[2]);
5552
0
            } else {
5553
0
                out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5554
0
            }
5555
0
        }
5556
0
        break;
5557
5558
0
    case INDEX_op_sub:
5559
0
        {
5560
0
            const TCGOutOpSubtract *out = &outop_sub;
5561
5562
            /*
5563
             * Constants should never appear in the second source operand.
5564
             * These are folded to add with negative constant.
5565
             */
5566
0
            tcg_debug_assert(!const_args[2]);
5567
0
            if (const_args[1]) {
5568
0
                out->out_rir(s, type, new_args[0], new_args[1], new_args[2]);
5569
0
            } else {
5570
0
                out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5571
0
            }
5572
0
        }
5573
0
        break;
5574
5575
0
    case INDEX_op_subb1o:
5576
0
        tcg_out_set_borrow(s);
5577
        /* fall through */
5578
0
    case INDEX_op_addci:
5579
0
    case INDEX_op_subbi:
5580
0
    case INDEX_op_subbio:
5581
0
    case INDEX_op_subbo:
5582
0
        {
5583
0
            const TCGOutOpAddSubCarry *out =
5584
0
                container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base);
5585
5586
0
            if (const_args[2]) {
5587
0
                if (const_args[1]) {
5588
0
                    out->out_rii(s, type, new_args[0],
5589
0
                                 new_args[1], new_args[2]);
5590
0
                } else {
5591
0
                    out->out_rri(s, type, new_args[0],
5592
0
                                 new_args[1], new_args[2]);
5593
0
                }
5594
0
            } else if (const_args[1]) {
5595
0
                out->out_rir(s, type, new_args[0], new_args[1], new_args[2]);
5596
0
            } else {
5597
0
                out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5598
0
            }
5599
0
        }
5600
0
        break;
5601
5602
0
    case INDEX_op_bswap64:
5603
0
    case INDEX_op_ext_i32_i64:
5604
0
    case INDEX_op_extu_i32_i64:
5605
0
    case INDEX_op_extrl_i64_i32:
5606
0
    case INDEX_op_extrh_i64_i32:
5607
0
    case INDEX_op_ctpop:
5608
0
    case INDEX_op_neg:
5609
0
    case INDEX_op_not:
5610
0
    case INDEX_op_revbit8:
5611
0
    case INDEX_op_revbit64:
5612
0
        {
5613
0
            const TCGOutOpUnary *out =
5614
0
                container_of(all_outop[op->opc], TCGOutOpUnary, base);
5615
5616
            /* Constants should have been folded. */
5617
0
            tcg_debug_assert(!const_args[1]);
5618
0
            out->out_rr(s, type, new_args[0], new_args[1]);
5619
0
        }
5620
0
        break;
5621
5622
0
    case INDEX_op_bswap16:
5623
0
    case INDEX_op_bswap32:
5624
0
    case INDEX_op_revbit32:
5625
0
        {
5626
0
            const TCGOutOpBswap *out =
5627
0
                container_of(all_outop[op->opc], TCGOutOpBswap, base);
5628
5629
0
            tcg_debug_assert(!const_args[1]);
5630
0
            out->out_rr(s, type, new_args[0], new_args[1], new_args[2]);
5631
0
        }
5632
0
        break;
5633
5634
0
    case INDEX_op_deposit:
5635
0
        {
5636
0
            const TCGOutOpDeposit *out = &outop_deposit;
5637
5638
0
            if (const_args[2]) {
5639
0
                tcg_debug_assert(!const_args[1]);
5640
0
                out->out_rri(s, type, new_args[0], new_args[1],
5641
0
                             new_args[2], new_args[3], new_args[4]);
5642
0
            } else if (const_args[1]) {
5643
0
                tcg_debug_assert(new_args[1] == 0);
5644
0
                tcg_debug_assert(!const_args[2]);
5645
0
                out->out_rzr(s, type, new_args[0], new_args[2],
5646
0
                             new_args[3], new_args[4]);
5647
0
            } else {
5648
0
                out->out_rrr(s, type, new_args[0], new_args[1],
5649
0
                             new_args[2], new_args[3], new_args[4]);
5650
0
            }
5651
0
        }
5652
0
        break;
5653
5654
0
    case INDEX_op_divs2:
5655
0
    case INDEX_op_divu2:
5656
0
        {
5657
0
            const TCGOutOpDivRem *out =
5658
0
                container_of(all_outop[op->opc], TCGOutOpDivRem, base);
5659
5660
            /* Only used by x86 and s390x, which use matching constraints. */
5661
0
            tcg_debug_assert(new_args[0] == new_args[2]);
5662
0
            tcg_debug_assert(new_args[1] == new_args[3]);
5663
0
            tcg_debug_assert(!const_args[4]);
5664
0
            out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]);
5665
0
        }
5666
0
        break;
5667
5668
0
    case INDEX_op_extract:
5669
0
    case INDEX_op_sextract:
5670
0
        {
5671
0
            const TCGOutOpExtract *out =
5672
0
                container_of(all_outop[op->opc], TCGOutOpExtract, base);
5673
5674
0
            tcg_debug_assert(!const_args[1]);
5675
0
            out->out_rr(s, type, new_args[0], new_args[1],
5676
0
                        new_args[2], new_args[3]);
5677
0
        }
5678
0
        break;
5679
5680
0
    case INDEX_op_extract2:
5681
0
        {
5682
0
            const TCGOutOpExtract2 *out = &outop_extract2;
5683
5684
0
            tcg_debug_assert(!const_args[1]);
5685
0
            tcg_debug_assert(!const_args[2]);
5686
0
            out->out_rrr(s, type, new_args[0], new_args[1],
5687
0
                         new_args[2], new_args[3]);
5688
0
        }
5689
0
        break;
5690
5691
0
    case INDEX_op_ld8u:
5692
0
    case INDEX_op_ld8s:
5693
0
    case INDEX_op_ld16u:
5694
0
    case INDEX_op_ld16s:
5695
0
    case INDEX_op_ld32u:
5696
0
    case INDEX_op_ld32s:
5697
0
    case INDEX_op_ld:
5698
0
        {
5699
0
            const TCGOutOpLoad *out =
5700
0
                container_of(all_outop[op->opc], TCGOutOpLoad, base);
5701
5702
0
            tcg_debug_assert(!const_args[1]);
5703
0
            out->out(s, type, new_args[0], new_args[1], new_args[2]);
5704
0
        }
5705
0
        break;
5706
5707
0
    case INDEX_op_muls2:
5708
0
    case INDEX_op_mulu2:
5709
0
        {
5710
0
            const TCGOutOpMul2 *out =
5711
0
                container_of(all_outop[op->opc], TCGOutOpMul2, base);
5712
5713
0
            tcg_debug_assert(!const_args[2]);
5714
0
            tcg_debug_assert(!const_args[3]);
5715
0
            out->out_rrrr(s, type, new_args[0], new_args[1],
5716
0
                          new_args[2], new_args[3]);
5717
0
        }
5718
0
        break;
5719
5720
0
    case INDEX_op_st32:
5721
        /* Use tcg_op_st w/ I32. */
5722
0
        type = TCG_TYPE_I32;
5723
        /* fall through */
5724
0
    case INDEX_op_st:
5725
0
    case INDEX_op_st8:
5726
0
    case INDEX_op_st16:
5727
0
        {
5728
0
            const TCGOutOpStore *out =
5729
0
                container_of(all_outop[op->opc], TCGOutOpStore, base);
5730
5731
0
            if (const_args[0]) {
5732
0
                out->out_i(s, type, new_args[0], new_args[1], new_args[2]);
5733
0
            } else {
5734
0
                out->out_r(s, type, new_args[0], new_args[1], new_args[2]);
5735
0
            }
5736
0
        }
5737
0
        break;
5738
5739
0
    case INDEX_op_qemu_ld:
5740
0
    case INDEX_op_qemu_st:
5741
0
        {
5742
0
            const TCGOutOpQemuLdSt *out =
5743
0
                container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base);
5744
5745
0
            out->out(s, type, new_args[0], new_args[1], new_args[2]);
5746
0
        }
5747
0
        break;
5748
5749
0
    case INDEX_op_qemu_ld2:
5750
0
    case INDEX_op_qemu_st2:
5751
0
        {
5752
0
            const TCGOutOpQemuLdSt2 *out =
5753
0
                container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base);
5754
5755
0
            out->out(s, type, new_args[0], new_args[1],
5756
0
                     new_args[2], new_args[3]);
5757
0
        }
5758
0
        break;
5759
5760
0
    case INDEX_op_brcond:
5761
0
        {
5762
0
            const TCGOutOpBrcond *out = &outop_brcond;
5763
0
            TCGCond cond = new_args[2];
5764
0
            TCGLabel *label = arg_label(new_args[3]);
5765
5766
0
            tcg_debug_assert(!const_args[0]);
5767
0
            if (const_args[1]) {
5768
0
                out->out_ri(s, type, cond, new_args[0], new_args[1], label);
5769
0
            } else {
5770
0
                out->out_rr(s, type, cond, new_args[0], new_args[1], label);
5771
0
            }
5772
0
        }
5773
0
        break;
5774
5775
0
    case INDEX_op_movcond:
5776
0
        {
5777
0
            const TCGOutOpMovcond *out = &outop_movcond;
5778
0
            TCGCond cond = new_args[5];
5779
5780
0
            tcg_debug_assert(!const_args[1]);
5781
0
            out->out(s, type, cond, new_args[0],
5782
0
                     new_args[1], new_args[2], const_args[2],
5783
0
                     new_args[3], const_args[3],
5784
0
                     new_args[4], const_args[4]);
5785
0
        }
5786
0
        break;
5787
5788
0
    case INDEX_op_setcond:
5789
0
    case INDEX_op_negsetcond:
5790
0
        {
5791
0
            const TCGOutOpSetcond *out =
5792
0
                container_of(all_outop[op->opc], TCGOutOpSetcond, base);
5793
0
            TCGCond cond = new_args[3];
5794
5795
0
            tcg_debug_assert(!const_args[1]);
5796
0
            if (const_args[2]) {
5797
0
                out->out_rri(s, type, cond,
5798
0
                             new_args[0], new_args[1], new_args[2]);
5799
0
            } else {
5800
0
                out->out_rrr(s, type, cond,
5801
0
                             new_args[0], new_args[1], new_args[2]);
5802
0
            }
5803
0
        }
5804
0
        break;
5805
5806
0
    case INDEX_op_lea:
5807
0
        {
5808
0
            const TCGOutOpLea *out = &outop_lea;
5809
0
            int sh = new_args[3];
5810
0
            tcg_target_long imm = new_args[4];
5811
5812
0
            tcg_debug_assert(!const_args[1]);
5813
0
            tcg_debug_assert(!const_args[2]);
5814
0
            tcg_debug_assert(TCG_TARGET_lea_sh_valid(type, sh));
5815
0
            tcg_debug_assert(TCG_TARGET_lea_imm_valid(type, imm));
5816
5817
0
            out->out(s, type, new_args[0], new_args[1], new_args[2], sh, imm);
5818
0
        }
5819
0
        break;
5820
5821
0
    case INDEX_op_goto_ptr:
5822
0
        tcg_debug_assert(!const_args[0]);
5823
0
        tcg_out_goto_ptr(s, new_args[0]);
5824
0
        break;
5825
5826
0
    default:
5827
0
        tcg_debug_assert(def->flags & TCG_OPF_VECTOR);
5828
0
        tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64,
5829
0
                       TCGOP_VECE(op), new_args, const_args);
5830
0
        break;
5831
0
    }
5832
5833
0
    if (def->flags & TCG_OPF_CARRY_IN) {
5834
0
        s->carry_live = false;
5835
0
    }
5836
0
    if (def->flags & TCG_OPF_CARRY_OUT) {
5837
0
        s->carry_live = true;
5838
0
    }
5839
5840
    /* move the outputs in the correct register if needed */
5841
0
    for(i = 0; i < nb_oargs; i++) {
5842
0
        ts = arg_temp(op->args[i]);
5843
5844
        /* ENV should not be modified.  */
5845
0
        tcg_debug_assert(!temp_readonly(ts));
5846
5847
0
        if (NEED_SYNC_ARG(i)) {
5848
0
            temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
5849
0
        } else if (IS_DEAD_ARG(i)) {
5850
0
            temp_dead(s, ts);
5851
0
        }
5852
0
    }
5853
0
}
5854
5855
static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
5856
                         TCGRegSet allocated_regs)
5857
0
{
5858
0
    if (ts->val_type == TEMP_VAL_REG) {
5859
0
        if (ts->reg != reg) {
5860
0
            tcg_reg_free(s, reg, allocated_regs);
5861
0
            if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5862
                /*
5863
                 * Cross register class move not supported.  Sync the
5864
                 * temp back to its slot and load from there.
5865
                 */
5866
0
                temp_sync(s, ts, allocated_regs, 0, 0);
5867
0
                tcg_out_ld(s, ts->type, reg,
5868
0
                           ts->mem_base->reg, ts->mem_offset);
5869
0
            }
5870
0
        }
5871
0
    } else {
5872
0
        TCGRegSet arg_set = 0;
5873
5874
0
        tcg_reg_free(s, reg, allocated_regs);
5875
0
        tcg_regset_set_reg(arg_set, reg);
5876
0
        temp_load(s, ts, arg_set, allocated_regs, 0);
5877
0
    }
5878
0
}
5879
5880
static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts,
5881
                         TCGRegSet allocated_regs)
5882
0
{
5883
    /*
5884
     * When the destination is on the stack, load up the temp and store.
5885
     * If there are many call-saved registers, the temp might live to
5886
     * see another use; otherwise it'll be discarded.
5887
     */
5888
0
    temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
5889
0
    tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
5890
0
               arg_slot_stk_ofs(arg_slot));
5891
0
}
5892
5893
static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
5894
                            TCGTemp *ts, TCGRegSet *allocated_regs)
5895
0
{
5896
0
    if (arg_slot_reg_p(l->arg_slot)) {
5897
0
        TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
5898
0
        load_arg_reg(s, reg, ts, *allocated_regs);
5899
0
        tcg_regset_set_reg(*allocated_regs, reg);
5900
0
    } else {
5901
0
        load_arg_stk(s, l->arg_slot, ts, *allocated_regs);
5902
0
    }
5903
0
}
5904
5905
static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base,
5906
                         intptr_t ref_off, TCGRegSet *allocated_regs)
5907
0
{
5908
0
    TCGReg reg;
5909
5910
0
    if (arg_slot_reg_p(arg_slot)) {
5911
0
        reg = tcg_target_call_iarg_regs[arg_slot];
5912
0
        tcg_reg_free(s, reg, *allocated_regs);
5913
0
        tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5914
0
        tcg_regset_set_reg(*allocated_regs, reg);
5915
0
    } else {
5916
0
        reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
5917
0
                            *allocated_regs, 0, false);
5918
0
        tcg_out_addi_ptr(s, reg, ref_base, ref_off);
5919
0
        tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
5920
0
                   arg_slot_stk_ofs(arg_slot));
5921
0
    }
5922
0
}
5923
5924
static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
5925
0
{
5926
0
    const int nb_oargs = TCGOP_CALLO(op);
5927
0
    const int nb_iargs = TCGOP_CALLI(op);
5928
0
    const TCGLifeData arg_life = op->life;
5929
0
    const TCGHelperInfo *info = tcg_call_info(op);
5930
0
    TCGRegSet allocated_regs = s->reserved_regs;
5931
0
    int i;
5932
5933
    /*
5934
     * Move inputs into place in reverse order,
5935
     * so that we place stacked arguments first.
5936
     */
5937
0
    for (i = nb_iargs - 1; i >= 0; --i) {
5938
0
        const TCGCallArgumentLoc *loc = &info->in[i];
5939
0
        TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
5940
5941
0
        switch (loc->kind) {
5942
0
        case TCG_CALL_ARG_NORMAL:
5943
0
        case TCG_CALL_ARG_EXTEND_U:
5944
0
        case TCG_CALL_ARG_EXTEND_S:
5945
0
            load_arg_normal(s, loc, ts, &allocated_regs);
5946
0
            break;
5947
0
        case TCG_CALL_ARG_BY_REF:
5948
0
            load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5949
0
            load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
5950
0
                         arg_slot_stk_ofs(loc->ref_slot),
5951
0
                         &allocated_regs);
5952
0
            break;
5953
0
        case TCG_CALL_ARG_BY_REF_N:
5954
0
            load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
5955
0
            break;
5956
0
        default:
5957
0
            g_assert_not_reached();
5958
0
        }
5959
0
    }
5960
5961
    /* Mark dead temporaries and free the associated registers.  */
5962
0
    for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
5963
0
        if (IS_DEAD_ARG(i)) {
5964
0
            temp_dead(s, arg_temp(op->args[i]));
5965
0
        }
5966
0
    }
5967
5968
    /* Clobber call registers.  */
5969
0
    for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5970
0
        if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
5971
0
            tcg_reg_free(s, i, allocated_regs);
5972
0
        }
5973
0
    }
5974
5975
    /*
5976
     * Save globals if they might be written by the helper,
5977
     * sync them if they might be read.
5978
     */
5979
0
    if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
5980
        /* Nothing to do */
5981
0
    } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
5982
0
        sync_globals(s, allocated_regs);
5983
0
    } else {
5984
0
        save_globals(s, allocated_regs);
5985
0
    }
5986
5987
    /*
5988
     * If the ABI passes a pointer to the returned struct as the first
5989
     * argument, load that now.  Pass a pointer to the output home slot.
5990
     */
5991
0
    if (info->out_kind == TCG_CALL_RET_BY_REF) {
5992
0
        TCGTemp *ts = arg_temp(op->args[0]);
5993
5994
0
        if (!ts->mem_allocated) {
5995
0
            temp_allocate_frame(s, ts);
5996
0
        }
5997
0
        load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
5998
0
    }
5999
6000
0
    tcg_out_call(s, tcg_call_func(op), info);
6001
6002
    /* Assign output registers and emit moves if needed.  */
6003
0
    switch (info->out_kind) {
6004
0
    case TCG_CALL_RET_NORMAL:
6005
0
        for (i = 0; i < nb_oargs; i++) {
6006
0
            TCGTemp *ts = arg_temp(op->args[i]);
6007
0
            TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
6008
6009
            /* ENV should not be modified.  */
6010
0
            tcg_debug_assert(!temp_readonly(ts));
6011
6012
0
            set_temp_val_reg(s, ts, reg);
6013
0
            ts->mem_coherent = 0;
6014
0
        }
6015
0
        break;
6016
6017
0
    case TCG_CALL_RET_BY_VEC:
6018
0
        {
6019
0
            TCGTemp *ts = arg_temp(op->args[0]);
6020
6021
0
            tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
6022
0
            tcg_debug_assert(ts->temp_subindex == 0);
6023
0
            if (!ts->mem_allocated) {
6024
0
                temp_allocate_frame(s, ts);
6025
0
            }
6026
0
            tcg_out_st(s, TCG_TYPE_V128,
6027
0
                       tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
6028
0
                       ts->mem_base->reg, ts->mem_offset);
6029
0
        }
6030
        /* fall through to mark all parts in memory */
6031
6032
0
    case TCG_CALL_RET_BY_REF:
6033
        /* The callee has performed a write through the reference. */
6034
0
        for (i = 0; i < nb_oargs; i++) {
6035
0
            TCGTemp *ts = arg_temp(op->args[i]);
6036
0
            ts->val_type = TEMP_VAL_MEM;
6037
0
        }
6038
0
        break;
6039
6040
0
    default:
6041
0
        g_assert_not_reached();
6042
0
    }
6043
6044
    /* Flush or discard output registers as needed. */
6045
0
    for (i = 0; i < nb_oargs; i++) {
6046
0
        TCGTemp *ts = arg_temp(op->args[i]);
6047
0
        if (NEED_SYNC_ARG(i)) {
6048
0
            temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
6049
0
        } else if (IS_DEAD_ARG(i)) {
6050
0
            temp_dead(s, ts);
6051
0
        }
6052
0
    }
6053
0
}
6054
6055
/**
6056
 * atom_and_align_for_opc:
6057
 * @s: tcg context
6058
 * @opc: memory operation code
6059
 * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations
6060
 * @allow_two_ops: true if we are prepared to issue two operations
6061
 *
6062
 * Return the alignment and atomicity to use for the inline fast path
6063
 * for the given memory operation.  The alignment may be larger than
6064
 * that specified in @opc, and the correct alignment will be diagnosed
6065
 * by the slow path helper.
6066
 *
6067
 * If @allow_two_ops, the host is prepared to test for 2x alignment,
6068
 * and issue two loads or stores for subalignment.
6069
 */
6070
static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
6071
                                           MemOp host_atom, bool allow_two_ops)
6072
0
{
6073
0
    MemOp align = memop_alignment_bits(opc);
6074
0
    MemOp size = opc & MO_SIZE;
6075
0
    MemOp half = size ? size - 1 : 0;
6076
0
    MemOp atom = opc & MO_ATOM_MASK;
6077
0
    MemOp atmax;
6078
6079
0
    switch (atom) {
6080
0
    case MO_ATOM_NONE:
6081
        /* The operation requires no specific atomicity. */
6082
0
        atmax = MO_8;
6083
0
        break;
6084
6085
0
    case MO_ATOM_IFALIGN:
6086
0
        atmax = size;
6087
0
        break;
6088
6089
0
    case MO_ATOM_IFALIGN_PAIR:
6090
0
        atmax = half;
6091
0
        break;
6092
6093
0
    case MO_ATOM_WITHIN16:
6094
0
        atmax = size;
6095
0
        if (size == MO_128) {
6096
            /* Misalignment implies !within16, and therefore no atomicity. */
6097
0
        } else if (host_atom != MO_ATOM_WITHIN16) {
6098
            /* The host does not implement within16, so require alignment. */
6099
0
            align = MAX(align, size);
6100
0
        }
6101
0
        break;
6102
6103
0
    case MO_ATOM_WITHIN16_PAIR:
6104
0
        atmax = size;
6105
        /*
6106
         * Misalignment implies !within16, and therefore half atomicity.
6107
         * Any host prepared for two operations can implement this with
6108
         * half alignment.
6109
         */
6110
0
        if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) {
6111
0
            align = MAX(align, half);
6112
0
        }
6113
0
        break;
6114
6115
0
    case MO_ATOM_SUBALIGN:
6116
0
        atmax = size;
6117
0
        if (host_atom != MO_ATOM_SUBALIGN) {
6118
            /* If unaligned but not odd, there are subobjects up to half. */
6119
0
            if (allow_two_ops) {
6120
0
                align = MAX(align, half);
6121
0
            } else {
6122
0
                align = MAX(align, size);
6123
0
            }
6124
0
        }
6125
0
        break;
6126
6127
0
    default:
6128
0
        g_assert_not_reached();
6129
0
    }
6130
6131
0
    return (TCGAtomAlign){ .atom = atmax, .align = align };
6132
0
}
6133
6134
/*
6135
 * Similarly for qemu_ld/st slow path helpers.
6136
 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
6137
 * using only the provided backend tcg_out_* functions.
6138
 */
6139
6140
static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot)
6141
0
{
6142
0
    int ofs = arg_slot_stk_ofs(slot);
6143
6144
    /*
6145
     * Each stack slot is TCG_TARGET_LONG_BITS.  If the host does not
6146
     * require extension to uint64_t, adjust the address for uint32_t.
6147
     */
6148
0
    if (HOST_BIG_ENDIAN && type == TCG_TYPE_I32) {
6149
0
        ofs += 4;
6150
0
    }
6151
0
    return ofs;
6152
0
}
6153
6154
static void tcg_out_helper_load_slots(TCGContext *s,
6155
                                      unsigned nmov, TCGMovExtend *mov,
6156
                                      const TCGLdstHelperParam *parm)
6157
0
{
6158
0
    unsigned i;
6159
0
    TCGReg dst3;
6160
6161
    /*
6162
     * Start from the end, storing to the stack first.
6163
     * This frees those registers, so we need not consider overlap.
6164
     */
6165
0
    for (i = nmov; i-- > 0; ) {
6166
0
        unsigned slot = mov[i].dst;
6167
6168
0
        if (arg_slot_reg_p(slot)) {
6169
0
            goto found_reg;
6170
0
        }
6171
6172
0
        TCGReg src = mov[i].src;
6173
0
        TCGType dst_type = mov[i].dst_type;
6174
0
        MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64;
6175
6176
        /* The argument is going onto the stack; extend into scratch. */
6177
0
        if ((mov[i].src_ext & MO_SIZE) != dst_mo) {
6178
0
            tcg_debug_assert(parm->ntmp != 0);
6179
0
            mov[i].dst = src = parm->tmp[0];
6180
0
            tcg_out_movext1(s, &mov[i]);
6181
0
        }
6182
6183
0
        tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK,
6184
0
                   tcg_out_helper_stk_ofs(dst_type, slot));
6185
0
    }
6186
0
    return;
6187
6188
0
 found_reg:
6189
    /*
6190
     * The remaining arguments are in registers.
6191
     * Convert slot numbers to argument registers.
6192
     */
6193
0
    nmov = i + 1;
6194
0
    for (i = 0; i < nmov; ++i) {
6195
0
        mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst];
6196
0
    }
6197
6198
0
    switch (nmov) {
6199
0
    case 4:
6200
        /* The backend must have provided enough temps for the worst case. */
6201
0
        tcg_debug_assert(parm->ntmp >= 2);
6202
6203
0
        dst3 = mov[3].dst;
6204
0
        for (unsigned j = 0; j < 3; ++j) {
6205
0
            if (dst3 == mov[j].src) {
6206
                /*
6207
                 * Conflict. Copy the source to a temporary, perform the
6208
                 * remaining moves, then the extension from our scratch
6209
                 * on the way out.
6210
                 */
6211
0
                TCGReg scratch = parm->tmp[1];
6212
6213
0
                tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src);
6214
0
                tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]);
6215
0
                tcg_out_movext1_new_src(s, &mov[3], scratch);
6216
0
                break;
6217
0
            }
6218
0
        }
6219
6220
        /* No conflicts: perform this move and continue. */
6221
0
        tcg_out_movext1(s, &mov[3]);
6222
        /* fall through */
6223
6224
0
    case 3:
6225
0
        tcg_out_movext3(s, mov, mov + 1, mov + 2,
6226
0
                        parm->ntmp ? parm->tmp[0] : -1);
6227
0
        break;
6228
0
    case 2:
6229
0
        tcg_out_movext2(s, mov, mov + 1,
6230
0
                        parm->ntmp ? parm->tmp[0] : -1);
6231
0
        break;
6232
0
    case 1:
6233
0
        tcg_out_movext1(s, mov);
6234
0
        break;
6235
0
    default:
6236
0
        g_assert_not_reached();
6237
0
    }
6238
0
}
6239
6240
static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot,
6241
                                    TCGType type, tcg_target_long imm,
6242
                                    const TCGLdstHelperParam *parm)
6243
0
{
6244
0
    if (arg_slot_reg_p(slot)) {
6245
0
        tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm);
6246
0
    } else {
6247
0
        int ofs = tcg_out_helper_stk_ofs(type, slot);
6248
0
        if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) {
6249
0
            tcg_debug_assert(parm->ntmp != 0);
6250
0
            tcg_out_movi(s, type, parm->tmp[0], imm);
6251
0
            tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs);
6252
0
        }
6253
0
    }
6254
0
}
6255
6256
static void tcg_out_helper_load_common_args(TCGContext *s,
6257
                                            const TCGLabelQemuLdst *ldst,
6258
                                            const TCGLdstHelperParam *parm,
6259
                                            const TCGHelperInfo *info,
6260
                                            unsigned next_arg)
6261
0
{
6262
0
    TCGMovExtend ptr_mov = {
6263
0
        .dst_type = TCG_TYPE_PTR,
6264
0
        .src_type = TCG_TYPE_PTR,
6265
0
        .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64
6266
0
    };
6267
0
    const TCGCallArgumentLoc *loc = &info->in[0];
6268
0
    TCGType type;
6269
0
    unsigned slot;
6270
0
    tcg_target_ulong imm;
6271
6272
    /*
6273
     * Handle env, which is always first.
6274
     */
6275
0
    ptr_mov.dst = loc->arg_slot;
6276
0
    ptr_mov.src = TCG_AREG0;
6277
0
    tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
6278
6279
    /*
6280
     * Handle oi.
6281
     */
6282
0
    imm = ldst->oi;
6283
0
    loc = &info->in[next_arg];
6284
0
    type = TCG_TYPE_I32;
6285
0
    switch (loc->kind) {
6286
0
    case TCG_CALL_ARG_NORMAL:
6287
0
        break;
6288
0
    case TCG_CALL_ARG_EXTEND_U:
6289
0
    case TCG_CALL_ARG_EXTEND_S:
6290
        /* No extension required for MemOpIdx. */
6291
0
        tcg_debug_assert(imm <= INT32_MAX);
6292
0
        type = TCG_TYPE_REG;
6293
0
        break;
6294
0
    default:
6295
0
        g_assert_not_reached();
6296
0
    }
6297
0
    tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm);
6298
0
    next_arg++;
6299
6300
    /*
6301
     * Handle ra.
6302
     */
6303
0
    loc = &info->in[next_arg];
6304
0
    slot = loc->arg_slot;
6305
0
    if (parm->ra_gen) {
6306
0
        int arg_reg = -1;
6307
0
        TCGReg ra_reg;
6308
6309
0
        if (arg_slot_reg_p(slot)) {
6310
0
            arg_reg = tcg_target_call_iarg_regs[slot];
6311
0
        }
6312
0
        ra_reg = parm->ra_gen(s, ldst, arg_reg);
6313
6314
0
        ptr_mov.dst = slot;
6315
0
        ptr_mov.src = ra_reg;
6316
0
        tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
6317
0
    } else {
6318
0
        imm = (uintptr_t)ldst->raddr;
6319
0
        tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm);
6320
0
    }
6321
0
}
6322
6323
static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov,
6324
                                       const TCGCallArgumentLoc *loc,
6325
                                       TCGType dst_type, TCGType src_type,
6326
                                       TCGReg lo, TCGReg hi)
6327
0
{
6328
0
    MemOp reg_mo;
6329
6330
0
    if (dst_type <= TCG_TYPE_REG) {
6331
0
        MemOp src_ext;
6332
6333
0
        switch (loc->kind) {
6334
0
        case TCG_CALL_ARG_NORMAL:
6335
0
            src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64;
6336
0
            break;
6337
0
        case TCG_CALL_ARG_EXTEND_U:
6338
0
            dst_type = TCG_TYPE_REG;
6339
0
            src_ext = MO_UL;
6340
0
            break;
6341
0
        case TCG_CALL_ARG_EXTEND_S:
6342
0
            dst_type = TCG_TYPE_REG;
6343
0
            src_ext = MO_SL;
6344
0
            break;
6345
0
        default:
6346
0
            g_assert_not_reached();
6347
0
        }
6348
6349
0
        mov[0].dst = loc->arg_slot;
6350
0
        mov[0].dst_type = dst_type;
6351
0
        mov[0].src = lo;
6352
0
        mov[0].src_type = src_type;
6353
0
        mov[0].src_ext = src_ext;
6354
0
        return 1;
6355
0
    }
6356
6357
0
    assert(dst_type == TCG_TYPE_I128);
6358
0
    reg_mo = MO_64;
6359
6360
0
    mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot;
6361
0
    mov[0].src = lo;
6362
0
    mov[0].dst_type = TCG_TYPE_REG;
6363
0
    mov[0].src_type = TCG_TYPE_REG;
6364
0
    mov[0].src_ext = reg_mo;
6365
6366
0
    mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot;
6367
0
    mov[1].src = hi;
6368
0
    mov[1].dst_type = TCG_TYPE_REG;
6369
0
    mov[1].src_type = TCG_TYPE_REG;
6370
0
    mov[1].src_ext = reg_mo;
6371
6372
0
    return 2;
6373
0
}
6374
6375
static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
6376
                                   const TCGLdstHelperParam *parm)
6377
0
{
6378
0
    const TCGHelperInfo *info;
6379
0
    const TCGCallArgumentLoc *loc;
6380
0
    TCGMovExtend mov[2];
6381
0
    unsigned next_arg, nmov;
6382
0
    MemOp mop = get_memop(ldst->oi);
6383
6384
0
    switch (mop & MO_SIZE) {
6385
0
    case MO_8:
6386
0
    case MO_16:
6387
0
    case MO_32:
6388
0
        info = &info_helper_ld32_mmu;
6389
0
        break;
6390
0
    case MO_64:
6391
0
        info = &info_helper_ld64_mmu;
6392
0
        break;
6393
0
    case MO_128:
6394
0
        info = &info_helper_ld128_mmu;
6395
0
        break;
6396
0
    default:
6397
0
        g_assert_not_reached();
6398
0
    }
6399
6400
    /* Defer env argument. */
6401
0
    next_arg = 1;
6402
6403
0
    loc = &info->in[next_arg];
6404
0
    nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6405
0
                                  ldst->addr_reg, -1);
6406
0
    tcg_out_helper_load_slots(s, nmov, mov, parm);
6407
0
    next_arg += nmov;
6408
6409
0
    switch (info->out_kind) {
6410
0
    case TCG_CALL_RET_NORMAL:
6411
0
    case TCG_CALL_RET_BY_VEC:
6412
0
        break;
6413
0
    case TCG_CALL_RET_BY_REF:
6414
        /*
6415
         * The return reference is in the first argument slot.
6416
         * We need memory in which to return: re-use the top of stack.
6417
         */
6418
0
        {
6419
0
            int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
6420
6421
0
            if (arg_slot_reg_p(0)) {
6422
0
                tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0],
6423
0
                                 TCG_REG_CALL_STACK, ofs_slot0);
6424
0
            } else {
6425
0
                tcg_debug_assert(parm->ntmp != 0);
6426
0
                tcg_out_addi_ptr(s, parm->tmp[0],
6427
0
                                 TCG_REG_CALL_STACK, ofs_slot0);
6428
0
                tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6429
0
                           TCG_REG_CALL_STACK, ofs_slot0);
6430
0
            }
6431
0
        }
6432
0
        break;
6433
0
    default:
6434
0
        g_assert_not_reached();
6435
0
    }
6436
6437
0
    tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
6438
0
}
6439
6440
static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst,
6441
                                  bool load_sign,
6442
                                  const TCGLdstHelperParam *parm)
6443
0
{
6444
0
    MemOp mop = get_memop(ldst->oi);
6445
0
    TCGMovExtend mov[2];
6446
0
    int ofs_slot0;
6447
6448
0
    switch (ldst->type) {
6449
0
    case TCG_TYPE_I32:
6450
0
    case TCG_TYPE_I64:
6451
0
        mov[0].dst = ldst->datalo_reg;
6452
0
        mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0);
6453
0
        mov[0].dst_type = ldst->type;
6454
0
        mov[0].src_type = TCG_TYPE_REG;
6455
6456
        /*
6457
         * If load_sign, then we allowed the helper to perform the
6458
         * appropriate sign extension to tcg_target_ulong, and all
6459
         * we need now is a plain move.
6460
         *
6461
         * If they do not, then we expect the relevant extension
6462
         * instruction to be no more expensive than a move, and
6463
         * we thus save the icache etc by only using one of two
6464
         * helper functions.
6465
         */
6466
0
        if (load_sign || !(mop & MO_SIGN)) {
6467
0
            if (ldst->type == TCG_TYPE_I32) {
6468
0
                mov[0].src_ext = MO_32;
6469
0
            } else {
6470
0
                mov[0].src_ext = MO_64;
6471
0
            }
6472
0
        } else {
6473
0
            mov[0].src_ext = mop & MO_SSIZE;
6474
0
        }
6475
0
        tcg_out_movext1(s, mov);
6476
0
        return;
6477
6478
0
    case TCG_TYPE_I128:
6479
0
        ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
6480
0
        switch (TCG_TARGET_CALL_RET_I128) {
6481
0
        case TCG_CALL_RET_NORMAL:
6482
0
            break;
6483
0
        case TCG_CALL_RET_BY_VEC:
6484
0
            tcg_out_st(s, TCG_TYPE_V128,
6485
0
                       tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
6486
0
                       TCG_REG_CALL_STACK, ofs_slot0);
6487
            /* fall through */
6488
0
        case TCG_CALL_RET_BY_REF:
6489
0
            tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg,
6490
0
                       TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN);
6491
0
            tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg,
6492
0
                       TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN);
6493
0
            return;
6494
0
        default:
6495
0
            g_assert_not_reached();
6496
0
        }
6497
0
        break;
6498
6499
0
    default:
6500
0
        g_assert_not_reached();
6501
0
    }
6502
6503
0
    mov[0].dst = ldst->datalo_reg;
6504
0
    mov[0].src =
6505
0
        tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN);
6506
0
    mov[0].dst_type = TCG_TYPE_REG;
6507
0
    mov[0].src_type = TCG_TYPE_REG;
6508
0
    mov[0].src_ext = MO_64;
6509
6510
0
    mov[1].dst = ldst->datahi_reg;
6511
0
    mov[1].src =
6512
0
        tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN);
6513
0
    mov[1].dst_type = TCG_TYPE_REG;
6514
0
    mov[1].src_type = TCG_TYPE_REG;
6515
0
    mov[1].src_ext = MO_64;
6516
6517
0
    tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1);
6518
0
}
6519
6520
static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
6521
                                   const TCGLdstHelperParam *parm)
6522
0
{
6523
0
    const TCGHelperInfo *info;
6524
0
    const TCGCallArgumentLoc *loc;
6525
0
    TCGMovExtend mov[4];
6526
0
    TCGType data_type;
6527
0
    unsigned next_arg, nmov, n;
6528
0
    MemOp mop = get_memop(ldst->oi);
6529
6530
0
    switch (mop & MO_SIZE) {
6531
0
    case MO_8:
6532
0
    case MO_16:
6533
0
    case MO_32:
6534
0
        info = &info_helper_st32_mmu;
6535
0
        data_type = TCG_TYPE_I32;
6536
0
        break;
6537
0
    case MO_64:
6538
0
        info = &info_helper_st64_mmu;
6539
0
        data_type = TCG_TYPE_I64;
6540
0
        break;
6541
0
    case MO_128:
6542
0
        info = &info_helper_st128_mmu;
6543
0
        data_type = TCG_TYPE_I128;
6544
0
        break;
6545
0
    default:
6546
0
        g_assert_not_reached();
6547
0
    }
6548
6549
    /* Defer env argument. */
6550
0
    next_arg = 1;
6551
0
    nmov = 0;
6552
6553
    /* Handle addr argument. */
6554
0
    loc = &info->in[next_arg];
6555
0
    tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
6556
0
    n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6557
0
                               ldst->addr_reg, -1);
6558
0
    next_arg += n;
6559
0
    nmov += n;
6560
6561
    /* Handle data argument. */
6562
0
    loc = &info->in[next_arg];
6563
0
    switch (loc->kind) {
6564
0
    case TCG_CALL_ARG_NORMAL:
6565
0
    case TCG_CALL_ARG_EXTEND_U:
6566
0
    case TCG_CALL_ARG_EXTEND_S:
6567
0
        n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type,
6568
0
                                   ldst->datalo_reg, ldst->datahi_reg);
6569
0
        next_arg += n;
6570
0
        nmov += n;
6571
0
        tcg_out_helper_load_slots(s, nmov, mov, parm);
6572
0
        break;
6573
6574
0
    case TCG_CALL_ARG_BY_REF:
6575
0
        tcg_debug_assert(data_type == TCG_TYPE_I128);
6576
0
        tcg_out_st(s, TCG_TYPE_I64,
6577
0
                   HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg,
6578
0
                   TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot));
6579
0
        tcg_out_st(s, TCG_TYPE_I64,
6580
0
                   HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg,
6581
0
                   TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot));
6582
6583
0
        tcg_out_helper_load_slots(s, nmov, mov, parm);
6584
6585
0
        if (arg_slot_reg_p(loc->arg_slot)) {
6586
0
            tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot],
6587
0
                             TCG_REG_CALL_STACK,
6588
0
                             arg_slot_stk_ofs(loc->ref_slot));
6589
0
        } else {
6590
0
            tcg_debug_assert(parm->ntmp != 0);
6591
0
            tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK,
6592
0
                             arg_slot_stk_ofs(loc->ref_slot));
6593
0
            tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6594
0
                       TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot));
6595
0
        }
6596
0
        next_arg += 2;
6597
0
        break;
6598
6599
0
    default:
6600
0
        g_assert_not_reached();
6601
0
    }
6602
6603
0
    tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
6604
0
}
6605
6606
int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
6607
0
{
6608
0
    int i, num_insns;
6609
0
    TCGOp *op;
6610
6611
0
    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
6612
0
                 && qemu_log_in_addr_range(pc_start))) {
6613
0
        FILE *logfile = qemu_log_trylock();
6614
0
        if (logfile) {
6615
0
            fprintf(logfile, "OP:\n");
6616
0
            tcg_dump_ops(s, logfile, false);
6617
0
            fprintf(logfile, "\n");
6618
0
            qemu_log_unlock(logfile);
6619
0
        }
6620
0
    }
6621
6622
#ifdef CONFIG_DEBUG_TCG
6623
    /* Ensure all labels referenced have been emitted.  */
6624
    {
6625
        TCGLabel *l;
6626
        bool error = false;
6627
6628
        QSIMPLEQ_FOREACH(l, &s->labels, next) {
6629
            if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
6630
                qemu_log_mask(CPU_LOG_TB_OP,
6631
                              "$L%d referenced but not present.\n", l->id);
6632
                error = true;
6633
            }
6634
        }
6635
        assert(!error);
6636
    }
6637
#endif
6638
6639
    /* Do not reuse any EBB that may be allocated within the TB. */
6640
0
    tcg_temp_ebb_reset_freed(s);
6641
6642
0
    tcg_optimize(s);
6643
6644
0
    reachable_code_pass(s);
6645
0
    liveness_pass_0(s);
6646
0
    liveness_pass_1(s);
6647
6648
0
    if (s->nb_indirects > 0) {
6649
0
        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
6650
0
                     && qemu_log_in_addr_range(pc_start))) {
6651
0
            FILE *logfile = qemu_log_trylock();
6652
0
            if (logfile) {
6653
0
                fprintf(logfile, "OP before indirect lowering:\n");
6654
0
                tcg_dump_ops(s, logfile, false);
6655
0
                fprintf(logfile, "\n");
6656
0
                qemu_log_unlock(logfile);
6657
0
            }
6658
0
        }
6659
6660
        /* Replace indirect temps with direct temps.  */
6661
0
        if (liveness_pass_2(s)) {
6662
            /* If changes were made, re-run liveness.  */
6663
0
            liveness_pass_1(s);
6664
0
        }
6665
0
    }
6666
6667
0
    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
6668
0
                 && qemu_log_in_addr_range(pc_start))) {
6669
0
        FILE *logfile = qemu_log_trylock();
6670
0
        if (logfile) {
6671
0
            fprintf(logfile, "OP after optimization and liveness analysis:\n");
6672
0
            tcg_dump_ops(s, logfile, true);
6673
0
            fprintf(logfile, "\n");
6674
0
            qemu_log_unlock(logfile);
6675
0
        }
6676
0
    }
6677
6678
    /* Initialize goto_tb jump offsets. */
6679
0
    tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
6680
0
    tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
6681
0
    tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
6682
0
    tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
6683
6684
0
    tcg_reg_alloc_start(s);
6685
6686
    /*
6687
     * Reset the buffer pointers when restarting after overflow.
6688
     * TODO: Move this into translate-all.c with the rest of the
6689
     * buffer management.  Having only this done here is confusing.
6690
     */
6691
0
    s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
6692
0
    s->code_ptr = s->code_buf;
6693
0
    s->data_gen_ptr = NULL;
6694
6695
0
    QSIMPLEQ_INIT(&s->ldst_labels);
6696
0
    s->pool_labels = NULL;
6697
6698
0
    s->gen_insn_data =
6699
0
        tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS);
6700
6701
0
    tcg_out_tb_start(s);
6702
6703
0
    num_insns = -1;
6704
0
    s->carry_live = false;
6705
0
    QTAILQ_FOREACH(op, &s->ops, link) {
6706
0
        TCGOpcode opc = op->opc;
6707
6708
0
        switch (opc) {
6709
0
        case INDEX_op_extrl_i64_i32:
6710
            /*
6711
             * If TCG_TYPE_I32 is represented in some canonical form,
6712
             * e.g. zero or sign-extended, then emit as a unary op.
6713
             * Otherwise we can treat this as a plain move.
6714
             * If the output dies, treat this as a plain move, because
6715
             * this will be implemented with a store.
6716
             */
6717
0
            if (TCG_TARGET_HAS_extr_i64_i32) {
6718
0
                TCGLifeData arg_life = op->life;
6719
0
                if (!IS_DEAD_ARG(0)) {
6720
0
                    goto do_default;
6721
0
                }
6722
0
            }
6723
            /* fall through */
6724
0
        case INDEX_op_mov:
6725
0
        case INDEX_op_mov_vec:
6726
0
            tcg_reg_alloc_mov(s, op);
6727
0
            break;
6728
0
        case INDEX_op_dup_vec:
6729
0
            tcg_reg_alloc_dup(s, op);
6730
0
            break;
6731
0
        case INDEX_op_insn_start:
6732
0
            assert_carry_dead(s);
6733
0
            if (num_insns >= 0) {
6734
0
                size_t off = tcg_current_code_size(s);
6735
0
                s->gen_insn_end_off[num_insns] = off;
6736
                /* Assert that we do not overflow our stored offset.  */
6737
0
                assert(s->gen_insn_end_off[num_insns] == off);
6738
0
            }
6739
0
            num_insns++;
6740
0
            for (i = 0; i < INSN_START_WORDS; ++i) {
6741
0
                s->gen_insn_data[num_insns * INSN_START_WORDS + i] =
6742
0
                    tcg_get_insn_start_param(op, i);
6743
0
            }
6744
0
            break;
6745
0
        case INDEX_op_discard:
6746
0
            temp_dead(s, arg_temp(op->args[0]));
6747
0
            break;
6748
0
        case INDEX_op_set_label:
6749
0
            tcg_reg_alloc_bb_end(s, s->reserved_regs);
6750
0
            tcg_out_label(s, arg_label(op->args[0]));
6751
0
            break;
6752
0
        case INDEX_op_call:
6753
0
            assert_carry_dead(s);
6754
0
            tcg_reg_alloc_call(s, op);
6755
0
            break;
6756
0
        case INDEX_op_exit_tb:
6757
0
            tcg_out_exit_tb(s, op->args[0]);
6758
0
            break;
6759
0
        case INDEX_op_goto_tb:
6760
0
            tcg_out_goto_tb(s, op->args[0]);
6761
0
            break;
6762
0
        case INDEX_op_br:
6763
0
            tcg_out_br(s, arg_label(op->args[0]));
6764
0
            break;
6765
0
        case INDEX_op_mb:
6766
0
            tcg_out_mb(s, op->args[0]);
6767
0
            break;
6768
0
        default:
6769
0
        do_default:
6770
            /* Sanity check that we've not introduced any unhandled opcodes. */
6771
0
            tcg_debug_assert(tcg_op_supported(opc, TCGOP_TYPE(op),
6772
0
                                              TCGOP_FLAGS(op)));
6773
            /* Note: in order to speed up the code, it would be much
6774
               faster to have specialized register allocator functions for
6775
               some common argument patterns */
6776
0
            tcg_reg_alloc_op(s, op);
6777
0
            break;
6778
0
        }
6779
        /* Test for (pending) buffer overflow.  The assumption is that any
6780
           one operation beginning below the high water mark cannot overrun
6781
           the buffer completely.  Thus we can test for overflow after
6782
           generating code without having to check during generation.  */
6783
0
        if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
6784
0
            return -1;
6785
0
        }
6786
        /* Test for TB overflow, as seen by gen_insn_end_off.  */
6787
0
        if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
6788
0
            return -2;
6789
0
        }
6790
0
    }
6791
0
    assert_carry_dead(s);
6792
6793
0
    tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
6794
0
    s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
6795
6796
    /* Generate TB finalization at the end of block */
6797
0
    i = tcg_out_ldst_finalize(s);
6798
0
    if (i < 0) {
6799
0
        return i;
6800
0
    }
6801
0
    i = tcg_out_pool_finalize(s);
6802
0
    if (i < 0) {
6803
0
        return i;
6804
0
    }
6805
0
    if (!tcg_resolve_relocs(s)) {
6806
0
        return -2;
6807
0
    }
6808
6809
0
#ifndef CONFIG_TCG_INTERPRETER
6810
    /* flush instruction cache */
6811
0
    flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
6812
0
                        (uintptr_t)s->code_buf,
6813
0
                        tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
6814
0
#endif
6815
6816
0
    return tcg_current_code_size(s);
6817
0
}
6818
6819
#ifdef ELF_HOST_MACHINE
6820
/* In order to use this feature, the backend needs to do three things:
6821
6822
   (1) Define ELF_HOST_MACHINE to indicate both what value to
6823
       put into the ELF image and to indicate support for the feature.
6824
6825
   (2) Define tcg_register_jit.  This should create a buffer containing
6826
       the contents of a .debug_frame section that describes the post-
6827
       prologue unwind info for the tcg machine.
6828
6829
   (3) Call tcg_register_jit_int, with the constructed .debug_frame.
6830
*/
6831
6832
/* Begin GDB interface.  THE FOLLOWING MUST MATCH GDB DOCS.  */
6833
typedef enum {
6834
    JIT_NOACTION = 0,
6835
    JIT_REGISTER_FN,
6836
    JIT_UNREGISTER_FN
6837
} jit_actions_t;
6838
6839
struct jit_code_entry {
6840
    struct jit_code_entry *next_entry;
6841
    struct jit_code_entry *prev_entry;
6842
    const void *symfile_addr;
6843
    uint64_t symfile_size;
6844
};
6845
6846
struct jit_descriptor {
6847
    uint32_t version;
6848
    uint32_t action_flag;
6849
    struct jit_code_entry *relevant_entry;
6850
    struct jit_code_entry *first_entry;
6851
};
6852
6853
void __jit_debug_register_code(void) __attribute__((noinline));
6854
void __jit_debug_register_code(void)
6855
0
{
6856
0
    asm("");
6857
0
}
6858
6859
/* Must statically initialize the version, because GDB may check
6860
   the version before we can set it.  */
6861
struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
6862
6863
/* End GDB interface.  */
6864
6865
static int find_string(const char *strtab, const char *str)
6866
0
{
6867
0
    const char *p = strtab + 1;
6868
6869
0
    while (1) {
6870
0
        if (strcmp(p, str) == 0) {
6871
0
            return p - strtab;
6872
0
        }
6873
0
        p += strlen(p) + 1;
6874
0
    }
6875
0
}
6876
6877
static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
6878
                                 const void *debug_frame,
6879
                                 size_t debug_frame_size)
6880
0
{
6881
0
    struct __attribute__((packed)) DebugInfo {
6882
0
        uint32_t  len;
6883
0
        uint16_t  version;
6884
0
        uint32_t  abbrev;
6885
0
        uint8_t   ptr_size;
6886
0
        uint8_t   cu_die;
6887
0
        uint16_t  cu_lang;
6888
0
        uintptr_t cu_low_pc;
6889
0
        uintptr_t cu_high_pc;
6890
0
        uint8_t   fn_die;
6891
0
        char      fn_name[16];
6892
0
        uintptr_t fn_low_pc;
6893
0
        uintptr_t fn_high_pc;
6894
0
        uint8_t   cu_eoc;
6895
0
    };
6896
6897
0
    struct ElfImage {
6898
0
        ElfW(Ehdr) ehdr;
6899
0
        ElfW(Phdr) phdr;
6900
0
        ElfW(Shdr) shdr[7];
6901
0
        ElfW(Sym)  sym[2];
6902
0
        struct DebugInfo di;
6903
0
        uint8_t    da[24];
6904
0
        char       str[80];
6905
0
    };
6906
6907
0
    struct ElfImage *img;
6908
6909
0
    static const struct ElfImage img_template = {
6910
0
        .ehdr = {
6911
0
            .e_ident[EI_MAG0] = ELFMAG0,
6912
0
            .e_ident[EI_MAG1] = ELFMAG1,
6913
0
            .e_ident[EI_MAG2] = ELFMAG2,
6914
0
            .e_ident[EI_MAG3] = ELFMAG3,
6915
0
            .e_ident[EI_CLASS] = ELF_CLASS,
6916
0
            .e_ident[EI_DATA] = ELF_DATA,
6917
0
            .e_ident[EI_VERSION] = EV_CURRENT,
6918
0
            .e_type = ET_EXEC,
6919
0
            .e_machine = ELF_HOST_MACHINE,
6920
0
            .e_version = EV_CURRENT,
6921
0
            .e_phoff = offsetof(struct ElfImage, phdr),
6922
0
            .e_shoff = offsetof(struct ElfImage, shdr),
6923
0
            .e_ehsize = sizeof(ElfW(Shdr)),
6924
0
            .e_phentsize = sizeof(ElfW(Phdr)),
6925
0
            .e_phnum = 1,
6926
0
            .e_shentsize = sizeof(ElfW(Shdr)),
6927
0
            .e_shnum = ARRAY_SIZE(img->shdr),
6928
0
            .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
6929
#ifdef ELF_HOST_FLAGS
6930
            .e_flags = ELF_HOST_FLAGS,
6931
#endif
6932
#ifdef ELF_OSABI
6933
            .e_ident[EI_OSABI] = ELF_OSABI,
6934
#endif
6935
0
        },
6936
0
        .phdr = {
6937
0
            .p_type = PT_LOAD,
6938
0
            .p_flags = PF_X,
6939
0
        },
6940
0
        .shdr = {
6941
0
            [0] = { .sh_type = SHT_NULL },
6942
            /* Trick: The contents of code_gen_buffer are not present in
6943
               this fake ELF file; that got allocated elsewhere.  Therefore
6944
               we mark .text as SHT_NOBITS (similar to .bss) so that readers
6945
               will not look for contents.  We can record any address.  */
6946
0
            [1] = { /* .text */
6947
0
                .sh_type = SHT_NOBITS,
6948
0
                .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
6949
0
            },
6950
0
            [2] = { /* .debug_info */
6951
0
                .sh_type = SHT_PROGBITS,
6952
0
                .sh_offset = offsetof(struct ElfImage, di),
6953
0
                .sh_size = sizeof(struct DebugInfo),
6954
0
            },
6955
0
            [3] = { /* .debug_abbrev */
6956
0
                .sh_type = SHT_PROGBITS,
6957
0
                .sh_offset = offsetof(struct ElfImage, da),
6958
0
                .sh_size = sizeof(img->da),
6959
0
            },
6960
0
            [4] = { /* .debug_frame */
6961
0
                .sh_type = SHT_PROGBITS,
6962
0
                .sh_offset = sizeof(struct ElfImage),
6963
0
            },
6964
0
            [5] = { /* .symtab */
6965
0
                .sh_type = SHT_SYMTAB,
6966
0
                .sh_offset = offsetof(struct ElfImage, sym),
6967
0
                .sh_size = sizeof(img->sym),
6968
0
                .sh_info = 1,
6969
0
                .sh_link = ARRAY_SIZE(img->shdr) - 1,
6970
0
                .sh_entsize = sizeof(ElfW(Sym)),
6971
0
            },
6972
0
            [6] = { /* .strtab */
6973
0
                .sh_type = SHT_STRTAB,
6974
0
                .sh_offset = offsetof(struct ElfImage, str),
6975
0
                .sh_size = sizeof(img->str),
6976
0
            }
6977
0
        },
6978
0
        .sym = {
6979
0
            [1] = { /* code_gen_buffer */
6980
0
                .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
6981
0
                .st_shndx = 1,
6982
0
            }
6983
0
        },
6984
0
        .di = {
6985
0
            .len = sizeof(struct DebugInfo) - 4,
6986
0
            .version = 2,
6987
0
            .ptr_size = sizeof(void *),
6988
0
            .cu_die = 1,
6989
0
            .cu_lang = 0x8001,  /* DW_LANG_Mips_Assembler */
6990
0
            .fn_die = 2,
6991
0
            .fn_name = "code_gen_buffer"
6992
0
        },
6993
0
        .da = {
6994
0
            1,          /* abbrev number (the cu) */
6995
0
            0x11, 1,    /* DW_TAG_compile_unit, has children */
6996
0
            0x13, 0x5,  /* DW_AT_language, DW_FORM_data2 */
6997
0
            0x11, 0x1,  /* DW_AT_low_pc, DW_FORM_addr */
6998
0
            0x12, 0x1,  /* DW_AT_high_pc, DW_FORM_addr */
6999
0
            0, 0,       /* end of abbrev */
7000
0
            2,          /* abbrev number (the fn) */
7001
0
            0x2e, 0,    /* DW_TAG_subprogram, no children */
7002
0
            0x3, 0x8,   /* DW_AT_name, DW_FORM_string */
7003
0
            0x11, 0x1,  /* DW_AT_low_pc, DW_FORM_addr */
7004
0
            0x12, 0x1,  /* DW_AT_high_pc, DW_FORM_addr */
7005
0
            0, 0,       /* end of abbrev */
7006
0
            0           /* no more abbrev */
7007
0
        },
7008
0
        .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
7009
0
               ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
7010
0
    };
7011
7012
    /* We only need a single jit entry; statically allocate it.  */
7013
0
    static struct jit_code_entry one_entry;
7014
7015
0
    uintptr_t buf = (uintptr_t)buf_ptr;
7016
0
    size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
7017
0
    DebugFrameHeader *dfh;
7018
7019
0
    img = g_malloc(img_size);
7020
0
    *img = img_template;
7021
7022
0
    img->phdr.p_vaddr = buf;
7023
0
    img->phdr.p_paddr = buf;
7024
0
    img->phdr.p_memsz = buf_size;
7025
7026
0
    img->shdr[1].sh_name = find_string(img->str, ".text");
7027
0
    img->shdr[1].sh_addr = buf;
7028
0
    img->shdr[1].sh_size = buf_size;
7029
7030
0
    img->shdr[2].sh_name = find_string(img->str, ".debug_info");
7031
0
    img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
7032
7033
0
    img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
7034
0
    img->shdr[4].sh_size = debug_frame_size;
7035
7036
0
    img->shdr[5].sh_name = find_string(img->str, ".symtab");
7037
0
    img->shdr[6].sh_name = find_string(img->str, ".strtab");
7038
7039
0
    img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
7040
0
    img->sym[1].st_value = buf;
7041
0
    img->sym[1].st_size = buf_size;
7042
7043
0
    img->di.cu_low_pc = buf;
7044
0
    img->di.cu_high_pc = buf + buf_size;
7045
0
    img->di.fn_low_pc = buf;
7046
0
    img->di.fn_high_pc = buf + buf_size;
7047
7048
0
    dfh = (DebugFrameHeader *)(img + 1);
7049
0
    memcpy(dfh, debug_frame, debug_frame_size);
7050
0
    dfh->fde.func_start = buf;
7051
0
    dfh->fde.func_len = buf_size;
7052
7053
#ifdef DEBUG_JIT
7054
    /* Enable this block to be able to debug the ELF image file creation.
7055
       One can use readelf, objdump, or other inspection utilities.  */
7056
    {
7057
        g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
7058
        FILE *f = fopen(jit, "w+b");
7059
        if (f) {
7060
            if (fwrite(img, img_size, 1, f) != img_size) {
7061
                /* Avoid stupid unused return value warning for fwrite.  */
7062
            }
7063
            fclose(f);
7064
        }
7065
    }
7066
#endif
7067
7068
0
    one_entry.symfile_addr = img;
7069
0
    one_entry.symfile_size = img_size;
7070
7071
0
    __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
7072
0
    __jit_debug_descriptor.relevant_entry = &one_entry;
7073
0
    __jit_debug_descriptor.first_entry = &one_entry;
7074
0
    __jit_debug_register_code();
7075
0
}
7076
#else
7077
/* No support for the feature.  Provide the entry point expected by exec.c,
7078
   and implement the internal function we declared earlier.  */
7079
7080
static void tcg_register_jit_int(const void *buf, size_t size,
7081
                                 const void *debug_frame,
7082
                                 size_t debug_frame_size)
7083
{
7084
}
7085
7086
void tcg_register_jit(const void *buf, size_t buf_size)
7087
{
7088
}
7089
#endif /* ELF_HOST_MACHINE */
7090
7091
#if !TCG_TARGET_MAYBE_vec
7092
void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
7093
{
7094
    g_assert_not_reached();
7095
}
7096
#endif