Line | Count | Source |
1 | | /* |
2 | | * Tiny Code Generator for QEMU |
3 | | * |
4 | | * Copyright (c) 2008 Fabrice Bellard |
5 | | * |
6 | | * Permission is hereby granted, free of charge, to any person obtaining a copy |
7 | | * of this software and associated documentation files (the "Software"), to deal |
8 | | * in the Software without restriction, including without limitation the rights |
9 | | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
10 | | * copies of the Software, and to permit persons to whom the Software is |
11 | | * furnished to do so, subject to the following conditions: |
12 | | * |
13 | | * The above copyright notice and this permission notice shall be included in |
14 | | * all copies or substantial portions of the Software. |
15 | | * |
16 | | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
19 | | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
20 | | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
21 | | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
22 | | * THE SOFTWARE. |
23 | | */ |
24 | | |
25 | | #include "qemu/osdep.h" |
26 | | |
27 | | /* Define to jump the ELF file used to communicate with GDB. */ |
28 | | #undef DEBUG_JIT |
29 | | |
30 | | #include "qemu/error-report.h" |
31 | | #include "qemu/cutils.h" |
32 | | #include "qemu/host-utils.h" |
33 | | #include "qemu/qemu-print.h" |
34 | | #include "qemu/cacheflush.h" |
35 | | #include "qemu/cacheinfo.h" |
36 | | #include "qemu/timer.h" |
37 | | #include "exec/target_page.h" |
38 | | #include "exec/translation-block.h" |
39 | | #include "exec/tlb-common.h" |
40 | | #include "tcg/startup.h" |
41 | | #include "tcg/tcg-op-common.h" |
42 | | |
43 | | #if UINTPTR_MAX == UINT32_MAX |
44 | | # define ELF_CLASS ELFCLASS32 |
45 | | #else |
46 | 0 | # define ELF_CLASS ELFCLASS64 |
47 | | #endif |
48 | | #if HOST_BIG_ENDIAN |
49 | | # define ELF_DATA ELFDATA2MSB |
50 | | #else |
51 | 0 | # define ELF_DATA ELFDATA2LSB |
52 | | #endif |
53 | | |
54 | | #include "elf.h" |
55 | | #include "exec/log.h" |
56 | | #include "tcg/tcg-ldst.h" |
57 | | #include "tcg/tcg-temp-internal.h" |
58 | | #include "tcg-internal.h" |
59 | | #include "tcg/perf.h" |
60 | | #include "tcg-has.h" |
61 | | #ifdef CONFIG_USER_ONLY |
62 | | #include "user/guest-base.h" |
63 | | #endif |
64 | | |
65 | | /* Forward declarations for functions declared in tcg-target.c.inc and |
66 | | used here. */ |
67 | | static void tcg_target_init(TCGContext *s); |
68 | | static void tcg_target_qemu_prologue(TCGContext *s); |
69 | | static bool patch_reloc(tcg_insn_unit *code_ptr, int type, |
70 | | intptr_t value, intptr_t addend); |
71 | | static void tcg_out_nop_fill(tcg_insn_unit *p, int count); |
72 | | |
73 | | typedef struct TCGLabelQemuLdst TCGLabelQemuLdst; |
74 | | static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l); |
75 | | static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l); |
76 | | |
77 | | /* The CIE and FDE header definitions will be common to all hosts. */ |
78 | | typedef struct { |
79 | | uint32_t len __attribute__((aligned((sizeof(void *))))); |
80 | | uint32_t id; |
81 | | uint8_t version; |
82 | | char augmentation[1]; |
83 | | uint8_t code_align; |
84 | | uint8_t data_align; |
85 | | uint8_t return_column; |
86 | | } DebugFrameCIE; |
87 | | |
88 | | typedef struct QEMU_PACKED { |
89 | | uint32_t len __attribute__((aligned((sizeof(void *))))); |
90 | | uint32_t cie_offset; |
91 | | uintptr_t func_start; |
92 | | uintptr_t func_len; |
93 | | } DebugFrameFDEHeader; |
94 | | |
95 | | typedef struct QEMU_PACKED { |
96 | | DebugFrameCIE cie; |
97 | | DebugFrameFDEHeader fde; |
98 | | } DebugFrameHeader; |
99 | | |
100 | | struct TCGLabelQemuLdst { |
101 | | bool is_ld; /* qemu_ld: true, qemu_st: false */ |
102 | | MemOpIdx oi; |
103 | | TCGType type; /* result type of a load */ |
104 | | TCGReg addr_reg; /* reg index for guest virtual addr */ |
105 | | TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ |
106 | | TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ |
107 | | const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ |
108 | | tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ |
109 | | QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; |
110 | | }; |
111 | | |
112 | | static void tcg_register_jit_int(const void *buf, size_t size, |
113 | | const void *debug_frame, |
114 | | size_t debug_frame_size) |
115 | | __attribute__((unused)); |
116 | | |
117 | | /* Forward declarations for functions declared and used in tcg-target.c.inc. */ |
118 | | static void tcg_out_tb_start(TCGContext *s); |
119 | | static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, |
120 | | intptr_t arg2); |
121 | | static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); |
122 | | static void tcg_out_movi(TCGContext *s, TCGType type, |
123 | | TCGReg ret, tcg_target_long arg); |
124 | | static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); |
125 | | static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); |
126 | | static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg); |
127 | | static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg); |
128 | | static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg); |
129 | | static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg); |
130 | | static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); |
131 | | static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); |
132 | | static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg); |
133 | | static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); |
134 | | static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2); |
135 | | static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); |
136 | | static void tcg_out_goto_tb(TCGContext *s, int which); |
137 | | static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest); |
138 | | static void tcg_out_mb(TCGContext *s, unsigned bar); |
139 | | static void tcg_out_br(TCGContext *s, TCGLabel *l); |
140 | | static void tcg_out_set_carry(TCGContext *s); |
141 | | static void tcg_out_set_borrow(TCGContext *s); |
142 | | #if TCG_TARGET_MAYBE_vec |
143 | | static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, |
144 | | TCGReg dst, TCGReg src); |
145 | | static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, |
146 | | TCGReg dst, TCGReg base, intptr_t offset); |
147 | | static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, |
148 | | TCGReg dst, int64_t arg); |
149 | | static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, |
150 | | unsigned vecl, unsigned vece, |
151 | | const TCGArg args[TCG_MAX_OP_ARGS], |
152 | | const int const_args[TCG_MAX_OP_ARGS]); |
153 | | #else |
154 | | static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, |
155 | | TCGReg dst, TCGReg src) |
156 | | { |
157 | | g_assert_not_reached(); |
158 | | } |
159 | | static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, |
160 | | TCGReg dst, TCGReg base, intptr_t offset) |
161 | | { |
162 | | g_assert_not_reached(); |
163 | | } |
164 | | static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, |
165 | | TCGReg dst, int64_t arg) |
166 | | { |
167 | | g_assert_not_reached(); |
168 | | } |
169 | | static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, |
170 | | unsigned vecl, unsigned vece, |
171 | | const TCGArg args[TCG_MAX_OP_ARGS], |
172 | | const int const_args[TCG_MAX_OP_ARGS]) |
173 | | { |
174 | | g_assert_not_reached(); |
175 | | } |
176 | | int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve) |
177 | | { |
178 | | return 0; |
179 | | } |
180 | | #endif |
181 | | static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, |
182 | | intptr_t arg2); |
183 | | static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, |
184 | | TCGReg base, intptr_t ofs); |
185 | | static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, |
186 | | const TCGHelperInfo *info); |
187 | | static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); |
188 | | static bool tcg_target_const_match(int64_t val, int ct, |
189 | | TCGType type, TCGCond cond, int vece); |
190 | | |
191 | | #ifndef CONFIG_USER_ONLY |
192 | 0 | #define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; }) |
193 | | #endif |
194 | | |
195 | | typedef struct TCGLdstHelperParam { |
196 | | TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg); |
197 | | unsigned ntmp; |
198 | | int tmp[3]; |
199 | | } TCGLdstHelperParam; |
200 | | |
201 | | static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, |
202 | | const TCGLdstHelperParam *p) |
203 | | __attribute__((unused)); |
204 | | static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l, |
205 | | bool load_sign, const TCGLdstHelperParam *p) |
206 | | __attribute__((unused)); |
207 | | static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, |
208 | | const TCGLdstHelperParam *p) |
209 | | __attribute__((unused)); |
210 | | |
211 | | static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = { |
212 | | [MO_UB] = helper_ldub_mmu, |
213 | | [MO_SB] = helper_ldsb_mmu, |
214 | | [MO_UW] = helper_lduw_mmu, |
215 | | [MO_SW] = helper_ldsw_mmu, |
216 | | [MO_UL] = helper_ldul_mmu, |
217 | | [MO_UQ] = helper_ldq_mmu, |
218 | | [MO_SL] = helper_ldsl_mmu, |
219 | | [MO_128] = helper_ld16_mmu, |
220 | | }; |
221 | | |
222 | | static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = { |
223 | | [MO_8] = helper_stb_mmu, |
224 | | [MO_16] = helper_stw_mmu, |
225 | | [MO_32] = helper_stl_mmu, |
226 | | [MO_64] = helper_stq_mmu, |
227 | | [MO_128] = helper_st16_mmu, |
228 | | }; |
229 | | |
230 | | typedef struct { |
231 | | MemOp atom; /* lg2 bits of atomicity required */ |
232 | | MemOp align; /* lg2 bits of alignment to use */ |
233 | | } TCGAtomAlign; |
234 | | |
235 | | static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, |
236 | | MemOp host_atom, bool allow_two_ops) |
237 | | __attribute__((unused)); |
238 | | |
239 | | TCGContext tcg_init_ctx; |
240 | | __thread TCGContext *tcg_ctx; |
241 | | |
242 | | TCGContext **tcg_ctxs; |
243 | | unsigned int tcg_cur_ctxs; |
244 | | unsigned int tcg_max_ctxs; |
245 | | TCGv_env tcg_env; |
246 | | const void *tcg_code_gen_epilogue; |
247 | | ptrdiff_t tcg_splitwx_diff; |
248 | | |
249 | | #ifndef CONFIG_TCG_INTERPRETER |
250 | | tcg_prologue_fn *tcg_qemu_tb_exec; |
251 | | #endif |
252 | | |
253 | | static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; |
254 | | static TCGRegSet tcg_target_call_clobber_regs; |
255 | | |
256 | | #if TCG_TARGET_INSN_UNIT_SIZE == 1 |
257 | | static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) |
258 | 0 | { |
259 | 0 | *s->code_ptr++ = v; |
260 | 0 | } |
261 | | |
262 | | static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, |
263 | | uint8_t v) |
264 | 0 | { |
265 | 0 | *p = v; |
266 | 0 | } |
267 | | #endif |
268 | | |
269 | | #if TCG_TARGET_INSN_UNIT_SIZE <= 2 |
270 | | static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) |
271 | 0 | { |
272 | 0 | if (TCG_TARGET_INSN_UNIT_SIZE == 2) { |
273 | 0 | *s->code_ptr++ = v; |
274 | 0 | } else { |
275 | 0 | tcg_insn_unit *p = s->code_ptr; |
276 | 0 | memcpy(p, &v, sizeof(v)); |
277 | 0 | s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); |
278 | 0 | } |
279 | 0 | } |
280 | | |
281 | | static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, |
282 | | uint16_t v) |
283 | 0 | { |
284 | 0 | if (TCG_TARGET_INSN_UNIT_SIZE == 2) { |
285 | 0 | *p = v; |
286 | 0 | } else { |
287 | 0 | memcpy(p, &v, sizeof(v)); |
288 | 0 | } |
289 | 0 | } |
290 | | #endif |
291 | | |
292 | | #if TCG_TARGET_INSN_UNIT_SIZE <= 4 |
293 | | static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) |
294 | 0 | { |
295 | 0 | if (TCG_TARGET_INSN_UNIT_SIZE == 4) { |
296 | 0 | *s->code_ptr++ = v; |
297 | 0 | } else { |
298 | 0 | tcg_insn_unit *p = s->code_ptr; |
299 | 0 | memcpy(p, &v, sizeof(v)); |
300 | 0 | s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); |
301 | 0 | } |
302 | 0 | } |
303 | | |
304 | | static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, |
305 | | uint32_t v) |
306 | 0 | { |
307 | 0 | if (TCG_TARGET_INSN_UNIT_SIZE == 4) { |
308 | 0 | *p = v; |
309 | 0 | } else { |
310 | 0 | memcpy(p, &v, sizeof(v)); |
311 | 0 | } |
312 | 0 | } |
313 | | #endif |
314 | | |
315 | | #if TCG_TARGET_INSN_UNIT_SIZE <= 8 |
316 | | static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) |
317 | 0 | { |
318 | 0 | if (TCG_TARGET_INSN_UNIT_SIZE == 8) { |
319 | 0 | *s->code_ptr++ = v; |
320 | 0 | } else { |
321 | 0 | tcg_insn_unit *p = s->code_ptr; |
322 | 0 | memcpy(p, &v, sizeof(v)); |
323 | 0 | s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); |
324 | 0 | } |
325 | 0 | } |
326 | | |
327 | | static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, |
328 | | uint64_t v) |
329 | 0 | { |
330 | 0 | if (TCG_TARGET_INSN_UNIT_SIZE == 8) { |
331 | 0 | *p = v; |
332 | 0 | } else { |
333 | 0 | memcpy(p, &v, sizeof(v)); |
334 | 0 | } |
335 | 0 | } |
336 | | #endif |
337 | | |
338 | | /* label relocation processing */ |
339 | | |
340 | | static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, |
341 | | TCGLabel *l, intptr_t addend) |
342 | 0 | { |
343 | 0 | TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); |
344 | |
|
345 | 0 | r->type = type; |
346 | 0 | r->ptr = code_ptr; |
347 | 0 | r->addend = addend; |
348 | 0 | QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); |
349 | 0 | } |
350 | | |
351 | | static void tcg_out_label(TCGContext *s, TCGLabel *l) |
352 | 0 | { |
353 | 0 | tcg_debug_assert(!l->has_value); |
354 | 0 | l->has_value = 1; |
355 | 0 | l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); |
356 | 0 | } |
357 | | |
358 | | TCGLabel *gen_new_label(void) |
359 | 0 | { |
360 | 0 | TCGContext *s = tcg_ctx; |
361 | 0 | TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); |
362 | |
|
363 | 0 | memset(l, 0, sizeof(TCGLabel)); |
364 | 0 | l->id = s->nb_labels++; |
365 | 0 | QSIMPLEQ_INIT(&l->branches); |
366 | 0 | QSIMPLEQ_INIT(&l->relocs); |
367 | |
|
368 | 0 | QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); |
369 | |
|
370 | 0 | return l; |
371 | 0 | } |
372 | | |
373 | | static bool tcg_resolve_relocs(TCGContext *s) |
374 | 0 | { |
375 | 0 | TCGLabel *l; |
376 | |
|
377 | 0 | QSIMPLEQ_FOREACH(l, &s->labels, next) { |
378 | 0 | TCGRelocation *r; |
379 | 0 | uintptr_t value = l->u.value; |
380 | |
|
381 | 0 | QSIMPLEQ_FOREACH(r, &l->relocs, next) { |
382 | 0 | if (!patch_reloc(r->ptr, r->type, value, r->addend)) { |
383 | 0 | return false; |
384 | 0 | } |
385 | 0 | } |
386 | 0 | } |
387 | 0 | return true; |
388 | 0 | } |
389 | | |
390 | | static void set_jmp_reset_offset(TCGContext *s, int which) |
391 | 0 | { |
392 | | /* |
393 | | * We will check for overflow at the end of the opcode loop in |
394 | | * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. |
395 | | */ |
396 | 0 | s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); |
397 | 0 | } |
398 | | |
399 | | static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) |
400 | 0 | { |
401 | | /* |
402 | | * We will check for overflow at the end of the opcode loop in |
403 | | * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. |
404 | | */ |
405 | 0 | s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); |
406 | 0 | } |
407 | | |
408 | | static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) |
409 | 0 | { |
410 | 0 | /* |
411 | 0 | * Return the read-execute version of the pointer, for the benefit |
412 | 0 | * of any pc-relative addressing mode. |
413 | 0 | */ |
414 | 0 | return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); |
415 | 0 | } |
416 | | |
417 | | static int __attribute__((unused)) |
418 | | tlb_mask_table_ofs(TCGContext *s, int which) |
419 | 0 | { |
420 | 0 | int fi = mmuidx_to_fast_index(which); |
421 | 0 | return (offsetof(CPUNegativeOffsetState, tlb.f[fi]) - |
422 | 0 | sizeof(CPUNegativeOffsetState)); |
423 | 0 | } |
424 | | |
425 | | /* Signal overflow, starting over with fewer guest insns. */ |
426 | | static G_NORETURN |
427 | | void tcg_raise_tb_overflow(TCGContext *s) |
428 | 0 | { |
429 | 0 | siglongjmp(s->jmp_trans, -2); |
430 | 0 | } |
431 | | |
432 | | /* |
433 | | * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext. |
434 | | * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg. |
435 | | * |
436 | | * However, tcg_out_helper_load_slots reuses this field to hold an |
437 | | * argument slot number (which may designate a argument register or an |
438 | | * argument stack slot), converting to TCGReg once all arguments that |
439 | | * are destined for the stack are processed. |
440 | | */ |
441 | | typedef struct TCGMovExtend { |
442 | | unsigned dst; |
443 | | TCGReg src; |
444 | | TCGType dst_type; |
445 | | TCGType src_type; |
446 | | MemOp src_ext; |
447 | | } TCGMovExtend; |
448 | | |
449 | | /** |
450 | | * tcg_out_movext -- move and extend |
451 | | * @s: tcg context |
452 | | * @dst_type: integral type for destination |
453 | | * @dst: destination register |
454 | | * @src_type: integral type for source |
455 | | * @src_ext: extension to apply to source |
456 | | * @src: source register |
457 | | * |
458 | | * Move or extend @src into @dst, depending on @src_ext and the types. |
459 | | */ |
460 | | static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst, |
461 | | TCGType src_type, MemOp src_ext, TCGReg src) |
462 | 0 | { |
463 | 0 | switch (src_ext) { |
464 | 0 | case MO_UB: |
465 | 0 | tcg_out_ext8u(s, dst, src); |
466 | 0 | break; |
467 | 0 | case MO_SB: |
468 | 0 | tcg_out_ext8s(s, dst_type, dst, src); |
469 | 0 | break; |
470 | 0 | case MO_UW: |
471 | 0 | tcg_out_ext16u(s, dst, src); |
472 | 0 | break; |
473 | 0 | case MO_SW: |
474 | 0 | tcg_out_ext16s(s, dst_type, dst, src); |
475 | 0 | break; |
476 | 0 | case MO_UL: |
477 | 0 | case MO_SL: |
478 | 0 | if (dst_type == TCG_TYPE_I32) { |
479 | 0 | if (src_type == TCG_TYPE_I32) { |
480 | 0 | tcg_out_mov(s, TCG_TYPE_I32, dst, src); |
481 | 0 | } else { |
482 | 0 | tcg_out_extrl_i64_i32(s, dst, src); |
483 | 0 | } |
484 | 0 | } else if (src_type == TCG_TYPE_I32) { |
485 | 0 | if (src_ext & MO_SIGN) { |
486 | 0 | tcg_out_exts_i32_i64(s, dst, src); |
487 | 0 | } else { |
488 | 0 | tcg_out_extu_i32_i64(s, dst, src); |
489 | 0 | } |
490 | 0 | } else { |
491 | 0 | if (src_ext & MO_SIGN) { |
492 | 0 | tcg_out_ext32s(s, dst, src); |
493 | 0 | } else { |
494 | 0 | tcg_out_ext32u(s, dst, src); |
495 | 0 | } |
496 | 0 | } |
497 | 0 | break; |
498 | 0 | case MO_UQ: |
499 | 0 | if (dst_type == TCG_TYPE_I32) { |
500 | 0 | tcg_out_extrl_i64_i32(s, dst, src); |
501 | 0 | } else { |
502 | 0 | tcg_out_mov(s, TCG_TYPE_I64, dst, src); |
503 | 0 | } |
504 | 0 | break; |
505 | 0 | default: |
506 | 0 | g_assert_not_reached(); |
507 | 0 | } |
508 | 0 | } |
509 | | |
510 | | /* Minor variations on a theme, using a structure. */ |
511 | | static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i, |
512 | | TCGReg src) |
513 | 0 | { |
514 | 0 | tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src); |
515 | 0 | } |
516 | | |
517 | | static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i) |
518 | 0 | { |
519 | 0 | tcg_out_movext1_new_src(s, i, i->src); |
520 | 0 | } |
521 | | |
522 | | /** |
523 | | * tcg_out_movext2 -- move and extend two pair |
524 | | * @s: tcg context |
525 | | * @i1: first move description |
526 | | * @i2: second move description |
527 | | * @scratch: temporary register, or -1 for none |
528 | | * |
529 | | * As tcg_out_movext, for both @i1 and @i2, caring for overlap |
530 | | * between the sources and destinations. |
531 | | */ |
532 | | |
533 | | static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1, |
534 | | const TCGMovExtend *i2, int scratch) |
535 | 0 | { |
536 | 0 | TCGReg src1 = i1->src; |
537 | 0 | TCGReg src2 = i2->src; |
538 | |
|
539 | 0 | if (i1->dst != src2) { |
540 | 0 | tcg_out_movext1(s, i1); |
541 | 0 | tcg_out_movext1(s, i2); |
542 | 0 | return; |
543 | 0 | } |
544 | 0 | if (i2->dst == src1) { |
545 | 0 | TCGType src1_type = i1->src_type; |
546 | 0 | TCGType src2_type = i2->src_type; |
547 | |
|
548 | 0 | if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) { |
549 | | /* The data is now in the correct registers, now extend. */ |
550 | 0 | src1 = i2->src; |
551 | 0 | src2 = i1->src; |
552 | 0 | } else { |
553 | 0 | tcg_debug_assert(scratch >= 0); |
554 | 0 | tcg_out_mov(s, src1_type, scratch, src1); |
555 | 0 | src1 = scratch; |
556 | 0 | } |
557 | 0 | } |
558 | 0 | tcg_out_movext1_new_src(s, i2, src2); |
559 | 0 | tcg_out_movext1_new_src(s, i1, src1); |
560 | 0 | } |
561 | | |
562 | | /** |
563 | | * tcg_out_movext3 -- move and extend three pair |
564 | | * @s: tcg context |
565 | | * @i1: first move description |
566 | | * @i2: second move description |
567 | | * @i3: third move description |
568 | | * @scratch: temporary register, or -1 for none |
569 | | * |
570 | | * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap |
571 | | * between the sources and destinations. |
572 | | */ |
573 | | |
574 | | static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1, |
575 | | const TCGMovExtend *i2, const TCGMovExtend *i3, |
576 | | int scratch) |
577 | 0 | { |
578 | 0 | TCGReg src1 = i1->src; |
579 | 0 | TCGReg src2 = i2->src; |
580 | 0 | TCGReg src3 = i3->src; |
581 | |
|
582 | 0 | if (i1->dst != src2 && i1->dst != src3) { |
583 | 0 | tcg_out_movext1(s, i1); |
584 | 0 | tcg_out_movext2(s, i2, i3, scratch); |
585 | 0 | return; |
586 | 0 | } |
587 | 0 | if (i2->dst != src1 && i2->dst != src3) { |
588 | 0 | tcg_out_movext1(s, i2); |
589 | 0 | tcg_out_movext2(s, i1, i3, scratch); |
590 | 0 | return; |
591 | 0 | } |
592 | 0 | if (i3->dst != src1 && i3->dst != src2) { |
593 | 0 | tcg_out_movext1(s, i3); |
594 | 0 | tcg_out_movext2(s, i1, i2, scratch); |
595 | 0 | return; |
596 | 0 | } |
597 | | |
598 | | /* |
599 | | * There is a cycle. Since there are only 3 nodes, the cycle is |
600 | | * either "clockwise" or "anti-clockwise", and can be solved with |
601 | | * a single scratch or two xchg. |
602 | | */ |
603 | 0 | if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) { |
604 | | /* "Clockwise" */ |
605 | 0 | if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) { |
606 | 0 | tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3); |
607 | | /* The data is now in the correct registers, now extend. */ |
608 | 0 | tcg_out_movext1_new_src(s, i1, i1->dst); |
609 | 0 | tcg_out_movext1_new_src(s, i2, i2->dst); |
610 | 0 | tcg_out_movext1_new_src(s, i3, i3->dst); |
611 | 0 | } else { |
612 | 0 | tcg_debug_assert(scratch >= 0); |
613 | 0 | tcg_out_mov(s, i1->src_type, scratch, src1); |
614 | 0 | tcg_out_movext1(s, i3); |
615 | 0 | tcg_out_movext1(s, i2); |
616 | 0 | tcg_out_movext1_new_src(s, i1, scratch); |
617 | 0 | } |
618 | 0 | } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) { |
619 | | /* "Anti-clockwise" */ |
620 | 0 | if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) { |
621 | 0 | tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2); |
622 | | /* The data is now in the correct registers, now extend. */ |
623 | 0 | tcg_out_movext1_new_src(s, i1, i1->dst); |
624 | 0 | tcg_out_movext1_new_src(s, i2, i2->dst); |
625 | 0 | tcg_out_movext1_new_src(s, i3, i3->dst); |
626 | 0 | } else { |
627 | 0 | tcg_debug_assert(scratch >= 0); |
628 | 0 | tcg_out_mov(s, i1->src_type, scratch, src1); |
629 | 0 | tcg_out_movext1(s, i2); |
630 | 0 | tcg_out_movext1(s, i3); |
631 | 0 | tcg_out_movext1_new_src(s, i1, scratch); |
632 | 0 | } |
633 | 0 | } else { |
634 | 0 | g_assert_not_reached(); |
635 | 0 | } |
636 | 0 | } |
637 | | |
638 | | /* |
639 | | * Allocate a new TCGLabelQemuLdst entry. |
640 | | */ |
641 | | |
642 | | __attribute__((unused)) |
643 | | static TCGLabelQemuLdst *new_ldst_label(TCGContext *s) |
644 | 0 | { |
645 | 0 | TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l)); |
646 | |
|
647 | 0 | memset(l, 0, sizeof(*l)); |
648 | 0 | QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next); |
649 | |
|
650 | 0 | return l; |
651 | 0 | } |
652 | | |
653 | | /* |
654 | | * Allocate new constant pool entries. |
655 | | */ |
656 | | |
657 | | typedef struct TCGLabelPoolData { |
658 | | struct TCGLabelPoolData *next; |
659 | | tcg_insn_unit *label; |
660 | | intptr_t addend; |
661 | | int rtype; |
662 | | unsigned nlong; |
663 | | tcg_target_ulong data[]; |
664 | | } TCGLabelPoolData; |
665 | | |
666 | | static TCGLabelPoolData *new_pool_alloc(TCGContext *s, int nlong, int rtype, |
667 | | tcg_insn_unit *label, intptr_t addend) |
668 | 0 | { |
669 | 0 | TCGLabelPoolData *n = tcg_malloc(sizeof(TCGLabelPoolData) |
670 | 0 | + sizeof(tcg_target_ulong) * nlong); |
671 | |
|
672 | 0 | n->label = label; |
673 | 0 | n->addend = addend; |
674 | 0 | n->rtype = rtype; |
675 | 0 | n->nlong = nlong; |
676 | 0 | return n; |
677 | 0 | } |
678 | | |
679 | | static void new_pool_insert(TCGContext *s, TCGLabelPoolData *n) |
680 | 0 | { |
681 | 0 | TCGLabelPoolData *i, **pp; |
682 | 0 | int nlong = n->nlong; |
683 | | |
684 | | /* Insertion sort on the pool. */ |
685 | 0 | for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) { |
686 | 0 | if (nlong > i->nlong) { |
687 | 0 | break; |
688 | 0 | } |
689 | 0 | if (nlong < i->nlong) { |
690 | 0 | continue; |
691 | 0 | } |
692 | 0 | if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) { |
693 | 0 | break; |
694 | 0 | } |
695 | 0 | } |
696 | 0 | n->next = *pp; |
697 | 0 | *pp = n; |
698 | 0 | } |
699 | | |
700 | | /* The "usual" for generic integer code. */ |
701 | | __attribute__((unused)) |
702 | | static void new_pool_label(TCGContext *s, tcg_target_ulong d, int rtype, |
703 | | tcg_insn_unit *label, intptr_t addend) |
704 | 0 | { |
705 | 0 | TCGLabelPoolData *n = new_pool_alloc(s, 1, rtype, label, addend); |
706 | 0 | n->data[0] = d; |
707 | 0 | new_pool_insert(s, n); |
708 | 0 | } |
709 | | |
710 | | /* For v64 or v128, depending on the host. */ |
711 | | __attribute__((unused)) |
712 | | static void new_pool_l2(TCGContext *s, int rtype, tcg_insn_unit *label, |
713 | | intptr_t addend, tcg_target_ulong d0, |
714 | | tcg_target_ulong d1) |
715 | 0 | { |
716 | 0 | TCGLabelPoolData *n = new_pool_alloc(s, 2, rtype, label, addend); |
717 | 0 | n->data[0] = d0; |
718 | 0 | n->data[1] = d1; |
719 | 0 | new_pool_insert(s, n); |
720 | 0 | } |
721 | | |
722 | | /* For v128 or v256, depending on the host. */ |
723 | | __attribute__((unused)) |
724 | | static void new_pool_l4(TCGContext *s, int rtype, tcg_insn_unit *label, |
725 | | intptr_t addend, tcg_target_ulong d0, |
726 | | tcg_target_ulong d1, tcg_target_ulong d2, |
727 | | tcg_target_ulong d3) |
728 | 0 | { |
729 | 0 | TCGLabelPoolData *n = new_pool_alloc(s, 4, rtype, label, addend); |
730 | 0 | n->data[0] = d0; |
731 | 0 | n->data[1] = d1; |
732 | 0 | n->data[2] = d2; |
733 | 0 | n->data[3] = d3; |
734 | 0 | new_pool_insert(s, n); |
735 | 0 | } |
736 | | |
737 | | /* For v256, for 32-bit host. */ |
738 | | __attribute__((unused)) |
739 | | static void new_pool_l8(TCGContext *s, int rtype, tcg_insn_unit *label, |
740 | | intptr_t addend, tcg_target_ulong d0, |
741 | | tcg_target_ulong d1, tcg_target_ulong d2, |
742 | | tcg_target_ulong d3, tcg_target_ulong d4, |
743 | | tcg_target_ulong d5, tcg_target_ulong d6, |
744 | | tcg_target_ulong d7) |
745 | 0 | { |
746 | 0 | TCGLabelPoolData *n = new_pool_alloc(s, 8, rtype, label, addend); |
747 | 0 | n->data[0] = d0; |
748 | 0 | n->data[1] = d1; |
749 | 0 | n->data[2] = d2; |
750 | 0 | n->data[3] = d3; |
751 | 0 | n->data[4] = d4; |
752 | 0 | n->data[5] = d5; |
753 | 0 | n->data[6] = d6; |
754 | 0 | n->data[7] = d7; |
755 | 0 | new_pool_insert(s, n); |
756 | 0 | } |
757 | | |
758 | | /* |
759 | | * Generate TB finalization at the end of block |
760 | | */ |
761 | | |
762 | | static int tcg_out_ldst_finalize(TCGContext *s) |
763 | 0 | { |
764 | 0 | TCGLabelQemuLdst *lb; |
765 | | |
766 | | /* qemu_ld/st slow paths */ |
767 | 0 | QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) { |
768 | 0 | if (lb->is_ld |
769 | 0 | ? !tcg_out_qemu_ld_slow_path(s, lb) |
770 | 0 | : !tcg_out_qemu_st_slow_path(s, lb)) { |
771 | 0 | return -2; |
772 | 0 | } |
773 | | |
774 | | /* |
775 | | * Test for (pending) buffer overflow. The assumption is that any |
776 | | * one operation beginning below the high water mark cannot overrun |
777 | | * the buffer completely. Thus we can test for overflow after |
778 | | * generating code without having to check during generation. |
779 | | */ |
780 | 0 | if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { |
781 | 0 | return -1; |
782 | 0 | } |
783 | 0 | } |
784 | 0 | return 0; |
785 | 0 | } |
786 | | |
787 | | static int tcg_out_pool_finalize(TCGContext *s) |
788 | 0 | { |
789 | 0 | TCGLabelPoolData *p = s->pool_labels; |
790 | 0 | TCGLabelPoolData *l = NULL; |
791 | 0 | void *a; |
792 | |
|
793 | 0 | if (p == NULL) { |
794 | 0 | return 0; |
795 | 0 | } |
796 | | |
797 | | /* |
798 | | * ??? Round up to qemu_icache_linesize, but then do not round |
799 | | * again when allocating the next TranslationBlock structure. |
800 | | */ |
801 | 0 | a = (void *)ROUND_UP((uintptr_t)s->code_ptr, |
802 | 0 | sizeof(tcg_target_ulong) * p->nlong); |
803 | 0 | tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr); |
804 | 0 | s->data_gen_ptr = a; |
805 | |
|
806 | 0 | for (; p != NULL; p = p->next) { |
807 | 0 | size_t size = sizeof(tcg_target_ulong) * p->nlong; |
808 | 0 | uintptr_t value; |
809 | |
|
810 | 0 | if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) { |
811 | 0 | if (unlikely(a > s->code_gen_highwater)) { |
812 | 0 | return -1; |
813 | 0 | } |
814 | 0 | memcpy(a, p->data, size); |
815 | 0 | a += size; |
816 | 0 | l = p; |
817 | 0 | } |
818 | | |
819 | 0 | value = (uintptr_t)tcg_splitwx_to_rx(a) - size; |
820 | 0 | if (!patch_reloc(p->label, p->rtype, value, p->addend)) { |
821 | 0 | return -2; |
822 | 0 | } |
823 | 0 | } |
824 | | |
825 | 0 | s->code_ptr = a; |
826 | 0 | return 0; |
827 | 0 | } |
828 | | |
829 | | #define C_PFX1(P, A) P##A |
830 | 0 | #define C_PFX2(P, A, B) P##A##_##B |
831 | 0 | #define C_PFX3(P, A, B, C) P##A##_##B##_##C |
832 | 0 | #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D |
833 | 0 | #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E |
834 | | #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F |
835 | | |
836 | | /* Define an enumeration for the various combinations. */ |
837 | | |
838 | | #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), |
839 | | #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), |
840 | | #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), |
841 | | #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), |
842 | | |
843 | | #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), |
844 | | #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), |
845 | | #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), |
846 | | #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), |
847 | | |
848 | | #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), |
849 | | #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1), |
850 | | #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1), |
851 | | |
852 | | #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), |
853 | | #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), |
854 | | #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), |
855 | | #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), |
856 | | #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4), |
857 | | |
858 | | typedef enum { |
859 | | C_Dynamic = -2, |
860 | | C_NotImplemented = -1, |
861 | | #include "tcg-target-con-set.h" |
862 | | } TCGConstraintSetIndex; |
863 | | |
864 | | static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode, TCGType, unsigned); |
865 | | |
866 | | #undef C_O0_I1 |
867 | | #undef C_O0_I2 |
868 | | #undef C_O0_I3 |
869 | | #undef C_O0_I4 |
870 | | #undef C_O1_I1 |
871 | | #undef C_O1_I2 |
872 | | #undef C_O1_I3 |
873 | | #undef C_O1_I4 |
874 | | #undef C_N1_I2 |
875 | | #undef C_N1O1_I1 |
876 | | #undef C_N2_I1 |
877 | | #undef C_O2_I1 |
878 | | #undef C_O2_I2 |
879 | | #undef C_O2_I3 |
880 | | #undef C_O2_I4 |
881 | | #undef C_N1_O1_I4 |
882 | | |
883 | | /* Put all of the constraint sets into an array, indexed by the enum. */ |
884 | | |
885 | | typedef struct TCGConstraintSet { |
886 | | uint8_t nb_oargs, nb_iargs; |
887 | | const char *args_ct_str[TCG_MAX_OP_ARGS]; |
888 | | } TCGConstraintSet; |
889 | | |
890 | | #define C_O0_I1(I1) { 0, 1, { #I1 } }, |
891 | | #define C_O0_I2(I1, I2) { 0, 2, { #I1, #I2 } }, |
892 | | #define C_O0_I3(I1, I2, I3) { 0, 3, { #I1, #I2, #I3 } }, |
893 | | #define C_O0_I4(I1, I2, I3, I4) { 0, 4, { #I1, #I2, #I3, #I4 } }, |
894 | | |
895 | | #define C_O1_I1(O1, I1) { 1, 1, { #O1, #I1 } }, |
896 | | #define C_O1_I2(O1, I1, I2) { 1, 2, { #O1, #I1, #I2 } }, |
897 | | #define C_O1_I3(O1, I1, I2, I3) { 1, 3, { #O1, #I1, #I2, #I3 } }, |
898 | | #define C_O1_I4(O1, I1, I2, I3, I4) { 1, 4, { #O1, #I1, #I2, #I3, #I4 } }, |
899 | | |
900 | | #define C_N1_I2(O1, I1, I2) { 1, 2, { "&" #O1, #I1, #I2 } }, |
901 | | #define C_N1O1_I1(O1, O2, I1) { 2, 1, { "&" #O1, #O2, #I1 } }, |
902 | | #define C_N2_I1(O1, O2, I1) { 2, 1, { "&" #O1, "&" #O2, #I1 } }, |
903 | | |
904 | | #define C_O2_I1(O1, O2, I1) { 2, 1, { #O1, #O2, #I1 } }, |
905 | | #define C_O2_I2(O1, O2, I1, I2) { 2, 2, { #O1, #O2, #I1, #I2 } }, |
906 | | #define C_O2_I3(O1, O2, I1, I2, I3) { 2, 3, { #O1, #O2, #I1, #I2, #I3 } }, |
907 | | #define C_O2_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { #O1, #O2, #I1, #I2, #I3, #I4 } }, |
908 | | #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { "&" #O1, #O2, #I1, #I2, #I3, #I4 } }, |
909 | | |
910 | | static const TCGConstraintSet constraint_sets[] = { |
911 | | #include "tcg-target-con-set.h" |
912 | | }; |
913 | | |
914 | | #undef C_O0_I1 |
915 | | #undef C_O0_I2 |
916 | | #undef C_O0_I3 |
917 | | #undef C_O0_I4 |
918 | | #undef C_O1_I1 |
919 | | #undef C_O1_I2 |
920 | | #undef C_O1_I3 |
921 | | #undef C_O1_I4 |
922 | | #undef C_N1_I2 |
923 | | #undef C_N1O1_I1 |
924 | | #undef C_N2_I1 |
925 | | #undef C_O2_I1 |
926 | | #undef C_O2_I2 |
927 | | #undef C_O2_I3 |
928 | | #undef C_O2_I4 |
929 | | #undef C_N1_O1_I4 |
930 | | |
931 | | /* Expand the enumerator to be returned from tcg_target_op_def(). */ |
932 | | |
933 | | #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) |
934 | 0 | #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) |
935 | | #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) |
936 | | #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) |
937 | | |
938 | 0 | #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) |
939 | 0 | #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) |
940 | 0 | #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) |
941 | 0 | #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) |
942 | | |
943 | 0 | #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) |
944 | | #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1) |
945 | | #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1) |
946 | | |
947 | | #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) |
948 | 0 | #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) |
949 | | #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) |
950 | | #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) |
951 | | #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4) |
952 | | |
953 | | /* |
954 | | * TCGOutOp is the base class for a set of structures that describe how |
955 | | * to generate code for a given TCGOpcode. |
956 | | * |
957 | | * @static_constraint: |
958 | | * C_NotImplemented: The TCGOpcode is not supported by the backend. |
959 | | * C_Dynamic: Use @dynamic_constraint to select a constraint set |
960 | | * based on any of @type, @flags, or host isa. |
961 | | * Otherwise: The register allocation constrains for the TCGOpcode. |
962 | | * |
963 | | * Subclasses of TCGOutOp will define a set of output routines that may |
964 | | * be used. Such routines will often be selected by the set of registers |
965 | | * and constants that come out of register allocation. The set of |
966 | | * routines that are provided will guide the set of constraints that are |
967 | | * legal. In particular, assume that tcg_optimize() has done its job in |
968 | | * swapping commutative operands and folding operations for which all |
969 | | * operands are constant. |
970 | | */ |
971 | | typedef struct TCGOutOp { |
972 | | TCGConstraintSetIndex static_constraint; |
973 | | TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags); |
974 | | } TCGOutOp; |
975 | | |
976 | | typedef struct TCGOutOpAddSubCarry { |
977 | | TCGOutOp base; |
978 | | void (*out_rrr)(TCGContext *s, TCGType type, |
979 | | TCGReg a0, TCGReg a1, TCGReg a2); |
980 | | void (*out_rri)(TCGContext *s, TCGType type, |
981 | | TCGReg a0, TCGReg a1, tcg_target_long a2); |
982 | | void (*out_rir)(TCGContext *s, TCGType type, |
983 | | TCGReg a0, tcg_target_long a1, TCGReg a2); |
984 | | void (*out_rii)(TCGContext *s, TCGType type, |
985 | | TCGReg a0, tcg_target_long a1, tcg_target_long a2); |
986 | | } TCGOutOpAddSubCarry; |
987 | | |
988 | | typedef struct TCGOutOpBinary { |
989 | | TCGOutOp base; |
990 | | void (*out_rrr)(TCGContext *s, TCGType type, |
991 | | TCGReg a0, TCGReg a1, TCGReg a2); |
992 | | void (*out_rri)(TCGContext *s, TCGType type, |
993 | | TCGReg a0, TCGReg a1, tcg_target_long a2); |
994 | | } TCGOutOpBinary; |
995 | | |
996 | | typedef struct TCGOutOpBrcond { |
997 | | TCGOutOp base; |
998 | | void (*out_rr)(TCGContext *s, TCGType type, TCGCond cond, |
999 | | TCGReg a1, TCGReg a2, TCGLabel *label); |
1000 | | void (*out_ri)(TCGContext *s, TCGType type, TCGCond cond, |
1001 | | TCGReg a1, tcg_target_long a2, TCGLabel *label); |
1002 | | } TCGOutOpBrcond; |
1003 | | |
1004 | | typedef struct TCGOutOpBswap { |
1005 | | TCGOutOp base; |
1006 | | void (*out_rr)(TCGContext *s, TCGType type, |
1007 | | TCGReg a0, TCGReg a1, unsigned flags); |
1008 | | } TCGOutOpBswap; |
1009 | | |
1010 | | typedef struct TCGOutOpDeposit { |
1011 | | TCGOutOp base; |
1012 | | void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, |
1013 | | TCGReg a2, unsigned ofs, unsigned len); |
1014 | | void (*out_rri)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, |
1015 | | tcg_target_long a2, unsigned ofs, unsigned len); |
1016 | | void (*out_rzr)(TCGContext *s, TCGType type, TCGReg a0, |
1017 | | TCGReg a2, unsigned ofs, unsigned len); |
1018 | | } TCGOutOpDeposit; |
1019 | | |
1020 | | typedef struct TCGOutOpDivRem { |
1021 | | TCGOutOp base; |
1022 | | void (*out_rr01r)(TCGContext *s, TCGType type, |
1023 | | TCGReg a0, TCGReg a1, TCGReg a4); |
1024 | | } TCGOutOpDivRem; |
1025 | | |
1026 | | typedef struct TCGOutOpExtract { |
1027 | | TCGOutOp base; |
1028 | | void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, |
1029 | | unsigned ofs, unsigned len); |
1030 | | } TCGOutOpExtract; |
1031 | | |
1032 | | typedef struct TCGOutOpExtract2 { |
1033 | | TCGOutOp base; |
1034 | | void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, |
1035 | | TCGReg a2, unsigned shr); |
1036 | | } TCGOutOpExtract2; |
1037 | | |
1038 | | typedef struct TCGOutOpLoad { |
1039 | | TCGOutOp base; |
1040 | | void (*out)(TCGContext *s, TCGType type, TCGReg dest, |
1041 | | TCGReg base, intptr_t offset); |
1042 | | } TCGOutOpLoad; |
1043 | | |
1044 | | typedef struct TCGOutOpMovcond { |
1045 | | TCGOutOp base; |
1046 | | void (*out)(TCGContext *s, TCGType type, TCGCond cond, |
1047 | | TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, |
1048 | | TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf); |
1049 | | } TCGOutOpMovcond; |
1050 | | |
1051 | | typedef struct TCGOutOpMul2 { |
1052 | | TCGOutOp base; |
1053 | | void (*out_rrrr)(TCGContext *s, TCGType type, |
1054 | | TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3); |
1055 | | } TCGOutOpMul2; |
1056 | | |
1057 | | typedef struct TCGOutOpQemuLdSt { |
1058 | | TCGOutOp base; |
1059 | | void (*out)(TCGContext *s, TCGType type, TCGReg dest, |
1060 | | TCGReg addr, MemOpIdx oi); |
1061 | | } TCGOutOpQemuLdSt; |
1062 | | |
1063 | | typedef struct TCGOutOpQemuLdSt2 { |
1064 | | TCGOutOp base; |
1065 | | void (*out)(TCGContext *s, TCGType type, TCGReg dlo, TCGReg dhi, |
1066 | | TCGReg addr, MemOpIdx oi); |
1067 | | } TCGOutOpQemuLdSt2; |
1068 | | |
1069 | | typedef struct TCGOutOpUnary { |
1070 | | TCGOutOp base; |
1071 | | void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1); |
1072 | | } TCGOutOpUnary; |
1073 | | |
1074 | | typedef struct TCGOutOpSetcond { |
1075 | | TCGOutOp base; |
1076 | | void (*out_rrr)(TCGContext *s, TCGType type, TCGCond cond, |
1077 | | TCGReg ret, TCGReg a1, TCGReg a2); |
1078 | | void (*out_rri)(TCGContext *s, TCGType type, TCGCond cond, |
1079 | | TCGReg ret, TCGReg a1, tcg_target_long a2); |
1080 | | } TCGOutOpSetcond; |
1081 | | |
1082 | | typedef struct TCGOutOpStore { |
1083 | | TCGOutOp base; |
1084 | | void (*out_r)(TCGContext *s, TCGType type, TCGReg data, |
1085 | | TCGReg base, intptr_t offset); |
1086 | | void (*out_i)(TCGContext *s, TCGType type, tcg_target_long data, |
1087 | | TCGReg base, intptr_t offset); |
1088 | | } TCGOutOpStore; |
1089 | | |
1090 | | typedef struct TCGOutOpSubtract { |
1091 | | TCGOutOp base; |
1092 | | void (*out_rrr)(TCGContext *s, TCGType type, |
1093 | | TCGReg a0, TCGReg a1, TCGReg a2); |
1094 | | void (*out_rir)(TCGContext *s, TCGType type, |
1095 | | TCGReg a0, tcg_target_long a1, TCGReg a2); |
1096 | | } TCGOutOpSubtract; |
1097 | | |
1098 | | typedef struct TCGOutOpLea { |
1099 | | TCGOutOp base; |
1100 | | void (*out)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, |
1101 | | TCGReg a2, unsigned sh, tcg_target_long imm); |
1102 | | } TCGOutOpLea; |
1103 | | |
1104 | | #include "tcg-target.c.inc" |
1105 | | |
1106 | | #ifndef CONFIG_TCG_INTERPRETER |
1107 | | /* Validate CPUTLBDescFast placement. */ |
1108 | | QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) - |
1109 | | sizeof(CPUNegativeOffsetState)) |
1110 | | < MIN_TLB_MASK_TABLE_OFS); |
1111 | | #endif |
1112 | | |
1113 | | /* |
1114 | | * We require these functions for slow-path function calls. |
1115 | | * Adapt them generically for opcode output. |
1116 | | */ |
1117 | | |
1118 | | static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) |
1119 | 0 | { |
1120 | 0 | tcg_out_exts_i32_i64(s, a0, a1); |
1121 | 0 | } |
1122 | | |
1123 | | static const TCGOutOpUnary outop_exts_i32_i64 = { |
1124 | | .base.static_constraint = C_O1_I1(r, r), |
1125 | | .out_rr = tgen_exts_i32_i64, |
1126 | | }; |
1127 | | |
1128 | | static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) |
1129 | 0 | { |
1130 | 0 | tcg_out_extu_i32_i64(s, a0, a1); |
1131 | 0 | } |
1132 | | |
1133 | | static const TCGOutOpUnary outop_extu_i32_i64 = { |
1134 | | .base.static_constraint = C_O1_I1(r, r), |
1135 | | .out_rr = tgen_extu_i32_i64, |
1136 | | }; |
1137 | | |
1138 | | static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) |
1139 | 0 | { |
1140 | 0 | tcg_out_extrl_i64_i32(s, a0, a1); |
1141 | 0 | } |
1142 | | |
1143 | | static const TCGOutOpUnary outop_extrl_i64_i32 = { |
1144 | | .base.static_constraint = C_O1_I1(r, r), |
1145 | | .out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL, |
1146 | | }; |
1147 | | |
1148 | | static const TCGOutOp outop_goto_ptr = { |
1149 | | .static_constraint = C_O0_I1(r), |
1150 | | }; |
1151 | | |
1152 | | static const TCGOutOpLoad outop_ld = { |
1153 | | .base.static_constraint = C_O1_I1(r, r), |
1154 | | .out = tcg_out_ld, |
1155 | | }; |
1156 | | |
1157 | | /* |
1158 | | * Register V as the TCGOutOp for O. |
1159 | | * This verifies that V is of type T, otherwise give a nice compiler error. |
1160 | | * This prevents trivial mistakes within each arch/tcg-target.c.inc. |
1161 | | */ |
1162 | | #define OUTOP(O, T, V) [O] = _Generic(V, T: &V.base) |
1163 | | |
1164 | | /* Register allocation descriptions for every TCGOpcode. */ |
1165 | | static const TCGOutOp * const all_outop[NB_OPS] = { |
1166 | | OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add), |
1167 | | /* addc1o is implemented with set_carry + addcio */ |
1168 | | OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio), |
1169 | | OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci), |
1170 | | OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio), |
1171 | | OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco), |
1172 | | OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and), |
1173 | | OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc), |
1174 | | OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond), |
1175 | | OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16), |
1176 | | OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32), |
1177 | | OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64), |
1178 | | OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz), |
1179 | | OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop), |
1180 | | OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz), |
1181 | | OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit), |
1182 | | OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs), |
1183 | | OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2), |
1184 | | OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu), |
1185 | | OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2), |
1186 | | OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv), |
1187 | | OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract), |
1188 | | OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2), |
1189 | | OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64), |
1190 | | OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32), |
1191 | | OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32), |
1192 | | OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64), |
1193 | | OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld), |
1194 | | OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s), |
1195 | | OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u), |
1196 | | OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s), |
1197 | | OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u), |
1198 | | OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s), |
1199 | | OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u), |
1200 | | OUTOP(INDEX_op_lea, TCGOutOpLea, outop_lea), |
1201 | | OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond), |
1202 | | OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul), |
1203 | | OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2), |
1204 | | OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh), |
1205 | | OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2), |
1206 | | OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh), |
1207 | | OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand), |
1208 | | OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg), |
1209 | | OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond), |
1210 | | OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor), |
1211 | | OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not), |
1212 | | OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or), |
1213 | | OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc), |
1214 | | OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld), |
1215 | | OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2), |
1216 | | OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st), |
1217 | | OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2), |
1218 | | OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems), |
1219 | | OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu), |
1220 | | OUTOP(INDEX_op_revbit8, TCGOutOpUnary, outop_revbit8), |
1221 | | OUTOP(INDEX_op_revbit32, TCGOutOpBswap, outop_revbit32), |
1222 | | OUTOP(INDEX_op_revbit64, TCGOutOpUnary, outop_revbit64), |
1223 | | OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl), |
1224 | | OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr), |
1225 | | OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar), |
1226 | | OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond), |
1227 | | OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract), |
1228 | | OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl), |
1229 | | OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr), |
1230 | | OUTOP(INDEX_op_smax, TCGOutOpBinary, outop_smax), |
1231 | | OUTOP(INDEX_op_smin, TCGOutOpBinary, outop_smin), |
1232 | | OUTOP(INDEX_op_st, TCGOutOpStore, outop_st), |
1233 | | OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8), |
1234 | | OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16), |
1235 | | OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st), |
1236 | | OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub), |
1237 | | /* subb1o is implemented with set_borrow + subbio */ |
1238 | | OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio), |
1239 | | OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi), |
1240 | | OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio), |
1241 | | OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo), |
1242 | | OUTOP(INDEX_op_umax, TCGOutOpBinary, outop_umax), |
1243 | | OUTOP(INDEX_op_umin, TCGOutOpBinary, outop_umin), |
1244 | | OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor), |
1245 | | |
1246 | | [INDEX_op_goto_ptr] = &outop_goto_ptr, |
1247 | | }; |
1248 | | |
1249 | | #undef OUTOP |
1250 | | |
1251 | | /* |
1252 | | * All TCG threads except the parent (i.e. the one that called tcg_context_init |
1253 | | * and registered the target's TCG globals) must register with this function |
1254 | | * before initiating translation. |
1255 | | * |
1256 | | * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation |
1257 | | * of tcg_region_init() for the reasoning behind this. |
1258 | | * |
1259 | | * In system-mode each caller registers its context in tcg_ctxs[]. Note that in |
1260 | | * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context |
1261 | | * is not used anymore for translation once this function is called. |
1262 | | * |
1263 | | * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that |
1264 | | * iterates over the array (e.g. tcg_code_size() the same for both system/user |
1265 | | * modes. |
1266 | | */ |
1267 | | #ifdef CONFIG_USER_ONLY |
1268 | | void tcg_register_thread(void) |
1269 | | { |
1270 | | tcg_ctx = &tcg_init_ctx; |
1271 | | } |
1272 | | #else |
1273 | | void tcg_register_thread(void) |
1274 | 0 | { |
1275 | 0 | TCGContext *s = g_malloc(sizeof(*s)); |
1276 | 0 | unsigned int i, n; |
1277 | |
|
1278 | 0 | *s = tcg_init_ctx; |
1279 | | |
1280 | | /* Relink mem_base. */ |
1281 | 0 | for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { |
1282 | 0 | if (tcg_init_ctx.temps[i].mem_base) { |
1283 | 0 | ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; |
1284 | 0 | tcg_debug_assert(b >= 0 && b < n); |
1285 | 0 | s->temps[i].mem_base = &s->temps[b]; |
1286 | 0 | } |
1287 | 0 | } |
1288 | | |
1289 | | /* Claim an entry in tcg_ctxs */ |
1290 | 0 | n = qatomic_fetch_inc(&tcg_cur_ctxs); |
1291 | 0 | g_assert(n < tcg_max_ctxs); |
1292 | 0 | qatomic_set(&tcg_ctxs[n], s); |
1293 | | |
1294 | 0 | if (n > 0) { |
1295 | 0 | tcg_region_thread_initial_alloc(s); |
1296 | 0 | } |
1297 | |
|
1298 | 0 | tcg_ctx = s; |
1299 | 0 | } |
1300 | | #endif /* !CONFIG_USER_ONLY */ |
1301 | | |
1302 | | /* pool based memory allocation */ |
1303 | | void *tcg_malloc_internal(TCGContext *s, int size) |
1304 | 0 | { |
1305 | 0 | TCGPool *p; |
1306 | 0 | int pool_size; |
1307 | |
|
1308 | 0 | if (size > TCG_POOL_CHUNK_SIZE) { |
1309 | | /* big malloc: insert a new pool (XXX: could optimize) */ |
1310 | 0 | p = g_malloc(sizeof(TCGPool) + size); |
1311 | 0 | p->size = size; |
1312 | 0 | p->next = s->pool_first_large; |
1313 | 0 | s->pool_first_large = p; |
1314 | 0 | return p->data; |
1315 | 0 | } else { |
1316 | 0 | p = s->pool_current; |
1317 | 0 | if (!p) { |
1318 | 0 | p = s->pool_first; |
1319 | 0 | if (!p) { |
1320 | 0 | goto new_pool; |
1321 | 0 | } |
1322 | 0 | } else { |
1323 | 0 | if (!p->next) { |
1324 | 0 | new_pool: |
1325 | 0 | pool_size = TCG_POOL_CHUNK_SIZE; |
1326 | 0 | p = g_malloc(sizeof(TCGPool) + pool_size); |
1327 | 0 | p->size = pool_size; |
1328 | 0 | p->next = NULL; |
1329 | 0 | if (s->pool_current) { |
1330 | 0 | s->pool_current->next = p; |
1331 | 0 | } else { |
1332 | 0 | s->pool_first = p; |
1333 | 0 | } |
1334 | 0 | } else { |
1335 | 0 | p = p->next; |
1336 | 0 | } |
1337 | 0 | } |
1338 | 0 | } |
1339 | 0 | s->pool_current = p; |
1340 | 0 | s->pool_cur = (uintptr_t)p->data + size; |
1341 | 0 | s->pool_end = (uintptr_t)p->data + p->size; |
1342 | 0 | return p->data; |
1343 | 0 | } |
1344 | | |
1345 | | void tcg_pool_reset(TCGContext *s) |
1346 | 0 | { |
1347 | 0 | TCGPool *p, *t; |
1348 | 0 | for (p = s->pool_first_large; p; p = t) { |
1349 | 0 | t = p->next; |
1350 | 0 | g_free(p); |
1351 | 0 | } |
1352 | 0 | s->pool_first_large = NULL; |
1353 | 0 | s->pool_cur = s->pool_end = 0; |
1354 | 0 | s->pool_current = NULL; |
1355 | 0 | } |
1356 | | |
1357 | | /* |
1358 | | * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions, |
1359 | | * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N. |
1360 | | * We only use these for layout in tcg_out_ld_helper_ret and |
1361 | | * tcg_out_st_helper_args, and share them between several of |
1362 | | * the helpers, with the end result that it's easier to build manually. |
1363 | | */ |
1364 | | |
1365 | | #define dh_typecode_ttl dh_typecode_i64 |
1366 | | |
1367 | | static TCGHelperInfo info_helper_ld32_mmu = { |
1368 | | .flags = TCG_CALL_NO_WG, |
1369 | | .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */ |
1370 | | | dh_typemask(env, 1) |
1371 | | | dh_typemask(i64, 2) /* uint64_t addr */ |
1372 | | | dh_typemask(i32, 3) /* unsigned oi */ |
1373 | | | dh_typemask(ptr, 4) /* uintptr_t ra */ |
1374 | | }; |
1375 | | |
1376 | | static TCGHelperInfo info_helper_ld64_mmu = { |
1377 | | .flags = TCG_CALL_NO_WG, |
1378 | | .typemask = dh_typemask(i64, 0) /* return uint64_t */ |
1379 | | | dh_typemask(env, 1) |
1380 | | | dh_typemask(i64, 2) /* uint64_t addr */ |
1381 | | | dh_typemask(i32, 3) /* unsigned oi */ |
1382 | | | dh_typemask(ptr, 4) /* uintptr_t ra */ |
1383 | | }; |
1384 | | |
1385 | | static TCGHelperInfo info_helper_ld128_mmu = { |
1386 | | .flags = TCG_CALL_NO_WG, |
1387 | | .typemask = dh_typemask(i128, 0) /* return Int128 */ |
1388 | | | dh_typemask(env, 1) |
1389 | | | dh_typemask(i64, 2) /* uint64_t addr */ |
1390 | | | dh_typemask(i32, 3) /* unsigned oi */ |
1391 | | | dh_typemask(ptr, 4) /* uintptr_t ra */ |
1392 | | }; |
1393 | | |
1394 | | static TCGHelperInfo info_helper_st32_mmu = { |
1395 | | .flags = TCG_CALL_NO_WG, |
1396 | | .typemask = dh_typemask(void, 0) |
1397 | | | dh_typemask(env, 1) |
1398 | | | dh_typemask(i64, 2) /* uint64_t addr */ |
1399 | | | dh_typemask(i32, 3) /* uint32_t data */ |
1400 | | | dh_typemask(i32, 4) /* unsigned oi */ |
1401 | | | dh_typemask(ptr, 5) /* uintptr_t ra */ |
1402 | | }; |
1403 | | |
1404 | | static TCGHelperInfo info_helper_st64_mmu = { |
1405 | | .flags = TCG_CALL_NO_WG, |
1406 | | .typemask = dh_typemask(void, 0) |
1407 | | | dh_typemask(env, 1) |
1408 | | | dh_typemask(i64, 2) /* uint64_t addr */ |
1409 | | | dh_typemask(i64, 3) /* uint64_t data */ |
1410 | | | dh_typemask(i32, 4) /* unsigned oi */ |
1411 | | | dh_typemask(ptr, 5) /* uintptr_t ra */ |
1412 | | }; |
1413 | | |
1414 | | static TCGHelperInfo info_helper_st128_mmu = { |
1415 | | .flags = TCG_CALL_NO_WG, |
1416 | | .typemask = dh_typemask(void, 0) |
1417 | | | dh_typemask(env, 1) |
1418 | | | dh_typemask(i64, 2) /* uint64_t addr */ |
1419 | | | dh_typemask(i128, 3) /* Int128 data */ |
1420 | | | dh_typemask(i32, 4) /* unsigned oi */ |
1421 | | | dh_typemask(ptr, 5) /* uintptr_t ra */ |
1422 | | }; |
1423 | | |
1424 | | #ifdef CONFIG_TCG_INTERPRETER |
1425 | | static ffi_type *typecode_to_ffi(int argmask) |
1426 | | { |
1427 | | /* |
1428 | | * libffi does not support __int128_t, so we have forced Int128 |
1429 | | * to use the structure definition instead of the builtin type. |
1430 | | */ |
1431 | | static ffi_type *ffi_type_i128_elements[3] = { |
1432 | | &ffi_type_uint64, |
1433 | | &ffi_type_uint64, |
1434 | | NULL |
1435 | | }; |
1436 | | static ffi_type ffi_type_i128 = { |
1437 | | .size = 16, |
1438 | | .alignment = __alignof__(Int128), |
1439 | | .type = FFI_TYPE_STRUCT, |
1440 | | .elements = ffi_type_i128_elements, |
1441 | | }; |
1442 | | |
1443 | | switch (argmask) { |
1444 | | case dh_typecode_void: |
1445 | | return &ffi_type_void; |
1446 | | case dh_typecode_i32: |
1447 | | return &ffi_type_uint32; |
1448 | | case dh_typecode_s32: |
1449 | | return &ffi_type_sint32; |
1450 | | case dh_typecode_i64: |
1451 | | return &ffi_type_uint64; |
1452 | | case dh_typecode_s64: |
1453 | | return &ffi_type_sint64; |
1454 | | case dh_typecode_ptr: |
1455 | | return &ffi_type_pointer; |
1456 | | case dh_typecode_i128: |
1457 | | return &ffi_type_i128; |
1458 | | } |
1459 | | g_assert_not_reached(); |
1460 | | } |
1461 | | |
1462 | | static ffi_cif *init_ffi_layout(TCGHelperInfo *info) |
1463 | | { |
1464 | | unsigned typemask = info->typemask; |
1465 | | struct { |
1466 | | ffi_cif cif; |
1467 | | ffi_type *args[]; |
1468 | | } *ca; |
1469 | | ffi_status status; |
1470 | | int nargs; |
1471 | | |
1472 | | /* Ignoring the return type, find the last non-zero field. */ |
1473 | | nargs = 32 - clz32(typemask >> 3); |
1474 | | nargs = DIV_ROUND_UP(nargs, 3); |
1475 | | assert(nargs <= MAX_CALL_IARGS); |
1476 | | |
1477 | | ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); |
1478 | | ca->cif.rtype = typecode_to_ffi(typemask & 7); |
1479 | | ca->cif.nargs = nargs; |
1480 | | |
1481 | | if (nargs != 0) { |
1482 | | ca->cif.arg_types = ca->args; |
1483 | | for (int j = 0; j < nargs; ++j) { |
1484 | | int typecode = extract32(typemask, (j + 1) * 3, 3); |
1485 | | ca->args[j] = typecode_to_ffi(typecode); |
1486 | | } |
1487 | | } |
1488 | | |
1489 | | status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, |
1490 | | ca->cif.rtype, ca->cif.arg_types); |
1491 | | assert(status == FFI_OK); |
1492 | | |
1493 | | return &ca->cif; |
1494 | | } |
1495 | | |
1496 | | #define HELPER_INFO_INIT(I) (&(I)->cif) |
1497 | | #define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I) |
1498 | | #else |
1499 | | #define HELPER_INFO_INIT(I) (&(I)->init) |
1500 | | #define HELPER_INFO_INIT_VAL(I) 1 |
1501 | | #endif /* CONFIG_TCG_INTERPRETER */ |
1502 | | |
1503 | | static inline bool arg_slot_reg_p(unsigned arg_slot) |
1504 | 0 | { |
1505 | | /* |
1506 | | * Split the sizeof away from the comparison to avoid Werror from |
1507 | | * "unsigned < 0 is always false", when iarg_regs is empty. |
1508 | | */ |
1509 | 0 | unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs); |
1510 | 0 | return arg_slot < nreg; |
1511 | 0 | } |
1512 | | |
1513 | | static inline int arg_slot_stk_ofs(unsigned arg_slot) |
1514 | 0 | { |
1515 | 0 | unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); |
1516 | 0 | unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); |
1517 | |
|
1518 | 0 | tcg_debug_assert(stk_slot < max); |
1519 | 0 | return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long); |
1520 | 0 | } |
1521 | | |
1522 | | typedef struct TCGCumulativeArgs { |
1523 | | int arg_idx; /* tcg_gen_callN args[] */ |
1524 | | int info_in_idx; /* TCGHelperInfo in[] */ |
1525 | | int arg_slot; /* regs+stack slot */ |
1526 | | int ref_slot; /* stack slots for references */ |
1527 | | } TCGCumulativeArgs; |
1528 | | |
1529 | | static void layout_arg_even(TCGCumulativeArgs *cum) |
1530 | 0 | { |
1531 | 0 | cum->arg_slot += cum->arg_slot & 1; |
1532 | 0 | } |
1533 | | |
1534 | | static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, |
1535 | | TCGCallArgumentKind kind) |
1536 | 0 | { |
1537 | 0 | TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; |
1538 | |
|
1539 | 0 | *loc = (TCGCallArgumentLoc){ |
1540 | 0 | .kind = kind, |
1541 | 0 | .arg_idx = cum->arg_idx, |
1542 | 0 | .arg_slot = cum->arg_slot, |
1543 | 0 | }; |
1544 | 0 | cum->info_in_idx++; |
1545 | 0 | cum->arg_slot++; |
1546 | 0 | } |
1547 | | |
1548 | | static void layout_arg_normal_n(TCGCumulativeArgs *cum, |
1549 | | TCGHelperInfo *info, int n) |
1550 | 0 | { |
1551 | 0 | TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; |
1552 | |
|
1553 | 0 | for (int i = 0; i < n; ++i) { |
1554 | | /* Layout all using the same arg_idx, adjusting the subindex. */ |
1555 | 0 | loc[i] = (TCGCallArgumentLoc){ |
1556 | 0 | .kind = TCG_CALL_ARG_NORMAL, |
1557 | 0 | .arg_idx = cum->arg_idx, |
1558 | 0 | .tmp_subindex = i, |
1559 | 0 | .arg_slot = cum->arg_slot + i, |
1560 | 0 | }; |
1561 | 0 | } |
1562 | 0 | cum->info_in_idx += n; |
1563 | 0 | cum->arg_slot += n; |
1564 | 0 | } |
1565 | | |
1566 | | static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) |
1567 | 0 | { |
1568 | 0 | TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; |
1569 | 0 | int n = 128 / TCG_TARGET_REG_BITS; |
1570 | 0 |
|
1571 | 0 | /* The first subindex carries the pointer. */ |
1572 | 0 | layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); |
1573 | 0 |
|
1574 | 0 | /* |
1575 | 0 | * The callee is allowed to clobber memory associated with |
1576 | 0 | * structure pass by-reference. Therefore we must make copies. |
1577 | 0 | * Allocate space from "ref_slot", which will be adjusted to |
1578 | 0 | * follow the parameters on the stack. |
1579 | 0 | */ |
1580 | 0 | loc[0].ref_slot = cum->ref_slot; |
1581 | 0 |
|
1582 | 0 | /* |
1583 | 0 | * Subsequent words also go into the reference slot, but |
1584 | 0 | * do not accumulate into the regular arguments. |
1585 | 0 | */ |
1586 | 0 | for (int i = 1; i < n; ++i) { |
1587 | 0 | loc[i] = (TCGCallArgumentLoc){ |
1588 | 0 | .kind = TCG_CALL_ARG_BY_REF_N, |
1589 | 0 | .arg_idx = cum->arg_idx, |
1590 | 0 | .tmp_subindex = i, |
1591 | 0 | .ref_slot = cum->ref_slot + i, |
1592 | 0 | }; |
1593 | 0 | } |
1594 | 0 | cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */ |
1595 | 0 | cum->ref_slot += n; |
1596 | 0 | } |
1597 | | |
1598 | | static void init_call_layout(TCGHelperInfo *info) |
1599 | 0 | { |
1600 | 0 | int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); |
1601 | 0 | int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); |
1602 | 0 | unsigned typemask = info->typemask; |
1603 | 0 | unsigned typecode; |
1604 | 0 | TCGCumulativeArgs cum = { }; |
1605 | | |
1606 | | /* |
1607 | | * Parse and place any function return value. |
1608 | | */ |
1609 | 0 | typecode = typemask & 7; |
1610 | 0 | switch (typecode) { |
1611 | 0 | case dh_typecode_void: |
1612 | 0 | info->nr_out = 0; |
1613 | 0 | break; |
1614 | 0 | case dh_typecode_i32: |
1615 | 0 | case dh_typecode_s32: |
1616 | 0 | case dh_typecode_i64: |
1617 | 0 | case dh_typecode_s64: |
1618 | 0 | case dh_typecode_ptr: |
1619 | 0 | info->nr_out = 1; |
1620 | 0 | info->out_kind = TCG_CALL_RET_NORMAL; |
1621 | 0 | break; |
1622 | 0 | case dh_typecode_i128: |
1623 | 0 | info->nr_out = 128 / TCG_TARGET_REG_BITS; |
1624 | 0 | info->out_kind = TCG_TARGET_CALL_RET_I128; |
1625 | 0 | switch (TCG_TARGET_CALL_RET_I128) { |
1626 | 0 | case TCG_CALL_RET_NORMAL: |
1627 | | /* Query the last register now to trigger any assert early. */ |
1628 | 0 | tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); |
1629 | 0 | break; |
1630 | 0 | case TCG_CALL_RET_BY_VEC: |
1631 | | /* Query the single register now to trigger any assert early. */ |
1632 | 0 | tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); |
1633 | 0 | break; |
1634 | 0 | case TCG_CALL_RET_BY_REF: |
1635 | | /* |
1636 | | * Allocate the first argument to the output. |
1637 | | * We don't need to store this anywhere, just make it |
1638 | | * unavailable for use in the input loop below. |
1639 | | */ |
1640 | 0 | cum.arg_slot = 1; |
1641 | 0 | break; |
1642 | 0 | default: |
1643 | 0 | qemu_build_not_reached(); |
1644 | 0 | } |
1645 | 0 | break; |
1646 | 0 | default: |
1647 | 0 | g_assert_not_reached(); |
1648 | 0 | } |
1649 | | |
1650 | | /* |
1651 | | * Parse and place function arguments. |
1652 | | */ |
1653 | 0 | for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { |
1654 | 0 | TCGCallArgumentKind kind; |
1655 | 0 | TCGType type; |
1656 | |
|
1657 | 0 | typecode = typemask & 7; |
1658 | 0 | switch (typecode) { |
1659 | 0 | case dh_typecode_i32: |
1660 | 0 | case dh_typecode_s32: |
1661 | 0 | type = TCG_TYPE_I32; |
1662 | 0 | break; |
1663 | 0 | case dh_typecode_i64: |
1664 | 0 | case dh_typecode_s64: |
1665 | 0 | type = TCG_TYPE_I64; |
1666 | 0 | break; |
1667 | 0 | case dh_typecode_ptr: |
1668 | 0 | type = TCG_TYPE_PTR; |
1669 | 0 | break; |
1670 | 0 | case dh_typecode_i128: |
1671 | 0 | type = TCG_TYPE_I128; |
1672 | 0 | break; |
1673 | 0 | default: |
1674 | 0 | g_assert_not_reached(); |
1675 | 0 | } |
1676 | | |
1677 | 0 | switch (type) { |
1678 | 0 | case TCG_TYPE_I32: |
1679 | 0 | switch (TCG_TARGET_CALL_ARG_I32) { |
1680 | 0 | case TCG_CALL_ARG_EVEN: |
1681 | 0 | layout_arg_even(&cum); |
1682 | | /* fall through */ |
1683 | 0 | case TCG_CALL_ARG_NORMAL: |
1684 | 0 | layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); |
1685 | 0 | break; |
1686 | 0 | case TCG_CALL_ARG_EXTEND: |
1687 | 0 | kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); |
1688 | 0 | layout_arg_1(&cum, info, kind); |
1689 | 0 | break; |
1690 | 0 | default: |
1691 | 0 | qemu_build_not_reached(); |
1692 | 0 | } |
1693 | 0 | break; |
1694 | | |
1695 | 0 | case TCG_TYPE_I64: |
1696 | 0 | switch (TCG_TARGET_CALL_ARG_I64) { |
1697 | 0 | case TCG_CALL_ARG_EVEN: |
1698 | 0 | layout_arg_even(&cum); |
1699 | | /* fall through */ |
1700 | 0 | case TCG_CALL_ARG_NORMAL: |
1701 | 0 | layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); |
1702 | 0 | break; |
1703 | 0 | default: |
1704 | 0 | qemu_build_not_reached(); |
1705 | 0 | } |
1706 | 0 | break; |
1707 | | |
1708 | 0 | case TCG_TYPE_I128: |
1709 | 0 | switch (TCG_TARGET_CALL_ARG_I128) { |
1710 | 0 | case TCG_CALL_ARG_EVEN: |
1711 | 0 | layout_arg_even(&cum); |
1712 | | /* fall through */ |
1713 | 0 | case TCG_CALL_ARG_NORMAL: |
1714 | 0 | layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); |
1715 | 0 | break; |
1716 | 0 | case TCG_CALL_ARG_BY_REF: |
1717 | 0 | layout_arg_by_ref(&cum, info); |
1718 | 0 | break; |
1719 | 0 | default: |
1720 | 0 | qemu_build_not_reached(); |
1721 | 0 | } |
1722 | 0 | break; |
1723 | | |
1724 | 0 | default: |
1725 | 0 | g_assert_not_reached(); |
1726 | 0 | } |
1727 | 0 | } |
1728 | 0 | info->nr_in = cum.info_in_idx; |
1729 | | |
1730 | | /* Validate that we didn't overrun the input array. */ |
1731 | 0 | assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); |
1732 | | /* Validate the backend has enough argument space. */ |
1733 | 0 | assert(cum.arg_slot <= max_reg_slots + max_stk_slots); |
1734 | | |
1735 | | /* |
1736 | | * Relocate the "ref_slot" area to the end of the parameters. |
1737 | | * Minimizing this stack offset helps code size for x86, |
1738 | | * which has a signed 8-bit offset encoding. |
1739 | | */ |
1740 | 0 | if (cum.ref_slot != 0) { |
1741 | 0 | int ref_base = 0; |
1742 | |
|
1743 | 0 | if (cum.arg_slot > max_reg_slots) { |
1744 | 0 | int align = __alignof(Int128) / sizeof(tcg_target_long); |
1745 | |
|
1746 | 0 | ref_base = cum.arg_slot - max_reg_slots; |
1747 | 0 | if (align > 1) { |
1748 | 0 | ref_base = ROUND_UP(ref_base, align); |
1749 | 0 | } |
1750 | 0 | } |
1751 | 0 | assert(ref_base + cum.ref_slot <= max_stk_slots); |
1752 | 0 | ref_base += max_reg_slots; |
1753 | |
|
1754 | 0 | if (ref_base != 0) { |
1755 | 0 | for (int i = cum.info_in_idx - 1; i >= 0; --i) { |
1756 | 0 | TCGCallArgumentLoc *loc = &info->in[i]; |
1757 | 0 | switch (loc->kind) { |
1758 | 0 | case TCG_CALL_ARG_BY_REF: |
1759 | 0 | case TCG_CALL_ARG_BY_REF_N: |
1760 | 0 | loc->ref_slot += ref_base; |
1761 | 0 | break; |
1762 | 0 | default: |
1763 | 0 | break; |
1764 | 0 | } |
1765 | 0 | } |
1766 | 0 | } |
1767 | 0 | } |
1768 | 0 | } |
1769 | | |
1770 | | static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; |
1771 | | static void process_constraint_sets(void); |
1772 | | static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, |
1773 | | TCGReg reg, const char *name); |
1774 | | |
1775 | | static void tcg_context_init(unsigned max_threads) |
1776 | 0 | { |
1777 | 0 | TCGContext *s = &tcg_init_ctx; |
1778 | 0 | int n, i; |
1779 | 0 | TCGTemp *ts; |
1780 | |
|
1781 | 0 | memset(s, 0, sizeof(*s)); |
1782 | 0 | s->nb_globals = 0; |
1783 | |
|
1784 | 0 | init_call_layout(&info_helper_ld32_mmu); |
1785 | 0 | init_call_layout(&info_helper_ld64_mmu); |
1786 | 0 | init_call_layout(&info_helper_ld128_mmu); |
1787 | 0 | init_call_layout(&info_helper_st32_mmu); |
1788 | 0 | init_call_layout(&info_helper_st64_mmu); |
1789 | 0 | init_call_layout(&info_helper_st128_mmu); |
1790 | |
|
1791 | 0 | tcg_target_init(s); |
1792 | 0 | process_constraint_sets(); |
1793 | | |
1794 | | /* Reverse the order of the saved registers, assuming they're all at |
1795 | | the start of tcg_target_reg_alloc_order. */ |
1796 | 0 | for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { |
1797 | 0 | int r = tcg_target_reg_alloc_order[n]; |
1798 | 0 | if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { |
1799 | 0 | break; |
1800 | 0 | } |
1801 | 0 | } |
1802 | 0 | for (i = 0; i < n; ++i) { |
1803 | 0 | indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; |
1804 | 0 | } |
1805 | 0 | for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { |
1806 | 0 | indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; |
1807 | 0 | } |
1808 | |
|
1809 | 0 | tcg_ctx = s; |
1810 | | /* |
1811 | | * In user-mode we simply share the init context among threads, since we |
1812 | | * use a single region. See the documentation tcg_region_init() for the |
1813 | | * reasoning behind this. |
1814 | | * In system-mode we will have at most max_threads TCG threads. |
1815 | | */ |
1816 | | #ifdef CONFIG_USER_ONLY |
1817 | | tcg_ctxs = &tcg_ctx; |
1818 | | tcg_cur_ctxs = 1; |
1819 | | tcg_max_ctxs = 1; |
1820 | | #else |
1821 | 0 | tcg_max_ctxs = max_threads; |
1822 | 0 | tcg_ctxs = g_new0(TCGContext *, max_threads); |
1823 | 0 | #endif |
1824 | |
|
1825 | 0 | tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); |
1826 | 0 | ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); |
1827 | 0 | tcg_env = temp_tcgv_ptr(ts); |
1828 | 0 | } |
1829 | | |
1830 | | void tcg_init(size_t tb_size, int splitwx, unsigned max_threads) |
1831 | 0 | { |
1832 | 0 | tcg_context_init(max_threads); |
1833 | 0 | tcg_region_init(tb_size, splitwx, max_threads); |
1834 | 0 | } |
1835 | | |
1836 | | /* |
1837 | | * Allocate TBs right before their corresponding translated code, making |
1838 | | * sure that TBs and code are on different cache lines. |
1839 | | */ |
1840 | | TranslationBlock *tcg_tb_alloc(TCGContext *s) |
1841 | 0 | { |
1842 | 0 | uintptr_t align = qemu_icache_linesize; |
1843 | 0 | TranslationBlock *tb; |
1844 | 0 | void *next; |
1845 | |
|
1846 | 0 | while (1) { |
1847 | 0 | tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); |
1848 | | |
1849 | | /* |
1850 | | * Note that code_gen_ptr can be NULL after vCPU hotplug. |
1851 | | * See tcg_region_thread_initial_alloc. |
1852 | | */ |
1853 | 0 | if (tb) { |
1854 | 0 | next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); |
1855 | 0 | if (next <= s->code_gen_highwater) { |
1856 | 0 | qatomic_set(&s->code_gen_ptr, next); |
1857 | 0 | return tb; |
1858 | 0 | } |
1859 | 0 | } |
1860 | 0 | if (!tcg_region_alloc(s)) { |
1861 | 0 | return NULL; |
1862 | 0 | } |
1863 | 0 | } |
1864 | 0 | } |
1865 | | |
1866 | | void tcg_prologue_init(void) |
1867 | 0 | { |
1868 | 0 | TCGContext *s = tcg_ctx; |
1869 | 0 | size_t prologue_size; |
1870 | |
|
1871 | 0 | s->code_ptr = s->code_gen_ptr; |
1872 | 0 | s->code_buf = s->code_gen_ptr; |
1873 | 0 | s->data_gen_ptr = NULL; |
1874 | |
|
1875 | 0 | #ifndef CONFIG_TCG_INTERPRETER |
1876 | 0 | tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); |
1877 | 0 | #endif |
1878 | |
|
1879 | 0 | s->pool_labels = NULL; |
1880 | |
|
1881 | 0 | qemu_thread_jit_write(); |
1882 | | /* Generate the prologue. */ |
1883 | 0 | tcg_target_qemu_prologue(s); |
1884 | | |
1885 | | /* Allow the prologue to put e.g. guest_base into a pool entry. */ |
1886 | 0 | { |
1887 | 0 | int result = tcg_out_pool_finalize(s); |
1888 | 0 | tcg_debug_assert(result == 0); |
1889 | 0 | } |
1890 | | |
1891 | 0 | prologue_size = tcg_current_code_size(s); |
1892 | 0 | perf_report_prologue(s->code_gen_ptr, prologue_size); |
1893 | |
|
1894 | 0 | #ifndef CONFIG_TCG_INTERPRETER |
1895 | 0 | flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), |
1896 | 0 | (uintptr_t)s->code_buf, prologue_size); |
1897 | 0 | #endif |
1898 | |
|
1899 | 0 | if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { |
1900 | 0 | FILE *logfile = qemu_log_trylock(); |
1901 | 0 | if (logfile) { |
1902 | 0 | fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); |
1903 | 0 | if (s->data_gen_ptr) { |
1904 | 0 | size_t code_size = s->data_gen_ptr - s->code_gen_ptr; |
1905 | 0 | size_t data_size = prologue_size - code_size; |
1906 | 0 | size_t i; |
1907 | |
|
1908 | 0 | disas(logfile, s->code_gen_ptr, code_size); |
1909 | |
|
1910 | 0 | for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { |
1911 | 0 | if (sizeof(tcg_target_ulong) == 8) { |
1912 | 0 | fprintf(logfile, |
1913 | 0 | "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", |
1914 | 0 | (uintptr_t)s->data_gen_ptr + i, |
1915 | 0 | *(uint64_t *)(s->data_gen_ptr + i)); |
1916 | 0 | } else { |
1917 | 0 | fprintf(logfile, |
1918 | 0 | "0x%08" PRIxPTR ": .long 0x%08x\n", |
1919 | 0 | (uintptr_t)s->data_gen_ptr + i, |
1920 | 0 | *(uint32_t *)(s->data_gen_ptr + i)); |
1921 | 0 | } |
1922 | 0 | } |
1923 | 0 | } else { |
1924 | 0 | disas(logfile, s->code_gen_ptr, prologue_size); |
1925 | 0 | } |
1926 | 0 | fprintf(logfile, "\n"); |
1927 | 0 | qemu_log_unlock(logfile); |
1928 | 0 | } |
1929 | 0 | } |
1930 | |
|
1931 | 0 | #ifndef CONFIG_TCG_INTERPRETER |
1932 | | /* |
1933 | | * Assert that goto_ptr is implemented completely, setting an epilogue. |
1934 | | * For tci, we use NULL as the signal to return from the interpreter, |
1935 | | * so skip this check. |
1936 | | */ |
1937 | 0 | tcg_debug_assert(tcg_code_gen_epilogue != NULL); |
1938 | 0 | #endif |
1939 | | |
1940 | 0 | tcg_region_prologue_set(s); |
1941 | 0 | } |
1942 | | |
1943 | | void tcg_func_start(TCGContext *s) |
1944 | 0 | { |
1945 | 0 | tcg_pool_reset(s); |
1946 | 0 | s->nb_temps = s->nb_globals; |
1947 | | |
1948 | | /* No temps have been previously allocated for size or locality. */ |
1949 | 0 | tcg_temp_ebb_reset_freed(s); |
1950 | | |
1951 | | /* No constant temps have been previously allocated. */ |
1952 | 0 | for (int i = 0; i < TCG_TYPE_COUNT; ++i) { |
1953 | 0 | if (s->const_table[i]) { |
1954 | 0 | g_hash_table_remove_all(s->const_table[i]); |
1955 | 0 | } |
1956 | 0 | } |
1957 | |
|
1958 | 0 | s->nb_ops = 0; |
1959 | 0 | s->nb_labels = 0; |
1960 | 0 | s->current_frame_offset = s->frame_start; |
1961 | |
|
1962 | | #ifdef CONFIG_DEBUG_TCG |
1963 | | s->goto_tb_issue_mask = 0; |
1964 | | #endif |
1965 | |
|
1966 | 0 | QTAILQ_INIT(&s->ops); |
1967 | 0 | QTAILQ_INIT(&s->free_ops); |
1968 | 0 | s->emit_before_op = NULL; |
1969 | 0 | QSIMPLEQ_INIT(&s->labels); |
1970 | |
|
1971 | 0 | tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); |
1972 | 0 | } |
1973 | | |
1974 | | static TCGTemp *tcg_temp_alloc(TCGContext *s) |
1975 | 0 | { |
1976 | 0 | int n = s->nb_temps++; |
1977 | |
|
1978 | 0 | if (n >= TCG_MAX_TEMPS) { |
1979 | 0 | tcg_raise_tb_overflow(s); |
1980 | 0 | } |
1981 | 0 | return memset(&s->temps[n], 0, sizeof(TCGTemp)); |
1982 | 0 | } |
1983 | | |
1984 | | static TCGTemp *tcg_global_alloc(TCGContext *s) |
1985 | 0 | { |
1986 | 0 | TCGTemp *ts; |
1987 | |
|
1988 | 0 | tcg_debug_assert(s->nb_globals == s->nb_temps); |
1989 | 0 | tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); |
1990 | 0 | s->nb_globals++; |
1991 | 0 | ts = tcg_temp_alloc(s); |
1992 | 0 | ts->kind = TEMP_GLOBAL; |
1993 | |
|
1994 | 0 | return ts; |
1995 | 0 | } |
1996 | | |
1997 | | static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, |
1998 | | TCGReg reg, const char *name) |
1999 | 0 | { |
2000 | 0 | TCGTemp *ts = tcg_global_alloc(s); |
2001 | |
|
2002 | 0 | ts->base_type = type; |
2003 | 0 | ts->type = type; |
2004 | 0 | ts->kind = TEMP_FIXED; |
2005 | 0 | ts->reg = reg; |
2006 | 0 | ts->name = name; |
2007 | 0 | tcg_regset_set_reg(s->reserved_regs, reg); |
2008 | |
|
2009 | 0 | return ts; |
2010 | 0 | } |
2011 | | |
2012 | | void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) |
2013 | 0 | { |
2014 | 0 | s->frame_start = start; |
2015 | 0 | s->frame_end = start + size; |
2016 | 0 | s->frame_temp |
2017 | 0 | = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); |
2018 | 0 | } |
2019 | | |
2020 | | static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset, |
2021 | | const char *name, TCGType type) |
2022 | 0 | { |
2023 | 0 | TCGContext *s = tcg_ctx; |
2024 | 0 | TCGTemp *base_ts = tcgv_ptr_temp(base); |
2025 | 0 | TCGTemp *ts = tcg_global_alloc(s); |
2026 | 0 | int indirect_reg = 0; |
2027 | |
|
2028 | 0 | switch (base_ts->kind) { |
2029 | 0 | case TEMP_FIXED: |
2030 | 0 | break; |
2031 | 0 | case TEMP_GLOBAL: |
2032 | | /* We do not support double-indirect registers. */ |
2033 | 0 | tcg_debug_assert(!base_ts->indirect_reg); |
2034 | 0 | base_ts->indirect_base = 1; |
2035 | 0 | s->nb_indirects += 1; |
2036 | 0 | indirect_reg = 1; |
2037 | 0 | break; |
2038 | 0 | default: |
2039 | 0 | g_assert_not_reached(); |
2040 | 0 | } |
2041 | | |
2042 | 0 | ts->base_type = type; |
2043 | 0 | ts->type = type; |
2044 | 0 | ts->indirect_reg = indirect_reg; |
2045 | 0 | ts->mem_allocated = 1; |
2046 | 0 | ts->mem_base = base_ts; |
2047 | 0 | ts->mem_offset = offset; |
2048 | 0 | ts->name = name; |
2049 | 0 | return ts; |
2050 | 0 | } |
2051 | | |
2052 | | TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name) |
2053 | 0 | { |
2054 | 0 | TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32); |
2055 | 0 | return temp_tcgv_i32(ts); |
2056 | 0 | } |
2057 | | |
2058 | | TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name) |
2059 | 0 | { |
2060 | 0 | TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64); |
2061 | 0 | return temp_tcgv_i64(ts); |
2062 | 0 | } |
2063 | | |
2064 | | TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name) |
2065 | 0 | { |
2066 | 0 | TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR); |
2067 | 0 | return temp_tcgv_ptr(ts); |
2068 | 0 | } |
2069 | | |
2070 | | TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) |
2071 | 0 | { |
2072 | 0 | TCGContext *s = tcg_ctx; |
2073 | 0 | TCGTemp *ts; |
2074 | 0 | int n; |
2075 | |
|
2076 | 0 | if (kind == TEMP_EBB) { |
2077 | 0 | int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); |
2078 | |
|
2079 | 0 | if (idx < TCG_MAX_TEMPS) { |
2080 | | /* There is already an available temp with the right type. */ |
2081 | 0 | clear_bit(idx, s->free_temps[type].l); |
2082 | |
|
2083 | 0 | ts = &s->temps[idx]; |
2084 | 0 | ts->temp_allocated = 1; |
2085 | 0 | tcg_debug_assert(ts->base_type == type); |
2086 | 0 | tcg_debug_assert(ts->kind == kind); |
2087 | 0 | return ts; |
2088 | 0 | } |
2089 | 0 | } else { |
2090 | 0 | tcg_debug_assert(kind == TEMP_TB); |
2091 | 0 | } |
2092 | | |
2093 | 0 | switch (type) { |
2094 | 0 | case TCG_TYPE_I32: |
2095 | 0 | case TCG_TYPE_I64: |
2096 | 0 | case TCG_TYPE_V64: |
2097 | 0 | case TCG_TYPE_V128: |
2098 | 0 | case TCG_TYPE_V256: |
2099 | 0 | n = 1; |
2100 | 0 | break; |
2101 | 0 | case TCG_TYPE_I128: |
2102 | 0 | n = 128 / TCG_TARGET_REG_BITS; |
2103 | 0 | break; |
2104 | 0 | default: |
2105 | 0 | g_assert_not_reached(); |
2106 | 0 | } |
2107 | | |
2108 | 0 | ts = tcg_temp_alloc(s); |
2109 | 0 | ts->base_type = type; |
2110 | 0 | ts->temp_allocated = 1; |
2111 | 0 | ts->kind = kind; |
2112 | |
|
2113 | 0 | if (n == 1) { |
2114 | 0 | ts->type = type; |
2115 | 0 | } else { |
2116 | 0 | ts->type = TCG_TYPE_REG; |
2117 | |
|
2118 | 0 | for (int i = 1; i < n; ++i) { |
2119 | 0 | TCGTemp *ts2 = tcg_temp_alloc(s); |
2120 | |
|
2121 | 0 | tcg_debug_assert(ts2 == ts + i); |
2122 | 0 | ts2->base_type = type; |
2123 | 0 | ts2->type = TCG_TYPE_REG; |
2124 | 0 | ts2->temp_allocated = 1; |
2125 | 0 | ts2->temp_subindex = i; |
2126 | 0 | ts2->kind = kind; |
2127 | 0 | } |
2128 | 0 | } |
2129 | 0 | return ts; |
2130 | 0 | } |
2131 | | |
2132 | | TCGv_i32 tcg_temp_new_i32(void) |
2133 | 0 | { |
2134 | 0 | return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB)); |
2135 | 0 | } |
2136 | | |
2137 | | TCGv_i32 tcg_temp_ebb_new_i32(void) |
2138 | 0 | { |
2139 | 0 | return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB)); |
2140 | 0 | } |
2141 | | |
2142 | | TCGv_i64 tcg_temp_new_i64(void) |
2143 | 0 | { |
2144 | 0 | return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB)); |
2145 | 0 | } |
2146 | | |
2147 | | TCGv_i64 tcg_temp_ebb_new_i64(void) |
2148 | 0 | { |
2149 | 0 | return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB)); |
2150 | 0 | } |
2151 | | |
2152 | | TCGv_ptr tcg_temp_new_ptr(void) |
2153 | 0 | { |
2154 | 0 | return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB)); |
2155 | 0 | } |
2156 | | |
2157 | | TCGv_ptr tcg_temp_ebb_new_ptr(void) |
2158 | 0 | { |
2159 | 0 | return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB)); |
2160 | 0 | } |
2161 | | |
2162 | | TCGv_i128 tcg_temp_new_i128(void) |
2163 | 0 | { |
2164 | 0 | return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB)); |
2165 | 0 | } |
2166 | | |
2167 | | TCGv_i128 tcg_temp_ebb_new_i128(void) |
2168 | 0 | { |
2169 | 0 | return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB)); |
2170 | 0 | } |
2171 | | |
2172 | | TCGv_vec tcg_temp_new_vec(TCGType type) |
2173 | 0 | { |
2174 | 0 | TCGTemp *t; |
2175 | |
|
2176 | | #ifdef CONFIG_DEBUG_TCG |
2177 | | switch (type) { |
2178 | | case TCG_TYPE_V64: |
2179 | | assert(TCG_TARGET_HAS_v64); |
2180 | | break; |
2181 | | case TCG_TYPE_V128: |
2182 | | assert(TCG_TARGET_HAS_v128); |
2183 | | break; |
2184 | | case TCG_TYPE_V256: |
2185 | | assert(TCG_TARGET_HAS_v256); |
2186 | | break; |
2187 | | default: |
2188 | | g_assert_not_reached(); |
2189 | | } |
2190 | | #endif |
2191 | |
|
2192 | 0 | t = tcg_temp_new_internal(type, TEMP_EBB); |
2193 | 0 | return temp_tcgv_vec(t); |
2194 | 0 | } |
2195 | | |
2196 | | /* Create a new temp of the same type as an existing temp. */ |
2197 | | TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) |
2198 | 0 | { |
2199 | 0 | TCGTemp *t = tcgv_vec_temp(match); |
2200 | |
|
2201 | 0 | tcg_debug_assert(t->temp_allocated != 0); |
2202 | | |
2203 | 0 | t = tcg_temp_new_internal(t->base_type, TEMP_EBB); |
2204 | 0 | return temp_tcgv_vec(t); |
2205 | 0 | } |
2206 | | |
2207 | | void tcg_temp_free_internal(TCGTemp *ts) |
2208 | 0 | { |
2209 | 0 | TCGContext *s = tcg_ctx; |
2210 | |
|
2211 | 0 | if (ts == NULL) { |
2212 | 0 | return; |
2213 | 0 | } |
2214 | | |
2215 | 0 | switch (ts->kind) { |
2216 | 0 | case TEMP_CONST: |
2217 | 0 | case TEMP_TB: |
2218 | | /* Silently ignore free. */ |
2219 | 0 | break; |
2220 | 0 | case TEMP_EBB: |
2221 | 0 | tcg_debug_assert(ts->temp_allocated != 0); |
2222 | 0 | ts->temp_allocated = 0; |
2223 | 0 | set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); |
2224 | 0 | break; |
2225 | 0 | default: |
2226 | | /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ |
2227 | 0 | g_assert_not_reached(); |
2228 | 0 | } |
2229 | 0 | } |
2230 | | |
2231 | | void tcg_temp_free_i32(TCGv_i32 arg) |
2232 | 0 | { |
2233 | 0 | tcg_temp_free_internal(tcgv_i32_temp(arg)); |
2234 | 0 | } |
2235 | | |
2236 | | void tcg_temp_free_i64(TCGv_i64 arg) |
2237 | 0 | { |
2238 | 0 | tcg_temp_free_internal(tcgv_i64_temp(arg)); |
2239 | 0 | } |
2240 | | |
2241 | | void tcg_temp_free_i128(TCGv_i128 arg) |
2242 | 0 | { |
2243 | 0 | tcg_temp_free_internal(tcgv_i128_temp(arg)); |
2244 | 0 | } |
2245 | | |
2246 | | void tcg_temp_free_ptr(TCGv_ptr arg) |
2247 | 0 | { |
2248 | 0 | tcg_temp_free_internal(tcgv_ptr_temp(arg)); |
2249 | 0 | } |
2250 | | |
2251 | | void tcg_temp_free_vec(TCGv_vec arg) |
2252 | 0 | { |
2253 | 0 | tcg_temp_free_internal(tcgv_vec_temp(arg)); |
2254 | 0 | } |
2255 | | |
2256 | | TCGTemp *tcg_constant_internal(TCGType type, int64_t val) |
2257 | 0 | { |
2258 | 0 | TCGContext *s = tcg_ctx; |
2259 | 0 | GHashTable *h = s->const_table[type]; |
2260 | 0 | TCGTemp *ts; |
2261 | |
|
2262 | 0 | if (h == NULL) { |
2263 | 0 | h = g_hash_table_new(g_int64_hash, g_int64_equal); |
2264 | 0 | s->const_table[type] = h; |
2265 | 0 | } |
2266 | |
|
2267 | 0 | if (type == TCG_TYPE_I32) { |
2268 | 0 | val = (int32_t)val; |
2269 | 0 | } |
2270 | |
|
2271 | 0 | ts = g_hash_table_lookup(h, &val); |
2272 | 0 | if (ts == NULL) { |
2273 | 0 | ts = tcg_temp_alloc(s); |
2274 | 0 | ts->base_type = type; |
2275 | 0 | ts->type = type; |
2276 | 0 | ts->kind = TEMP_CONST; |
2277 | 0 | ts->temp_allocated = 1; |
2278 | 0 | ts->val = val; |
2279 | 0 | g_hash_table_insert(h, &ts->val, ts); |
2280 | 0 | } |
2281 | |
|
2282 | 0 | return ts; |
2283 | 0 | } |
2284 | | |
2285 | | TCGv_i32 tcg_constant_i32(int32_t val) |
2286 | 0 | { |
2287 | 0 | return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val)); |
2288 | 0 | } |
2289 | | |
2290 | | TCGv_i64 tcg_constant_i64(int64_t val) |
2291 | 0 | { |
2292 | 0 | return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val)); |
2293 | 0 | } |
2294 | | |
2295 | | TCGv_vaddr tcg_constant_vaddr(uintptr_t val) |
2296 | 0 | { |
2297 | 0 | return temp_tcgv_vaddr(tcg_constant_internal(TCG_TYPE_PTR, val)); |
2298 | 0 | } |
2299 | | |
2300 | | TCGv_ptr tcg_constant_ptr_int(intptr_t val) |
2301 | 0 | { |
2302 | 0 | return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val)); |
2303 | 0 | } |
2304 | | |
2305 | | TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) |
2306 | 0 | { |
2307 | 0 | val = dup_const(vece, val); |
2308 | 0 | return temp_tcgv_vec(tcg_constant_internal(type, val)); |
2309 | 0 | } |
2310 | | |
2311 | | TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) |
2312 | 0 | { |
2313 | 0 | TCGTemp *t = tcgv_vec_temp(match); |
2314 | |
|
2315 | 0 | tcg_debug_assert(t->temp_allocated != 0); |
2316 | 0 | return tcg_constant_vec(t->base_type, vece, val); |
2317 | 0 | } |
2318 | | |
2319 | | #ifdef CONFIG_DEBUG_TCG |
2320 | | size_t temp_idx(TCGTemp *ts) |
2321 | | { |
2322 | | ptrdiff_t n = ts - tcg_ctx->temps; |
2323 | | assert(n >= 0 && n < tcg_ctx->nb_temps); |
2324 | | return n; |
2325 | | } |
2326 | | |
2327 | | TCGTemp *tcgv_i32_temp(TCGv_i32 v) |
2328 | | { |
2329 | | uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps); |
2330 | | |
2331 | | assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps); |
2332 | | assert(o % sizeof(TCGTemp) == 0); |
2333 | | |
2334 | | return (void *)tcg_ctx + (uintptr_t)v; |
2335 | | } |
2336 | | #endif /* CONFIG_DEBUG_TCG */ |
2337 | | |
2338 | | /* |
2339 | | * Return true if OP may appear in the opcode stream with TYPE. |
2340 | | * Test the runtime variable that controls each opcode. |
2341 | | */ |
2342 | | bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags) |
2343 | 0 | { |
2344 | 0 | bool has_type; |
2345 | |
|
2346 | 0 | switch (type) { |
2347 | 0 | case TCG_TYPE_I32: |
2348 | 0 | case TCG_TYPE_I64: |
2349 | 0 | has_type = true; |
2350 | 0 | break; |
2351 | 0 | case TCG_TYPE_V64: |
2352 | 0 | has_type = TCG_TARGET_HAS_v64; |
2353 | 0 | break; |
2354 | 0 | case TCG_TYPE_V128: |
2355 | 0 | has_type = TCG_TARGET_HAS_v128; |
2356 | 0 | break; |
2357 | 0 | case TCG_TYPE_V256: |
2358 | 0 | has_type = TCG_TARGET_HAS_v256; |
2359 | 0 | break; |
2360 | 0 | default: |
2361 | 0 | has_type = false; |
2362 | 0 | break; |
2363 | 0 | } |
2364 | | |
2365 | 0 | switch (op) { |
2366 | 0 | case INDEX_op_discard: |
2367 | 0 | case INDEX_op_set_label: |
2368 | 0 | case INDEX_op_call: |
2369 | 0 | case INDEX_op_br: |
2370 | 0 | case INDEX_op_mb: |
2371 | 0 | case INDEX_op_insn_start: |
2372 | 0 | case INDEX_op_exit_tb: |
2373 | 0 | case INDEX_op_goto_tb: |
2374 | 0 | case INDEX_op_goto_ptr: |
2375 | 0 | return true; |
2376 | | |
2377 | 0 | case INDEX_op_qemu_ld: |
2378 | 0 | case INDEX_op_qemu_st: |
2379 | 0 | tcg_debug_assert(type <= TCG_TYPE_REG); |
2380 | 0 | return true; |
2381 | | |
2382 | 0 | case INDEX_op_qemu_ld2: |
2383 | 0 | case INDEX_op_qemu_st2: |
2384 | 0 | tcg_debug_assert(type == TCG_TYPE_I128); |
2385 | 0 | goto do_lookup; |
2386 | | |
2387 | 0 | case INDEX_op_add: |
2388 | 0 | case INDEX_op_and: |
2389 | 0 | case INDEX_op_brcond: |
2390 | 0 | case INDEX_op_deposit: |
2391 | 0 | case INDEX_op_extract: |
2392 | 0 | case INDEX_op_ld8u: |
2393 | 0 | case INDEX_op_ld8s: |
2394 | 0 | case INDEX_op_ld16u: |
2395 | 0 | case INDEX_op_ld16s: |
2396 | 0 | case INDEX_op_ld: |
2397 | 0 | case INDEX_op_mov: |
2398 | 0 | case INDEX_op_movcond: |
2399 | 0 | case INDEX_op_negsetcond: |
2400 | 0 | case INDEX_op_or: |
2401 | 0 | case INDEX_op_setcond: |
2402 | 0 | case INDEX_op_sextract: |
2403 | 0 | case INDEX_op_st8: |
2404 | 0 | case INDEX_op_st16: |
2405 | 0 | case INDEX_op_st: |
2406 | 0 | case INDEX_op_xor: |
2407 | 0 | return has_type; |
2408 | | |
2409 | 0 | case INDEX_op_ld32u: |
2410 | 0 | case INDEX_op_ld32s: |
2411 | 0 | case INDEX_op_st32: |
2412 | 0 | case INDEX_op_ext_i32_i64: |
2413 | 0 | case INDEX_op_extu_i32_i64: |
2414 | 0 | case INDEX_op_extrl_i64_i32: |
2415 | 0 | case INDEX_op_extrh_i64_i32: |
2416 | 0 | return true; |
2417 | | |
2418 | 0 | case INDEX_op_mov_vec: |
2419 | 0 | case INDEX_op_dup_vec: |
2420 | 0 | case INDEX_op_dupm_vec: |
2421 | 0 | case INDEX_op_ld_vec: |
2422 | 0 | case INDEX_op_st_vec: |
2423 | 0 | case INDEX_op_add_vec: |
2424 | 0 | case INDEX_op_sub_vec: |
2425 | 0 | case INDEX_op_and_vec: |
2426 | 0 | case INDEX_op_or_vec: |
2427 | 0 | case INDEX_op_xor_vec: |
2428 | 0 | case INDEX_op_cmp_vec: |
2429 | 0 | return has_type; |
2430 | 0 | case INDEX_op_not_vec: |
2431 | 0 | return has_type && TCG_TARGET_HAS_not_vec; |
2432 | 0 | case INDEX_op_neg_vec: |
2433 | 0 | return has_type && TCG_TARGET_HAS_neg_vec; |
2434 | 0 | case INDEX_op_abs_vec: |
2435 | 0 | return has_type && TCG_TARGET_HAS_abs_vec; |
2436 | 0 | case INDEX_op_andc_vec: |
2437 | 0 | return has_type && TCG_TARGET_HAS_andc_vec; |
2438 | 0 | case INDEX_op_orc_vec: |
2439 | 0 | return has_type && TCG_TARGET_HAS_orc_vec; |
2440 | 0 | case INDEX_op_nand_vec: |
2441 | 0 | return has_type && TCG_TARGET_HAS_nand_vec; |
2442 | 0 | case INDEX_op_nor_vec: |
2443 | 0 | return has_type && TCG_TARGET_HAS_nor_vec; |
2444 | 0 | case INDEX_op_eqv_vec: |
2445 | 0 | return has_type && TCG_TARGET_HAS_eqv_vec; |
2446 | 0 | case INDEX_op_mul_vec: |
2447 | 0 | return has_type && TCG_TARGET_HAS_mul_vec; |
2448 | 0 | case INDEX_op_shli_vec: |
2449 | 0 | case INDEX_op_shri_vec: |
2450 | 0 | case INDEX_op_sari_vec: |
2451 | 0 | return has_type && TCG_TARGET_HAS_shi_vec; |
2452 | 0 | case INDEX_op_shls_vec: |
2453 | 0 | case INDEX_op_shrs_vec: |
2454 | 0 | case INDEX_op_sars_vec: |
2455 | 0 | return has_type && TCG_TARGET_HAS_shs_vec; |
2456 | 0 | case INDEX_op_shlv_vec: |
2457 | 0 | case INDEX_op_shrv_vec: |
2458 | 0 | case INDEX_op_sarv_vec: |
2459 | 0 | return has_type && TCG_TARGET_HAS_shv_vec; |
2460 | 0 | case INDEX_op_rotli_vec: |
2461 | 0 | return has_type && TCG_TARGET_HAS_roti_vec; |
2462 | 0 | case INDEX_op_rotls_vec: |
2463 | 0 | return has_type && TCG_TARGET_HAS_rots_vec; |
2464 | 0 | case INDEX_op_rotlv_vec: |
2465 | 0 | case INDEX_op_rotrv_vec: |
2466 | 0 | return has_type && TCG_TARGET_HAS_rotv_vec; |
2467 | 0 | case INDEX_op_ssadd_vec: |
2468 | 0 | case INDEX_op_usadd_vec: |
2469 | 0 | case INDEX_op_sssub_vec: |
2470 | 0 | case INDEX_op_ussub_vec: |
2471 | 0 | return has_type && TCG_TARGET_HAS_sat_vec; |
2472 | 0 | case INDEX_op_smin_vec: |
2473 | 0 | case INDEX_op_umin_vec: |
2474 | 0 | case INDEX_op_smax_vec: |
2475 | 0 | case INDEX_op_umax_vec: |
2476 | 0 | return has_type && TCG_TARGET_HAS_minmax_vec; |
2477 | 0 | case INDEX_op_bitsel_vec: |
2478 | 0 | return has_type && TCG_TARGET_HAS_bitsel_vec; |
2479 | 0 | case INDEX_op_cmpsel_vec: |
2480 | 0 | return has_type && TCG_TARGET_HAS_cmpsel_vec; |
2481 | | |
2482 | 0 | default: |
2483 | 0 | if (op < INDEX_op_last_generic) { |
2484 | 0 | const TCGOutOp *outop; |
2485 | 0 | TCGConstraintSetIndex con_set; |
2486 | |
|
2487 | 0 | if (!has_type) { |
2488 | 0 | return false; |
2489 | 0 | } |
2490 | | |
2491 | 0 | do_lookup: |
2492 | 0 | outop = all_outop[op]; |
2493 | 0 | tcg_debug_assert(outop != NULL); |
2494 | | |
2495 | 0 | con_set = outop->static_constraint; |
2496 | 0 | if (con_set == C_Dynamic) { |
2497 | 0 | con_set = outop->dynamic_constraint(type, flags); |
2498 | 0 | } |
2499 | 0 | if (con_set >= 0) { |
2500 | 0 | return true; |
2501 | 0 | } |
2502 | 0 | tcg_debug_assert(con_set == C_NotImplemented); |
2503 | 0 | return false; |
2504 | 0 | } |
2505 | 0 | tcg_debug_assert(op < NB_OPS); |
2506 | 0 | return true; |
2507 | | |
2508 | 0 | case INDEX_op_last_generic: |
2509 | 0 | g_assert_not_reached(); |
2510 | 0 | } |
2511 | 0 | } |
2512 | | |
2513 | | bool tcg_op_deposit_valid(TCGType type, unsigned ofs, unsigned len) |
2514 | 0 | { |
2515 | 0 | unsigned width; |
2516 | |
|
2517 | 0 | tcg_debug_assert(type == TCG_TYPE_I32 || type == TCG_TYPE_I64); |
2518 | 0 | width = (type == TCG_TYPE_I32 ? 32 : 64); |
2519 | |
|
2520 | 0 | tcg_debug_assert(ofs < width); |
2521 | 0 | tcg_debug_assert(len > 0); |
2522 | 0 | tcg_debug_assert(len <= width - ofs); |
2523 | | |
2524 | 0 | return TCG_TARGET_deposit_valid(type, ofs, len); |
2525 | 0 | } |
2526 | | |
2527 | | static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); |
2528 | | |
2529 | | static void tcg_gen_callN(void *func, TCGHelperInfo *info, |
2530 | | TCGTemp *ret, TCGTemp **args) |
2531 | 0 | { |
2532 | 0 | TCGv_i64 extend_free[MAX_CALL_IARGS]; |
2533 | 0 | int n_extend = 0; |
2534 | 0 | TCGOp *op; |
2535 | 0 | int i, n, pi = 0, total_args; |
2536 | |
|
2537 | 0 | if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) { |
2538 | 0 | init_call_layout(info); |
2539 | 0 | g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info)); |
2540 | 0 | } |
2541 | |
|
2542 | 0 | total_args = info->nr_out + info->nr_in + 2; |
2543 | 0 | op = tcg_op_alloc(INDEX_op_call, total_args); |
2544 | |
|
2545 | 0 | #ifdef CONFIG_PLUGIN |
2546 | | /* Flag helpers that may affect guest state */ |
2547 | 0 | if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { |
2548 | 0 | tcg_ctx->plugin_insn->calls_helpers = true; |
2549 | 0 | } |
2550 | 0 | #endif |
2551 | |
|
2552 | 0 | TCGOP_CALLO(op) = n = info->nr_out; |
2553 | 0 | switch (n) { |
2554 | 0 | case 0: |
2555 | 0 | tcg_debug_assert(ret == NULL); |
2556 | 0 | break; |
2557 | 0 | case 1: |
2558 | 0 | tcg_debug_assert(ret != NULL); |
2559 | 0 | op->args[pi++] = temp_arg(ret); |
2560 | 0 | break; |
2561 | 0 | case 2: |
2562 | 0 | case 4: |
2563 | 0 | tcg_debug_assert(ret != NULL); |
2564 | 0 | tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); |
2565 | 0 | tcg_debug_assert(ret->temp_subindex == 0); |
2566 | 0 | for (i = 0; i < n; ++i) { |
2567 | 0 | op->args[pi++] = temp_arg(ret + i); |
2568 | 0 | } |
2569 | 0 | break; |
2570 | 0 | default: |
2571 | 0 | g_assert_not_reached(); |
2572 | 0 | } |
2573 | | |
2574 | 0 | TCGOP_CALLI(op) = n = info->nr_in; |
2575 | 0 | for (i = 0; i < n; i++) { |
2576 | 0 | const TCGCallArgumentLoc *loc = &info->in[i]; |
2577 | 0 | TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; |
2578 | |
|
2579 | 0 | switch (loc->kind) { |
2580 | 0 | case TCG_CALL_ARG_NORMAL: |
2581 | 0 | case TCG_CALL_ARG_BY_REF: |
2582 | 0 | case TCG_CALL_ARG_BY_REF_N: |
2583 | 0 | op->args[pi++] = temp_arg(ts); |
2584 | 0 | break; |
2585 | | |
2586 | 0 | case TCG_CALL_ARG_EXTEND_U: |
2587 | 0 | case TCG_CALL_ARG_EXTEND_S: |
2588 | 0 | { |
2589 | 0 | TCGv_i64 temp = tcg_temp_ebb_new_i64(); |
2590 | 0 | TCGv_i32 orig = temp_tcgv_i32(ts); |
2591 | |
|
2592 | 0 | if (loc->kind == TCG_CALL_ARG_EXTEND_S) { |
2593 | 0 | tcg_gen_ext_i32_i64(temp, orig); |
2594 | 0 | } else { |
2595 | 0 | tcg_gen_extu_i32_i64(temp, orig); |
2596 | 0 | } |
2597 | 0 | op->args[pi++] = tcgv_i64_arg(temp); |
2598 | 0 | extend_free[n_extend++] = temp; |
2599 | 0 | } |
2600 | 0 | break; |
2601 | | |
2602 | 0 | default: |
2603 | 0 | g_assert_not_reached(); |
2604 | 0 | } |
2605 | 0 | } |
2606 | 0 | op->args[pi++] = (uintptr_t)func; |
2607 | 0 | op->args[pi++] = (uintptr_t)info; |
2608 | 0 | tcg_debug_assert(pi == total_args); |
2609 | | |
2610 | 0 | if (tcg_ctx->emit_before_op) { |
2611 | 0 | QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); |
2612 | 0 | } else { |
2613 | 0 | QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); |
2614 | 0 | } |
2615 | |
|
2616 | 0 | tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); |
2617 | 0 | for (i = 0; i < n_extend; ++i) { |
2618 | 0 | tcg_temp_free_i64(extend_free[i]); |
2619 | 0 | } |
2620 | 0 | } |
2621 | | |
2622 | | void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret) |
2623 | 0 | { |
2624 | 0 | tcg_gen_callN(func, info, ret, NULL); |
2625 | 0 | } |
2626 | | |
2627 | | void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1) |
2628 | 0 | { |
2629 | 0 | tcg_gen_callN(func, info, ret, &t1); |
2630 | 0 | } |
2631 | | |
2632 | | void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret, |
2633 | | TCGTemp *t1, TCGTemp *t2) |
2634 | 0 | { |
2635 | 0 | TCGTemp *args[2] = { t1, t2 }; |
2636 | 0 | tcg_gen_callN(func, info, ret, args); |
2637 | 0 | } |
2638 | | |
2639 | | void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret, |
2640 | | TCGTemp *t1, TCGTemp *t2, TCGTemp *t3) |
2641 | 0 | { |
2642 | 0 | TCGTemp *args[3] = { t1, t2, t3 }; |
2643 | 0 | tcg_gen_callN(func, info, ret, args); |
2644 | 0 | } |
2645 | | |
2646 | | void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret, |
2647 | | TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4) |
2648 | 0 | { |
2649 | 0 | TCGTemp *args[4] = { t1, t2, t3, t4 }; |
2650 | 0 | tcg_gen_callN(func, info, ret, args); |
2651 | 0 | } |
2652 | | |
2653 | | void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, |
2654 | | TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5) |
2655 | 0 | { |
2656 | 0 | TCGTemp *args[5] = { t1, t2, t3, t4, t5 }; |
2657 | 0 | tcg_gen_callN(func, info, ret, args); |
2658 | 0 | } |
2659 | | |
2660 | | void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret, |
2661 | | TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, |
2662 | | TCGTemp *t4, TCGTemp *t5, TCGTemp *t6) |
2663 | 0 | { |
2664 | 0 | TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 }; |
2665 | 0 | tcg_gen_callN(func, info, ret, args); |
2666 | 0 | } |
2667 | | |
2668 | | void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, |
2669 | | TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, |
2670 | | TCGTemp *t5, TCGTemp *t6, TCGTemp *t7) |
2671 | 0 | { |
2672 | 0 | TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 }; |
2673 | 0 | tcg_gen_callN(func, info, ret, args); |
2674 | 0 | } |
2675 | | |
2676 | | static void tcg_reg_alloc_start(TCGContext *s) |
2677 | 0 | { |
2678 | 0 | int i, n; |
2679 | |
|
2680 | 0 | for (i = 0, n = s->nb_temps; i < n; i++) { |
2681 | 0 | TCGTemp *ts = &s->temps[i]; |
2682 | 0 | TCGTempVal val = TEMP_VAL_MEM; |
2683 | |
|
2684 | 0 | switch (ts->kind) { |
2685 | 0 | case TEMP_CONST: |
2686 | 0 | val = TEMP_VAL_CONST; |
2687 | 0 | break; |
2688 | 0 | case TEMP_FIXED: |
2689 | 0 | val = TEMP_VAL_REG; |
2690 | 0 | break; |
2691 | 0 | case TEMP_GLOBAL: |
2692 | 0 | break; |
2693 | 0 | case TEMP_EBB: |
2694 | 0 | val = TEMP_VAL_DEAD; |
2695 | | /* fall through */ |
2696 | 0 | case TEMP_TB: |
2697 | 0 | ts->mem_allocated = 0; |
2698 | 0 | break; |
2699 | 0 | default: |
2700 | 0 | g_assert_not_reached(); |
2701 | 0 | } |
2702 | 0 | ts->val_type = val; |
2703 | 0 | } |
2704 | | |
2705 | 0 | memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); |
2706 | 0 | } |
2707 | | |
2708 | | static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, |
2709 | | TCGTemp *ts) |
2710 | 0 | { |
2711 | 0 | int idx = temp_idx(ts); |
2712 | |
|
2713 | 0 | switch (ts->kind) { |
2714 | 0 | case TEMP_FIXED: |
2715 | 0 | case TEMP_GLOBAL: |
2716 | 0 | pstrcpy(buf, buf_size, ts->name); |
2717 | 0 | break; |
2718 | 0 | case TEMP_TB: |
2719 | 0 | snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); |
2720 | 0 | break; |
2721 | 0 | case TEMP_EBB: |
2722 | 0 | snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); |
2723 | 0 | break; |
2724 | 0 | case TEMP_CONST: |
2725 | 0 | switch (ts->type) { |
2726 | 0 | case TCG_TYPE_I32: |
2727 | 0 | snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); |
2728 | 0 | break; |
2729 | 0 | case TCG_TYPE_I64: |
2730 | 0 | snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); |
2731 | 0 | break; |
2732 | 0 | case TCG_TYPE_V64: |
2733 | 0 | case TCG_TYPE_V128: |
2734 | 0 | case TCG_TYPE_V256: |
2735 | 0 | snprintf(buf, buf_size, "v%d$0x%" PRIx64, |
2736 | 0 | 64 << (ts->type - TCG_TYPE_V64), ts->val); |
2737 | 0 | break; |
2738 | 0 | default: |
2739 | 0 | g_assert_not_reached(); |
2740 | 0 | } |
2741 | 0 | break; |
2742 | 0 | } |
2743 | 0 | return buf; |
2744 | 0 | } |
2745 | | |
2746 | | static char *tcg_get_arg_str(TCGContext *s, char *buf, |
2747 | | int buf_size, TCGArg arg) |
2748 | 0 | { |
2749 | 0 | return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); |
2750 | 0 | } |
2751 | | |
2752 | | static const char * const cond_name[] = |
2753 | | { |
2754 | | [TCG_COND_NEVER] = "never", |
2755 | | [TCG_COND_ALWAYS] = "always", |
2756 | | [TCG_COND_EQ] = "eq", |
2757 | | [TCG_COND_NE] = "ne", |
2758 | | [TCG_COND_LT] = "lt", |
2759 | | [TCG_COND_GE] = "ge", |
2760 | | [TCG_COND_LE] = "le", |
2761 | | [TCG_COND_GT] = "gt", |
2762 | | [TCG_COND_LTU] = "ltu", |
2763 | | [TCG_COND_GEU] = "geu", |
2764 | | [TCG_COND_LEU] = "leu", |
2765 | | [TCG_COND_GTU] = "gtu", |
2766 | | [TCG_COND_TSTEQ] = "tsteq", |
2767 | | [TCG_COND_TSTNE] = "tstne", |
2768 | | }; |
2769 | | |
2770 | | static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] = |
2771 | | { |
2772 | | [MO_UB] = "ub", |
2773 | | [MO_SB] = "sb", |
2774 | | [MO_LEUW] = "leuw", |
2775 | | [MO_LESW] = "lesw", |
2776 | | [MO_LEUL] = "leul", |
2777 | | [MO_LESL] = "lesl", |
2778 | | [MO_LEUQ] = "leq", |
2779 | | [MO_BEUW] = "beuw", |
2780 | | [MO_BESW] = "besw", |
2781 | | [MO_BEUL] = "beul", |
2782 | | [MO_BESL] = "besl", |
2783 | | [MO_BEUQ] = "beq", |
2784 | | [MO_128 + MO_BE] = "beo", |
2785 | | [MO_128 + MO_LE] = "leo", |
2786 | | }; |
2787 | | |
2788 | | static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { |
2789 | | [MO_UNALN >> MO_ASHIFT] = "un+", |
2790 | | [MO_ALIGN >> MO_ASHIFT] = "al+", |
2791 | | [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", |
2792 | | [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", |
2793 | | [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", |
2794 | | [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", |
2795 | | [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", |
2796 | | [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", |
2797 | | }; |
2798 | | |
2799 | | static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = { |
2800 | | [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "", |
2801 | | [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+", |
2802 | | [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+", |
2803 | | [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+", |
2804 | | [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+", |
2805 | | [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+", |
2806 | | }; |
2807 | | |
2808 | | static const char bswap_flag_name[][6] = { |
2809 | | [TCG_BSWAP_IZ] = "iz", |
2810 | | [TCG_BSWAP_OZ] = "oz", |
2811 | | [TCG_BSWAP_OS] = "os", |
2812 | | [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", |
2813 | | [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", |
2814 | | }; |
2815 | | |
2816 | | #ifdef CONFIG_PLUGIN |
2817 | | static const char * const plugin_from_name[] = { |
2818 | | "from-tb", |
2819 | | "from-insn", |
2820 | | "after-insn", |
2821 | | "after-tb", |
2822 | | }; |
2823 | | #endif |
2824 | | |
2825 | | static inline bool tcg_regset_single(TCGRegSet d) |
2826 | 0 | { |
2827 | 0 | return (d & (d - 1)) == 0; |
2828 | 0 | } |
2829 | | |
2830 | | static inline TCGReg tcg_regset_first(TCGRegSet d) |
2831 | 0 | { |
2832 | 0 | if (TCG_TARGET_NB_REGS <= 32) { |
2833 | 0 | return ctz32(d); |
2834 | 0 | } else { |
2835 | 0 | return ctz64(d); |
2836 | 0 | } |
2837 | 0 | } |
2838 | | |
2839 | | /* Return only the number of characters output -- no error return. */ |
2840 | | #define ne_fprintf(...) \ |
2841 | 0 | ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) |
2842 | | |
2843 | | void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) |
2844 | 0 | { |
2845 | 0 | char buf[128]; |
2846 | 0 | TCGOp *op; |
2847 | |
|
2848 | 0 | QTAILQ_FOREACH(op, &s->ops, link) { |
2849 | 0 | int i, k, nb_oargs, nb_iargs, nb_cargs; |
2850 | 0 | const TCGOpDef *def; |
2851 | 0 | TCGOpcode c; |
2852 | 0 | int col = 0; |
2853 | |
|
2854 | 0 | c = op->opc; |
2855 | 0 | def = &tcg_op_defs[c]; |
2856 | |
|
2857 | 0 | if (c == INDEX_op_insn_start) { |
2858 | 0 | nb_oargs = 0; |
2859 | 0 | col += ne_fprintf(f, "\n ----"); |
2860 | |
|
2861 | 0 | for (i = 0, k = INSN_START_WORDS; i < k; ++i) { |
2862 | 0 | col += ne_fprintf(f, " %016" PRIx64, |
2863 | 0 | tcg_get_insn_start_param(op, i)); |
2864 | 0 | } |
2865 | 0 | } else if (c == INDEX_op_call) { |
2866 | 0 | const TCGHelperInfo *info = tcg_call_info(op); |
2867 | 0 | void *func = tcg_call_func(op); |
2868 | | |
2869 | | /* variable number of arguments */ |
2870 | 0 | nb_oargs = TCGOP_CALLO(op); |
2871 | 0 | nb_iargs = TCGOP_CALLI(op); |
2872 | 0 | nb_cargs = def->nb_cargs; |
2873 | |
|
2874 | 0 | col += ne_fprintf(f, " %s ", def->name); |
2875 | | |
2876 | | /* |
2877 | | * Print the function name from TCGHelperInfo, if available. |
2878 | | * Note that plugins have a template function for the info, |
2879 | | * but the actual function pointer comes from the plugin. |
2880 | | */ |
2881 | 0 | if (func == info->func) { |
2882 | 0 | col += ne_fprintf(f, "%s", info->name); |
2883 | 0 | } else { |
2884 | 0 | col += ne_fprintf(f, "plugin(%p)", func); |
2885 | 0 | } |
2886 | |
|
2887 | 0 | col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); |
2888 | 0 | for (i = 0; i < nb_oargs; i++) { |
2889 | 0 | col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), |
2890 | 0 | op->args[i])); |
2891 | 0 | } |
2892 | 0 | for (i = 0; i < nb_iargs; i++) { |
2893 | 0 | TCGArg arg = op->args[nb_oargs + i]; |
2894 | 0 | const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); |
2895 | 0 | col += ne_fprintf(f, ",%s", t); |
2896 | 0 | } |
2897 | 0 | } else { |
2898 | 0 | if (def->flags & TCG_OPF_INT) { |
2899 | 0 | col += ne_fprintf(f, " %s_i%d ", |
2900 | 0 | def->name, |
2901 | 0 | tcg_type_bits(TCGOP_TYPE(op))); |
2902 | 0 | } else if (def->flags & TCG_OPF_VECTOR) { |
2903 | 0 | col += ne_fprintf(f, "%s v%d,e%d,", |
2904 | 0 | def->name, |
2905 | 0 | tcg_type_bits(TCGOP_TYPE(op)), |
2906 | 0 | 8 << TCGOP_VECE(op)); |
2907 | 0 | } else { |
2908 | 0 | col += ne_fprintf(f, " %s ", def->name); |
2909 | 0 | } |
2910 | |
|
2911 | 0 | nb_oargs = def->nb_oargs; |
2912 | 0 | nb_iargs = def->nb_iargs; |
2913 | 0 | nb_cargs = def->nb_cargs; |
2914 | |
|
2915 | 0 | k = 0; |
2916 | 0 | for (i = 0; i < nb_oargs; i++) { |
2917 | 0 | const char *sep = k ? "," : ""; |
2918 | 0 | col += ne_fprintf(f, "%s%s", sep, |
2919 | 0 | tcg_get_arg_str(s, buf, sizeof(buf), |
2920 | 0 | op->args[k++])); |
2921 | 0 | } |
2922 | 0 | for (i = 0; i < nb_iargs; i++) { |
2923 | 0 | const char *sep = k ? "," : ""; |
2924 | 0 | col += ne_fprintf(f, "%s%s", sep, |
2925 | 0 | tcg_get_arg_str(s, buf, sizeof(buf), |
2926 | 0 | op->args[k++])); |
2927 | 0 | } |
2928 | 0 | switch (c) { |
2929 | 0 | case INDEX_op_brcond: |
2930 | 0 | case INDEX_op_setcond: |
2931 | 0 | case INDEX_op_negsetcond: |
2932 | 0 | case INDEX_op_movcond: |
2933 | 0 | case INDEX_op_cmp_vec: |
2934 | 0 | case INDEX_op_cmpsel_vec: |
2935 | 0 | if (op->args[k] < ARRAY_SIZE(cond_name) |
2936 | 0 | && cond_name[op->args[k]]) { |
2937 | 0 | col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); |
2938 | 0 | } else { |
2939 | 0 | col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); |
2940 | 0 | } |
2941 | 0 | i = 1; |
2942 | 0 | break; |
2943 | 0 | case INDEX_op_qemu_ld: |
2944 | 0 | case INDEX_op_qemu_st: |
2945 | 0 | case INDEX_op_qemu_ld2: |
2946 | 0 | case INDEX_op_qemu_st2: |
2947 | 0 | { |
2948 | 0 | const char *s_al, *s_tlb, *s_op, *s_at; |
2949 | 0 | MemOpIdx oi = op->args[k++]; |
2950 | 0 | MemOp mop = get_memop(oi); |
2951 | 0 | unsigned ix = get_mmuidx(oi); |
2952 | |
|
2953 | 0 | s_tlb = mop & MO_ALIGN_TLB_ONLY ? "tlb+" : ""; |
2954 | 0 | s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT]; |
2955 | 0 | s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)]; |
2956 | 0 | s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT]; |
2957 | 0 | mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | |
2958 | 0 | MO_ATOM_MASK | MO_ALIGN_TLB_ONLY); |
2959 | | |
2960 | | /* If all fields are accounted for, print symbolically. */ |
2961 | 0 | if (!mop && s_al && s_op && s_at) { |
2962 | 0 | col += ne_fprintf(f, ",%s%s%s%s,%u", |
2963 | 0 | s_at, s_al, s_tlb, s_op, ix); |
2964 | 0 | } else { |
2965 | 0 | mop = get_memop(oi); |
2966 | 0 | col += ne_fprintf(f, ",$0x%x,%u", mop, ix); |
2967 | 0 | } |
2968 | 0 | i = 1; |
2969 | 0 | } |
2970 | 0 | break; |
2971 | 0 | case INDEX_op_bswap16: |
2972 | 0 | case INDEX_op_bswap32: |
2973 | 0 | case INDEX_op_bswap64: |
2974 | 0 | case INDEX_op_revbit32: |
2975 | 0 | { |
2976 | 0 | TCGArg flags = op->args[k]; |
2977 | 0 | const char *name = NULL; |
2978 | |
|
2979 | 0 | if (flags < ARRAY_SIZE(bswap_flag_name)) { |
2980 | 0 | name = bswap_flag_name[flags]; |
2981 | 0 | } |
2982 | 0 | if (name && name[0]) { |
2983 | 0 | col += ne_fprintf(f, ",%s", name); |
2984 | 0 | } else { |
2985 | 0 | col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); |
2986 | 0 | } |
2987 | 0 | i = k = 1; |
2988 | 0 | } |
2989 | 0 | break; |
2990 | 0 | #ifdef CONFIG_PLUGIN |
2991 | 0 | case INDEX_op_plugin_cb: |
2992 | 0 | { |
2993 | 0 | TCGArg from = op->args[k++]; |
2994 | 0 | const char *name = NULL; |
2995 | |
|
2996 | 0 | if (from < ARRAY_SIZE(plugin_from_name)) { |
2997 | 0 | name = plugin_from_name[from]; |
2998 | 0 | } |
2999 | 0 | if (name) { |
3000 | 0 | col += ne_fprintf(f, "%s", name); |
3001 | 0 | } else { |
3002 | 0 | col += ne_fprintf(f, "$0x%" TCG_PRIlx, from); |
3003 | 0 | } |
3004 | 0 | i = 1; |
3005 | 0 | } |
3006 | 0 | break; |
3007 | 0 | #endif |
3008 | 0 | default: |
3009 | 0 | i = 0; |
3010 | 0 | break; |
3011 | 0 | } |
3012 | 0 | switch (c) { |
3013 | 0 | case INDEX_op_set_label: |
3014 | 0 | case INDEX_op_br: |
3015 | 0 | case INDEX_op_brcond: |
3016 | 0 | col += ne_fprintf(f, "%s$L%d", k ? "," : "", |
3017 | 0 | arg_label(op->args[k])->id); |
3018 | 0 | i++, k++; |
3019 | 0 | break; |
3020 | 0 | case INDEX_op_mb: |
3021 | 0 | { |
3022 | 0 | TCGBar membar = op->args[k]; |
3023 | 0 | const char *b_op, *m_op; |
3024 | |
|
3025 | 0 | switch (membar & TCG_BAR_SC) { |
3026 | 0 | case 0: |
3027 | 0 | b_op = "none"; |
3028 | 0 | break; |
3029 | 0 | case TCG_BAR_LDAQ: |
3030 | 0 | b_op = "acq"; |
3031 | 0 | break; |
3032 | 0 | case TCG_BAR_STRL: |
3033 | 0 | b_op = "rel"; |
3034 | 0 | break; |
3035 | 0 | case TCG_BAR_SC: |
3036 | 0 | b_op = "seq"; |
3037 | 0 | break; |
3038 | 0 | default: |
3039 | 0 | g_assert_not_reached(); |
3040 | 0 | } |
3041 | | |
3042 | 0 | switch (membar & TCG_MO_ALL) { |
3043 | 0 | case 0: |
3044 | 0 | m_op = "none"; |
3045 | 0 | break; |
3046 | 0 | case TCG_MO_LD_LD: |
3047 | 0 | m_op = "rr"; |
3048 | 0 | break; |
3049 | 0 | case TCG_MO_LD_ST: |
3050 | 0 | m_op = "rw"; |
3051 | 0 | break; |
3052 | 0 | case TCG_MO_ST_LD: |
3053 | 0 | m_op = "wr"; |
3054 | 0 | break; |
3055 | 0 | case TCG_MO_ST_ST: |
3056 | 0 | m_op = "ww"; |
3057 | 0 | break; |
3058 | 0 | case TCG_MO_LD_LD | TCG_MO_LD_ST: |
3059 | 0 | m_op = "rr+rw"; |
3060 | 0 | break; |
3061 | 0 | case TCG_MO_LD_LD | TCG_MO_ST_LD: |
3062 | 0 | m_op = "rr+wr"; |
3063 | 0 | break; |
3064 | 0 | case TCG_MO_LD_LD | TCG_MO_ST_ST: |
3065 | 0 | m_op = "rr+ww"; |
3066 | 0 | break; |
3067 | 0 | case TCG_MO_LD_ST | TCG_MO_ST_LD: |
3068 | 0 | m_op = "rw+wr"; |
3069 | 0 | break; |
3070 | 0 | case TCG_MO_LD_ST | TCG_MO_ST_ST: |
3071 | 0 | m_op = "rw+ww"; |
3072 | 0 | break; |
3073 | 0 | case TCG_MO_ST_LD | TCG_MO_ST_ST: |
3074 | 0 | m_op = "wr+ww"; |
3075 | 0 | break; |
3076 | 0 | case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: |
3077 | 0 | m_op = "rr+rw+wr"; |
3078 | 0 | break; |
3079 | 0 | case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: |
3080 | 0 | m_op = "rr+rw+ww"; |
3081 | 0 | break; |
3082 | 0 | case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: |
3083 | 0 | m_op = "rr+wr+ww"; |
3084 | 0 | break; |
3085 | 0 | case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: |
3086 | 0 | m_op = "rw+wr+ww"; |
3087 | 0 | break; |
3088 | 0 | case TCG_MO_ALL: |
3089 | 0 | m_op = "all"; |
3090 | 0 | break; |
3091 | 0 | default: |
3092 | 0 | g_assert_not_reached(); |
3093 | 0 | } |
3094 | | |
3095 | 0 | col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); |
3096 | 0 | i++, k++; |
3097 | 0 | } |
3098 | 0 | break; |
3099 | 0 | default: |
3100 | 0 | break; |
3101 | 0 | } |
3102 | 0 | for (; i < nb_cargs; i++, k++) { |
3103 | 0 | col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", |
3104 | 0 | op->args[k]); |
3105 | 0 | } |
3106 | 0 | } |
3107 | | |
3108 | 0 | if (have_prefs || op->life) { |
3109 | 0 | for (; col < 40; ++col) { |
3110 | 0 | putc(' ', f); |
3111 | 0 | } |
3112 | 0 | } |
3113 | |
|
3114 | 0 | if (op->life) { |
3115 | 0 | unsigned life = op->life; |
3116 | |
|
3117 | 0 | if (life & (SYNC_ARG * 3)) { |
3118 | 0 | ne_fprintf(f, " sync:"); |
3119 | 0 | for (i = 0; i < 2; ++i) { |
3120 | 0 | if (life & (SYNC_ARG << i)) { |
3121 | 0 | ne_fprintf(f, " %d", i); |
3122 | 0 | } |
3123 | 0 | } |
3124 | 0 | } |
3125 | 0 | life /= DEAD_ARG; |
3126 | 0 | if (life) { |
3127 | 0 | ne_fprintf(f, " dead:"); |
3128 | 0 | for (i = 0; life; ++i, life >>= 1) { |
3129 | 0 | if (life & 1) { |
3130 | 0 | ne_fprintf(f, " %d", i); |
3131 | 0 | } |
3132 | 0 | } |
3133 | 0 | } |
3134 | 0 | } |
3135 | |
|
3136 | 0 | if (have_prefs) { |
3137 | 0 | for (i = 0; i < nb_oargs; ++i) { |
3138 | 0 | TCGRegSet set = output_pref(op, i); |
3139 | |
|
3140 | 0 | if (i == 0) { |
3141 | 0 | ne_fprintf(f, " pref="); |
3142 | 0 | } else { |
3143 | 0 | ne_fprintf(f, ","); |
3144 | 0 | } |
3145 | 0 | if (set == 0) { |
3146 | 0 | ne_fprintf(f, "none"); |
3147 | 0 | } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { |
3148 | 0 | ne_fprintf(f, "all"); |
3149 | | #ifdef CONFIG_DEBUG_TCG |
3150 | | } else if (tcg_regset_single(set)) { |
3151 | | TCGReg reg = tcg_regset_first(set); |
3152 | | ne_fprintf(f, "%s", tcg_target_reg_names[reg]); |
3153 | | #endif |
3154 | 0 | } else if (TCG_TARGET_NB_REGS <= 32) { |
3155 | 0 | ne_fprintf(f, "0x%x", (uint32_t)set); |
3156 | 0 | } else { |
3157 | 0 | ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); |
3158 | 0 | } |
3159 | 0 | } |
3160 | 0 | } |
3161 | |
|
3162 | 0 | putc('\n', f); |
3163 | 0 | } |
3164 | 0 | } |
3165 | | |
3166 | | /* we give more priority to constraints with less registers */ |
3167 | | static int get_constraint_priority(const TCGArgConstraint *arg_ct, int k) |
3168 | 0 | { |
3169 | 0 | int n; |
3170 | |
|
3171 | 0 | arg_ct += k; |
3172 | 0 | n = ctpop64(arg_ct->regs); |
3173 | | |
3174 | | /* |
3175 | | * Sort constraints of a single register first, which includes output |
3176 | | * aliases (which must exactly match the input already allocated). |
3177 | | */ |
3178 | 0 | if (n == 1 || arg_ct->oalias) { |
3179 | 0 | return INT_MAX; |
3180 | 0 | } |
3181 | | |
3182 | | /* |
3183 | | * Sort register pairs next, first then second immediately after. |
3184 | | * Arbitrarily sort multiple pairs by the index of the first reg; |
3185 | | * there shouldn't be many pairs. |
3186 | | */ |
3187 | 0 | switch (arg_ct->pair) { |
3188 | 0 | case 1: |
3189 | 0 | case 3: |
3190 | 0 | return (k + 1) * 2; |
3191 | 0 | case 2: |
3192 | 0 | return (arg_ct->pair_index + 1) * 2 - 1; |
3193 | 0 | } |
3194 | | |
3195 | | /* Finally, sort by decreasing register count. */ |
3196 | 0 | assert(n > 1); |
3197 | 0 | return -n; |
3198 | 0 | } |
3199 | | |
3200 | | /* sort from highest priority to lowest */ |
3201 | | static void sort_constraints(TCGArgConstraint *a, int start, int n) |
3202 | 0 | { |
3203 | 0 | int i, j; |
3204 | |
|
3205 | 0 | for (i = 0; i < n; i++) { |
3206 | 0 | a[start + i].sort_index = start + i; |
3207 | 0 | } |
3208 | 0 | if (n <= 1) { |
3209 | 0 | return; |
3210 | 0 | } |
3211 | 0 | for (i = 0; i < n - 1; i++) { |
3212 | 0 | for (j = i + 1; j < n; j++) { |
3213 | 0 | int p1 = get_constraint_priority(a, a[start + i].sort_index); |
3214 | 0 | int p2 = get_constraint_priority(a, a[start + j].sort_index); |
3215 | 0 | if (p1 < p2) { |
3216 | 0 | int tmp = a[start + i].sort_index; |
3217 | 0 | a[start + i].sort_index = a[start + j].sort_index; |
3218 | 0 | a[start + j].sort_index = tmp; |
3219 | 0 | } |
3220 | 0 | } |
3221 | 0 | } |
3222 | 0 | } |
3223 | | |
3224 | | static const TCGArgConstraint empty_cts[TCG_MAX_OP_ARGS]; |
3225 | | static TCGArgConstraint all_cts[ARRAY_SIZE(constraint_sets)][TCG_MAX_OP_ARGS]; |
3226 | | |
3227 | | static void process_constraint_sets(void) |
3228 | 0 | { |
3229 | 0 | for (size_t c = 0; c < ARRAY_SIZE(constraint_sets); ++c) { |
3230 | 0 | const TCGConstraintSet *tdefs = &constraint_sets[c]; |
3231 | 0 | TCGArgConstraint *args_ct = all_cts[c]; |
3232 | 0 | int nb_oargs = tdefs->nb_oargs; |
3233 | 0 | int nb_iargs = tdefs->nb_iargs; |
3234 | 0 | int nb_args = nb_oargs + nb_iargs; |
3235 | 0 | bool saw_alias_pair = false; |
3236 | |
|
3237 | 0 | for (int i = 0; i < nb_args; i++) { |
3238 | 0 | const char *ct_str = tdefs->args_ct_str[i]; |
3239 | 0 | bool input_p = i >= nb_oargs; |
3240 | 0 | int o; |
3241 | |
|
3242 | 0 | switch (*ct_str) { |
3243 | 0 | case '0' ... '9': |
3244 | 0 | o = *ct_str - '0'; |
3245 | 0 | tcg_debug_assert(input_p); |
3246 | 0 | tcg_debug_assert(o < nb_oargs); |
3247 | 0 | tcg_debug_assert(args_ct[o].regs != 0); |
3248 | 0 | tcg_debug_assert(!args_ct[o].oalias); |
3249 | 0 | args_ct[i] = args_ct[o]; |
3250 | | /* The output sets oalias. */ |
3251 | 0 | args_ct[o].oalias = 1; |
3252 | 0 | args_ct[o].alias_index = i; |
3253 | | /* The input sets ialias. */ |
3254 | 0 | args_ct[i].ialias = 1; |
3255 | 0 | args_ct[i].alias_index = o; |
3256 | 0 | if (args_ct[i].pair) { |
3257 | 0 | saw_alias_pair = true; |
3258 | 0 | } |
3259 | 0 | tcg_debug_assert(ct_str[1] == '\0'); |
3260 | 0 | continue; |
3261 | | |
3262 | 0 | case '&': |
3263 | 0 | tcg_debug_assert(!input_p); |
3264 | 0 | args_ct[i].newreg = true; |
3265 | 0 | ct_str++; |
3266 | 0 | break; |
3267 | | |
3268 | 0 | case 'p': /* plus */ |
3269 | | /* Allocate to the register after the previous. */ |
3270 | 0 | tcg_debug_assert(i > (input_p ? nb_oargs : 0)); |
3271 | 0 | o = i - 1; |
3272 | 0 | tcg_debug_assert(!args_ct[o].pair); |
3273 | 0 | tcg_debug_assert(!args_ct[o].ct); |
3274 | 0 | args_ct[i] = (TCGArgConstraint){ |
3275 | 0 | .pair = 2, |
3276 | 0 | .pair_index = o, |
3277 | 0 | .regs = args_ct[o].regs << 1, |
3278 | 0 | .newreg = args_ct[o].newreg, |
3279 | 0 | }; |
3280 | 0 | args_ct[o].pair = 1; |
3281 | 0 | args_ct[o].pair_index = i; |
3282 | 0 | tcg_debug_assert(ct_str[1] == '\0'); |
3283 | 0 | continue; |
3284 | | |
3285 | 0 | case 'm': /* minus */ |
3286 | | /* Allocate to the register before the previous. */ |
3287 | 0 | tcg_debug_assert(i > (input_p ? nb_oargs : 0)); |
3288 | 0 | o = i - 1; |
3289 | 0 | tcg_debug_assert(!args_ct[o].pair); |
3290 | 0 | tcg_debug_assert(!args_ct[o].ct); |
3291 | 0 | args_ct[i] = (TCGArgConstraint){ |
3292 | 0 | .pair = 1, |
3293 | 0 | .pair_index = o, |
3294 | 0 | .regs = args_ct[o].regs >> 1, |
3295 | 0 | .newreg = args_ct[o].newreg, |
3296 | 0 | }; |
3297 | 0 | args_ct[o].pair = 2; |
3298 | 0 | args_ct[o].pair_index = i; |
3299 | 0 | tcg_debug_assert(ct_str[1] == '\0'); |
3300 | 0 | continue; |
3301 | 0 | } |
3302 | | |
3303 | 0 | do { |
3304 | 0 | switch (*ct_str) { |
3305 | 0 | case 'i': |
3306 | 0 | args_ct[i].ct |= TCG_CT_CONST; |
3307 | 0 | break; |
3308 | 0 | #ifdef TCG_REG_ZERO |
3309 | 0 | case 'z': |
3310 | 0 | args_ct[i].ct |= TCG_CT_REG_ZERO; |
3311 | 0 | break; |
3312 | 0 | #endif |
3313 | | |
3314 | | /* Include all of the target-specific constraints. */ |
3315 | | |
3316 | 0 | #undef CONST |
3317 | 0 | #define CONST(CASE, MASK) \ |
3318 | 0 | case CASE: args_ct[i].ct |= MASK; break; |
3319 | 0 | #define REGS(CASE, MASK) \ |
3320 | 0 | case CASE: args_ct[i].regs |= MASK; break; |
3321 | | |
3322 | 0 | #include "tcg-target-con-str.h" |
3323 | | |
3324 | 0 | #undef REGS |
3325 | 0 | #undef CONST |
3326 | 0 | default: |
3327 | 0 | case '0' ... '9': |
3328 | 0 | case '&': |
3329 | 0 | case 'p': |
3330 | 0 | case 'm': |
3331 | | /* Typo in TCGConstraintSet constraint. */ |
3332 | 0 | g_assert_not_reached(); |
3333 | 0 | } |
3334 | 0 | } while (*++ct_str != '\0'); |
3335 | 0 | } |
3336 | | |
3337 | | /* |
3338 | | * Fix up output pairs that are aliased with inputs. |
3339 | | * When we created the alias, we copied pair from the output. |
3340 | | * There are three cases: |
3341 | | * (1a) Pairs of inputs alias pairs of outputs. |
3342 | | * (1b) One input aliases the first of a pair of outputs. |
3343 | | * (2) One input aliases the second of a pair of outputs. |
3344 | | * |
3345 | | * Case 1a is handled by making sure that the pair_index'es are |
3346 | | * properly updated so that they appear the same as a pair of inputs. |
3347 | | * |
3348 | | * Case 1b is handled by setting the pair_index of the input to |
3349 | | * itself, simply so it doesn't point to an unrelated argument. |
3350 | | * Since we don't encounter the "second" during the input allocation |
3351 | | * phase, nothing happens with the second half of the input pair. |
3352 | | * |
3353 | | * Case 2 is handled by setting the second input to pair=3, the |
3354 | | * first output to pair=3, and the pair_index'es to match. |
3355 | | */ |
3356 | 0 | if (saw_alias_pair) { |
3357 | 0 | for (int i = nb_oargs; i < nb_args; i++) { |
3358 | 0 | int o, o2, i2; |
3359 | | |
3360 | | /* |
3361 | | * Since [0-9pm] must be alone in the constraint string, |
3362 | | * the only way they can both be set is if the pair comes |
3363 | | * from the output alias. |
3364 | | */ |
3365 | 0 | if (!args_ct[i].ialias) { |
3366 | 0 | continue; |
3367 | 0 | } |
3368 | 0 | switch (args_ct[i].pair) { |
3369 | 0 | case 0: |
3370 | 0 | break; |
3371 | 0 | case 1: |
3372 | 0 | o = args_ct[i].alias_index; |
3373 | 0 | o2 = args_ct[o].pair_index; |
3374 | 0 | tcg_debug_assert(args_ct[o].pair == 1); |
3375 | 0 | tcg_debug_assert(args_ct[o2].pair == 2); |
3376 | 0 | if (args_ct[o2].oalias) { |
3377 | | /* Case 1a */ |
3378 | 0 | i2 = args_ct[o2].alias_index; |
3379 | 0 | tcg_debug_assert(args_ct[i2].pair == 2); |
3380 | 0 | args_ct[i2].pair_index = i; |
3381 | 0 | args_ct[i].pair_index = i2; |
3382 | 0 | } else { |
3383 | | /* Case 1b */ |
3384 | 0 | args_ct[i].pair_index = i; |
3385 | 0 | } |
3386 | 0 | break; |
3387 | 0 | case 2: |
3388 | 0 | o = args_ct[i].alias_index; |
3389 | 0 | o2 = args_ct[o].pair_index; |
3390 | 0 | tcg_debug_assert(args_ct[o].pair == 2); |
3391 | 0 | tcg_debug_assert(args_ct[o2].pair == 1); |
3392 | 0 | if (args_ct[o2].oalias) { |
3393 | | /* Case 1a */ |
3394 | 0 | i2 = args_ct[o2].alias_index; |
3395 | 0 | tcg_debug_assert(args_ct[i2].pair == 1); |
3396 | 0 | args_ct[i2].pair_index = i; |
3397 | 0 | args_ct[i].pair_index = i2; |
3398 | 0 | } else { |
3399 | | /* Case 2 */ |
3400 | 0 | args_ct[i].pair = 3; |
3401 | 0 | args_ct[o2].pair = 3; |
3402 | 0 | args_ct[i].pair_index = o2; |
3403 | 0 | args_ct[o2].pair_index = i; |
3404 | 0 | } |
3405 | 0 | break; |
3406 | 0 | default: |
3407 | 0 | g_assert_not_reached(); |
3408 | 0 | } |
3409 | 0 | } |
3410 | 0 | } |
3411 | | |
3412 | | /* sort the constraints (XXX: this is just an heuristic) */ |
3413 | 0 | sort_constraints(args_ct, 0, nb_oargs); |
3414 | 0 | sort_constraints(args_ct, nb_oargs, nb_iargs); |
3415 | 0 | } |
3416 | 0 | } |
3417 | | |
3418 | | static const TCGArgConstraint *op_args_ct(TCGOpcode opc, TCGType type, |
3419 | | unsigned flags) |
3420 | 0 | { |
3421 | 0 | const TCGOpDef *def = &tcg_op_defs[opc]; |
3422 | 0 | const TCGOutOp *outop = all_outop[opc]; |
3423 | 0 | TCGConstraintSetIndex con_set; |
3424 | |
|
3425 | 0 | if (def->flags & TCG_OPF_NOT_PRESENT) { |
3426 | 0 | return empty_cts; |
3427 | 0 | } |
3428 | | |
3429 | 0 | if (outop) { |
3430 | 0 | con_set = outop->static_constraint; |
3431 | 0 | if (con_set == C_Dynamic) { |
3432 | 0 | con_set = outop->dynamic_constraint(type, flags); |
3433 | 0 | } |
3434 | 0 | } else { |
3435 | 0 | con_set = tcg_target_op_def(opc, type, flags); |
3436 | 0 | } |
3437 | 0 | tcg_debug_assert(con_set >= 0); |
3438 | 0 | tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); |
3439 | | |
3440 | | /* The constraint arguments must match TCGOpcode arguments. */ |
3441 | 0 | tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs); |
3442 | 0 | tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs); |
3443 | | |
3444 | 0 | return all_cts[con_set]; |
3445 | 0 | } |
3446 | | |
3447 | | static const TCGArgConstraint *opcode_args_ct(const TCGOp *op) |
3448 | 0 | { |
3449 | 0 | return op_args_ct(op->opc, TCGOP_TYPE(op), TCGOP_FLAGS(op)); |
3450 | 0 | } |
3451 | | |
3452 | | bool tcg_op_imm_match(TCGOpcode opc, TCGType type, tcg_target_ulong imm) |
3453 | 0 | { |
3454 | 0 | const TCGArgConstraint *args_ct = op_args_ct(opc, type, 0); |
3455 | 0 | const TCGOpDef *def = &tcg_op_defs[opc]; |
3456 | |
|
3457 | 0 | tcg_debug_assert(def->nb_oargs == 1); |
3458 | 0 | tcg_debug_assert(def->nb_iargs == 2); |
3459 | 0 | return tcg_target_const_match(imm, args_ct[2].ct, type, 0, 0); |
3460 | 0 | } |
3461 | | |
3462 | | static void remove_label_use(TCGOp *op, int idx) |
3463 | 0 | { |
3464 | 0 | TCGLabel *label = arg_label(op->args[idx]); |
3465 | 0 | TCGLabelUse *use; |
3466 | |
|
3467 | 0 | QSIMPLEQ_FOREACH(use, &label->branches, next) { |
3468 | 0 | if (use->op == op) { |
3469 | 0 | QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); |
3470 | 0 | return; |
3471 | 0 | } |
3472 | 0 | } |
3473 | 0 | g_assert_not_reached(); |
3474 | 0 | } |
3475 | | |
3476 | | void tcg_op_remove(TCGContext *s, TCGOp *op) |
3477 | 0 | { |
3478 | 0 | switch (op->opc) { |
3479 | 0 | case INDEX_op_br: |
3480 | 0 | remove_label_use(op, 0); |
3481 | 0 | break; |
3482 | 0 | case INDEX_op_brcond: |
3483 | 0 | remove_label_use(op, 3); |
3484 | 0 | break; |
3485 | 0 | default: |
3486 | 0 | break; |
3487 | 0 | } |
3488 | | |
3489 | 0 | QTAILQ_REMOVE(&s->ops, op, link); |
3490 | 0 | QTAILQ_INSERT_TAIL(&s->free_ops, op, link); |
3491 | 0 | s->nb_ops--; |
3492 | 0 | } |
3493 | | |
3494 | | void tcg_remove_ops_after(TCGOp *op) |
3495 | 0 | { |
3496 | 0 | TCGContext *s = tcg_ctx; |
3497 | |
|
3498 | 0 | while (true) { |
3499 | 0 | TCGOp *last = tcg_last_op(); |
3500 | 0 | if (last == op) { |
3501 | 0 | return; |
3502 | 0 | } |
3503 | 0 | tcg_op_remove(s, last); |
3504 | 0 | } |
3505 | 0 | } |
3506 | | |
3507 | | static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) |
3508 | 0 | { |
3509 | 0 | TCGContext *s = tcg_ctx; |
3510 | 0 | TCGOp *op = NULL; |
3511 | |
|
3512 | 0 | if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { |
3513 | 0 | QTAILQ_FOREACH(op, &s->free_ops, link) { |
3514 | 0 | if (nargs <= op->nargs) { |
3515 | 0 | QTAILQ_REMOVE(&s->free_ops, op, link); |
3516 | 0 | nargs = op->nargs; |
3517 | 0 | goto found; |
3518 | 0 | } |
3519 | 0 | } |
3520 | 0 | } |
3521 | | |
3522 | | /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ |
3523 | 0 | nargs = MAX(4, nargs); |
3524 | 0 | op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); |
3525 | |
|
3526 | 0 | found: |
3527 | 0 | memset(op, 0, offsetof(TCGOp, link)); |
3528 | 0 | op->opc = opc; |
3529 | 0 | op->nargs = nargs; |
3530 | | |
3531 | | /* Check for bitfield overflow. */ |
3532 | 0 | tcg_debug_assert(op->nargs == nargs); |
3533 | | |
3534 | 0 | s->nb_ops++; |
3535 | 0 | return op; |
3536 | 0 | } |
3537 | | |
3538 | | TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) |
3539 | 0 | { |
3540 | 0 | TCGOp *op = tcg_op_alloc(opc, nargs); |
3541 | |
|
3542 | 0 | if (tcg_ctx->emit_before_op) { |
3543 | 0 | QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); |
3544 | 0 | } else { |
3545 | 0 | QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); |
3546 | 0 | } |
3547 | 0 | return op; |
3548 | 0 | } |
3549 | | |
3550 | | TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, |
3551 | | TCGOpcode opc, TCGType type, unsigned nargs) |
3552 | 0 | { |
3553 | 0 | TCGOp *new_op = tcg_op_alloc(opc, nargs); |
3554 | |
|
3555 | 0 | TCGOP_TYPE(new_op) = type; |
3556 | 0 | QTAILQ_INSERT_BEFORE(old_op, new_op, link); |
3557 | 0 | return new_op; |
3558 | 0 | } |
3559 | | |
3560 | | TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, |
3561 | | TCGOpcode opc, TCGType type, unsigned nargs) |
3562 | 0 | { |
3563 | 0 | TCGOp *new_op = tcg_op_alloc(opc, nargs); |
3564 | |
|
3565 | 0 | TCGOP_TYPE(new_op) = type; |
3566 | 0 | QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); |
3567 | 0 | return new_op; |
3568 | 0 | } |
3569 | | |
3570 | | static void move_label_uses(TCGLabel *to, TCGLabel *from) |
3571 | 0 | { |
3572 | 0 | TCGLabelUse *u; |
3573 | |
|
3574 | 0 | QSIMPLEQ_FOREACH(u, &from->branches, next) { |
3575 | 0 | TCGOp *op = u->op; |
3576 | 0 | switch (op->opc) { |
3577 | 0 | case INDEX_op_br: |
3578 | 0 | op->args[0] = label_arg(to); |
3579 | 0 | break; |
3580 | 0 | case INDEX_op_brcond: |
3581 | 0 | op->args[3] = label_arg(to); |
3582 | 0 | break; |
3583 | 0 | default: |
3584 | 0 | g_assert_not_reached(); |
3585 | 0 | } |
3586 | 0 | } |
3587 | | |
3588 | 0 | QSIMPLEQ_CONCAT(&to->branches, &from->branches); |
3589 | 0 | } |
3590 | | |
3591 | | /* Reachable analysis : remove unreachable code. */ |
3592 | | static void __attribute__((noinline)) |
3593 | | reachable_code_pass(TCGContext *s) |
3594 | 0 | { |
3595 | 0 | TCGOp *op, *op_next, *op_prev; |
3596 | 0 | bool dead = false; |
3597 | |
|
3598 | 0 | QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { |
3599 | 0 | bool remove = dead; |
3600 | 0 | TCGLabel *label; |
3601 | |
|
3602 | 0 | switch (op->opc) { |
3603 | 0 | case INDEX_op_set_label: |
3604 | 0 | label = arg_label(op->args[0]); |
3605 | | |
3606 | | /* |
3607 | | * Note that the first op in the TB is always a load, |
3608 | | * so there is always something before a label. |
3609 | | */ |
3610 | 0 | op_prev = QTAILQ_PREV(op, link); |
3611 | | |
3612 | | /* |
3613 | | * If we find two sequential labels, move all branches to |
3614 | | * reference the second label and remove the first label. |
3615 | | * Do this before branch to next optimization, so that the |
3616 | | * middle label is out of the way. |
3617 | | */ |
3618 | 0 | if (op_prev->opc == INDEX_op_set_label) { |
3619 | 0 | move_label_uses(label, arg_label(op_prev->args[0])); |
3620 | 0 | tcg_op_remove(s, op_prev); |
3621 | 0 | op_prev = QTAILQ_PREV(op, link); |
3622 | 0 | } |
3623 | | |
3624 | | /* |
3625 | | * Optimization can fold conditional branches to unconditional. |
3626 | | * If we find a label which is preceded by an unconditional |
3627 | | * branch to next, remove the branch. We couldn't do this when |
3628 | | * processing the branch because any dead code between the branch |
3629 | | * and label had not yet been removed. |
3630 | | */ |
3631 | 0 | if (op_prev->opc == INDEX_op_br && |
3632 | 0 | label == arg_label(op_prev->args[0])) { |
3633 | 0 | tcg_op_remove(s, op_prev); |
3634 | | /* Fall through means insns become live again. */ |
3635 | 0 | dead = false; |
3636 | 0 | } |
3637 | |
|
3638 | 0 | if (QSIMPLEQ_EMPTY(&label->branches)) { |
3639 | | /* |
3640 | | * While there is an occasional backward branch, virtually |
3641 | | * all branches generated by the translators are forward. |
3642 | | * Which means that generally we will have already removed |
3643 | | * all references to the label that will be, and there is |
3644 | | * little to be gained by iterating. |
3645 | | */ |
3646 | 0 | remove = true; |
3647 | 0 | } else { |
3648 | | /* Once we see a label, insns become live again. */ |
3649 | 0 | dead = false; |
3650 | 0 | remove = false; |
3651 | 0 | } |
3652 | 0 | break; |
3653 | | |
3654 | 0 | case INDEX_op_br: |
3655 | 0 | case INDEX_op_exit_tb: |
3656 | 0 | case INDEX_op_goto_ptr: |
3657 | | /* Unconditional branches; everything following is dead. */ |
3658 | 0 | dead = true; |
3659 | 0 | break; |
3660 | | |
3661 | 0 | case INDEX_op_call: |
3662 | | /* Notice noreturn helper calls, raising exceptions. */ |
3663 | 0 | if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { |
3664 | 0 | dead = true; |
3665 | 0 | } |
3666 | 0 | break; |
3667 | | |
3668 | 0 | case INDEX_op_insn_start: |
3669 | | /* Never remove -- we need to keep these for unwind. */ |
3670 | 0 | remove = false; |
3671 | 0 | break; |
3672 | | |
3673 | 0 | default: |
3674 | 0 | break; |
3675 | 0 | } |
3676 | | |
3677 | 0 | if (remove) { |
3678 | 0 | tcg_op_remove(s, op); |
3679 | 0 | } |
3680 | 0 | } |
3681 | 0 | } |
3682 | | |
3683 | 0 | #define TS_DEAD 1 |
3684 | 0 | #define TS_MEM 2 |
3685 | | |
3686 | 0 | #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) |
3687 | 0 | #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) |
3688 | | |
3689 | | /* For liveness_pass_1, the register preferences for a given temp. */ |
3690 | | static inline TCGRegSet *la_temp_pref(TCGTemp *ts) |
3691 | 0 | { |
3692 | 0 | return ts->state_ptr; |
3693 | 0 | } |
3694 | | |
3695 | | /* For liveness_pass_1, reset the preferences for a given temp to the |
3696 | | * maximal regset for its type. |
3697 | | */ |
3698 | | static inline void la_reset_pref(TCGTemp *ts) |
3699 | 0 | { |
3700 | 0 | *la_temp_pref(ts) |
3701 | 0 | = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); |
3702 | 0 | } |
3703 | | |
3704 | | /* liveness analysis: end of function: all temps are dead, and globals |
3705 | | should be in memory. */ |
3706 | | static void la_func_end(TCGContext *s, int ng, int nt) |
3707 | 0 | { |
3708 | 0 | int i; |
3709 | |
|
3710 | 0 | for (i = 0; i < ng; ++i) { |
3711 | 0 | s->temps[i].state = TS_DEAD | TS_MEM; |
3712 | 0 | la_reset_pref(&s->temps[i]); |
3713 | 0 | } |
3714 | 0 | for (i = ng; i < nt; ++i) { |
3715 | 0 | s->temps[i].state = TS_DEAD; |
3716 | 0 | la_reset_pref(&s->temps[i]); |
3717 | 0 | } |
3718 | 0 | } |
3719 | | |
3720 | | /* liveness analysis: end of basic block: all temps are dead, globals |
3721 | | and local temps should be in memory. */ |
3722 | | static void la_bb_end(TCGContext *s, int ng, int nt) |
3723 | 0 | { |
3724 | 0 | int i; |
3725 | |
|
3726 | 0 | for (i = 0; i < nt; ++i) { |
3727 | 0 | TCGTemp *ts = &s->temps[i]; |
3728 | 0 | int state; |
3729 | |
|
3730 | 0 | switch (ts->kind) { |
3731 | 0 | case TEMP_FIXED: |
3732 | 0 | case TEMP_GLOBAL: |
3733 | 0 | case TEMP_TB: |
3734 | 0 | state = TS_DEAD | TS_MEM; |
3735 | 0 | break; |
3736 | 0 | case TEMP_EBB: |
3737 | 0 | case TEMP_CONST: |
3738 | 0 | state = TS_DEAD; |
3739 | 0 | break; |
3740 | 0 | default: |
3741 | 0 | g_assert_not_reached(); |
3742 | 0 | } |
3743 | 0 | ts->state = state; |
3744 | 0 | la_reset_pref(ts); |
3745 | 0 | } |
3746 | 0 | } |
3747 | | |
3748 | | /* liveness analysis: sync globals back to memory. */ |
3749 | | static void la_global_sync(TCGContext *s, int ng) |
3750 | 0 | { |
3751 | 0 | int i; |
3752 | |
|
3753 | 0 | for (i = 0; i < ng; ++i) { |
3754 | 0 | int state = s->temps[i].state; |
3755 | 0 | s->temps[i].state = state | TS_MEM; |
3756 | 0 | if (state == TS_DEAD) { |
3757 | | /* If the global was previously dead, reset prefs. */ |
3758 | 0 | la_reset_pref(&s->temps[i]); |
3759 | 0 | } |
3760 | 0 | } |
3761 | 0 | } |
3762 | | |
3763 | | /* |
3764 | | * liveness analysis: conditional branch: all temps are dead unless |
3765 | | * explicitly live-across-conditional-branch, globals and local temps |
3766 | | * should be synced. |
3767 | | */ |
3768 | | static void la_bb_sync(TCGContext *s, int ng, int nt) |
3769 | 0 | { |
3770 | 0 | la_global_sync(s, ng); |
3771 | |
|
3772 | 0 | for (int i = ng; i < nt; ++i) { |
3773 | 0 | TCGTemp *ts = &s->temps[i]; |
3774 | 0 | int state; |
3775 | |
|
3776 | 0 | switch (ts->kind) { |
3777 | 0 | case TEMP_TB: |
3778 | 0 | state = ts->state; |
3779 | 0 | ts->state = state | TS_MEM; |
3780 | 0 | if (state != TS_DEAD) { |
3781 | 0 | continue; |
3782 | 0 | } |
3783 | 0 | break; |
3784 | 0 | case TEMP_EBB: |
3785 | 0 | case TEMP_CONST: |
3786 | 0 | continue; |
3787 | 0 | default: |
3788 | 0 | g_assert_not_reached(); |
3789 | 0 | } |
3790 | 0 | la_reset_pref(&s->temps[i]); |
3791 | 0 | } |
3792 | 0 | } |
3793 | | |
3794 | | /* liveness analysis: sync globals back to memory and kill. */ |
3795 | | static void la_global_kill(TCGContext *s, int ng) |
3796 | 0 | { |
3797 | 0 | int i; |
3798 | |
|
3799 | 0 | for (i = 0; i < ng; i++) { |
3800 | 0 | s->temps[i].state = TS_DEAD | TS_MEM; |
3801 | 0 | la_reset_pref(&s->temps[i]); |
3802 | 0 | } |
3803 | 0 | } |
3804 | | |
3805 | | /* liveness analysis: note live globals crossing calls. */ |
3806 | | static void la_cross_call(TCGContext *s, int nt) |
3807 | 0 | { |
3808 | 0 | TCGRegSet mask = ~tcg_target_call_clobber_regs; |
3809 | 0 | int i; |
3810 | |
|
3811 | 0 | for (i = 0; i < nt; i++) { |
3812 | 0 | TCGTemp *ts = &s->temps[i]; |
3813 | 0 | if (!(ts->state & TS_DEAD)) { |
3814 | 0 | TCGRegSet *pset = la_temp_pref(ts); |
3815 | 0 | TCGRegSet set = *pset; |
3816 | |
|
3817 | 0 | set &= mask; |
3818 | | /* If the combination is not possible, restart. */ |
3819 | 0 | if (set == 0) { |
3820 | 0 | set = tcg_target_available_regs[ts->type] & mask; |
3821 | 0 | } |
3822 | 0 | *pset = set; |
3823 | 0 | } |
3824 | 0 | } |
3825 | 0 | } |
3826 | | |
3827 | | /* |
3828 | | * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce |
3829 | | * to TEMP_EBB, if possible. |
3830 | | */ |
3831 | | static void __attribute__((noinline)) |
3832 | | liveness_pass_0(TCGContext *s) |
3833 | 0 | { |
3834 | 0 | void * const multiple_ebb = (void *)(uintptr_t)-1; |
3835 | 0 | int nb_temps = s->nb_temps; |
3836 | 0 | TCGOp *op, *ebb; |
3837 | |
|
3838 | 0 | for (int i = s->nb_globals; i < nb_temps; ++i) { |
3839 | 0 | s->temps[i].state_ptr = NULL; |
3840 | 0 | } |
3841 | | |
3842 | | /* |
3843 | | * Represent each EBB by the op at which it begins. In the case of |
3844 | | * the first EBB, this is the first op, otherwise it is a label. |
3845 | | * Collect the uses of each TEMP_TB: NULL for unused, EBB for use |
3846 | | * within a single EBB, else MULTIPLE_EBB. |
3847 | | */ |
3848 | 0 | ebb = QTAILQ_FIRST(&s->ops); |
3849 | 0 | QTAILQ_FOREACH(op, &s->ops, link) { |
3850 | 0 | const TCGOpDef *def; |
3851 | 0 | int nb_oargs, nb_iargs; |
3852 | |
|
3853 | 0 | switch (op->opc) { |
3854 | 0 | case INDEX_op_set_label: |
3855 | 0 | ebb = op; |
3856 | 0 | continue; |
3857 | 0 | case INDEX_op_discard: |
3858 | 0 | continue; |
3859 | 0 | case INDEX_op_call: |
3860 | 0 | nb_oargs = TCGOP_CALLO(op); |
3861 | 0 | nb_iargs = TCGOP_CALLI(op); |
3862 | 0 | break; |
3863 | 0 | default: |
3864 | 0 | def = &tcg_op_defs[op->opc]; |
3865 | 0 | nb_oargs = def->nb_oargs; |
3866 | 0 | nb_iargs = def->nb_iargs; |
3867 | 0 | break; |
3868 | 0 | } |
3869 | | |
3870 | 0 | for (int i = 0; i < nb_oargs + nb_iargs; ++i) { |
3871 | 0 | TCGTemp *ts = arg_temp(op->args[i]); |
3872 | |
|
3873 | 0 | if (ts->kind != TEMP_TB) { |
3874 | 0 | continue; |
3875 | 0 | } |
3876 | 0 | if (ts->state_ptr == NULL) { |
3877 | 0 | ts->state_ptr = ebb; |
3878 | 0 | } else if (ts->state_ptr != ebb) { |
3879 | 0 | ts->state_ptr = multiple_ebb; |
3880 | 0 | } |
3881 | 0 | } |
3882 | 0 | } |
3883 | | |
3884 | | /* |
3885 | | * For TEMP_TB that turned out not to be used beyond one EBB, |
3886 | | * reduce the liveness to TEMP_EBB. |
3887 | | */ |
3888 | 0 | for (int i = s->nb_globals; i < nb_temps; ++i) { |
3889 | 0 | TCGTemp *ts = &s->temps[i]; |
3890 | 0 | if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { |
3891 | 0 | ts->kind = TEMP_EBB; |
3892 | 0 | } |
3893 | 0 | } |
3894 | 0 | } |
3895 | | |
3896 | | static void assert_carry_dead(TCGContext *s) |
3897 | 0 | { |
3898 | | /* |
3899 | | * Carry operations can be separated by a few insns like mov, |
3900 | | * load or store, but they should always be "close", and |
3901 | | * carry-out operations should always be paired with carry-in. |
3902 | | * At various boundaries, carry must have been consumed. |
3903 | | */ |
3904 | 0 | tcg_debug_assert(!s->carry_live); |
3905 | 0 | } |
3906 | | |
3907 | | /* Liveness analysis : update the opc_arg_life array to tell if a |
3908 | | given input arguments is dead. Instructions updating dead |
3909 | | temporaries are removed. */ |
3910 | | static void __attribute__((noinline)) |
3911 | | liveness_pass_1(TCGContext *s) |
3912 | 0 | { |
3913 | 0 | int nb_globals = s->nb_globals; |
3914 | 0 | int nb_temps = s->nb_temps; |
3915 | 0 | TCGOp *op, *op_prev; |
3916 | 0 | TCGRegSet *prefs; |
3917 | |
|
3918 | 0 | prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); |
3919 | 0 | for (int i = 0; i < nb_temps; ++i) { |
3920 | 0 | s->temps[i].state_ptr = prefs + i; |
3921 | 0 | } |
3922 | | |
3923 | | /* ??? Should be redundant with the exit_tb that ends the TB. */ |
3924 | 0 | la_func_end(s, nb_globals, nb_temps); |
3925 | |
|
3926 | 0 | s->carry_live = false; |
3927 | 0 | QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { |
3928 | 0 | int nb_iargs, nb_oargs; |
3929 | 0 | TCGOpcode opc_new, opc_new2; |
3930 | 0 | TCGLifeData arg_life = 0; |
3931 | 0 | TCGTemp *ts; |
3932 | 0 | TCGOpcode opc = op->opc; |
3933 | 0 | const TCGOpDef *def; |
3934 | 0 | const TCGArgConstraint *args_ct; |
3935 | |
|
3936 | 0 | switch (opc) { |
3937 | 0 | case INDEX_op_call: |
3938 | 0 | assert_carry_dead(s); |
3939 | 0 | { |
3940 | 0 | const TCGHelperInfo *info = tcg_call_info(op); |
3941 | 0 | int call_flags = tcg_call_flags(op); |
3942 | |
|
3943 | 0 | nb_oargs = TCGOP_CALLO(op); |
3944 | 0 | nb_iargs = TCGOP_CALLI(op); |
3945 | | |
3946 | | /* pure functions can be removed if their result is unused */ |
3947 | 0 | if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { |
3948 | 0 | for (int i = 0; i < nb_oargs; i++) { |
3949 | 0 | ts = arg_temp(op->args[i]); |
3950 | 0 | if (ts->state != TS_DEAD) { |
3951 | 0 | goto do_not_remove_call; |
3952 | 0 | } |
3953 | 0 | } |
3954 | 0 | goto do_remove; |
3955 | 0 | } |
3956 | 0 | do_not_remove_call: |
3957 | | |
3958 | | /* Output args are dead. */ |
3959 | 0 | for (int i = 0; i < nb_oargs; i++) { |
3960 | 0 | ts = arg_temp(op->args[i]); |
3961 | 0 | if (ts->state & TS_DEAD) { |
3962 | 0 | arg_life |= DEAD_ARG << i; |
3963 | 0 | } |
3964 | 0 | if (ts->state & TS_MEM) { |
3965 | 0 | arg_life |= SYNC_ARG << i; |
3966 | 0 | } |
3967 | 0 | ts->state = TS_DEAD; |
3968 | 0 | la_reset_pref(ts); |
3969 | 0 | } |
3970 | | |
3971 | | /* Not used -- it will be tcg_target_call_oarg_reg(). */ |
3972 | 0 | memset(op->output_pref, 0, sizeof(op->output_pref)); |
3973 | |
|
3974 | 0 | if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | |
3975 | 0 | TCG_CALL_NO_READ_GLOBALS))) { |
3976 | 0 | la_global_kill(s, nb_globals); |
3977 | 0 | } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { |
3978 | 0 | la_global_sync(s, nb_globals); |
3979 | 0 | } |
3980 | | |
3981 | | /* Record arguments that die in this helper. */ |
3982 | 0 | for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) { |
3983 | 0 | ts = arg_temp(op->args[i]); |
3984 | 0 | if (ts->state & TS_DEAD) { |
3985 | 0 | arg_life |= DEAD_ARG << i; |
3986 | 0 | } |
3987 | 0 | } |
3988 | | |
3989 | | /* For all live registers, remove call-clobbered prefs. */ |
3990 | 0 | la_cross_call(s, nb_temps); |
3991 | | |
3992 | | /* |
3993 | | * Input arguments are live for preceding opcodes. |
3994 | | * |
3995 | | * For those arguments that die, and will be allocated in |
3996 | | * registers, clear the register set for that arg, to be |
3997 | | * filled in below. For args that will be on the stack, |
3998 | | * reset to any available reg. Process arguments in reverse |
3999 | | * order so that if a temp is used more than once, the stack |
4000 | | * reset to max happens before the register reset to 0. |
4001 | | */ |
4002 | 0 | for (int i = nb_iargs - 1; i >= 0; i--) { |
4003 | 0 | const TCGCallArgumentLoc *loc = &info->in[i]; |
4004 | 0 | ts = arg_temp(op->args[nb_oargs + i]); |
4005 | |
|
4006 | 0 | if (ts->state & TS_DEAD) { |
4007 | 0 | switch (loc->kind) { |
4008 | 0 | case TCG_CALL_ARG_NORMAL: |
4009 | 0 | case TCG_CALL_ARG_EXTEND_U: |
4010 | 0 | case TCG_CALL_ARG_EXTEND_S: |
4011 | 0 | if (arg_slot_reg_p(loc->arg_slot)) { |
4012 | 0 | *la_temp_pref(ts) = 0; |
4013 | 0 | break; |
4014 | 0 | } |
4015 | | /* fall through */ |
4016 | 0 | default: |
4017 | 0 | *la_temp_pref(ts) = |
4018 | 0 | tcg_target_available_regs[ts->type]; |
4019 | 0 | break; |
4020 | 0 | } |
4021 | 0 | ts->state &= ~TS_DEAD; |
4022 | 0 | } |
4023 | 0 | } |
4024 | | |
4025 | | /* |
4026 | | * For each input argument, add its input register to prefs. |
4027 | | * If a temp is used once, this produces a single set bit; |
4028 | | * if a temp is used multiple times, this produces a set. |
4029 | | */ |
4030 | 0 | for (int i = 0; i < nb_iargs; i++) { |
4031 | 0 | const TCGCallArgumentLoc *loc = &info->in[i]; |
4032 | 0 | ts = arg_temp(op->args[nb_oargs + i]); |
4033 | |
|
4034 | 0 | switch (loc->kind) { |
4035 | 0 | case TCG_CALL_ARG_NORMAL: |
4036 | 0 | case TCG_CALL_ARG_EXTEND_U: |
4037 | 0 | case TCG_CALL_ARG_EXTEND_S: |
4038 | 0 | if (arg_slot_reg_p(loc->arg_slot)) { |
4039 | 0 | tcg_regset_set_reg(*la_temp_pref(ts), |
4040 | 0 | tcg_target_call_iarg_regs[loc->arg_slot]); |
4041 | 0 | } |
4042 | 0 | break; |
4043 | 0 | default: |
4044 | 0 | break; |
4045 | 0 | } |
4046 | 0 | } |
4047 | 0 | } |
4048 | 0 | break; |
4049 | 0 | case INDEX_op_insn_start: |
4050 | 0 | assert_carry_dead(s); |
4051 | 0 | break; |
4052 | 0 | case INDEX_op_discard: |
4053 | | /* mark the temporary as dead */ |
4054 | 0 | ts = arg_temp(op->args[0]); |
4055 | 0 | ts->state = TS_DEAD; |
4056 | 0 | la_reset_pref(ts); |
4057 | 0 | break; |
4058 | | |
4059 | 0 | case INDEX_op_muls2: |
4060 | 0 | opc_new = INDEX_op_mul; |
4061 | 0 | opc_new2 = INDEX_op_mulsh; |
4062 | 0 | goto do_mul2; |
4063 | 0 | case INDEX_op_mulu2: |
4064 | 0 | opc_new = INDEX_op_mul; |
4065 | 0 | opc_new2 = INDEX_op_muluh; |
4066 | 0 | do_mul2: |
4067 | 0 | assert_carry_dead(s); |
4068 | 0 | if (arg_temp(op->args[1])->state == TS_DEAD) { |
4069 | 0 | if (arg_temp(op->args[0])->state == TS_DEAD) { |
4070 | | /* Both parts of the operation are dead. */ |
4071 | 0 | goto do_remove; |
4072 | 0 | } |
4073 | | /* The high part of the operation is dead; generate the low. */ |
4074 | 0 | op->opc = opc = opc_new; |
4075 | 0 | op->args[1] = op->args[2]; |
4076 | 0 | op->args[2] = op->args[3]; |
4077 | 0 | } else if (arg_temp(op->args[0])->state == TS_DEAD && |
4078 | 0 | tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) { |
4079 | | /* The low part of the operation is dead; generate the high. */ |
4080 | 0 | op->opc = opc = opc_new2; |
4081 | 0 | op->args[0] = op->args[1]; |
4082 | 0 | op->args[1] = op->args[2]; |
4083 | 0 | op->args[2] = op->args[3]; |
4084 | 0 | } else { |
4085 | 0 | goto do_not_remove; |
4086 | 0 | } |
4087 | | /* Mark the single-word operation live. */ |
4088 | 0 | goto do_not_remove; |
4089 | | |
4090 | 0 | case INDEX_op_addco: |
4091 | 0 | if (s->carry_live) { |
4092 | 0 | goto do_not_remove; |
4093 | 0 | } |
4094 | 0 | op->opc = opc = INDEX_op_add; |
4095 | 0 | goto do_default; |
4096 | | |
4097 | 0 | case INDEX_op_addcio: |
4098 | 0 | if (s->carry_live) { |
4099 | 0 | goto do_not_remove; |
4100 | 0 | } |
4101 | 0 | op->opc = opc = INDEX_op_addci; |
4102 | 0 | goto do_default; |
4103 | | |
4104 | 0 | case INDEX_op_subbo: |
4105 | 0 | if (s->carry_live) { |
4106 | 0 | goto do_not_remove; |
4107 | 0 | } |
4108 | | /* Lower to sub, but this may also require canonicalization. */ |
4109 | 0 | op->opc = opc = INDEX_op_sub; |
4110 | 0 | ts = arg_temp(op->args[2]); |
4111 | 0 | if (ts->kind == TEMP_CONST) { |
4112 | 0 | ts = tcg_constant_internal(ts->type, -ts->val); |
4113 | 0 | if (ts->state_ptr == NULL) { |
4114 | 0 | tcg_debug_assert(temp_idx(ts) == nb_temps); |
4115 | 0 | nb_temps++; |
4116 | 0 | ts->state_ptr = tcg_malloc(sizeof(TCGRegSet)); |
4117 | 0 | ts->state = TS_DEAD; |
4118 | 0 | la_reset_pref(ts); |
4119 | 0 | } |
4120 | 0 | op->args[2] = temp_arg(ts); |
4121 | 0 | op->opc = opc = INDEX_op_add; |
4122 | 0 | } |
4123 | 0 | goto do_default; |
4124 | | |
4125 | 0 | case INDEX_op_subbio: |
4126 | 0 | if (s->carry_live) { |
4127 | 0 | goto do_not_remove; |
4128 | 0 | } |
4129 | 0 | op->opc = opc = INDEX_op_subbi; |
4130 | 0 | goto do_default; |
4131 | | |
4132 | 0 | case INDEX_op_addc1o: |
4133 | 0 | if (s->carry_live) { |
4134 | 0 | goto do_not_remove; |
4135 | 0 | } |
4136 | | /* Lower to add, add +1. */ |
4137 | 0 | op_prev = tcg_op_insert_before(s, op, INDEX_op_add, |
4138 | 0 | TCGOP_TYPE(op), 3); |
4139 | 0 | op_prev->args[0] = op->args[0]; |
4140 | 0 | op_prev->args[1] = op->args[1]; |
4141 | 0 | op_prev->args[2] = op->args[2]; |
4142 | 0 | op->opc = opc = INDEX_op_add; |
4143 | 0 | op->args[1] = op->args[0]; |
4144 | 0 | ts = arg_temp(op->args[0]); |
4145 | 0 | ts = tcg_constant_internal(ts->type, 1); |
4146 | 0 | op->args[2] = temp_arg(ts); |
4147 | 0 | goto do_default; |
4148 | | |
4149 | 0 | case INDEX_op_subb1o: |
4150 | 0 | if (s->carry_live) { |
4151 | 0 | goto do_not_remove; |
4152 | 0 | } |
4153 | | /* Lower to sub, add -1. */ |
4154 | 0 | op_prev = tcg_op_insert_before(s, op, INDEX_op_sub, |
4155 | 0 | TCGOP_TYPE(op), 3); |
4156 | 0 | op_prev->args[0] = op->args[0]; |
4157 | 0 | op_prev->args[1] = op->args[1]; |
4158 | 0 | op_prev->args[2] = op->args[2]; |
4159 | 0 | op->opc = opc = INDEX_op_add; |
4160 | 0 | op->args[1] = op->args[0]; |
4161 | 0 | ts = arg_temp(op->args[0]); |
4162 | 0 | ts = tcg_constant_internal(ts->type, -1); |
4163 | 0 | op->args[2] = temp_arg(ts); |
4164 | 0 | goto do_default; |
4165 | | |
4166 | 0 | default: |
4167 | 0 | do_default: |
4168 | | /* |
4169 | | * Test if the operation can be removed because all |
4170 | | * its outputs are dead. We assume that nb_oargs == 0 |
4171 | | * implies side effects. |
4172 | | */ |
4173 | 0 | def = &tcg_op_defs[opc]; |
4174 | 0 | if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) { |
4175 | 0 | for (int i = def->nb_oargs - 1; i >= 0; i--) { |
4176 | 0 | if (arg_temp(op->args[i])->state != TS_DEAD) { |
4177 | 0 | goto do_not_remove; |
4178 | 0 | } |
4179 | 0 | } |
4180 | 0 | goto do_remove; |
4181 | 0 | } |
4182 | 0 | goto do_not_remove; |
4183 | | |
4184 | 0 | do_remove: |
4185 | 0 | tcg_op_remove(s, op); |
4186 | 0 | break; |
4187 | | |
4188 | 0 | do_not_remove: |
4189 | 0 | def = &tcg_op_defs[opc]; |
4190 | 0 | nb_iargs = def->nb_iargs; |
4191 | 0 | nb_oargs = def->nb_oargs; |
4192 | |
|
4193 | 0 | for (int i = 0; i < nb_oargs; i++) { |
4194 | 0 | ts = arg_temp(op->args[i]); |
4195 | | |
4196 | | /* Remember the preference of the uses that followed. */ |
4197 | 0 | if (i < ARRAY_SIZE(op->output_pref)) { |
4198 | 0 | op->output_pref[i] = *la_temp_pref(ts); |
4199 | 0 | } |
4200 | | |
4201 | | /* Output args are dead. */ |
4202 | 0 | if (ts->state & TS_DEAD) { |
4203 | 0 | arg_life |= DEAD_ARG << i; |
4204 | 0 | } |
4205 | 0 | if (ts->state & TS_MEM) { |
4206 | 0 | arg_life |= SYNC_ARG << i; |
4207 | 0 | } |
4208 | 0 | ts->state = TS_DEAD; |
4209 | 0 | la_reset_pref(ts); |
4210 | 0 | } |
4211 | | |
4212 | | /* If end of basic block, update. */ |
4213 | 0 | if (def->flags & TCG_OPF_BB_EXIT) { |
4214 | 0 | assert_carry_dead(s); |
4215 | 0 | la_func_end(s, nb_globals, nb_temps); |
4216 | 0 | } else if (def->flags & TCG_OPF_COND_BRANCH) { |
4217 | 0 | assert_carry_dead(s); |
4218 | 0 | la_bb_sync(s, nb_globals, nb_temps); |
4219 | 0 | } else if (def->flags & TCG_OPF_BB_END) { |
4220 | 0 | assert_carry_dead(s); |
4221 | 0 | la_bb_end(s, nb_globals, nb_temps); |
4222 | 0 | } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { |
4223 | 0 | assert_carry_dead(s); |
4224 | 0 | la_global_sync(s, nb_globals); |
4225 | 0 | if (def->flags & TCG_OPF_CALL_CLOBBER) { |
4226 | 0 | la_cross_call(s, nb_temps); |
4227 | 0 | } |
4228 | 0 | } |
4229 | | |
4230 | | /* Record arguments that die in this opcode. */ |
4231 | 0 | for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { |
4232 | 0 | ts = arg_temp(op->args[i]); |
4233 | 0 | if (ts->state & TS_DEAD) { |
4234 | 0 | arg_life |= DEAD_ARG << i; |
4235 | 0 | } |
4236 | 0 | } |
4237 | 0 | if (def->flags & TCG_OPF_CARRY_OUT) { |
4238 | 0 | s->carry_live = false; |
4239 | 0 | } |
4240 | | |
4241 | | /* Input arguments are live for preceding opcodes. */ |
4242 | 0 | for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { |
4243 | 0 | ts = arg_temp(op->args[i]); |
4244 | 0 | if (ts->state & TS_DEAD) { |
4245 | | /* For operands that were dead, initially allow |
4246 | | all regs for the type. */ |
4247 | 0 | *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; |
4248 | 0 | ts->state &= ~TS_DEAD; |
4249 | 0 | } |
4250 | 0 | } |
4251 | 0 | if (def->flags & TCG_OPF_CARRY_IN) { |
4252 | 0 | s->carry_live = true; |
4253 | 0 | } |
4254 | | |
4255 | | /* Incorporate constraints for this operand. */ |
4256 | 0 | switch (opc) { |
4257 | 0 | case INDEX_op_mov: |
4258 | | /* Note that these are TCG_OPF_NOT_PRESENT and do not |
4259 | | have proper constraints. That said, special case |
4260 | | moves to propagate preferences backward. */ |
4261 | 0 | if (IS_DEAD_ARG(1)) { |
4262 | 0 | *la_temp_pref(arg_temp(op->args[0])) |
4263 | 0 | = *la_temp_pref(arg_temp(op->args[1])); |
4264 | 0 | } |
4265 | 0 | break; |
4266 | | |
4267 | 0 | default: |
4268 | 0 | args_ct = opcode_args_ct(op); |
4269 | 0 | for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { |
4270 | 0 | const TCGArgConstraint *ct = &args_ct[i]; |
4271 | 0 | TCGRegSet set, *pset; |
4272 | |
|
4273 | 0 | ts = arg_temp(op->args[i]); |
4274 | 0 | pset = la_temp_pref(ts); |
4275 | 0 | set = *pset; |
4276 | |
|
4277 | 0 | set &= ct->regs; |
4278 | 0 | if (ct->ialias) { |
4279 | 0 | set &= output_pref(op, ct->alias_index); |
4280 | 0 | } |
4281 | | /* If the combination is not possible, restart. */ |
4282 | 0 | if (set == 0) { |
4283 | 0 | set = ct->regs; |
4284 | 0 | } |
4285 | 0 | *pset = set; |
4286 | 0 | } |
4287 | 0 | break; |
4288 | 0 | } |
4289 | 0 | break; |
4290 | 0 | } |
4291 | 0 | op->life = arg_life; |
4292 | 0 | } |
4293 | 0 | assert_carry_dead(s); |
4294 | 0 | } |
4295 | | |
4296 | | /* Liveness analysis: Convert indirect regs to direct temporaries. */ |
4297 | | static bool __attribute__((noinline)) |
4298 | | liveness_pass_2(TCGContext *s) |
4299 | 0 | { |
4300 | 0 | int nb_globals = s->nb_globals; |
4301 | 0 | int nb_temps, i; |
4302 | 0 | bool changes = false; |
4303 | 0 | TCGOp *op, *op_next; |
4304 | | |
4305 | | /* Create a temporary for each indirect global. */ |
4306 | 0 | for (i = 0; i < nb_globals; ++i) { |
4307 | 0 | TCGTemp *its = &s->temps[i]; |
4308 | 0 | if (its->indirect_reg) { |
4309 | 0 | TCGTemp *dts = tcg_temp_alloc(s); |
4310 | 0 | dts->type = its->type; |
4311 | 0 | dts->base_type = its->base_type; |
4312 | 0 | dts->temp_subindex = its->temp_subindex; |
4313 | 0 | dts->kind = TEMP_EBB; |
4314 | 0 | its->state_ptr = dts; |
4315 | 0 | } else { |
4316 | 0 | its->state_ptr = NULL; |
4317 | 0 | } |
4318 | | /* All globals begin dead. */ |
4319 | 0 | its->state = TS_DEAD; |
4320 | 0 | } |
4321 | 0 | for (nb_temps = s->nb_temps; i < nb_temps; ++i) { |
4322 | 0 | TCGTemp *its = &s->temps[i]; |
4323 | 0 | its->state_ptr = NULL; |
4324 | 0 | its->state = TS_DEAD; |
4325 | 0 | } |
4326 | |
|
4327 | 0 | QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { |
4328 | 0 | TCGOpcode opc = op->opc; |
4329 | 0 | const TCGOpDef *def = &tcg_op_defs[opc]; |
4330 | 0 | TCGLifeData arg_life = op->life; |
4331 | 0 | int nb_iargs, nb_oargs, call_flags; |
4332 | 0 | TCGTemp *arg_ts, *dir_ts; |
4333 | |
|
4334 | 0 | if (opc == INDEX_op_call) { |
4335 | 0 | nb_oargs = TCGOP_CALLO(op); |
4336 | 0 | nb_iargs = TCGOP_CALLI(op); |
4337 | 0 | call_flags = tcg_call_flags(op); |
4338 | 0 | } else { |
4339 | 0 | nb_iargs = def->nb_iargs; |
4340 | 0 | nb_oargs = def->nb_oargs; |
4341 | | |
4342 | | /* Set flags similar to how calls require. */ |
4343 | 0 | if (def->flags & TCG_OPF_COND_BRANCH) { |
4344 | | /* Like reading globals: sync_globals */ |
4345 | 0 | call_flags = TCG_CALL_NO_WRITE_GLOBALS; |
4346 | 0 | } else if (def->flags & TCG_OPF_BB_END) { |
4347 | | /* Like writing globals: save_globals */ |
4348 | 0 | call_flags = 0; |
4349 | 0 | } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { |
4350 | | /* Like reading globals: sync_globals */ |
4351 | 0 | call_flags = TCG_CALL_NO_WRITE_GLOBALS; |
4352 | 0 | } else { |
4353 | | /* No effect on globals. */ |
4354 | 0 | call_flags = (TCG_CALL_NO_READ_GLOBALS | |
4355 | 0 | TCG_CALL_NO_WRITE_GLOBALS); |
4356 | 0 | } |
4357 | 0 | } |
4358 | | |
4359 | | /* Make sure that input arguments are available. */ |
4360 | 0 | for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { |
4361 | 0 | arg_ts = arg_temp(op->args[i]); |
4362 | 0 | dir_ts = arg_ts->state_ptr; |
4363 | 0 | if (dir_ts && arg_ts->state == TS_DEAD) { |
4364 | 0 | TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld, |
4365 | 0 | arg_ts->type, 3); |
4366 | |
|
4367 | 0 | lop->args[0] = temp_arg(dir_ts); |
4368 | 0 | lop->args[1] = temp_arg(arg_ts->mem_base); |
4369 | 0 | lop->args[2] = arg_ts->mem_offset; |
4370 | | |
4371 | | /* Loaded, but synced with memory. */ |
4372 | 0 | arg_ts->state = TS_MEM; |
4373 | 0 | } |
4374 | 0 | } |
4375 | | |
4376 | | /* Perform input replacement, and mark inputs that became dead. |
4377 | | No action is required except keeping temp_state up to date |
4378 | | so that we reload when needed. */ |
4379 | 0 | for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { |
4380 | 0 | arg_ts = arg_temp(op->args[i]); |
4381 | 0 | dir_ts = arg_ts->state_ptr; |
4382 | 0 | if (dir_ts) { |
4383 | 0 | op->args[i] = temp_arg(dir_ts); |
4384 | 0 | changes = true; |
4385 | 0 | if (IS_DEAD_ARG(i)) { |
4386 | 0 | arg_ts->state = TS_DEAD; |
4387 | 0 | } |
4388 | 0 | } |
4389 | 0 | } |
4390 | | |
4391 | | /* Liveness analysis should ensure that the following are |
4392 | | all correct, for call sites and basic block end points. */ |
4393 | 0 | if (call_flags & TCG_CALL_NO_READ_GLOBALS) { |
4394 | | /* Nothing to do */ |
4395 | 0 | } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { |
4396 | 0 | for (i = 0; i < nb_globals; ++i) { |
4397 | | /* Liveness should see that globals are synced back, |
4398 | | that is, either TS_DEAD or TS_MEM. */ |
4399 | 0 | arg_ts = &s->temps[i]; |
4400 | 0 | tcg_debug_assert(arg_ts->state_ptr == 0 |
4401 | 0 | || arg_ts->state != 0); |
4402 | 0 | } |
4403 | 0 | } else { |
4404 | 0 | for (i = 0; i < nb_globals; ++i) { |
4405 | | /* Liveness should see that globals are saved back, |
4406 | | that is, TS_DEAD, waiting to be reloaded. */ |
4407 | 0 | arg_ts = &s->temps[i]; |
4408 | 0 | tcg_debug_assert(arg_ts->state_ptr == 0 |
4409 | 0 | || arg_ts->state == TS_DEAD); |
4410 | 0 | } |
4411 | 0 | } |
4412 | | |
4413 | | /* Outputs become available. */ |
4414 | 0 | if (opc == INDEX_op_mov) { |
4415 | 0 | arg_ts = arg_temp(op->args[0]); |
4416 | 0 | dir_ts = arg_ts->state_ptr; |
4417 | 0 | if (dir_ts) { |
4418 | 0 | op->args[0] = temp_arg(dir_ts); |
4419 | 0 | changes = true; |
4420 | | |
4421 | | /* The output is now live and modified. */ |
4422 | 0 | arg_ts->state = 0; |
4423 | |
|
4424 | 0 | if (NEED_SYNC_ARG(0)) { |
4425 | 0 | TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, |
4426 | 0 | arg_ts->type, 3); |
4427 | 0 | TCGTemp *out_ts = dir_ts; |
4428 | |
|
4429 | 0 | if (IS_DEAD_ARG(0)) { |
4430 | 0 | out_ts = arg_temp(op->args[1]); |
4431 | 0 | arg_ts->state = TS_DEAD; |
4432 | 0 | tcg_op_remove(s, op); |
4433 | 0 | } else { |
4434 | 0 | arg_ts->state = TS_MEM; |
4435 | 0 | } |
4436 | |
|
4437 | 0 | sop->args[0] = temp_arg(out_ts); |
4438 | 0 | sop->args[1] = temp_arg(arg_ts->mem_base); |
4439 | 0 | sop->args[2] = arg_ts->mem_offset; |
4440 | 0 | } else { |
4441 | 0 | tcg_debug_assert(!IS_DEAD_ARG(0)); |
4442 | 0 | } |
4443 | 0 | } |
4444 | 0 | } else { |
4445 | 0 | for (i = 0; i < nb_oargs; i++) { |
4446 | 0 | arg_ts = arg_temp(op->args[i]); |
4447 | 0 | dir_ts = arg_ts->state_ptr; |
4448 | 0 | if (!dir_ts) { |
4449 | 0 | continue; |
4450 | 0 | } |
4451 | 0 | op->args[i] = temp_arg(dir_ts); |
4452 | 0 | changes = true; |
4453 | | |
4454 | | /* The output is now live and modified. */ |
4455 | 0 | arg_ts->state = 0; |
4456 | | |
4457 | | /* Sync outputs upon their last write. */ |
4458 | 0 | if (NEED_SYNC_ARG(i)) { |
4459 | 0 | TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, |
4460 | 0 | arg_ts->type, 3); |
4461 | |
|
4462 | 0 | sop->args[0] = temp_arg(dir_ts); |
4463 | 0 | sop->args[1] = temp_arg(arg_ts->mem_base); |
4464 | 0 | sop->args[2] = arg_ts->mem_offset; |
4465 | |
|
4466 | 0 | arg_ts->state = TS_MEM; |
4467 | 0 | } |
4468 | | /* Drop outputs that are dead. */ |
4469 | 0 | if (IS_DEAD_ARG(i)) { |
4470 | 0 | arg_ts->state = TS_DEAD; |
4471 | 0 | } |
4472 | 0 | } |
4473 | 0 | } |
4474 | 0 | } |
4475 | | |
4476 | 0 | return changes; |
4477 | 0 | } |
4478 | | |
4479 | | static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) |
4480 | 0 | { |
4481 | 0 | intptr_t off; |
4482 | 0 | int size, align; |
4483 | | |
4484 | | /* When allocating an object, look at the full type. */ |
4485 | 0 | size = tcg_type_size(ts->base_type); |
4486 | 0 | switch (ts->base_type) { |
4487 | 0 | case TCG_TYPE_I32: |
4488 | 0 | align = 4; |
4489 | 0 | break; |
4490 | 0 | case TCG_TYPE_I64: |
4491 | 0 | case TCG_TYPE_V64: |
4492 | 0 | align = 8; |
4493 | 0 | break; |
4494 | 0 | case TCG_TYPE_I128: |
4495 | 0 | case TCG_TYPE_V128: |
4496 | 0 | case TCG_TYPE_V256: |
4497 | | /* |
4498 | | * Note that we do not require aligned storage for V256, |
4499 | | * and that we provide alignment for I128 to match V128, |
4500 | | * even if that's above what the host ABI requires. |
4501 | | */ |
4502 | 0 | align = 16; |
4503 | 0 | break; |
4504 | 0 | default: |
4505 | 0 | g_assert_not_reached(); |
4506 | 0 | } |
4507 | | |
4508 | | /* |
4509 | | * Assume the stack is sufficiently aligned. |
4510 | | * This affects e.g. ARM NEON, where we have 8 byte stack alignment |
4511 | | * and do not require 16 byte vector alignment. This seems slightly |
4512 | | * easier than fully parameterizing the above switch statement. |
4513 | | */ |
4514 | 0 | align = MIN(TCG_TARGET_STACK_ALIGN, align); |
4515 | 0 | off = ROUND_UP(s->current_frame_offset, align); |
4516 | | |
4517 | | /* If we've exhausted the stack frame, restart with a smaller TB. */ |
4518 | 0 | if (off + size > s->frame_end) { |
4519 | 0 | tcg_raise_tb_overflow(s); |
4520 | 0 | } |
4521 | 0 | s->current_frame_offset = off + size; |
4522 | | #if defined(__sparc__) |
4523 | | off += TCG_TARGET_STACK_BIAS; |
4524 | | #endif |
4525 | | |
4526 | | /* If the object was subdivided, assign memory to all the parts. */ |
4527 | 0 | if (ts->base_type != ts->type) { |
4528 | 0 | int part_size = tcg_type_size(ts->type); |
4529 | 0 | int part_count = size / part_size; |
4530 | | |
4531 | | /* |
4532 | | * Each part is allocated sequentially in tcg_temp_new_internal. |
4533 | | * Jump back to the first part by subtracting the current index. |
4534 | | */ |
4535 | 0 | ts -= ts->temp_subindex; |
4536 | 0 | for (int i = 0; i < part_count; ++i) { |
4537 | 0 | ts[i].mem_offset = off + i * part_size; |
4538 | 0 | ts[i].mem_base = s->frame_temp; |
4539 | 0 | ts[i].mem_allocated = 1; |
4540 | 0 | } |
4541 | 0 | } else { |
4542 | 0 | ts->mem_offset = off; |
4543 | 0 | ts->mem_base = s->frame_temp; |
4544 | 0 | ts->mem_allocated = 1; |
4545 | 0 | } |
4546 | 0 | } |
4547 | | |
4548 | | /* Assign @reg to @ts, and update reg_to_temp[]. */ |
4549 | | static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) |
4550 | 0 | { |
4551 | 0 | if (ts->val_type == TEMP_VAL_REG) { |
4552 | 0 | TCGReg old = ts->reg; |
4553 | 0 | tcg_debug_assert(s->reg_to_temp[old] == ts); |
4554 | 0 | if (old == reg) { |
4555 | 0 | return; |
4556 | 0 | } |
4557 | 0 | s->reg_to_temp[old] = NULL; |
4558 | 0 | } |
4559 | 0 | tcg_debug_assert(s->reg_to_temp[reg] == NULL); |
4560 | 0 | s->reg_to_temp[reg] = ts; |
4561 | 0 | ts->val_type = TEMP_VAL_REG; |
4562 | 0 | ts->reg = reg; |
4563 | 0 | } |
4564 | | |
4565 | | /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ |
4566 | | static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) |
4567 | 0 | { |
4568 | 0 | tcg_debug_assert(type != TEMP_VAL_REG); |
4569 | 0 | if (ts->val_type == TEMP_VAL_REG) { |
4570 | 0 | TCGReg reg = ts->reg; |
4571 | 0 | tcg_debug_assert(s->reg_to_temp[reg] == ts); |
4572 | 0 | s->reg_to_temp[reg] = NULL; |
4573 | 0 | } |
4574 | 0 | ts->val_type = type; |
4575 | 0 | } |
4576 | | |
4577 | | static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); |
4578 | | |
4579 | | /* Mark a temporary as free or dead. If 'free_or_dead' is negative, |
4580 | | mark it free; otherwise mark it dead. */ |
4581 | | static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) |
4582 | 0 | { |
4583 | 0 | TCGTempVal new_type; |
4584 | |
|
4585 | 0 | switch (ts->kind) { |
4586 | 0 | case TEMP_FIXED: |
4587 | 0 | return; |
4588 | 0 | case TEMP_GLOBAL: |
4589 | 0 | case TEMP_TB: |
4590 | 0 | new_type = TEMP_VAL_MEM; |
4591 | 0 | break; |
4592 | 0 | case TEMP_EBB: |
4593 | 0 | new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; |
4594 | 0 | break; |
4595 | 0 | case TEMP_CONST: |
4596 | 0 | new_type = TEMP_VAL_CONST; |
4597 | 0 | break; |
4598 | 0 | default: |
4599 | 0 | g_assert_not_reached(); |
4600 | 0 | } |
4601 | 0 | set_temp_val_nonreg(s, ts, new_type); |
4602 | 0 | } |
4603 | | |
4604 | | /* Mark a temporary as dead. */ |
4605 | | static inline void temp_dead(TCGContext *s, TCGTemp *ts) |
4606 | 0 | { |
4607 | 0 | temp_free_or_dead(s, ts, 1); |
4608 | 0 | } |
4609 | | |
4610 | | /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary |
4611 | | registers needs to be allocated to store a constant. If 'free_or_dead' |
4612 | | is non-zero, subsequently release the temporary; if it is positive, the |
4613 | | temp is dead; if it is negative, the temp is free. */ |
4614 | | static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, |
4615 | | TCGRegSet preferred_regs, int free_or_dead) |
4616 | 0 | { |
4617 | 0 | if (!temp_readonly(ts) && !ts->mem_coherent) { |
4618 | 0 | if (!ts->mem_allocated) { |
4619 | 0 | temp_allocate_frame(s, ts); |
4620 | 0 | } |
4621 | 0 | switch (ts->val_type) { |
4622 | 0 | case TEMP_VAL_CONST: |
4623 | | /* If we're going to free the temp immediately, then we won't |
4624 | | require it later in a register, so attempt to store the |
4625 | | constant to memory directly. */ |
4626 | 0 | if (free_or_dead |
4627 | 0 | && tcg_out_sti(s, ts->type, ts->val, |
4628 | 0 | ts->mem_base->reg, ts->mem_offset)) { |
4629 | 0 | break; |
4630 | 0 | } |
4631 | 0 | temp_load(s, ts, tcg_target_available_regs[ts->type], |
4632 | 0 | allocated_regs, preferred_regs); |
4633 | | /* fallthrough */ |
4634 | |
|
4635 | 0 | case TEMP_VAL_REG: |
4636 | 0 | tcg_out_st(s, ts->type, ts->reg, |
4637 | 0 | ts->mem_base->reg, ts->mem_offset); |
4638 | 0 | break; |
4639 | | |
4640 | 0 | case TEMP_VAL_MEM: |
4641 | 0 | break; |
4642 | | |
4643 | 0 | case TEMP_VAL_DEAD: |
4644 | 0 | default: |
4645 | 0 | g_assert_not_reached(); |
4646 | 0 | } |
4647 | 0 | ts->mem_coherent = 1; |
4648 | 0 | } |
4649 | 0 | if (free_or_dead) { |
4650 | 0 | temp_free_or_dead(s, ts, free_or_dead); |
4651 | 0 | } |
4652 | 0 | } |
4653 | | |
4654 | | /* free register 'reg' by spilling the corresponding temporary if necessary */ |
4655 | | static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) |
4656 | 0 | { |
4657 | 0 | TCGTemp *ts = s->reg_to_temp[reg]; |
4658 | 0 | if (ts != NULL) { |
4659 | 0 | temp_sync(s, ts, allocated_regs, 0, -1); |
4660 | 0 | } |
4661 | 0 | } |
4662 | | |
4663 | | /** |
4664 | | * tcg_reg_alloc: |
4665 | | * @required_regs: Set of registers in which we must allocate. |
4666 | | * @allocated_regs: Set of registers which must be avoided. |
4667 | | * @preferred_regs: Set of registers we should prefer. |
4668 | | * @rev: True if we search the registers in "indirect" order. |
4669 | | * |
4670 | | * The allocated register must be in @required_regs & ~@allocated_regs, |
4671 | | * but if we can put it in @preferred_regs we may save a move later. |
4672 | | */ |
4673 | | static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, |
4674 | | TCGRegSet allocated_regs, |
4675 | | TCGRegSet preferred_regs, bool rev) |
4676 | 0 | { |
4677 | 0 | int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); |
4678 | 0 | TCGRegSet reg_ct[2]; |
4679 | 0 | const int *order; |
4680 | |
|
4681 | 0 | reg_ct[1] = required_regs & ~allocated_regs; |
4682 | 0 | tcg_debug_assert(reg_ct[1] != 0); |
4683 | 0 | reg_ct[0] = reg_ct[1] & preferred_regs; |
4684 | | |
4685 | | /* Skip the preferred_regs option if it cannot be satisfied, |
4686 | | or if the preference made no difference. */ |
4687 | 0 | f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; |
4688 | |
|
4689 | 0 | order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; |
4690 | | |
4691 | | /* Try free registers, preferences first. */ |
4692 | 0 | for (j = f; j < 2; j++) { |
4693 | 0 | TCGRegSet set = reg_ct[j]; |
4694 | |
|
4695 | 0 | if (tcg_regset_single(set)) { |
4696 | | /* One register in the set. */ |
4697 | 0 | TCGReg reg = tcg_regset_first(set); |
4698 | 0 | if (s->reg_to_temp[reg] == NULL) { |
4699 | 0 | return reg; |
4700 | 0 | } |
4701 | 0 | } else { |
4702 | 0 | for (i = 0; i < n; i++) { |
4703 | 0 | TCGReg reg = order[i]; |
4704 | 0 | if (s->reg_to_temp[reg] == NULL && |
4705 | 0 | tcg_regset_test_reg(set, reg)) { |
4706 | 0 | return reg; |
4707 | 0 | } |
4708 | 0 | } |
4709 | 0 | } |
4710 | 0 | } |
4711 | | |
4712 | | /* We must spill something. */ |
4713 | 0 | for (j = f; j < 2; j++) { |
4714 | 0 | TCGRegSet set = reg_ct[j]; |
4715 | |
|
4716 | 0 | if (tcg_regset_single(set)) { |
4717 | | /* One register in the set. */ |
4718 | 0 | TCGReg reg = tcg_regset_first(set); |
4719 | 0 | tcg_reg_free(s, reg, allocated_regs); |
4720 | 0 | return reg; |
4721 | 0 | } else { |
4722 | 0 | for (i = 0; i < n; i++) { |
4723 | 0 | TCGReg reg = order[i]; |
4724 | 0 | if (tcg_regset_test_reg(set, reg)) { |
4725 | 0 | tcg_reg_free(s, reg, allocated_regs); |
4726 | 0 | return reg; |
4727 | 0 | } |
4728 | 0 | } |
4729 | 0 | } |
4730 | 0 | } |
4731 | | |
4732 | 0 | g_assert_not_reached(); |
4733 | 0 | } |
4734 | | |
4735 | | static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, |
4736 | | TCGRegSet allocated_regs, |
4737 | | TCGRegSet preferred_regs, bool rev) |
4738 | 0 | { |
4739 | 0 | int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); |
4740 | 0 | TCGRegSet reg_ct[2]; |
4741 | 0 | const int *order; |
4742 | | |
4743 | | /* Ensure that if I is not in allocated_regs, I+1 is not either. */ |
4744 | 0 | reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); |
4745 | 0 | tcg_debug_assert(reg_ct[1] != 0); |
4746 | 0 | reg_ct[0] = reg_ct[1] & preferred_regs; |
4747 | |
|
4748 | 0 | order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; |
4749 | | |
4750 | | /* |
4751 | | * Skip the preferred_regs option if it cannot be satisfied, |
4752 | | * or if the preference made no difference. |
4753 | | */ |
4754 | 0 | k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; |
4755 | | |
4756 | | /* |
4757 | | * Minimize the number of flushes by looking for 2 free registers first, |
4758 | | * then a single flush, then two flushes. |
4759 | | */ |
4760 | 0 | for (fmin = 2; fmin >= 0; fmin--) { |
4761 | 0 | for (j = k; j < 2; j++) { |
4762 | 0 | TCGRegSet set = reg_ct[j]; |
4763 | |
|
4764 | 0 | for (i = 0; i < n; i++) { |
4765 | 0 | TCGReg reg = order[i]; |
4766 | |
|
4767 | 0 | if (tcg_regset_test_reg(set, reg)) { |
4768 | 0 | int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; |
4769 | 0 | if (f >= fmin) { |
4770 | 0 | tcg_reg_free(s, reg, allocated_regs); |
4771 | 0 | tcg_reg_free(s, reg + 1, allocated_regs); |
4772 | 0 | return reg; |
4773 | 0 | } |
4774 | 0 | } |
4775 | 0 | } |
4776 | 0 | } |
4777 | 0 | } |
4778 | 0 | g_assert_not_reached(); |
4779 | 0 | } |
4780 | | |
4781 | | /* Make sure the temporary is in a register. If needed, allocate the register |
4782 | | from DESIRED while avoiding ALLOCATED. */ |
4783 | | static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, |
4784 | | TCGRegSet allocated_regs, TCGRegSet preferred_regs) |
4785 | 0 | { |
4786 | 0 | TCGReg reg; |
4787 | |
|
4788 | 0 | switch (ts->val_type) { |
4789 | 0 | case TEMP_VAL_REG: |
4790 | 0 | return; |
4791 | 0 | case TEMP_VAL_CONST: |
4792 | 0 | reg = tcg_reg_alloc(s, desired_regs, allocated_regs, |
4793 | 0 | preferred_regs, ts->indirect_base); |
4794 | 0 | if (ts->type <= TCG_TYPE_I64) { |
4795 | 0 | tcg_out_movi(s, ts->type, reg, ts->val); |
4796 | 0 | } else { |
4797 | 0 | uint64_t val = ts->val; |
4798 | 0 | MemOp vece = MO_64; |
4799 | | |
4800 | | /* |
4801 | | * Find the minimal vector element that matches the constant. |
4802 | | * The targets will, in general, have to do this search anyway, |
4803 | | * do this generically. |
4804 | | */ |
4805 | 0 | if (val == dup_const(MO_8, val)) { |
4806 | 0 | vece = MO_8; |
4807 | 0 | } else if (val == dup_const(MO_16, val)) { |
4808 | 0 | vece = MO_16; |
4809 | 0 | } else if (val == dup_const(MO_32, val)) { |
4810 | 0 | vece = MO_32; |
4811 | 0 | } |
4812 | | |
4813 | 0 | tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); |
4814 | 0 | } |
4815 | 0 | ts->mem_coherent = 0; |
4816 | 0 | break; |
4817 | 0 | case TEMP_VAL_MEM: |
4818 | 0 | if (!ts->mem_allocated) { |
4819 | 0 | temp_allocate_frame(s, ts); |
4820 | 0 | } |
4821 | 0 | reg = tcg_reg_alloc(s, desired_regs, allocated_regs, |
4822 | 0 | preferred_regs, ts->indirect_base); |
4823 | 0 | tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); |
4824 | 0 | ts->mem_coherent = 1; |
4825 | 0 | break; |
4826 | 0 | case TEMP_VAL_DEAD: |
4827 | 0 | default: |
4828 | 0 | g_assert_not_reached(); |
4829 | 0 | } |
4830 | 0 | set_temp_val_reg(s, ts, reg); |
4831 | 0 | } |
4832 | | |
4833 | | /* Save a temporary to memory. 'allocated_regs' is used in case a |
4834 | | temporary registers needs to be allocated to store a constant. */ |
4835 | | static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) |
4836 | 0 | { |
4837 | | /* The liveness analysis already ensures that globals are back |
4838 | | in memory. Keep an tcg_debug_assert for safety. */ |
4839 | 0 | tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); |
4840 | 0 | } |
4841 | | |
4842 | | /* save globals to their canonical location and assume they can be |
4843 | | modified be the following code. 'allocated_regs' is used in case a |
4844 | | temporary registers needs to be allocated to store a constant. */ |
4845 | | static void save_globals(TCGContext *s, TCGRegSet allocated_regs) |
4846 | 0 | { |
4847 | 0 | int i, n; |
4848 | |
|
4849 | 0 | for (i = 0, n = s->nb_globals; i < n; i++) { |
4850 | 0 | temp_save(s, &s->temps[i], allocated_regs); |
4851 | 0 | } |
4852 | 0 | } |
4853 | | |
4854 | | /* sync globals to their canonical location and assume they can be |
4855 | | read by the following code. 'allocated_regs' is used in case a |
4856 | | temporary registers needs to be allocated to store a constant. */ |
4857 | | static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) |
4858 | 0 | { |
4859 | 0 | int i, n; |
4860 | |
|
4861 | 0 | for (i = 0, n = s->nb_globals; i < n; i++) { |
4862 | 0 | TCGTemp *ts = &s->temps[i]; |
4863 | 0 | tcg_debug_assert(ts->val_type != TEMP_VAL_REG |
4864 | 0 | || ts->kind == TEMP_FIXED |
4865 | 0 | || ts->mem_coherent); |
4866 | 0 | } |
4867 | 0 | } |
4868 | | |
4869 | | /* at the end of a basic block, we assume all temporaries are dead and |
4870 | | all globals are stored at their canonical location. */ |
4871 | | static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) |
4872 | 0 | { |
4873 | 0 | assert_carry_dead(s); |
4874 | 0 | for (int i = s->nb_globals; i < s->nb_temps; i++) { |
4875 | 0 | TCGTemp *ts = &s->temps[i]; |
4876 | |
|
4877 | 0 | switch (ts->kind) { |
4878 | 0 | case TEMP_TB: |
4879 | 0 | temp_save(s, ts, allocated_regs); |
4880 | 0 | break; |
4881 | 0 | case TEMP_EBB: |
4882 | | /* The liveness analysis already ensures that temps are dead. |
4883 | | Keep an tcg_debug_assert for safety. */ |
4884 | 0 | tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); |
4885 | 0 | break; |
4886 | 0 | case TEMP_CONST: |
4887 | | /* Similarly, we should have freed any allocated register. */ |
4888 | 0 | tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); |
4889 | 0 | break; |
4890 | 0 | default: |
4891 | 0 | g_assert_not_reached(); |
4892 | 0 | } |
4893 | 0 | } |
4894 | | |
4895 | 0 | save_globals(s, allocated_regs); |
4896 | 0 | } |
4897 | | |
4898 | | /* |
4899 | | * At a conditional branch, we assume all temporaries are dead unless |
4900 | | * explicitly live-across-conditional-branch; all globals and local |
4901 | | * temps are synced to their location. |
4902 | | */ |
4903 | | static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) |
4904 | 0 | { |
4905 | 0 | assert_carry_dead(s); |
4906 | 0 | sync_globals(s, allocated_regs); |
4907 | |
|
4908 | 0 | for (int i = s->nb_globals; i < s->nb_temps; i++) { |
4909 | 0 | TCGTemp *ts = &s->temps[i]; |
4910 | | /* |
4911 | | * The liveness analysis already ensures that temps are dead. |
4912 | | * Keep tcg_debug_asserts for safety. |
4913 | | */ |
4914 | 0 | switch (ts->kind) { |
4915 | 0 | case TEMP_TB: |
4916 | 0 | tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); |
4917 | 0 | break; |
4918 | 0 | case TEMP_EBB: |
4919 | 0 | case TEMP_CONST: |
4920 | 0 | break; |
4921 | 0 | default: |
4922 | 0 | g_assert_not_reached(); |
4923 | 0 | } |
4924 | 0 | } |
4925 | 0 | } |
4926 | | |
4927 | | /* |
4928 | | * Specialized code generation for INDEX_op_mov_* with a constant. |
4929 | | */ |
4930 | | static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, |
4931 | | tcg_target_ulong val, TCGLifeData arg_life, |
4932 | | TCGRegSet preferred_regs) |
4933 | 0 | { |
4934 | | /* ENV should not be modified. */ |
4935 | 0 | tcg_debug_assert(!temp_readonly(ots)); |
4936 | | |
4937 | | /* The movi is not explicitly generated here. */ |
4938 | 0 | set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); |
4939 | 0 | ots->val = val; |
4940 | 0 | ots->mem_coherent = 0; |
4941 | 0 | if (NEED_SYNC_ARG(0)) { |
4942 | 0 | temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); |
4943 | 0 | } else if (IS_DEAD_ARG(0)) { |
4944 | 0 | temp_dead(s, ots); |
4945 | 0 | } |
4946 | 0 | } |
4947 | | |
4948 | | /* |
4949 | | * Specialized code generation for INDEX_op_mov_*. |
4950 | | */ |
4951 | | static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) |
4952 | 0 | { |
4953 | 0 | const TCGLifeData arg_life = op->life; |
4954 | 0 | TCGRegSet allocated_regs, preferred_regs; |
4955 | 0 | TCGTemp *ts, *ots; |
4956 | 0 | TCGType otype, itype; |
4957 | 0 | TCGReg oreg, ireg; |
4958 | |
|
4959 | 0 | allocated_regs = s->reserved_regs; |
4960 | 0 | preferred_regs = output_pref(op, 0); |
4961 | 0 | ots = arg_temp(op->args[0]); |
4962 | 0 | ts = arg_temp(op->args[1]); |
4963 | | |
4964 | | /* ENV should not be modified. */ |
4965 | 0 | tcg_debug_assert(!temp_readonly(ots)); |
4966 | | |
4967 | | /* Note that otype != itype for no-op truncation. */ |
4968 | 0 | otype = ots->type; |
4969 | 0 | itype = ts->type; |
4970 | |
|
4971 | 0 | if (ts->val_type == TEMP_VAL_CONST) { |
4972 | | /* propagate constant or generate sti */ |
4973 | 0 | tcg_target_ulong val = ts->val; |
4974 | 0 | if (IS_DEAD_ARG(1)) { |
4975 | 0 | temp_dead(s, ts); |
4976 | 0 | } |
4977 | 0 | tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); |
4978 | 0 | return; |
4979 | 0 | } |
4980 | | |
4981 | | /* If the source value is in memory we're going to be forced |
4982 | | to have it in a register in order to perform the copy. Copy |
4983 | | the SOURCE value into its own register first, that way we |
4984 | | don't have to reload SOURCE the next time it is used. */ |
4985 | 0 | if (ts->val_type == TEMP_VAL_MEM) { |
4986 | 0 | temp_load(s, ts, tcg_target_available_regs[itype], |
4987 | 0 | allocated_regs, preferred_regs); |
4988 | 0 | } |
4989 | 0 | tcg_debug_assert(ts->val_type == TEMP_VAL_REG); |
4990 | 0 | ireg = ts->reg; |
4991 | |
|
4992 | 0 | if (IS_DEAD_ARG(0)) { |
4993 | | /* mov to a non-saved dead register makes no sense (even with |
4994 | | liveness analysis disabled). */ |
4995 | 0 | tcg_debug_assert(NEED_SYNC_ARG(0)); |
4996 | 0 | if (!ots->mem_allocated) { |
4997 | 0 | temp_allocate_frame(s, ots); |
4998 | 0 | } |
4999 | 0 | tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); |
5000 | 0 | if (IS_DEAD_ARG(1)) { |
5001 | 0 | temp_dead(s, ts); |
5002 | 0 | } |
5003 | 0 | temp_dead(s, ots); |
5004 | 0 | return; |
5005 | 0 | } |
5006 | | |
5007 | 0 | if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { |
5008 | | /* |
5009 | | * The mov can be suppressed. Kill input first, so that it |
5010 | | * is unlinked from reg_to_temp, then set the output to the |
5011 | | * reg that we saved from the input. |
5012 | | */ |
5013 | 0 | temp_dead(s, ts); |
5014 | 0 | oreg = ireg; |
5015 | 0 | } else { |
5016 | 0 | if (ots->val_type == TEMP_VAL_REG) { |
5017 | 0 | oreg = ots->reg; |
5018 | 0 | } else { |
5019 | | /* Make sure to not spill the input register during allocation. */ |
5020 | 0 | oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], |
5021 | 0 | allocated_regs | ((TCGRegSet)1 << ireg), |
5022 | 0 | preferred_regs, ots->indirect_base); |
5023 | 0 | } |
5024 | 0 | if (!tcg_out_mov(s, otype, oreg, ireg)) { |
5025 | | /* |
5026 | | * Cross register class move not supported. |
5027 | | * Store the source register into the destination slot |
5028 | | * and leave the destination temp as TEMP_VAL_MEM. |
5029 | | */ |
5030 | 0 | assert(!temp_readonly(ots)); |
5031 | 0 | if (!ts->mem_allocated) { |
5032 | 0 | temp_allocate_frame(s, ots); |
5033 | 0 | } |
5034 | 0 | tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); |
5035 | 0 | set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); |
5036 | 0 | ots->mem_coherent = 1; |
5037 | 0 | return; |
5038 | 0 | } |
5039 | 0 | } |
5040 | 0 | set_temp_val_reg(s, ots, oreg); |
5041 | 0 | ots->mem_coherent = 0; |
5042 | |
|
5043 | 0 | if (NEED_SYNC_ARG(0)) { |
5044 | 0 | temp_sync(s, ots, allocated_regs, 0, 0); |
5045 | 0 | } |
5046 | 0 | } |
5047 | | |
5048 | | /* |
5049 | | * Specialized code generation for INDEX_op_dup_vec. |
5050 | | */ |
5051 | | static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) |
5052 | 0 | { |
5053 | 0 | const TCGLifeData arg_life = op->life; |
5054 | 0 | TCGRegSet dup_out_regs, dup_in_regs; |
5055 | 0 | const TCGArgConstraint *dup_args_ct; |
5056 | 0 | TCGTemp *its, *ots; |
5057 | 0 | TCGType itype, vtype; |
5058 | 0 | unsigned vece; |
5059 | 0 | int lowpart_ofs; |
5060 | 0 | bool ok; |
5061 | |
|
5062 | 0 | ots = arg_temp(op->args[0]); |
5063 | 0 | its = arg_temp(op->args[1]); |
5064 | | |
5065 | | /* ENV should not be modified. */ |
5066 | 0 | tcg_debug_assert(!temp_readonly(ots)); |
5067 | | |
5068 | 0 | itype = its->type; |
5069 | 0 | vece = TCGOP_VECE(op); |
5070 | 0 | vtype = TCGOP_TYPE(op); |
5071 | |
|
5072 | 0 | if (its->val_type == TEMP_VAL_CONST) { |
5073 | | /* Propagate constant via movi -> dupi. */ |
5074 | 0 | tcg_target_ulong val = dup_const(vece, its->val); |
5075 | 0 | if (IS_DEAD_ARG(1)) { |
5076 | 0 | temp_dead(s, its); |
5077 | 0 | } |
5078 | 0 | tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); |
5079 | 0 | return; |
5080 | 0 | } |
5081 | | |
5082 | 0 | dup_args_ct = opcode_args_ct(op); |
5083 | 0 | dup_out_regs = dup_args_ct[0].regs; |
5084 | 0 | dup_in_regs = dup_args_ct[1].regs; |
5085 | | |
5086 | | /* Allocate the output register now. */ |
5087 | 0 | if (ots->val_type != TEMP_VAL_REG) { |
5088 | 0 | TCGRegSet allocated_regs = s->reserved_regs; |
5089 | 0 | TCGReg oreg; |
5090 | |
|
5091 | 0 | if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { |
5092 | | /* Make sure to not spill the input register. */ |
5093 | 0 | tcg_regset_set_reg(allocated_regs, its->reg); |
5094 | 0 | } |
5095 | 0 | oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, |
5096 | 0 | output_pref(op, 0), ots->indirect_base); |
5097 | 0 | set_temp_val_reg(s, ots, oreg); |
5098 | 0 | } |
5099 | |
|
5100 | 0 | switch (its->val_type) { |
5101 | 0 | case TEMP_VAL_REG: |
5102 | | /* |
5103 | | * The dup constriaints must be broad, covering all possible VECE. |
5104 | | * However, tcg_op_dup_vec() gets to see the VECE and we allow it |
5105 | | * to fail, indicating that extra moves are required for that case. |
5106 | | */ |
5107 | 0 | if (tcg_regset_test_reg(dup_in_regs, its->reg)) { |
5108 | 0 | if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { |
5109 | 0 | goto done; |
5110 | 0 | } |
5111 | | /* Try again from memory or a vector input register. */ |
5112 | 0 | } |
5113 | 0 | if (!its->mem_coherent) { |
5114 | | /* |
5115 | | * The input register is not synced, and so an extra store |
5116 | | * would be required to use memory. Attempt an integer-vector |
5117 | | * register move first. We do not have a TCGRegSet for this. |
5118 | | */ |
5119 | 0 | if (tcg_out_mov(s, itype, ots->reg, its->reg)) { |
5120 | 0 | break; |
5121 | 0 | } |
5122 | | /* Sync the temp back to its slot and load from there. */ |
5123 | 0 | temp_sync(s, its, s->reserved_regs, 0, 0); |
5124 | 0 | } |
5125 | | /* fall through */ |
5126 | | |
5127 | 0 | case TEMP_VAL_MEM: |
5128 | 0 | lowpart_ofs = 0; |
5129 | 0 | if (HOST_BIG_ENDIAN) { |
5130 | 0 | lowpart_ofs = tcg_type_size(itype) - (1 << vece); |
5131 | 0 | } |
5132 | 0 | if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, |
5133 | 0 | its->mem_offset + lowpart_ofs)) { |
5134 | 0 | goto done; |
5135 | 0 | } |
5136 | | /* Load the input into the destination vector register. */ |
5137 | 0 | tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); |
5138 | 0 | break; |
5139 | | |
5140 | 0 | default: |
5141 | 0 | g_assert_not_reached(); |
5142 | 0 | } |
5143 | | |
5144 | | /* We now have a vector input register, so dup must succeed. */ |
5145 | 0 | ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); |
5146 | 0 | tcg_debug_assert(ok); |
5147 | | |
5148 | 0 | done: |
5149 | 0 | ots->mem_coherent = 0; |
5150 | 0 | if (IS_DEAD_ARG(1)) { |
5151 | 0 | temp_dead(s, its); |
5152 | 0 | } |
5153 | 0 | if (NEED_SYNC_ARG(0)) { |
5154 | 0 | temp_sync(s, ots, s->reserved_regs, 0, 0); |
5155 | 0 | } |
5156 | 0 | if (IS_DEAD_ARG(0)) { |
5157 | 0 | temp_dead(s, ots); |
5158 | 0 | } |
5159 | 0 | } |
5160 | | |
5161 | | static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) |
5162 | 0 | { |
5163 | 0 | const TCGLifeData arg_life = op->life; |
5164 | 0 | const TCGOpDef * const def = &tcg_op_defs[op->opc]; |
5165 | 0 | TCGRegSet i_allocated_regs; |
5166 | 0 | TCGRegSet o_allocated_regs; |
5167 | 0 | int i, k, nb_iargs, nb_oargs; |
5168 | 0 | TCGReg reg; |
5169 | 0 | TCGArg arg; |
5170 | 0 | const TCGArgConstraint *args_ct; |
5171 | 0 | const TCGArgConstraint *arg_ct; |
5172 | 0 | TCGTemp *ts; |
5173 | 0 | TCGArg new_args[TCG_MAX_OP_ARGS]; |
5174 | 0 | int const_args[TCG_MAX_OP_ARGS]; |
5175 | 0 | TCGCond op_cond; |
5176 | |
|
5177 | 0 | if (def->flags & TCG_OPF_CARRY_IN) { |
5178 | 0 | tcg_debug_assert(s->carry_live); |
5179 | 0 | } |
5180 | | |
5181 | 0 | nb_oargs = def->nb_oargs; |
5182 | 0 | nb_iargs = def->nb_iargs; |
5183 | | |
5184 | | /* copy constants */ |
5185 | 0 | memcpy(new_args + nb_oargs + nb_iargs, |
5186 | 0 | op->args + nb_oargs + nb_iargs, |
5187 | 0 | sizeof(TCGArg) * def->nb_cargs); |
5188 | |
|
5189 | 0 | i_allocated_regs = s->reserved_regs; |
5190 | 0 | o_allocated_regs = s->reserved_regs; |
5191 | |
|
5192 | 0 | switch (op->opc) { |
5193 | 0 | case INDEX_op_brcond: |
5194 | 0 | op_cond = op->args[2]; |
5195 | 0 | break; |
5196 | 0 | case INDEX_op_setcond: |
5197 | 0 | case INDEX_op_negsetcond: |
5198 | 0 | case INDEX_op_cmp_vec: |
5199 | 0 | op_cond = op->args[3]; |
5200 | 0 | break; |
5201 | 0 | case INDEX_op_movcond: |
5202 | 0 | case INDEX_op_cmpsel_vec: |
5203 | 0 | op_cond = op->args[5]; |
5204 | 0 | break; |
5205 | 0 | default: |
5206 | | /* No condition within opcode. */ |
5207 | 0 | op_cond = TCG_COND_ALWAYS; |
5208 | 0 | break; |
5209 | 0 | } |
5210 | | |
5211 | 0 | args_ct = opcode_args_ct(op); |
5212 | | |
5213 | | /* satisfy input constraints */ |
5214 | 0 | for (k = 0; k < nb_iargs; k++) { |
5215 | 0 | TCGRegSet i_preferred_regs, i_required_regs; |
5216 | 0 | bool allocate_new_reg, copyto_new_reg; |
5217 | 0 | TCGTemp *ts2; |
5218 | 0 | int i1, i2; |
5219 | |
|
5220 | 0 | i = args_ct[nb_oargs + k].sort_index; |
5221 | 0 | arg = op->args[i]; |
5222 | 0 | arg_ct = &args_ct[i]; |
5223 | 0 | ts = arg_temp(arg); |
5224 | |
|
5225 | 0 | if (ts->val_type == TEMP_VAL_CONST) { |
5226 | 0 | #ifdef TCG_REG_ZERO |
5227 | 0 | if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) { |
5228 | | /* Hardware zero register: indicate register via non-const. */ |
5229 | 0 | const_args[i] = 0; |
5230 | 0 | new_args[i] = TCG_REG_ZERO; |
5231 | 0 | continue; |
5232 | 0 | } |
5233 | 0 | #endif |
5234 | | |
5235 | 0 | if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type, |
5236 | 0 | op_cond, TCGOP_VECE(op))) { |
5237 | | /* constant is OK for instruction */ |
5238 | 0 | const_args[i] = 1; |
5239 | 0 | new_args[i] = ts->val; |
5240 | 0 | continue; |
5241 | 0 | } |
5242 | 0 | } |
5243 | | |
5244 | 0 | reg = ts->reg; |
5245 | 0 | i_preferred_regs = 0; |
5246 | 0 | i_required_regs = arg_ct->regs; |
5247 | 0 | allocate_new_reg = false; |
5248 | 0 | copyto_new_reg = false; |
5249 | |
|
5250 | 0 | switch (arg_ct->pair) { |
5251 | 0 | case 0: /* not paired */ |
5252 | 0 | if (arg_ct->ialias) { |
5253 | 0 | i_preferred_regs = output_pref(op, arg_ct->alias_index); |
5254 | | |
5255 | | /* |
5256 | | * If the input is readonly, then it cannot also be an |
5257 | | * output and aliased to itself. If the input is not |
5258 | | * dead after the instruction, we must allocate a new |
5259 | | * register and move it. |
5260 | | */ |
5261 | 0 | if (temp_readonly(ts) || !IS_DEAD_ARG(i) |
5262 | 0 | || args_ct[arg_ct->alias_index].newreg) { |
5263 | 0 | allocate_new_reg = true; |
5264 | 0 | } else if (ts->val_type == TEMP_VAL_REG) { |
5265 | | /* |
5266 | | * Check if the current register has already been |
5267 | | * allocated for another input. |
5268 | | */ |
5269 | 0 | allocate_new_reg = |
5270 | 0 | tcg_regset_test_reg(i_allocated_regs, reg); |
5271 | 0 | } |
5272 | 0 | } |
5273 | 0 | if (!allocate_new_reg) { |
5274 | 0 | temp_load(s, ts, i_required_regs, i_allocated_regs, |
5275 | 0 | i_preferred_regs); |
5276 | 0 | reg = ts->reg; |
5277 | 0 | allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); |
5278 | 0 | } |
5279 | 0 | if (allocate_new_reg) { |
5280 | | /* |
5281 | | * Allocate a new register matching the constraint |
5282 | | * and move the temporary register into it. |
5283 | | */ |
5284 | 0 | temp_load(s, ts, tcg_target_available_regs[ts->type], |
5285 | 0 | i_allocated_regs, 0); |
5286 | 0 | reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, |
5287 | 0 | i_preferred_regs, ts->indirect_base); |
5288 | 0 | copyto_new_reg = true; |
5289 | 0 | } |
5290 | 0 | break; |
5291 | | |
5292 | 0 | case 1: |
5293 | | /* First of an input pair; if i1 == i2, the second is an output. */ |
5294 | 0 | i1 = i; |
5295 | 0 | i2 = arg_ct->pair_index; |
5296 | 0 | ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; |
5297 | | |
5298 | | /* |
5299 | | * It is easier to default to allocating a new pair |
5300 | | * and to identify a few cases where it's not required. |
5301 | | */ |
5302 | 0 | if (arg_ct->ialias) { |
5303 | 0 | i_preferred_regs = output_pref(op, arg_ct->alias_index); |
5304 | 0 | if (IS_DEAD_ARG(i1) && |
5305 | 0 | IS_DEAD_ARG(i2) && |
5306 | 0 | !temp_readonly(ts) && |
5307 | 0 | ts->val_type == TEMP_VAL_REG && |
5308 | 0 | ts->reg < TCG_TARGET_NB_REGS - 1 && |
5309 | 0 | tcg_regset_test_reg(i_required_regs, reg) && |
5310 | 0 | !tcg_regset_test_reg(i_allocated_regs, reg) && |
5311 | 0 | !tcg_regset_test_reg(i_allocated_regs, reg + 1) && |
5312 | 0 | (ts2 |
5313 | 0 | ? ts2->val_type == TEMP_VAL_REG && |
5314 | 0 | ts2->reg == reg + 1 && |
5315 | 0 | !temp_readonly(ts2) |
5316 | 0 | : s->reg_to_temp[reg + 1] == NULL)) { |
5317 | 0 | break; |
5318 | 0 | } |
5319 | 0 | } else { |
5320 | | /* Without aliasing, the pair must also be an input. */ |
5321 | 0 | tcg_debug_assert(ts2); |
5322 | 0 | if (ts->val_type == TEMP_VAL_REG && |
5323 | 0 | ts2->val_type == TEMP_VAL_REG && |
5324 | 0 | ts2->reg == reg + 1 && |
5325 | 0 | tcg_regset_test_reg(i_required_regs, reg)) { |
5326 | 0 | break; |
5327 | 0 | } |
5328 | 0 | } |
5329 | 0 | reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, |
5330 | 0 | 0, ts->indirect_base); |
5331 | 0 | goto do_pair; |
5332 | | |
5333 | 0 | case 2: /* pair second */ |
5334 | 0 | reg = new_args[arg_ct->pair_index] + 1; |
5335 | 0 | goto do_pair; |
5336 | | |
5337 | 0 | case 3: /* ialias with second output, no first input */ |
5338 | 0 | tcg_debug_assert(arg_ct->ialias); |
5339 | 0 | i_preferred_regs = output_pref(op, arg_ct->alias_index); |
5340 | |
|
5341 | 0 | if (IS_DEAD_ARG(i) && |
5342 | 0 | !temp_readonly(ts) && |
5343 | 0 | ts->val_type == TEMP_VAL_REG && |
5344 | 0 | reg > 0 && |
5345 | 0 | s->reg_to_temp[reg - 1] == NULL && |
5346 | 0 | tcg_regset_test_reg(i_required_regs, reg) && |
5347 | 0 | !tcg_regset_test_reg(i_allocated_regs, reg) && |
5348 | 0 | !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { |
5349 | 0 | tcg_regset_set_reg(i_allocated_regs, reg - 1); |
5350 | 0 | break; |
5351 | 0 | } |
5352 | 0 | reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, |
5353 | 0 | i_allocated_regs, 0, |
5354 | 0 | ts->indirect_base); |
5355 | 0 | tcg_regset_set_reg(i_allocated_regs, reg); |
5356 | 0 | reg += 1; |
5357 | 0 | goto do_pair; |
5358 | | |
5359 | 0 | do_pair: |
5360 | | /* |
5361 | | * If an aliased input is not dead after the instruction, |
5362 | | * we must allocate a new register and move it. |
5363 | | */ |
5364 | 0 | if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { |
5365 | 0 | TCGRegSet t_allocated_regs = i_allocated_regs; |
5366 | | |
5367 | | /* |
5368 | | * Because of the alias, and the continued life, make sure |
5369 | | * that the temp is somewhere *other* than the reg pair, |
5370 | | * and we get a copy in reg. |
5371 | | */ |
5372 | 0 | tcg_regset_set_reg(t_allocated_regs, reg); |
5373 | 0 | tcg_regset_set_reg(t_allocated_regs, reg + 1); |
5374 | 0 | if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { |
5375 | | /* If ts was already in reg, copy it somewhere else. */ |
5376 | 0 | TCGReg nr; |
5377 | 0 | bool ok; |
5378 | |
|
5379 | 0 | tcg_debug_assert(ts->kind != TEMP_FIXED); |
5380 | 0 | nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], |
5381 | 0 | t_allocated_regs, 0, ts->indirect_base); |
5382 | 0 | ok = tcg_out_mov(s, ts->type, nr, reg); |
5383 | 0 | tcg_debug_assert(ok); |
5384 | | |
5385 | 0 | set_temp_val_reg(s, ts, nr); |
5386 | 0 | } else { |
5387 | 0 | temp_load(s, ts, tcg_target_available_regs[ts->type], |
5388 | 0 | t_allocated_regs, 0); |
5389 | 0 | copyto_new_reg = true; |
5390 | 0 | } |
5391 | 0 | } else { |
5392 | | /* Preferably allocate to reg, otherwise copy. */ |
5393 | 0 | i_required_regs = (TCGRegSet)1 << reg; |
5394 | 0 | temp_load(s, ts, i_required_regs, i_allocated_regs, |
5395 | 0 | i_preferred_regs); |
5396 | 0 | copyto_new_reg = ts->reg != reg; |
5397 | 0 | } |
5398 | 0 | break; |
5399 | | |
5400 | 0 | default: |
5401 | 0 | g_assert_not_reached(); |
5402 | 0 | } |
5403 | | |
5404 | 0 | if (copyto_new_reg) { |
5405 | 0 | if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { |
5406 | | /* |
5407 | | * Cross register class move not supported. Sync the |
5408 | | * temp back to its slot and load from there. |
5409 | | */ |
5410 | 0 | temp_sync(s, ts, i_allocated_regs, 0, 0); |
5411 | 0 | tcg_out_ld(s, ts->type, reg, |
5412 | 0 | ts->mem_base->reg, ts->mem_offset); |
5413 | 0 | } |
5414 | 0 | } |
5415 | 0 | new_args[i] = reg; |
5416 | 0 | const_args[i] = 0; |
5417 | 0 | tcg_regset_set_reg(i_allocated_regs, reg); |
5418 | 0 | } |
5419 | | |
5420 | | /* mark dead temporaries and free the associated registers */ |
5421 | 0 | for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { |
5422 | 0 | if (IS_DEAD_ARG(i)) { |
5423 | 0 | temp_dead(s, arg_temp(op->args[i])); |
5424 | 0 | } |
5425 | 0 | } |
5426 | |
|
5427 | 0 | if (def->flags & TCG_OPF_COND_BRANCH) { |
5428 | 0 | tcg_reg_alloc_cbranch(s, i_allocated_regs); |
5429 | 0 | } else if (def->flags & TCG_OPF_BB_END) { |
5430 | 0 | tcg_reg_alloc_bb_end(s, i_allocated_regs); |
5431 | 0 | } else { |
5432 | 0 | if (def->flags & TCG_OPF_CALL_CLOBBER) { |
5433 | 0 | assert_carry_dead(s); |
5434 | | /* XXX: permit generic clobber register list ? */ |
5435 | 0 | for (i = 0; i < TCG_TARGET_NB_REGS; i++) { |
5436 | 0 | if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { |
5437 | 0 | tcg_reg_free(s, i, i_allocated_regs); |
5438 | 0 | } |
5439 | 0 | } |
5440 | 0 | } |
5441 | 0 | if (def->flags & TCG_OPF_SIDE_EFFECTS) { |
5442 | | /* sync globals if the op has side effects and might trigger |
5443 | | an exception. */ |
5444 | 0 | sync_globals(s, i_allocated_regs); |
5445 | 0 | } |
5446 | | |
5447 | | /* satisfy the output constraints */ |
5448 | 0 | for (k = 0; k < nb_oargs; k++) { |
5449 | 0 | i = args_ct[k].sort_index; |
5450 | 0 | arg = op->args[i]; |
5451 | 0 | arg_ct = &args_ct[i]; |
5452 | 0 | ts = arg_temp(arg); |
5453 | | |
5454 | | /* ENV should not be modified. */ |
5455 | 0 | tcg_debug_assert(!temp_readonly(ts)); |
5456 | | |
5457 | 0 | switch (arg_ct->pair) { |
5458 | 0 | case 0: /* not paired */ |
5459 | 0 | if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { |
5460 | 0 | reg = new_args[arg_ct->alias_index]; |
5461 | 0 | } else if (arg_ct->newreg) { |
5462 | 0 | reg = tcg_reg_alloc(s, arg_ct->regs, |
5463 | 0 | i_allocated_regs | o_allocated_regs, |
5464 | 0 | output_pref(op, k), ts->indirect_base); |
5465 | 0 | } else { |
5466 | 0 | reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, |
5467 | 0 | output_pref(op, k), ts->indirect_base); |
5468 | 0 | } |
5469 | 0 | break; |
5470 | | |
5471 | 0 | case 1: /* first of pair */ |
5472 | 0 | if (arg_ct->oalias) { |
5473 | 0 | reg = new_args[arg_ct->alias_index]; |
5474 | 0 | } else if (arg_ct->newreg) { |
5475 | 0 | reg = tcg_reg_alloc_pair(s, arg_ct->regs, |
5476 | 0 | i_allocated_regs | o_allocated_regs, |
5477 | 0 | output_pref(op, k), |
5478 | 0 | ts->indirect_base); |
5479 | 0 | } else { |
5480 | 0 | reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, |
5481 | 0 | output_pref(op, k), |
5482 | 0 | ts->indirect_base); |
5483 | 0 | } |
5484 | 0 | break; |
5485 | | |
5486 | 0 | case 2: /* second of pair */ |
5487 | 0 | if (arg_ct->oalias) { |
5488 | 0 | reg = new_args[arg_ct->alias_index]; |
5489 | 0 | } else { |
5490 | 0 | reg = new_args[arg_ct->pair_index] + 1; |
5491 | 0 | } |
5492 | 0 | break; |
5493 | | |
5494 | 0 | case 3: /* first of pair, aliasing with a second input */ |
5495 | 0 | tcg_debug_assert(!arg_ct->newreg); |
5496 | 0 | reg = new_args[arg_ct->pair_index] - 1; |
5497 | 0 | break; |
5498 | | |
5499 | 0 | default: |
5500 | 0 | g_assert_not_reached(); |
5501 | 0 | } |
5502 | 0 | tcg_regset_set_reg(o_allocated_regs, reg); |
5503 | 0 | set_temp_val_reg(s, ts, reg); |
5504 | 0 | ts->mem_coherent = 0; |
5505 | 0 | new_args[i] = reg; |
5506 | 0 | } |
5507 | 0 | } |
5508 | | |
5509 | | /* emit instruction */ |
5510 | 0 | TCGType type = TCGOP_TYPE(op); |
5511 | 0 | switch (op->opc) { |
5512 | 0 | case INDEX_op_addc1o: |
5513 | 0 | tcg_out_set_carry(s); |
5514 | | /* fall through */ |
5515 | 0 | case INDEX_op_add: |
5516 | 0 | case INDEX_op_addcio: |
5517 | 0 | case INDEX_op_addco: |
5518 | 0 | case INDEX_op_and: |
5519 | 0 | case INDEX_op_andc: |
5520 | 0 | case INDEX_op_clz: |
5521 | 0 | case INDEX_op_ctz: |
5522 | 0 | case INDEX_op_divs: |
5523 | 0 | case INDEX_op_divu: |
5524 | 0 | case INDEX_op_eqv: |
5525 | 0 | case INDEX_op_mul: |
5526 | 0 | case INDEX_op_mulsh: |
5527 | 0 | case INDEX_op_muluh: |
5528 | 0 | case INDEX_op_nand: |
5529 | 0 | case INDEX_op_nor: |
5530 | 0 | case INDEX_op_or: |
5531 | 0 | case INDEX_op_orc: |
5532 | 0 | case INDEX_op_rems: |
5533 | 0 | case INDEX_op_remu: |
5534 | 0 | case INDEX_op_rotl: |
5535 | 0 | case INDEX_op_rotr: |
5536 | 0 | case INDEX_op_sar: |
5537 | 0 | case INDEX_op_shl: |
5538 | 0 | case INDEX_op_shr: |
5539 | 0 | case INDEX_op_smax: |
5540 | 0 | case INDEX_op_smin: |
5541 | 0 | case INDEX_op_umax: |
5542 | 0 | case INDEX_op_umin: |
5543 | 0 | case INDEX_op_xor: |
5544 | 0 | { |
5545 | 0 | const TCGOutOpBinary *out = |
5546 | 0 | container_of(all_outop[op->opc], TCGOutOpBinary, base); |
5547 | | |
5548 | | /* Constants should never appear in the first source operand. */ |
5549 | 0 | tcg_debug_assert(!const_args[1]); |
5550 | 0 | if (const_args[2]) { |
5551 | 0 | out->out_rri(s, type, new_args[0], new_args[1], new_args[2]); |
5552 | 0 | } else { |
5553 | 0 | out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); |
5554 | 0 | } |
5555 | 0 | } |
5556 | 0 | break; |
5557 | | |
5558 | 0 | case INDEX_op_sub: |
5559 | 0 | { |
5560 | 0 | const TCGOutOpSubtract *out = &outop_sub; |
5561 | | |
5562 | | /* |
5563 | | * Constants should never appear in the second source operand. |
5564 | | * These are folded to add with negative constant. |
5565 | | */ |
5566 | 0 | tcg_debug_assert(!const_args[2]); |
5567 | 0 | if (const_args[1]) { |
5568 | 0 | out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); |
5569 | 0 | } else { |
5570 | 0 | out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); |
5571 | 0 | } |
5572 | 0 | } |
5573 | 0 | break; |
5574 | | |
5575 | 0 | case INDEX_op_subb1o: |
5576 | 0 | tcg_out_set_borrow(s); |
5577 | | /* fall through */ |
5578 | 0 | case INDEX_op_addci: |
5579 | 0 | case INDEX_op_subbi: |
5580 | 0 | case INDEX_op_subbio: |
5581 | 0 | case INDEX_op_subbo: |
5582 | 0 | { |
5583 | 0 | const TCGOutOpAddSubCarry *out = |
5584 | 0 | container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base); |
5585 | |
|
5586 | 0 | if (const_args[2]) { |
5587 | 0 | if (const_args[1]) { |
5588 | 0 | out->out_rii(s, type, new_args[0], |
5589 | 0 | new_args[1], new_args[2]); |
5590 | 0 | } else { |
5591 | 0 | out->out_rri(s, type, new_args[0], |
5592 | 0 | new_args[1], new_args[2]); |
5593 | 0 | } |
5594 | 0 | } else if (const_args[1]) { |
5595 | 0 | out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); |
5596 | 0 | } else { |
5597 | 0 | out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); |
5598 | 0 | } |
5599 | 0 | } |
5600 | 0 | break; |
5601 | | |
5602 | 0 | case INDEX_op_bswap64: |
5603 | 0 | case INDEX_op_ext_i32_i64: |
5604 | 0 | case INDEX_op_extu_i32_i64: |
5605 | 0 | case INDEX_op_extrl_i64_i32: |
5606 | 0 | case INDEX_op_extrh_i64_i32: |
5607 | 0 | case INDEX_op_ctpop: |
5608 | 0 | case INDEX_op_neg: |
5609 | 0 | case INDEX_op_not: |
5610 | 0 | case INDEX_op_revbit8: |
5611 | 0 | case INDEX_op_revbit64: |
5612 | 0 | { |
5613 | 0 | const TCGOutOpUnary *out = |
5614 | 0 | container_of(all_outop[op->opc], TCGOutOpUnary, base); |
5615 | | |
5616 | | /* Constants should have been folded. */ |
5617 | 0 | tcg_debug_assert(!const_args[1]); |
5618 | 0 | out->out_rr(s, type, new_args[0], new_args[1]); |
5619 | 0 | } |
5620 | 0 | break; |
5621 | | |
5622 | 0 | case INDEX_op_bswap16: |
5623 | 0 | case INDEX_op_bswap32: |
5624 | 0 | case INDEX_op_revbit32: |
5625 | 0 | { |
5626 | 0 | const TCGOutOpBswap *out = |
5627 | 0 | container_of(all_outop[op->opc], TCGOutOpBswap, base); |
5628 | |
|
5629 | 0 | tcg_debug_assert(!const_args[1]); |
5630 | 0 | out->out_rr(s, type, new_args[0], new_args[1], new_args[2]); |
5631 | 0 | } |
5632 | 0 | break; |
5633 | | |
5634 | 0 | case INDEX_op_deposit: |
5635 | 0 | { |
5636 | 0 | const TCGOutOpDeposit *out = &outop_deposit; |
5637 | |
|
5638 | 0 | if (const_args[2]) { |
5639 | 0 | tcg_debug_assert(!const_args[1]); |
5640 | 0 | out->out_rri(s, type, new_args[0], new_args[1], |
5641 | 0 | new_args[2], new_args[3], new_args[4]); |
5642 | 0 | } else if (const_args[1]) { |
5643 | 0 | tcg_debug_assert(new_args[1] == 0); |
5644 | 0 | tcg_debug_assert(!const_args[2]); |
5645 | 0 | out->out_rzr(s, type, new_args[0], new_args[2], |
5646 | 0 | new_args[3], new_args[4]); |
5647 | 0 | } else { |
5648 | 0 | out->out_rrr(s, type, new_args[0], new_args[1], |
5649 | 0 | new_args[2], new_args[3], new_args[4]); |
5650 | 0 | } |
5651 | 0 | } |
5652 | 0 | break; |
5653 | | |
5654 | 0 | case INDEX_op_divs2: |
5655 | 0 | case INDEX_op_divu2: |
5656 | 0 | { |
5657 | 0 | const TCGOutOpDivRem *out = |
5658 | 0 | container_of(all_outop[op->opc], TCGOutOpDivRem, base); |
5659 | | |
5660 | | /* Only used by x86 and s390x, which use matching constraints. */ |
5661 | 0 | tcg_debug_assert(new_args[0] == new_args[2]); |
5662 | 0 | tcg_debug_assert(new_args[1] == new_args[3]); |
5663 | 0 | tcg_debug_assert(!const_args[4]); |
5664 | 0 | out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]); |
5665 | 0 | } |
5666 | 0 | break; |
5667 | | |
5668 | 0 | case INDEX_op_extract: |
5669 | 0 | case INDEX_op_sextract: |
5670 | 0 | { |
5671 | 0 | const TCGOutOpExtract *out = |
5672 | 0 | container_of(all_outop[op->opc], TCGOutOpExtract, base); |
5673 | |
|
5674 | 0 | tcg_debug_assert(!const_args[1]); |
5675 | 0 | out->out_rr(s, type, new_args[0], new_args[1], |
5676 | 0 | new_args[2], new_args[3]); |
5677 | 0 | } |
5678 | 0 | break; |
5679 | | |
5680 | 0 | case INDEX_op_extract2: |
5681 | 0 | { |
5682 | 0 | const TCGOutOpExtract2 *out = &outop_extract2; |
5683 | |
|
5684 | 0 | tcg_debug_assert(!const_args[1]); |
5685 | 0 | tcg_debug_assert(!const_args[2]); |
5686 | 0 | out->out_rrr(s, type, new_args[0], new_args[1], |
5687 | 0 | new_args[2], new_args[3]); |
5688 | 0 | } |
5689 | 0 | break; |
5690 | | |
5691 | 0 | case INDEX_op_ld8u: |
5692 | 0 | case INDEX_op_ld8s: |
5693 | 0 | case INDEX_op_ld16u: |
5694 | 0 | case INDEX_op_ld16s: |
5695 | 0 | case INDEX_op_ld32u: |
5696 | 0 | case INDEX_op_ld32s: |
5697 | 0 | case INDEX_op_ld: |
5698 | 0 | { |
5699 | 0 | const TCGOutOpLoad *out = |
5700 | 0 | container_of(all_outop[op->opc], TCGOutOpLoad, base); |
5701 | |
|
5702 | 0 | tcg_debug_assert(!const_args[1]); |
5703 | 0 | out->out(s, type, new_args[0], new_args[1], new_args[2]); |
5704 | 0 | } |
5705 | 0 | break; |
5706 | | |
5707 | 0 | case INDEX_op_muls2: |
5708 | 0 | case INDEX_op_mulu2: |
5709 | 0 | { |
5710 | 0 | const TCGOutOpMul2 *out = |
5711 | 0 | container_of(all_outop[op->opc], TCGOutOpMul2, base); |
5712 | |
|
5713 | 0 | tcg_debug_assert(!const_args[2]); |
5714 | 0 | tcg_debug_assert(!const_args[3]); |
5715 | 0 | out->out_rrrr(s, type, new_args[0], new_args[1], |
5716 | 0 | new_args[2], new_args[3]); |
5717 | 0 | } |
5718 | 0 | break; |
5719 | | |
5720 | 0 | case INDEX_op_st32: |
5721 | | /* Use tcg_op_st w/ I32. */ |
5722 | 0 | type = TCG_TYPE_I32; |
5723 | | /* fall through */ |
5724 | 0 | case INDEX_op_st: |
5725 | 0 | case INDEX_op_st8: |
5726 | 0 | case INDEX_op_st16: |
5727 | 0 | { |
5728 | 0 | const TCGOutOpStore *out = |
5729 | 0 | container_of(all_outop[op->opc], TCGOutOpStore, base); |
5730 | |
|
5731 | 0 | if (const_args[0]) { |
5732 | 0 | out->out_i(s, type, new_args[0], new_args[1], new_args[2]); |
5733 | 0 | } else { |
5734 | 0 | out->out_r(s, type, new_args[0], new_args[1], new_args[2]); |
5735 | 0 | } |
5736 | 0 | } |
5737 | 0 | break; |
5738 | | |
5739 | 0 | case INDEX_op_qemu_ld: |
5740 | 0 | case INDEX_op_qemu_st: |
5741 | 0 | { |
5742 | 0 | const TCGOutOpQemuLdSt *out = |
5743 | 0 | container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base); |
5744 | |
|
5745 | 0 | out->out(s, type, new_args[0], new_args[1], new_args[2]); |
5746 | 0 | } |
5747 | 0 | break; |
5748 | | |
5749 | 0 | case INDEX_op_qemu_ld2: |
5750 | 0 | case INDEX_op_qemu_st2: |
5751 | 0 | { |
5752 | 0 | const TCGOutOpQemuLdSt2 *out = |
5753 | 0 | container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base); |
5754 | |
|
5755 | 0 | out->out(s, type, new_args[0], new_args[1], |
5756 | 0 | new_args[2], new_args[3]); |
5757 | 0 | } |
5758 | 0 | break; |
5759 | | |
5760 | 0 | case INDEX_op_brcond: |
5761 | 0 | { |
5762 | 0 | const TCGOutOpBrcond *out = &outop_brcond; |
5763 | 0 | TCGCond cond = new_args[2]; |
5764 | 0 | TCGLabel *label = arg_label(new_args[3]); |
5765 | |
|
5766 | 0 | tcg_debug_assert(!const_args[0]); |
5767 | 0 | if (const_args[1]) { |
5768 | 0 | out->out_ri(s, type, cond, new_args[0], new_args[1], label); |
5769 | 0 | } else { |
5770 | 0 | out->out_rr(s, type, cond, new_args[0], new_args[1], label); |
5771 | 0 | } |
5772 | 0 | } |
5773 | 0 | break; |
5774 | | |
5775 | 0 | case INDEX_op_movcond: |
5776 | 0 | { |
5777 | 0 | const TCGOutOpMovcond *out = &outop_movcond; |
5778 | 0 | TCGCond cond = new_args[5]; |
5779 | |
|
5780 | 0 | tcg_debug_assert(!const_args[1]); |
5781 | 0 | out->out(s, type, cond, new_args[0], |
5782 | 0 | new_args[1], new_args[2], const_args[2], |
5783 | 0 | new_args[3], const_args[3], |
5784 | 0 | new_args[4], const_args[4]); |
5785 | 0 | } |
5786 | 0 | break; |
5787 | | |
5788 | 0 | case INDEX_op_setcond: |
5789 | 0 | case INDEX_op_negsetcond: |
5790 | 0 | { |
5791 | 0 | const TCGOutOpSetcond *out = |
5792 | 0 | container_of(all_outop[op->opc], TCGOutOpSetcond, base); |
5793 | 0 | TCGCond cond = new_args[3]; |
5794 | |
|
5795 | 0 | tcg_debug_assert(!const_args[1]); |
5796 | 0 | if (const_args[2]) { |
5797 | 0 | out->out_rri(s, type, cond, |
5798 | 0 | new_args[0], new_args[1], new_args[2]); |
5799 | 0 | } else { |
5800 | 0 | out->out_rrr(s, type, cond, |
5801 | 0 | new_args[0], new_args[1], new_args[2]); |
5802 | 0 | } |
5803 | 0 | } |
5804 | 0 | break; |
5805 | | |
5806 | 0 | case INDEX_op_lea: |
5807 | 0 | { |
5808 | 0 | const TCGOutOpLea *out = &outop_lea; |
5809 | 0 | int sh = new_args[3]; |
5810 | 0 | tcg_target_long imm = new_args[4]; |
5811 | |
|
5812 | 0 | tcg_debug_assert(!const_args[1]); |
5813 | 0 | tcg_debug_assert(!const_args[2]); |
5814 | 0 | tcg_debug_assert(TCG_TARGET_lea_sh_valid(type, sh)); |
5815 | 0 | tcg_debug_assert(TCG_TARGET_lea_imm_valid(type, imm)); |
5816 | | |
5817 | 0 | out->out(s, type, new_args[0], new_args[1], new_args[2], sh, imm); |
5818 | 0 | } |
5819 | 0 | break; |
5820 | | |
5821 | 0 | case INDEX_op_goto_ptr: |
5822 | 0 | tcg_debug_assert(!const_args[0]); |
5823 | 0 | tcg_out_goto_ptr(s, new_args[0]); |
5824 | 0 | break; |
5825 | | |
5826 | 0 | default: |
5827 | 0 | tcg_debug_assert(def->flags & TCG_OPF_VECTOR); |
5828 | 0 | tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64, |
5829 | 0 | TCGOP_VECE(op), new_args, const_args); |
5830 | 0 | break; |
5831 | 0 | } |
5832 | | |
5833 | 0 | if (def->flags & TCG_OPF_CARRY_IN) { |
5834 | 0 | s->carry_live = false; |
5835 | 0 | } |
5836 | 0 | if (def->flags & TCG_OPF_CARRY_OUT) { |
5837 | 0 | s->carry_live = true; |
5838 | 0 | } |
5839 | | |
5840 | | /* move the outputs in the correct register if needed */ |
5841 | 0 | for(i = 0; i < nb_oargs; i++) { |
5842 | 0 | ts = arg_temp(op->args[i]); |
5843 | | |
5844 | | /* ENV should not be modified. */ |
5845 | 0 | tcg_debug_assert(!temp_readonly(ts)); |
5846 | | |
5847 | 0 | if (NEED_SYNC_ARG(i)) { |
5848 | 0 | temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); |
5849 | 0 | } else if (IS_DEAD_ARG(i)) { |
5850 | 0 | temp_dead(s, ts); |
5851 | 0 | } |
5852 | 0 | } |
5853 | 0 | } |
5854 | | |
5855 | | static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, |
5856 | | TCGRegSet allocated_regs) |
5857 | 0 | { |
5858 | 0 | if (ts->val_type == TEMP_VAL_REG) { |
5859 | 0 | if (ts->reg != reg) { |
5860 | 0 | tcg_reg_free(s, reg, allocated_regs); |
5861 | 0 | if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { |
5862 | | /* |
5863 | | * Cross register class move not supported. Sync the |
5864 | | * temp back to its slot and load from there. |
5865 | | */ |
5866 | 0 | temp_sync(s, ts, allocated_regs, 0, 0); |
5867 | 0 | tcg_out_ld(s, ts->type, reg, |
5868 | 0 | ts->mem_base->reg, ts->mem_offset); |
5869 | 0 | } |
5870 | 0 | } |
5871 | 0 | } else { |
5872 | 0 | TCGRegSet arg_set = 0; |
5873 | |
|
5874 | 0 | tcg_reg_free(s, reg, allocated_regs); |
5875 | 0 | tcg_regset_set_reg(arg_set, reg); |
5876 | 0 | temp_load(s, ts, arg_set, allocated_regs, 0); |
5877 | 0 | } |
5878 | 0 | } |
5879 | | |
5880 | | static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts, |
5881 | | TCGRegSet allocated_regs) |
5882 | 0 | { |
5883 | | /* |
5884 | | * When the destination is on the stack, load up the temp and store. |
5885 | | * If there are many call-saved registers, the temp might live to |
5886 | | * see another use; otherwise it'll be discarded. |
5887 | | */ |
5888 | 0 | temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); |
5889 | 0 | tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, |
5890 | 0 | arg_slot_stk_ofs(arg_slot)); |
5891 | 0 | } |
5892 | | |
5893 | | static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, |
5894 | | TCGTemp *ts, TCGRegSet *allocated_regs) |
5895 | 0 | { |
5896 | 0 | if (arg_slot_reg_p(l->arg_slot)) { |
5897 | 0 | TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; |
5898 | 0 | load_arg_reg(s, reg, ts, *allocated_regs); |
5899 | 0 | tcg_regset_set_reg(*allocated_regs, reg); |
5900 | 0 | } else { |
5901 | 0 | load_arg_stk(s, l->arg_slot, ts, *allocated_regs); |
5902 | 0 | } |
5903 | 0 | } |
5904 | | |
5905 | | static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base, |
5906 | | intptr_t ref_off, TCGRegSet *allocated_regs) |
5907 | 0 | { |
5908 | 0 | TCGReg reg; |
5909 | |
|
5910 | 0 | if (arg_slot_reg_p(arg_slot)) { |
5911 | 0 | reg = tcg_target_call_iarg_regs[arg_slot]; |
5912 | 0 | tcg_reg_free(s, reg, *allocated_regs); |
5913 | 0 | tcg_out_addi_ptr(s, reg, ref_base, ref_off); |
5914 | 0 | tcg_regset_set_reg(*allocated_regs, reg); |
5915 | 0 | } else { |
5916 | 0 | reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], |
5917 | 0 | *allocated_regs, 0, false); |
5918 | 0 | tcg_out_addi_ptr(s, reg, ref_base, ref_off); |
5919 | 0 | tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, |
5920 | 0 | arg_slot_stk_ofs(arg_slot)); |
5921 | 0 | } |
5922 | 0 | } |
5923 | | |
5924 | | static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) |
5925 | 0 | { |
5926 | 0 | const int nb_oargs = TCGOP_CALLO(op); |
5927 | 0 | const int nb_iargs = TCGOP_CALLI(op); |
5928 | 0 | const TCGLifeData arg_life = op->life; |
5929 | 0 | const TCGHelperInfo *info = tcg_call_info(op); |
5930 | 0 | TCGRegSet allocated_regs = s->reserved_regs; |
5931 | 0 | int i; |
5932 | | |
5933 | | /* |
5934 | | * Move inputs into place in reverse order, |
5935 | | * so that we place stacked arguments first. |
5936 | | */ |
5937 | 0 | for (i = nb_iargs - 1; i >= 0; --i) { |
5938 | 0 | const TCGCallArgumentLoc *loc = &info->in[i]; |
5939 | 0 | TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); |
5940 | |
|
5941 | 0 | switch (loc->kind) { |
5942 | 0 | case TCG_CALL_ARG_NORMAL: |
5943 | 0 | case TCG_CALL_ARG_EXTEND_U: |
5944 | 0 | case TCG_CALL_ARG_EXTEND_S: |
5945 | 0 | load_arg_normal(s, loc, ts, &allocated_regs); |
5946 | 0 | break; |
5947 | 0 | case TCG_CALL_ARG_BY_REF: |
5948 | 0 | load_arg_stk(s, loc->ref_slot, ts, allocated_regs); |
5949 | 0 | load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, |
5950 | 0 | arg_slot_stk_ofs(loc->ref_slot), |
5951 | 0 | &allocated_regs); |
5952 | 0 | break; |
5953 | 0 | case TCG_CALL_ARG_BY_REF_N: |
5954 | 0 | load_arg_stk(s, loc->ref_slot, ts, allocated_regs); |
5955 | 0 | break; |
5956 | 0 | default: |
5957 | 0 | g_assert_not_reached(); |
5958 | 0 | } |
5959 | 0 | } |
5960 | | |
5961 | | /* Mark dead temporaries and free the associated registers. */ |
5962 | 0 | for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { |
5963 | 0 | if (IS_DEAD_ARG(i)) { |
5964 | 0 | temp_dead(s, arg_temp(op->args[i])); |
5965 | 0 | } |
5966 | 0 | } |
5967 | | |
5968 | | /* Clobber call registers. */ |
5969 | 0 | for (i = 0; i < TCG_TARGET_NB_REGS; i++) { |
5970 | 0 | if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { |
5971 | 0 | tcg_reg_free(s, i, allocated_regs); |
5972 | 0 | } |
5973 | 0 | } |
5974 | | |
5975 | | /* |
5976 | | * Save globals if they might be written by the helper, |
5977 | | * sync them if they might be read. |
5978 | | */ |
5979 | 0 | if (info->flags & TCG_CALL_NO_READ_GLOBALS) { |
5980 | | /* Nothing to do */ |
5981 | 0 | } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { |
5982 | 0 | sync_globals(s, allocated_regs); |
5983 | 0 | } else { |
5984 | 0 | save_globals(s, allocated_regs); |
5985 | 0 | } |
5986 | | |
5987 | | /* |
5988 | | * If the ABI passes a pointer to the returned struct as the first |
5989 | | * argument, load that now. Pass a pointer to the output home slot. |
5990 | | */ |
5991 | 0 | if (info->out_kind == TCG_CALL_RET_BY_REF) { |
5992 | 0 | TCGTemp *ts = arg_temp(op->args[0]); |
5993 | |
|
5994 | 0 | if (!ts->mem_allocated) { |
5995 | 0 | temp_allocate_frame(s, ts); |
5996 | 0 | } |
5997 | 0 | load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); |
5998 | 0 | } |
5999 | |
|
6000 | 0 | tcg_out_call(s, tcg_call_func(op), info); |
6001 | | |
6002 | | /* Assign output registers and emit moves if needed. */ |
6003 | 0 | switch (info->out_kind) { |
6004 | 0 | case TCG_CALL_RET_NORMAL: |
6005 | 0 | for (i = 0; i < nb_oargs; i++) { |
6006 | 0 | TCGTemp *ts = arg_temp(op->args[i]); |
6007 | 0 | TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); |
6008 | | |
6009 | | /* ENV should not be modified. */ |
6010 | 0 | tcg_debug_assert(!temp_readonly(ts)); |
6011 | | |
6012 | 0 | set_temp_val_reg(s, ts, reg); |
6013 | 0 | ts->mem_coherent = 0; |
6014 | 0 | } |
6015 | 0 | break; |
6016 | | |
6017 | 0 | case TCG_CALL_RET_BY_VEC: |
6018 | 0 | { |
6019 | 0 | TCGTemp *ts = arg_temp(op->args[0]); |
6020 | |
|
6021 | 0 | tcg_debug_assert(ts->base_type == TCG_TYPE_I128); |
6022 | 0 | tcg_debug_assert(ts->temp_subindex == 0); |
6023 | 0 | if (!ts->mem_allocated) { |
6024 | 0 | temp_allocate_frame(s, ts); |
6025 | 0 | } |
6026 | 0 | tcg_out_st(s, TCG_TYPE_V128, |
6027 | 0 | tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), |
6028 | 0 | ts->mem_base->reg, ts->mem_offset); |
6029 | 0 | } |
6030 | | /* fall through to mark all parts in memory */ |
6031 | | |
6032 | 0 | case TCG_CALL_RET_BY_REF: |
6033 | | /* The callee has performed a write through the reference. */ |
6034 | 0 | for (i = 0; i < nb_oargs; i++) { |
6035 | 0 | TCGTemp *ts = arg_temp(op->args[i]); |
6036 | 0 | ts->val_type = TEMP_VAL_MEM; |
6037 | 0 | } |
6038 | 0 | break; |
6039 | | |
6040 | 0 | default: |
6041 | 0 | g_assert_not_reached(); |
6042 | 0 | } |
6043 | | |
6044 | | /* Flush or discard output registers as needed. */ |
6045 | 0 | for (i = 0; i < nb_oargs; i++) { |
6046 | 0 | TCGTemp *ts = arg_temp(op->args[i]); |
6047 | 0 | if (NEED_SYNC_ARG(i)) { |
6048 | 0 | temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); |
6049 | 0 | } else if (IS_DEAD_ARG(i)) { |
6050 | 0 | temp_dead(s, ts); |
6051 | 0 | } |
6052 | 0 | } |
6053 | 0 | } |
6054 | | |
6055 | | /** |
6056 | | * atom_and_align_for_opc: |
6057 | | * @s: tcg context |
6058 | | * @opc: memory operation code |
6059 | | * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations |
6060 | | * @allow_two_ops: true if we are prepared to issue two operations |
6061 | | * |
6062 | | * Return the alignment and atomicity to use for the inline fast path |
6063 | | * for the given memory operation. The alignment may be larger than |
6064 | | * that specified in @opc, and the correct alignment will be diagnosed |
6065 | | * by the slow path helper. |
6066 | | * |
6067 | | * If @allow_two_ops, the host is prepared to test for 2x alignment, |
6068 | | * and issue two loads or stores for subalignment. |
6069 | | */ |
6070 | | static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, |
6071 | | MemOp host_atom, bool allow_two_ops) |
6072 | 0 | { |
6073 | 0 | MemOp align = memop_alignment_bits(opc); |
6074 | 0 | MemOp size = opc & MO_SIZE; |
6075 | 0 | MemOp half = size ? size - 1 : 0; |
6076 | 0 | MemOp atom = opc & MO_ATOM_MASK; |
6077 | 0 | MemOp atmax; |
6078 | |
|
6079 | 0 | switch (atom) { |
6080 | 0 | case MO_ATOM_NONE: |
6081 | | /* The operation requires no specific atomicity. */ |
6082 | 0 | atmax = MO_8; |
6083 | 0 | break; |
6084 | | |
6085 | 0 | case MO_ATOM_IFALIGN: |
6086 | 0 | atmax = size; |
6087 | 0 | break; |
6088 | | |
6089 | 0 | case MO_ATOM_IFALIGN_PAIR: |
6090 | 0 | atmax = half; |
6091 | 0 | break; |
6092 | | |
6093 | 0 | case MO_ATOM_WITHIN16: |
6094 | 0 | atmax = size; |
6095 | 0 | if (size == MO_128) { |
6096 | | /* Misalignment implies !within16, and therefore no atomicity. */ |
6097 | 0 | } else if (host_atom != MO_ATOM_WITHIN16) { |
6098 | | /* The host does not implement within16, so require alignment. */ |
6099 | 0 | align = MAX(align, size); |
6100 | 0 | } |
6101 | 0 | break; |
6102 | | |
6103 | 0 | case MO_ATOM_WITHIN16_PAIR: |
6104 | 0 | atmax = size; |
6105 | | /* |
6106 | | * Misalignment implies !within16, and therefore half atomicity. |
6107 | | * Any host prepared for two operations can implement this with |
6108 | | * half alignment. |
6109 | | */ |
6110 | 0 | if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) { |
6111 | 0 | align = MAX(align, half); |
6112 | 0 | } |
6113 | 0 | break; |
6114 | | |
6115 | 0 | case MO_ATOM_SUBALIGN: |
6116 | 0 | atmax = size; |
6117 | 0 | if (host_atom != MO_ATOM_SUBALIGN) { |
6118 | | /* If unaligned but not odd, there are subobjects up to half. */ |
6119 | 0 | if (allow_two_ops) { |
6120 | 0 | align = MAX(align, half); |
6121 | 0 | } else { |
6122 | 0 | align = MAX(align, size); |
6123 | 0 | } |
6124 | 0 | } |
6125 | 0 | break; |
6126 | | |
6127 | 0 | default: |
6128 | 0 | g_assert_not_reached(); |
6129 | 0 | } |
6130 | | |
6131 | 0 | return (TCGAtomAlign){ .atom = atmax, .align = align }; |
6132 | 0 | } |
6133 | | |
6134 | | /* |
6135 | | * Similarly for qemu_ld/st slow path helpers. |
6136 | | * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously, |
6137 | | * using only the provided backend tcg_out_* functions. |
6138 | | */ |
6139 | | |
6140 | | static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot) |
6141 | 0 | { |
6142 | 0 | int ofs = arg_slot_stk_ofs(slot); |
6143 | | |
6144 | | /* |
6145 | | * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not |
6146 | | * require extension to uint64_t, adjust the address for uint32_t. |
6147 | | */ |
6148 | 0 | if (HOST_BIG_ENDIAN && type == TCG_TYPE_I32) { |
6149 | 0 | ofs += 4; |
6150 | 0 | } |
6151 | 0 | return ofs; |
6152 | 0 | } |
6153 | | |
6154 | | static void tcg_out_helper_load_slots(TCGContext *s, |
6155 | | unsigned nmov, TCGMovExtend *mov, |
6156 | | const TCGLdstHelperParam *parm) |
6157 | 0 | { |
6158 | 0 | unsigned i; |
6159 | 0 | TCGReg dst3; |
6160 | | |
6161 | | /* |
6162 | | * Start from the end, storing to the stack first. |
6163 | | * This frees those registers, so we need not consider overlap. |
6164 | | */ |
6165 | 0 | for (i = nmov; i-- > 0; ) { |
6166 | 0 | unsigned slot = mov[i].dst; |
6167 | |
|
6168 | 0 | if (arg_slot_reg_p(slot)) { |
6169 | 0 | goto found_reg; |
6170 | 0 | } |
6171 | | |
6172 | 0 | TCGReg src = mov[i].src; |
6173 | 0 | TCGType dst_type = mov[i].dst_type; |
6174 | 0 | MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64; |
6175 | | |
6176 | | /* The argument is going onto the stack; extend into scratch. */ |
6177 | 0 | if ((mov[i].src_ext & MO_SIZE) != dst_mo) { |
6178 | 0 | tcg_debug_assert(parm->ntmp != 0); |
6179 | 0 | mov[i].dst = src = parm->tmp[0]; |
6180 | 0 | tcg_out_movext1(s, &mov[i]); |
6181 | 0 | } |
6182 | | |
6183 | 0 | tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK, |
6184 | 0 | tcg_out_helper_stk_ofs(dst_type, slot)); |
6185 | 0 | } |
6186 | 0 | return; |
6187 | | |
6188 | 0 | found_reg: |
6189 | | /* |
6190 | | * The remaining arguments are in registers. |
6191 | | * Convert slot numbers to argument registers. |
6192 | | */ |
6193 | 0 | nmov = i + 1; |
6194 | 0 | for (i = 0; i < nmov; ++i) { |
6195 | 0 | mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst]; |
6196 | 0 | } |
6197 | |
|
6198 | 0 | switch (nmov) { |
6199 | 0 | case 4: |
6200 | | /* The backend must have provided enough temps for the worst case. */ |
6201 | 0 | tcg_debug_assert(parm->ntmp >= 2); |
6202 | | |
6203 | 0 | dst3 = mov[3].dst; |
6204 | 0 | for (unsigned j = 0; j < 3; ++j) { |
6205 | 0 | if (dst3 == mov[j].src) { |
6206 | | /* |
6207 | | * Conflict. Copy the source to a temporary, perform the |
6208 | | * remaining moves, then the extension from our scratch |
6209 | | * on the way out. |
6210 | | */ |
6211 | 0 | TCGReg scratch = parm->tmp[1]; |
6212 | |
|
6213 | 0 | tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src); |
6214 | 0 | tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]); |
6215 | 0 | tcg_out_movext1_new_src(s, &mov[3], scratch); |
6216 | 0 | break; |
6217 | 0 | } |
6218 | 0 | } |
6219 | | |
6220 | | /* No conflicts: perform this move and continue. */ |
6221 | 0 | tcg_out_movext1(s, &mov[3]); |
6222 | | /* fall through */ |
6223 | |
|
6224 | 0 | case 3: |
6225 | 0 | tcg_out_movext3(s, mov, mov + 1, mov + 2, |
6226 | 0 | parm->ntmp ? parm->tmp[0] : -1); |
6227 | 0 | break; |
6228 | 0 | case 2: |
6229 | 0 | tcg_out_movext2(s, mov, mov + 1, |
6230 | 0 | parm->ntmp ? parm->tmp[0] : -1); |
6231 | 0 | break; |
6232 | 0 | case 1: |
6233 | 0 | tcg_out_movext1(s, mov); |
6234 | 0 | break; |
6235 | 0 | default: |
6236 | 0 | g_assert_not_reached(); |
6237 | 0 | } |
6238 | 0 | } |
6239 | | |
6240 | | static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot, |
6241 | | TCGType type, tcg_target_long imm, |
6242 | | const TCGLdstHelperParam *parm) |
6243 | 0 | { |
6244 | 0 | if (arg_slot_reg_p(slot)) { |
6245 | 0 | tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm); |
6246 | 0 | } else { |
6247 | 0 | int ofs = tcg_out_helper_stk_ofs(type, slot); |
6248 | 0 | if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) { |
6249 | 0 | tcg_debug_assert(parm->ntmp != 0); |
6250 | 0 | tcg_out_movi(s, type, parm->tmp[0], imm); |
6251 | 0 | tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs); |
6252 | 0 | } |
6253 | 0 | } |
6254 | 0 | } |
6255 | | |
6256 | | static void tcg_out_helper_load_common_args(TCGContext *s, |
6257 | | const TCGLabelQemuLdst *ldst, |
6258 | | const TCGLdstHelperParam *parm, |
6259 | | const TCGHelperInfo *info, |
6260 | | unsigned next_arg) |
6261 | 0 | { |
6262 | 0 | TCGMovExtend ptr_mov = { |
6263 | 0 | .dst_type = TCG_TYPE_PTR, |
6264 | 0 | .src_type = TCG_TYPE_PTR, |
6265 | 0 | .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64 |
6266 | 0 | }; |
6267 | 0 | const TCGCallArgumentLoc *loc = &info->in[0]; |
6268 | 0 | TCGType type; |
6269 | 0 | unsigned slot; |
6270 | 0 | tcg_target_ulong imm; |
6271 | | |
6272 | | /* |
6273 | | * Handle env, which is always first. |
6274 | | */ |
6275 | 0 | ptr_mov.dst = loc->arg_slot; |
6276 | 0 | ptr_mov.src = TCG_AREG0; |
6277 | 0 | tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); |
6278 | | |
6279 | | /* |
6280 | | * Handle oi. |
6281 | | */ |
6282 | 0 | imm = ldst->oi; |
6283 | 0 | loc = &info->in[next_arg]; |
6284 | 0 | type = TCG_TYPE_I32; |
6285 | 0 | switch (loc->kind) { |
6286 | 0 | case TCG_CALL_ARG_NORMAL: |
6287 | 0 | break; |
6288 | 0 | case TCG_CALL_ARG_EXTEND_U: |
6289 | 0 | case TCG_CALL_ARG_EXTEND_S: |
6290 | | /* No extension required for MemOpIdx. */ |
6291 | 0 | tcg_debug_assert(imm <= INT32_MAX); |
6292 | 0 | type = TCG_TYPE_REG; |
6293 | 0 | break; |
6294 | 0 | default: |
6295 | 0 | g_assert_not_reached(); |
6296 | 0 | } |
6297 | 0 | tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm); |
6298 | 0 | next_arg++; |
6299 | | |
6300 | | /* |
6301 | | * Handle ra. |
6302 | | */ |
6303 | 0 | loc = &info->in[next_arg]; |
6304 | 0 | slot = loc->arg_slot; |
6305 | 0 | if (parm->ra_gen) { |
6306 | 0 | int arg_reg = -1; |
6307 | 0 | TCGReg ra_reg; |
6308 | |
|
6309 | 0 | if (arg_slot_reg_p(slot)) { |
6310 | 0 | arg_reg = tcg_target_call_iarg_regs[slot]; |
6311 | 0 | } |
6312 | 0 | ra_reg = parm->ra_gen(s, ldst, arg_reg); |
6313 | |
|
6314 | 0 | ptr_mov.dst = slot; |
6315 | 0 | ptr_mov.src = ra_reg; |
6316 | 0 | tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); |
6317 | 0 | } else { |
6318 | 0 | imm = (uintptr_t)ldst->raddr; |
6319 | 0 | tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm); |
6320 | 0 | } |
6321 | 0 | } |
6322 | | |
6323 | | static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov, |
6324 | | const TCGCallArgumentLoc *loc, |
6325 | | TCGType dst_type, TCGType src_type, |
6326 | | TCGReg lo, TCGReg hi) |
6327 | 0 | { |
6328 | 0 | MemOp reg_mo; |
6329 | |
|
6330 | 0 | if (dst_type <= TCG_TYPE_REG) { |
6331 | 0 | MemOp src_ext; |
6332 | |
|
6333 | 0 | switch (loc->kind) { |
6334 | 0 | case TCG_CALL_ARG_NORMAL: |
6335 | 0 | src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64; |
6336 | 0 | break; |
6337 | 0 | case TCG_CALL_ARG_EXTEND_U: |
6338 | 0 | dst_type = TCG_TYPE_REG; |
6339 | 0 | src_ext = MO_UL; |
6340 | 0 | break; |
6341 | 0 | case TCG_CALL_ARG_EXTEND_S: |
6342 | 0 | dst_type = TCG_TYPE_REG; |
6343 | 0 | src_ext = MO_SL; |
6344 | 0 | break; |
6345 | 0 | default: |
6346 | 0 | g_assert_not_reached(); |
6347 | 0 | } |
6348 | | |
6349 | 0 | mov[0].dst = loc->arg_slot; |
6350 | 0 | mov[0].dst_type = dst_type; |
6351 | 0 | mov[0].src = lo; |
6352 | 0 | mov[0].src_type = src_type; |
6353 | 0 | mov[0].src_ext = src_ext; |
6354 | 0 | return 1; |
6355 | 0 | } |
6356 | | |
6357 | 0 | assert(dst_type == TCG_TYPE_I128); |
6358 | 0 | reg_mo = MO_64; |
6359 | |
|
6360 | 0 | mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot; |
6361 | 0 | mov[0].src = lo; |
6362 | 0 | mov[0].dst_type = TCG_TYPE_REG; |
6363 | 0 | mov[0].src_type = TCG_TYPE_REG; |
6364 | 0 | mov[0].src_ext = reg_mo; |
6365 | |
|
6366 | 0 | mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot; |
6367 | 0 | mov[1].src = hi; |
6368 | 0 | mov[1].dst_type = TCG_TYPE_REG; |
6369 | 0 | mov[1].src_type = TCG_TYPE_REG; |
6370 | 0 | mov[1].src_ext = reg_mo; |
6371 | |
|
6372 | 0 | return 2; |
6373 | 0 | } |
6374 | | |
6375 | | static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, |
6376 | | const TCGLdstHelperParam *parm) |
6377 | 0 | { |
6378 | 0 | const TCGHelperInfo *info; |
6379 | 0 | const TCGCallArgumentLoc *loc; |
6380 | 0 | TCGMovExtend mov[2]; |
6381 | 0 | unsigned next_arg, nmov; |
6382 | 0 | MemOp mop = get_memop(ldst->oi); |
6383 | |
|
6384 | 0 | switch (mop & MO_SIZE) { |
6385 | 0 | case MO_8: |
6386 | 0 | case MO_16: |
6387 | 0 | case MO_32: |
6388 | 0 | info = &info_helper_ld32_mmu; |
6389 | 0 | break; |
6390 | 0 | case MO_64: |
6391 | 0 | info = &info_helper_ld64_mmu; |
6392 | 0 | break; |
6393 | 0 | case MO_128: |
6394 | 0 | info = &info_helper_ld128_mmu; |
6395 | 0 | break; |
6396 | 0 | default: |
6397 | 0 | g_assert_not_reached(); |
6398 | 0 | } |
6399 | | |
6400 | | /* Defer env argument. */ |
6401 | 0 | next_arg = 1; |
6402 | |
|
6403 | 0 | loc = &info->in[next_arg]; |
6404 | 0 | nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, |
6405 | 0 | ldst->addr_reg, -1); |
6406 | 0 | tcg_out_helper_load_slots(s, nmov, mov, parm); |
6407 | 0 | next_arg += nmov; |
6408 | |
|
6409 | 0 | switch (info->out_kind) { |
6410 | 0 | case TCG_CALL_RET_NORMAL: |
6411 | 0 | case TCG_CALL_RET_BY_VEC: |
6412 | 0 | break; |
6413 | 0 | case TCG_CALL_RET_BY_REF: |
6414 | | /* |
6415 | | * The return reference is in the first argument slot. |
6416 | | * We need memory in which to return: re-use the top of stack. |
6417 | | */ |
6418 | 0 | { |
6419 | 0 | int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; |
6420 | |
|
6421 | 0 | if (arg_slot_reg_p(0)) { |
6422 | 0 | tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0], |
6423 | 0 | TCG_REG_CALL_STACK, ofs_slot0); |
6424 | 0 | } else { |
6425 | 0 | tcg_debug_assert(parm->ntmp != 0); |
6426 | 0 | tcg_out_addi_ptr(s, parm->tmp[0], |
6427 | 0 | TCG_REG_CALL_STACK, ofs_slot0); |
6428 | 0 | tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], |
6429 | 0 | TCG_REG_CALL_STACK, ofs_slot0); |
6430 | 0 | } |
6431 | 0 | } |
6432 | 0 | break; |
6433 | 0 | default: |
6434 | 0 | g_assert_not_reached(); |
6435 | 0 | } |
6436 | | |
6437 | 0 | tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); |
6438 | 0 | } |
6439 | | |
6440 | | static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst, |
6441 | | bool load_sign, |
6442 | | const TCGLdstHelperParam *parm) |
6443 | 0 | { |
6444 | 0 | MemOp mop = get_memop(ldst->oi); |
6445 | 0 | TCGMovExtend mov[2]; |
6446 | 0 | int ofs_slot0; |
6447 | |
|
6448 | 0 | switch (ldst->type) { |
6449 | 0 | case TCG_TYPE_I32: |
6450 | 0 | case TCG_TYPE_I64: |
6451 | 0 | mov[0].dst = ldst->datalo_reg; |
6452 | 0 | mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0); |
6453 | 0 | mov[0].dst_type = ldst->type; |
6454 | 0 | mov[0].src_type = TCG_TYPE_REG; |
6455 | | |
6456 | | /* |
6457 | | * If load_sign, then we allowed the helper to perform the |
6458 | | * appropriate sign extension to tcg_target_ulong, and all |
6459 | | * we need now is a plain move. |
6460 | | * |
6461 | | * If they do not, then we expect the relevant extension |
6462 | | * instruction to be no more expensive than a move, and |
6463 | | * we thus save the icache etc by only using one of two |
6464 | | * helper functions. |
6465 | | */ |
6466 | 0 | if (load_sign || !(mop & MO_SIGN)) { |
6467 | 0 | if (ldst->type == TCG_TYPE_I32) { |
6468 | 0 | mov[0].src_ext = MO_32; |
6469 | 0 | } else { |
6470 | 0 | mov[0].src_ext = MO_64; |
6471 | 0 | } |
6472 | 0 | } else { |
6473 | 0 | mov[0].src_ext = mop & MO_SSIZE; |
6474 | 0 | } |
6475 | 0 | tcg_out_movext1(s, mov); |
6476 | 0 | return; |
6477 | | |
6478 | 0 | case TCG_TYPE_I128: |
6479 | 0 | ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; |
6480 | 0 | switch (TCG_TARGET_CALL_RET_I128) { |
6481 | 0 | case TCG_CALL_RET_NORMAL: |
6482 | 0 | break; |
6483 | 0 | case TCG_CALL_RET_BY_VEC: |
6484 | 0 | tcg_out_st(s, TCG_TYPE_V128, |
6485 | 0 | tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), |
6486 | 0 | TCG_REG_CALL_STACK, ofs_slot0); |
6487 | | /* fall through */ |
6488 | 0 | case TCG_CALL_RET_BY_REF: |
6489 | 0 | tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg, |
6490 | 0 | TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN); |
6491 | 0 | tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg, |
6492 | 0 | TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN); |
6493 | 0 | return; |
6494 | 0 | default: |
6495 | 0 | g_assert_not_reached(); |
6496 | 0 | } |
6497 | 0 | break; |
6498 | | |
6499 | 0 | default: |
6500 | 0 | g_assert_not_reached(); |
6501 | 0 | } |
6502 | | |
6503 | 0 | mov[0].dst = ldst->datalo_reg; |
6504 | 0 | mov[0].src = |
6505 | 0 | tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN); |
6506 | 0 | mov[0].dst_type = TCG_TYPE_REG; |
6507 | 0 | mov[0].src_type = TCG_TYPE_REG; |
6508 | 0 | mov[0].src_ext = MO_64; |
6509 | |
|
6510 | 0 | mov[1].dst = ldst->datahi_reg; |
6511 | 0 | mov[1].src = |
6512 | 0 | tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN); |
6513 | 0 | mov[1].dst_type = TCG_TYPE_REG; |
6514 | 0 | mov[1].src_type = TCG_TYPE_REG; |
6515 | 0 | mov[1].src_ext = MO_64; |
6516 | |
|
6517 | 0 | tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); |
6518 | 0 | } |
6519 | | |
6520 | | static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, |
6521 | | const TCGLdstHelperParam *parm) |
6522 | 0 | { |
6523 | 0 | const TCGHelperInfo *info; |
6524 | 0 | const TCGCallArgumentLoc *loc; |
6525 | 0 | TCGMovExtend mov[4]; |
6526 | 0 | TCGType data_type; |
6527 | 0 | unsigned next_arg, nmov, n; |
6528 | 0 | MemOp mop = get_memop(ldst->oi); |
6529 | |
|
6530 | 0 | switch (mop & MO_SIZE) { |
6531 | 0 | case MO_8: |
6532 | 0 | case MO_16: |
6533 | 0 | case MO_32: |
6534 | 0 | info = &info_helper_st32_mmu; |
6535 | 0 | data_type = TCG_TYPE_I32; |
6536 | 0 | break; |
6537 | 0 | case MO_64: |
6538 | 0 | info = &info_helper_st64_mmu; |
6539 | 0 | data_type = TCG_TYPE_I64; |
6540 | 0 | break; |
6541 | 0 | case MO_128: |
6542 | 0 | info = &info_helper_st128_mmu; |
6543 | 0 | data_type = TCG_TYPE_I128; |
6544 | 0 | break; |
6545 | 0 | default: |
6546 | 0 | g_assert_not_reached(); |
6547 | 0 | } |
6548 | | |
6549 | | /* Defer env argument. */ |
6550 | 0 | next_arg = 1; |
6551 | 0 | nmov = 0; |
6552 | | |
6553 | | /* Handle addr argument. */ |
6554 | 0 | loc = &info->in[next_arg]; |
6555 | 0 | tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); |
6556 | 0 | n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, |
6557 | 0 | ldst->addr_reg, -1); |
6558 | 0 | next_arg += n; |
6559 | 0 | nmov += n; |
6560 | | |
6561 | | /* Handle data argument. */ |
6562 | 0 | loc = &info->in[next_arg]; |
6563 | 0 | switch (loc->kind) { |
6564 | 0 | case TCG_CALL_ARG_NORMAL: |
6565 | 0 | case TCG_CALL_ARG_EXTEND_U: |
6566 | 0 | case TCG_CALL_ARG_EXTEND_S: |
6567 | 0 | n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, |
6568 | 0 | ldst->datalo_reg, ldst->datahi_reg); |
6569 | 0 | next_arg += n; |
6570 | 0 | nmov += n; |
6571 | 0 | tcg_out_helper_load_slots(s, nmov, mov, parm); |
6572 | 0 | break; |
6573 | | |
6574 | 0 | case TCG_CALL_ARG_BY_REF: |
6575 | 0 | tcg_debug_assert(data_type == TCG_TYPE_I128); |
6576 | 0 | tcg_out_st(s, TCG_TYPE_I64, |
6577 | 0 | HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg, |
6578 | 0 | TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot)); |
6579 | 0 | tcg_out_st(s, TCG_TYPE_I64, |
6580 | 0 | HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg, |
6581 | 0 | TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot)); |
6582 | |
|
6583 | 0 | tcg_out_helper_load_slots(s, nmov, mov, parm); |
6584 | |
|
6585 | 0 | if (arg_slot_reg_p(loc->arg_slot)) { |
6586 | 0 | tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot], |
6587 | 0 | TCG_REG_CALL_STACK, |
6588 | 0 | arg_slot_stk_ofs(loc->ref_slot)); |
6589 | 0 | } else { |
6590 | 0 | tcg_debug_assert(parm->ntmp != 0); |
6591 | 0 | tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK, |
6592 | 0 | arg_slot_stk_ofs(loc->ref_slot)); |
6593 | 0 | tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], |
6594 | 0 | TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot)); |
6595 | 0 | } |
6596 | 0 | next_arg += 2; |
6597 | 0 | break; |
6598 | | |
6599 | 0 | default: |
6600 | 0 | g_assert_not_reached(); |
6601 | 0 | } |
6602 | | |
6603 | 0 | tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); |
6604 | 0 | } |
6605 | | |
6606 | | int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) |
6607 | 0 | { |
6608 | 0 | int i, num_insns; |
6609 | 0 | TCGOp *op; |
6610 | |
|
6611 | 0 | if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) |
6612 | 0 | && qemu_log_in_addr_range(pc_start))) { |
6613 | 0 | FILE *logfile = qemu_log_trylock(); |
6614 | 0 | if (logfile) { |
6615 | 0 | fprintf(logfile, "OP:\n"); |
6616 | 0 | tcg_dump_ops(s, logfile, false); |
6617 | 0 | fprintf(logfile, "\n"); |
6618 | 0 | qemu_log_unlock(logfile); |
6619 | 0 | } |
6620 | 0 | } |
6621 | |
|
6622 | | #ifdef CONFIG_DEBUG_TCG |
6623 | | /* Ensure all labels referenced have been emitted. */ |
6624 | | { |
6625 | | TCGLabel *l; |
6626 | | bool error = false; |
6627 | | |
6628 | | QSIMPLEQ_FOREACH(l, &s->labels, next) { |
6629 | | if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { |
6630 | | qemu_log_mask(CPU_LOG_TB_OP, |
6631 | | "$L%d referenced but not present.\n", l->id); |
6632 | | error = true; |
6633 | | } |
6634 | | } |
6635 | | assert(!error); |
6636 | | } |
6637 | | #endif |
6638 | | |
6639 | | /* Do not reuse any EBB that may be allocated within the TB. */ |
6640 | 0 | tcg_temp_ebb_reset_freed(s); |
6641 | |
|
6642 | 0 | tcg_optimize(s); |
6643 | |
|
6644 | 0 | reachable_code_pass(s); |
6645 | 0 | liveness_pass_0(s); |
6646 | 0 | liveness_pass_1(s); |
6647 | |
|
6648 | 0 | if (s->nb_indirects > 0) { |
6649 | 0 | if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) |
6650 | 0 | && qemu_log_in_addr_range(pc_start))) { |
6651 | 0 | FILE *logfile = qemu_log_trylock(); |
6652 | 0 | if (logfile) { |
6653 | 0 | fprintf(logfile, "OP before indirect lowering:\n"); |
6654 | 0 | tcg_dump_ops(s, logfile, false); |
6655 | 0 | fprintf(logfile, "\n"); |
6656 | 0 | qemu_log_unlock(logfile); |
6657 | 0 | } |
6658 | 0 | } |
6659 | | |
6660 | | /* Replace indirect temps with direct temps. */ |
6661 | 0 | if (liveness_pass_2(s)) { |
6662 | | /* If changes were made, re-run liveness. */ |
6663 | 0 | liveness_pass_1(s); |
6664 | 0 | } |
6665 | 0 | } |
6666 | |
|
6667 | 0 | if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) |
6668 | 0 | && qemu_log_in_addr_range(pc_start))) { |
6669 | 0 | FILE *logfile = qemu_log_trylock(); |
6670 | 0 | if (logfile) { |
6671 | 0 | fprintf(logfile, "OP after optimization and liveness analysis:\n"); |
6672 | 0 | tcg_dump_ops(s, logfile, true); |
6673 | 0 | fprintf(logfile, "\n"); |
6674 | 0 | qemu_log_unlock(logfile); |
6675 | 0 | } |
6676 | 0 | } |
6677 | | |
6678 | | /* Initialize goto_tb jump offsets. */ |
6679 | 0 | tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; |
6680 | 0 | tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; |
6681 | 0 | tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; |
6682 | 0 | tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; |
6683 | |
|
6684 | 0 | tcg_reg_alloc_start(s); |
6685 | | |
6686 | | /* |
6687 | | * Reset the buffer pointers when restarting after overflow. |
6688 | | * TODO: Move this into translate-all.c with the rest of the |
6689 | | * buffer management. Having only this done here is confusing. |
6690 | | */ |
6691 | 0 | s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); |
6692 | 0 | s->code_ptr = s->code_buf; |
6693 | 0 | s->data_gen_ptr = NULL; |
6694 | |
|
6695 | 0 | QSIMPLEQ_INIT(&s->ldst_labels); |
6696 | 0 | s->pool_labels = NULL; |
6697 | |
|
6698 | 0 | s->gen_insn_data = |
6699 | 0 | tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS); |
6700 | |
|
6701 | 0 | tcg_out_tb_start(s); |
6702 | |
|
6703 | 0 | num_insns = -1; |
6704 | 0 | s->carry_live = false; |
6705 | 0 | QTAILQ_FOREACH(op, &s->ops, link) { |
6706 | 0 | TCGOpcode opc = op->opc; |
6707 | |
|
6708 | 0 | switch (opc) { |
6709 | 0 | case INDEX_op_extrl_i64_i32: |
6710 | | /* |
6711 | | * If TCG_TYPE_I32 is represented in some canonical form, |
6712 | | * e.g. zero or sign-extended, then emit as a unary op. |
6713 | | * Otherwise we can treat this as a plain move. |
6714 | | * If the output dies, treat this as a plain move, because |
6715 | | * this will be implemented with a store. |
6716 | | */ |
6717 | 0 | if (TCG_TARGET_HAS_extr_i64_i32) { |
6718 | 0 | TCGLifeData arg_life = op->life; |
6719 | 0 | if (!IS_DEAD_ARG(0)) { |
6720 | 0 | goto do_default; |
6721 | 0 | } |
6722 | 0 | } |
6723 | | /* fall through */ |
6724 | 0 | case INDEX_op_mov: |
6725 | 0 | case INDEX_op_mov_vec: |
6726 | 0 | tcg_reg_alloc_mov(s, op); |
6727 | 0 | break; |
6728 | 0 | case INDEX_op_dup_vec: |
6729 | 0 | tcg_reg_alloc_dup(s, op); |
6730 | 0 | break; |
6731 | 0 | case INDEX_op_insn_start: |
6732 | 0 | assert_carry_dead(s); |
6733 | 0 | if (num_insns >= 0) { |
6734 | 0 | size_t off = tcg_current_code_size(s); |
6735 | 0 | s->gen_insn_end_off[num_insns] = off; |
6736 | | /* Assert that we do not overflow our stored offset. */ |
6737 | 0 | assert(s->gen_insn_end_off[num_insns] == off); |
6738 | 0 | } |
6739 | 0 | num_insns++; |
6740 | 0 | for (i = 0; i < INSN_START_WORDS; ++i) { |
6741 | 0 | s->gen_insn_data[num_insns * INSN_START_WORDS + i] = |
6742 | 0 | tcg_get_insn_start_param(op, i); |
6743 | 0 | } |
6744 | 0 | break; |
6745 | 0 | case INDEX_op_discard: |
6746 | 0 | temp_dead(s, arg_temp(op->args[0])); |
6747 | 0 | break; |
6748 | 0 | case INDEX_op_set_label: |
6749 | 0 | tcg_reg_alloc_bb_end(s, s->reserved_regs); |
6750 | 0 | tcg_out_label(s, arg_label(op->args[0])); |
6751 | 0 | break; |
6752 | 0 | case INDEX_op_call: |
6753 | 0 | assert_carry_dead(s); |
6754 | 0 | tcg_reg_alloc_call(s, op); |
6755 | 0 | break; |
6756 | 0 | case INDEX_op_exit_tb: |
6757 | 0 | tcg_out_exit_tb(s, op->args[0]); |
6758 | 0 | break; |
6759 | 0 | case INDEX_op_goto_tb: |
6760 | 0 | tcg_out_goto_tb(s, op->args[0]); |
6761 | 0 | break; |
6762 | 0 | case INDEX_op_br: |
6763 | 0 | tcg_out_br(s, arg_label(op->args[0])); |
6764 | 0 | break; |
6765 | 0 | case INDEX_op_mb: |
6766 | 0 | tcg_out_mb(s, op->args[0]); |
6767 | 0 | break; |
6768 | 0 | default: |
6769 | 0 | do_default: |
6770 | | /* Sanity check that we've not introduced any unhandled opcodes. */ |
6771 | 0 | tcg_debug_assert(tcg_op_supported(opc, TCGOP_TYPE(op), |
6772 | 0 | TCGOP_FLAGS(op))); |
6773 | | /* Note: in order to speed up the code, it would be much |
6774 | | faster to have specialized register allocator functions for |
6775 | | some common argument patterns */ |
6776 | 0 | tcg_reg_alloc_op(s, op); |
6777 | 0 | break; |
6778 | 0 | } |
6779 | | /* Test for (pending) buffer overflow. The assumption is that any |
6780 | | one operation beginning below the high water mark cannot overrun |
6781 | | the buffer completely. Thus we can test for overflow after |
6782 | | generating code without having to check during generation. */ |
6783 | 0 | if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { |
6784 | 0 | return -1; |
6785 | 0 | } |
6786 | | /* Test for TB overflow, as seen by gen_insn_end_off. */ |
6787 | 0 | if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { |
6788 | 0 | return -2; |
6789 | 0 | } |
6790 | 0 | } |
6791 | 0 | assert_carry_dead(s); |
6792 | |
|
6793 | 0 | tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); |
6794 | 0 | s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); |
6795 | | |
6796 | | /* Generate TB finalization at the end of block */ |
6797 | 0 | i = tcg_out_ldst_finalize(s); |
6798 | 0 | if (i < 0) { |
6799 | 0 | return i; |
6800 | 0 | } |
6801 | 0 | i = tcg_out_pool_finalize(s); |
6802 | 0 | if (i < 0) { |
6803 | 0 | return i; |
6804 | 0 | } |
6805 | 0 | if (!tcg_resolve_relocs(s)) { |
6806 | 0 | return -2; |
6807 | 0 | } |
6808 | | |
6809 | 0 | #ifndef CONFIG_TCG_INTERPRETER |
6810 | | /* flush instruction cache */ |
6811 | 0 | flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), |
6812 | 0 | (uintptr_t)s->code_buf, |
6813 | 0 | tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); |
6814 | 0 | #endif |
6815 | |
|
6816 | 0 | return tcg_current_code_size(s); |
6817 | 0 | } |
6818 | | |
6819 | | #ifdef ELF_HOST_MACHINE |
6820 | | /* In order to use this feature, the backend needs to do three things: |
6821 | | |
6822 | | (1) Define ELF_HOST_MACHINE to indicate both what value to |
6823 | | put into the ELF image and to indicate support for the feature. |
6824 | | |
6825 | | (2) Define tcg_register_jit. This should create a buffer containing |
6826 | | the contents of a .debug_frame section that describes the post- |
6827 | | prologue unwind info for the tcg machine. |
6828 | | |
6829 | | (3) Call tcg_register_jit_int, with the constructed .debug_frame. |
6830 | | */ |
6831 | | |
6832 | | /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ |
6833 | | typedef enum { |
6834 | | JIT_NOACTION = 0, |
6835 | | JIT_REGISTER_FN, |
6836 | | JIT_UNREGISTER_FN |
6837 | | } jit_actions_t; |
6838 | | |
6839 | | struct jit_code_entry { |
6840 | | struct jit_code_entry *next_entry; |
6841 | | struct jit_code_entry *prev_entry; |
6842 | | const void *symfile_addr; |
6843 | | uint64_t symfile_size; |
6844 | | }; |
6845 | | |
6846 | | struct jit_descriptor { |
6847 | | uint32_t version; |
6848 | | uint32_t action_flag; |
6849 | | struct jit_code_entry *relevant_entry; |
6850 | | struct jit_code_entry *first_entry; |
6851 | | }; |
6852 | | |
6853 | | void __jit_debug_register_code(void) __attribute__((noinline)); |
6854 | | void __jit_debug_register_code(void) |
6855 | 0 | { |
6856 | 0 | asm(""); |
6857 | 0 | } |
6858 | | |
6859 | | /* Must statically initialize the version, because GDB may check |
6860 | | the version before we can set it. */ |
6861 | | struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; |
6862 | | |
6863 | | /* End GDB interface. */ |
6864 | | |
6865 | | static int find_string(const char *strtab, const char *str) |
6866 | 0 | { |
6867 | 0 | const char *p = strtab + 1; |
6868 | |
|
6869 | 0 | while (1) { |
6870 | 0 | if (strcmp(p, str) == 0) { |
6871 | 0 | return p - strtab; |
6872 | 0 | } |
6873 | 0 | p += strlen(p) + 1; |
6874 | 0 | } |
6875 | 0 | } |
6876 | | |
6877 | | static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, |
6878 | | const void *debug_frame, |
6879 | | size_t debug_frame_size) |
6880 | 0 | { |
6881 | 0 | struct __attribute__((packed)) DebugInfo { |
6882 | 0 | uint32_t len; |
6883 | 0 | uint16_t version; |
6884 | 0 | uint32_t abbrev; |
6885 | 0 | uint8_t ptr_size; |
6886 | 0 | uint8_t cu_die; |
6887 | 0 | uint16_t cu_lang; |
6888 | 0 | uintptr_t cu_low_pc; |
6889 | 0 | uintptr_t cu_high_pc; |
6890 | 0 | uint8_t fn_die; |
6891 | 0 | char fn_name[16]; |
6892 | 0 | uintptr_t fn_low_pc; |
6893 | 0 | uintptr_t fn_high_pc; |
6894 | 0 | uint8_t cu_eoc; |
6895 | 0 | }; |
6896 | |
|
6897 | 0 | struct ElfImage { |
6898 | 0 | ElfW(Ehdr) ehdr; |
6899 | 0 | ElfW(Phdr) phdr; |
6900 | 0 | ElfW(Shdr) shdr[7]; |
6901 | 0 | ElfW(Sym) sym[2]; |
6902 | 0 | struct DebugInfo di; |
6903 | 0 | uint8_t da[24]; |
6904 | 0 | char str[80]; |
6905 | 0 | }; |
6906 | |
|
6907 | 0 | struct ElfImage *img; |
6908 | |
|
6909 | 0 | static const struct ElfImage img_template = { |
6910 | 0 | .ehdr = { |
6911 | 0 | .e_ident[EI_MAG0] = ELFMAG0, |
6912 | 0 | .e_ident[EI_MAG1] = ELFMAG1, |
6913 | 0 | .e_ident[EI_MAG2] = ELFMAG2, |
6914 | 0 | .e_ident[EI_MAG3] = ELFMAG3, |
6915 | 0 | .e_ident[EI_CLASS] = ELF_CLASS, |
6916 | 0 | .e_ident[EI_DATA] = ELF_DATA, |
6917 | 0 | .e_ident[EI_VERSION] = EV_CURRENT, |
6918 | 0 | .e_type = ET_EXEC, |
6919 | 0 | .e_machine = ELF_HOST_MACHINE, |
6920 | 0 | .e_version = EV_CURRENT, |
6921 | 0 | .e_phoff = offsetof(struct ElfImage, phdr), |
6922 | 0 | .e_shoff = offsetof(struct ElfImage, shdr), |
6923 | 0 | .e_ehsize = sizeof(ElfW(Shdr)), |
6924 | 0 | .e_phentsize = sizeof(ElfW(Phdr)), |
6925 | 0 | .e_phnum = 1, |
6926 | 0 | .e_shentsize = sizeof(ElfW(Shdr)), |
6927 | 0 | .e_shnum = ARRAY_SIZE(img->shdr), |
6928 | 0 | .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, |
6929 | | #ifdef ELF_HOST_FLAGS |
6930 | | .e_flags = ELF_HOST_FLAGS, |
6931 | | #endif |
6932 | | #ifdef ELF_OSABI |
6933 | | .e_ident[EI_OSABI] = ELF_OSABI, |
6934 | | #endif |
6935 | 0 | }, |
6936 | 0 | .phdr = { |
6937 | 0 | .p_type = PT_LOAD, |
6938 | 0 | .p_flags = PF_X, |
6939 | 0 | }, |
6940 | 0 | .shdr = { |
6941 | 0 | [0] = { .sh_type = SHT_NULL }, |
6942 | | /* Trick: The contents of code_gen_buffer are not present in |
6943 | | this fake ELF file; that got allocated elsewhere. Therefore |
6944 | | we mark .text as SHT_NOBITS (similar to .bss) so that readers |
6945 | | will not look for contents. We can record any address. */ |
6946 | 0 | [1] = { /* .text */ |
6947 | 0 | .sh_type = SHT_NOBITS, |
6948 | 0 | .sh_flags = SHF_EXECINSTR | SHF_ALLOC, |
6949 | 0 | }, |
6950 | 0 | [2] = { /* .debug_info */ |
6951 | 0 | .sh_type = SHT_PROGBITS, |
6952 | 0 | .sh_offset = offsetof(struct ElfImage, di), |
6953 | 0 | .sh_size = sizeof(struct DebugInfo), |
6954 | 0 | }, |
6955 | 0 | [3] = { /* .debug_abbrev */ |
6956 | 0 | .sh_type = SHT_PROGBITS, |
6957 | 0 | .sh_offset = offsetof(struct ElfImage, da), |
6958 | 0 | .sh_size = sizeof(img->da), |
6959 | 0 | }, |
6960 | 0 | [4] = { /* .debug_frame */ |
6961 | 0 | .sh_type = SHT_PROGBITS, |
6962 | 0 | .sh_offset = sizeof(struct ElfImage), |
6963 | 0 | }, |
6964 | 0 | [5] = { /* .symtab */ |
6965 | 0 | .sh_type = SHT_SYMTAB, |
6966 | 0 | .sh_offset = offsetof(struct ElfImage, sym), |
6967 | 0 | .sh_size = sizeof(img->sym), |
6968 | 0 | .sh_info = 1, |
6969 | 0 | .sh_link = ARRAY_SIZE(img->shdr) - 1, |
6970 | 0 | .sh_entsize = sizeof(ElfW(Sym)), |
6971 | 0 | }, |
6972 | 0 | [6] = { /* .strtab */ |
6973 | 0 | .sh_type = SHT_STRTAB, |
6974 | 0 | .sh_offset = offsetof(struct ElfImage, str), |
6975 | 0 | .sh_size = sizeof(img->str), |
6976 | 0 | } |
6977 | 0 | }, |
6978 | 0 | .sym = { |
6979 | 0 | [1] = { /* code_gen_buffer */ |
6980 | 0 | .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), |
6981 | 0 | .st_shndx = 1, |
6982 | 0 | } |
6983 | 0 | }, |
6984 | 0 | .di = { |
6985 | 0 | .len = sizeof(struct DebugInfo) - 4, |
6986 | 0 | .version = 2, |
6987 | 0 | .ptr_size = sizeof(void *), |
6988 | 0 | .cu_die = 1, |
6989 | 0 | .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ |
6990 | 0 | .fn_die = 2, |
6991 | 0 | .fn_name = "code_gen_buffer" |
6992 | 0 | }, |
6993 | 0 | .da = { |
6994 | 0 | 1, /* abbrev number (the cu) */ |
6995 | 0 | 0x11, 1, /* DW_TAG_compile_unit, has children */ |
6996 | 0 | 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ |
6997 | 0 | 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ |
6998 | 0 | 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ |
6999 | 0 | 0, 0, /* end of abbrev */ |
7000 | 0 | 2, /* abbrev number (the fn) */ |
7001 | 0 | 0x2e, 0, /* DW_TAG_subprogram, no children */ |
7002 | 0 | 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ |
7003 | 0 | 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ |
7004 | 0 | 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ |
7005 | 0 | 0, 0, /* end of abbrev */ |
7006 | 0 | 0 /* no more abbrev */ |
7007 | 0 | }, |
7008 | 0 | .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" |
7009 | 0 | ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", |
7010 | 0 | }; |
7011 | | |
7012 | | /* We only need a single jit entry; statically allocate it. */ |
7013 | 0 | static struct jit_code_entry one_entry; |
7014 | |
|
7015 | 0 | uintptr_t buf = (uintptr_t)buf_ptr; |
7016 | 0 | size_t img_size = sizeof(struct ElfImage) + debug_frame_size; |
7017 | 0 | DebugFrameHeader *dfh; |
7018 | |
|
7019 | 0 | img = g_malloc(img_size); |
7020 | 0 | *img = img_template; |
7021 | |
|
7022 | 0 | img->phdr.p_vaddr = buf; |
7023 | 0 | img->phdr.p_paddr = buf; |
7024 | 0 | img->phdr.p_memsz = buf_size; |
7025 | |
|
7026 | 0 | img->shdr[1].sh_name = find_string(img->str, ".text"); |
7027 | 0 | img->shdr[1].sh_addr = buf; |
7028 | 0 | img->shdr[1].sh_size = buf_size; |
7029 | |
|
7030 | 0 | img->shdr[2].sh_name = find_string(img->str, ".debug_info"); |
7031 | 0 | img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); |
7032 | |
|
7033 | 0 | img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); |
7034 | 0 | img->shdr[4].sh_size = debug_frame_size; |
7035 | |
|
7036 | 0 | img->shdr[5].sh_name = find_string(img->str, ".symtab"); |
7037 | 0 | img->shdr[6].sh_name = find_string(img->str, ".strtab"); |
7038 | |
|
7039 | 0 | img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); |
7040 | 0 | img->sym[1].st_value = buf; |
7041 | 0 | img->sym[1].st_size = buf_size; |
7042 | |
|
7043 | 0 | img->di.cu_low_pc = buf; |
7044 | 0 | img->di.cu_high_pc = buf + buf_size; |
7045 | 0 | img->di.fn_low_pc = buf; |
7046 | 0 | img->di.fn_high_pc = buf + buf_size; |
7047 | |
|
7048 | 0 | dfh = (DebugFrameHeader *)(img + 1); |
7049 | 0 | memcpy(dfh, debug_frame, debug_frame_size); |
7050 | 0 | dfh->fde.func_start = buf; |
7051 | 0 | dfh->fde.func_len = buf_size; |
7052 | |
|
7053 | | #ifdef DEBUG_JIT |
7054 | | /* Enable this block to be able to debug the ELF image file creation. |
7055 | | One can use readelf, objdump, or other inspection utilities. */ |
7056 | | { |
7057 | | g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); |
7058 | | FILE *f = fopen(jit, "w+b"); |
7059 | | if (f) { |
7060 | | if (fwrite(img, img_size, 1, f) != img_size) { |
7061 | | /* Avoid stupid unused return value warning for fwrite. */ |
7062 | | } |
7063 | | fclose(f); |
7064 | | } |
7065 | | } |
7066 | | #endif |
7067 | |
|
7068 | 0 | one_entry.symfile_addr = img; |
7069 | 0 | one_entry.symfile_size = img_size; |
7070 | |
|
7071 | 0 | __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; |
7072 | 0 | __jit_debug_descriptor.relevant_entry = &one_entry; |
7073 | 0 | __jit_debug_descriptor.first_entry = &one_entry; |
7074 | 0 | __jit_debug_register_code(); |
7075 | 0 | } |
7076 | | #else |
7077 | | /* No support for the feature. Provide the entry point expected by exec.c, |
7078 | | and implement the internal function we declared earlier. */ |
7079 | | |
7080 | | static void tcg_register_jit_int(const void *buf, size_t size, |
7081 | | const void *debug_frame, |
7082 | | size_t debug_frame_size) |
7083 | | { |
7084 | | } |
7085 | | |
7086 | | void tcg_register_jit(const void *buf, size_t buf_size) |
7087 | | { |
7088 | | } |
7089 | | #endif /* ELF_HOST_MACHINE */ |
7090 | | |
7091 | | #if !TCG_TARGET_MAYBE_vec |
7092 | | void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) |
7093 | | { |
7094 | | g_assert_not_reached(); |
7095 | | } |
7096 | | #endif |