Coverage Report

Created: 2026-11-20 08:13

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qemu/accel/tcg/translator.c
Line
Count
Source
1
/*
2
 * Generic intermediate code generation.
3
 *
4
 * Copyright (C) 2016-2017 LluĂ­s Vilanova <vilanova@ac.upc.edu>
5
 *
6
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
7
 * See the COPYING file in the top-level directory.
8
 */
9
10
#include "qemu/osdep.h"
11
#include "qemu/bswap.h"
12
#include "qemu/log.h"
13
#include "qemu/error-report.h"
14
#include "accel/tcg/cpu-ldst-common.h"
15
#include "accel/tcg/cpu-mmu-index.h"
16
#include "exec/target_page.h"
17
#include "exec/translator.h"
18
#ifdef CONFIG_USER_ONLY
19
#include "gdbstub/user.h"
20
#endif
21
#include "exec/plugin-gen.h"
22
#include "tcg/tcg-op-common.h"
23
#include "internal-common.h"
24
#include "disas/disas.h"
25
#include "tb-internal.h"
26
27
#ifndef CONFIG_USER_ONLY
28
static void set_can_do_io(DisasContextBase *db, bool val)
29
0
{
30
0
    QEMU_BUILD_BUG_ON(sizeof_field(CPUState, neg.can_do_io) != 1);
31
0
    tcg_gen_st8_i32(tcg_constant_i32(val), tcg_env,
32
0
                    offsetof(CPUState, neg.can_do_io) - sizeof(CPUState));
33
0
}
34
#endif
35
36
bool translator_io_start(DisasContextBase *db)
37
0
{
38
    /*
39
     * Ensure that this instruction will be the last in the TB.
40
     * The target may override this to something more forceful.
41
     */
42
0
    if (db->is_jmp == DISAS_NEXT) {
43
0
        db->is_jmp = DISAS_TOO_MANY;
44
0
    }
45
0
    return true;
46
0
}
47
48
static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
49
0
{
50
0
    TCGv_i32 count = NULL;
51
0
    TCGOp *icount_start_insn = NULL;
52
53
0
    if ((cflags & CF_USE_ICOUNT) || !(cflags & CF_NOIRQ)) {
54
0
        count = tcg_temp_new_i32();
55
0
        tcg_gen_ld_i32(count, tcg_env,
56
0
                       offsetof(CPUState, neg.icount_decr.u32) -
57
0
                       sizeof(CPUState));
58
0
    }
59
60
0
    if (cflags & CF_USE_ICOUNT) {
61
        /*
62
         * We emit a sub with a dummy immediate argument. Keep the insn index
63
         * of the sub so that we later (when we know the actual insn count)
64
         * can update the argument with the actual insn count.
65
         */
66
0
        tcg_gen_sub_i32(count, count, tcg_constant_i32(0));
67
0
        icount_start_insn = tcg_last_op();
68
0
    }
69
70
    /*
71
     * Emit the check against icount_decr.u32 to see if we should exit
72
     * unless we suppress the check with CF_NOIRQ. If we are using
73
     * icount and have suppressed interruption the higher level code
74
     * should have ensured we don't run more instructions than the
75
     * budget.
76
     */
77
0
    if (cflags & CF_NOIRQ) {
78
0
        tcg_ctx->exitreq_label = NULL;
79
0
    } else {
80
0
        tcg_ctx->exitreq_label = gen_new_label();
81
0
        tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label);
82
0
    }
83
84
0
    if (cflags & CF_USE_ICOUNT) {
85
0
        tcg_gen_st16_i32(count, tcg_env,
86
0
                         offsetof(CPUState, neg.icount_decr.u16.low) -
87
0
                         sizeof(CPUState));
88
0
    }
89
90
0
    return icount_start_insn;
91
0
}
92
93
static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
94
                       TCGOp *icount_start_insn, int num_insns)
95
0
{
96
0
    if (cflags & CF_USE_ICOUNT) {
97
        /*
98
         * Update the num_insn immediate parameter now that we know
99
         * the actual insn count.
100
         */
101
0
        tcg_set_insn_param(icount_start_insn, 2,
102
0
                           tcgv_i32_arg(tcg_constant_i32(num_insns)));
103
0
    }
104
105
0
    if (tcg_ctx->exitreq_label) {
106
0
        gen_set_label(tcg_ctx->exitreq_label);
107
0
        tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED);
108
0
    }
109
0
}
110
111
bool translator_is_same_page(const DisasContextBase *db, vaddr addr)
112
0
{
113
0
    return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0;
114
0
}
115
116
/*
117
 * Whether a direct jump may be chained to a destination outside the page
118
 * the TB started in.
119
 *
120
 * In user-only mode there are no page tables.  Every mmap, mprotect and
121
 * munmap goes through page_set_flags(), which calls tb_invalidate_phys_range()
122
 * whenever a change in flags so warrants, and tb_phys_invalidate() unlinks
123
 * incoming jumps.  A cross-page link is therefore broken whenever the
124
 * destination page's permissions change.
125
 *
126
 * What the same-page rule also provides is that execution cannot enter a page
127
 * without a TB lookup, and so without check_for_breakpoints(), which is what
128
 * makes a breakpoint set after a block was translated take effect.  Nothing
129
 * invalidates on breakpoint insertion, so a link established beforehand would
130
 * jump straight over it.  In user-only mode breakpoints only ever come from
131
 * gdb -- BP_CPU is g_assert_not_reached() there and the guest has no way to
132
 * ask for one -- and gdb has to be requested with -g before the first block
133
 * is translated, so a run that has no gdbstub can never acquire a breakpoint.
134
 */
135
static bool use_cross_page_goto_tb(void)
136
0
{
137
#ifdef CONFIG_USER_ONLY
138
    return !gdb_may_set_breakpoints();
139
#else
140
0
    return false;
141
0
#endif
142
0
}
143
144
bool translator_use_goto_tb(DisasContextBase *db, vaddr dest)
145
0
{
146
    /* Suppress goto_tb if requested. */
147
0
    if (tb_cflags(db->tb) & CF_NO_GOTO_TB) {
148
0
        return false;
149
0
    }
150
151
    /* Check for the dest on the same page as the start of the TB.  */
152
0
    return use_cross_page_goto_tb() || translator_is_same_page(db, dest);
153
0
}
154
155
void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
156
                     vaddr pc, void *host_pc, const TranslatorOps *ops,
157
                     DisasContextBase *db, TCGType addr_type)
158
0
{
159
0
    uint32_t cflags = tb_cflags(tb);
160
0
    TCGOp *icount_start_insn;
161
0
    bool plugin_enabled;
162
0
#ifndef CONFIG_USER_ONLY
163
0
    TCGOp *first_insn_start = NULL;
164
0
#endif
165
166
0
    tcg_ctx->addr_type = addr_type;
167
168
    /* Initialize DisasContext */
169
0
    db->tb = tb;
170
0
    db->pc_first = pc;
171
0
    db->pc_next = pc;
172
0
    db->pc_second_page = -1;
173
0
    db->is_jmp = DISAS_NEXT;
174
0
    db->num_insns = 0;
175
0
    db->max_insns = *max_insns;
176
0
    db->insn_start = NULL;
177
0
    db->fake_insn = false;
178
0
    db->host_addr[0] = host_pc;
179
0
    db->host_addr[1] = NULL;
180
0
    db->record_start = 0;
181
0
    db->record_len = 0;
182
0
    db->code_mmuidx = cpu_mmu_index(cpu, true);
183
184
0
    ops->init_disas_context(db, cpu);
185
0
    tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */
186
187
    /* Start translating.  */
188
0
    icount_start_insn = gen_tb_start(db, cflags);
189
0
    ops->tb_start(db, cpu);
190
0
    tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */
191
192
0
    plugin_enabled = plugin_gen_tb_start(cpu, db);
193
0
    db->plugin_enabled = plugin_enabled;
194
195
0
    while (true) {
196
0
        *max_insns = ++db->num_insns;
197
0
        ops->insn_start(db, cpu);
198
0
        db->insn_start = tcg_last_op();
199
0
#ifndef CONFIG_USER_ONLY
200
0
        if (first_insn_start == NULL) {
201
0
            first_insn_start = db->insn_start;
202
0
        }
203
0
#endif
204
0
        tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */
205
206
0
        if (plugin_enabled) {
207
0
            plugin_gen_insn_start(cpu, db);
208
0
        }
209
210
        /*
211
         * Disassemble one instruction.  The translate_insn hook should
212
         * update db->pc_next and db->is_jmp to indicate what should be
213
         * done next -- either exiting this loop or locate the start of
214
         * the next instruction.
215
         */
216
0
        ops->translate_insn(db, cpu);
217
218
        /*
219
         * We can't instrument after instructions that change control
220
         * flow although this only really affects post-load operations.
221
         *
222
         * Calling plugin_gen_insn_end() before we possibly stop translation
223
         * is important. Even if this ends up as dead code, plugin generation
224
         * needs to see a matching plugin_gen_insn_{start,end}() pair in order
225
         * to accurately track instrumented helpers that might access memory.
226
         */
227
0
        if (plugin_enabled) {
228
0
            plugin_gen_insn_end();
229
0
        }
230
231
        /* Stop translation if translate_insn so indicated.  */
232
0
        if (db->is_jmp != DISAS_NEXT) {
233
0
            break;
234
0
        }
235
236
        /* Stop translation if the output buffer is full,
237
           or we have executed all of the allowed instructions.  */
238
0
        if (tcg_op_buf_full() || db->num_insns >= db->max_insns) {
239
0
            db->is_jmp = DISAS_TOO_MANY;
240
0
            break;
241
0
        }
242
0
    }
243
244
    /* Emit code to exit the TB, as indicated by db->is_jmp.  */
245
0
    ops->tb_stop(db, cpu);
246
0
    gen_tb_end(tb, cflags, icount_start_insn, db->num_insns);
247
248
0
#ifndef CONFIG_USER_ONLY
249
    /*
250
     * Manage can_do_io for the translation block: set to false before
251
     * the first insn and set to true before the last insn.
252
     *
253
     * Nothing reads can_do_io in user-only builds.  There is no MMIO
254
     * there, and every reader (cputlb.c, watchpoint.c, icount) is in
255
     * system_ss, so skip the two stores per TB entirely.
256
     */
257
0
    if (db->num_insns == 1) {
258
0
        tcg_debug_assert(first_insn_start == db->insn_start);
259
0
    } else {
260
0
        tcg_debug_assert(first_insn_start != db->insn_start);
261
0
        tcg_ctx->emit_before_op = first_insn_start;
262
0
        set_can_do_io(db, false);
263
0
    }
264
0
    tcg_ctx->emit_before_op = db->insn_start;
265
0
    set_can_do_io(db, true);
266
0
    tcg_ctx->emit_before_op = NULL;
267
0
#endif
268
269
    /* May be used by disas_log or plugin callbacks. */
270
0
    tb->size = db->pc_next - db->pc_first;
271
0
    tb->icount = db->num_insns;
272
273
0
    if (plugin_enabled) {
274
0
        plugin_gen_tb_end(cpu, db->num_insns);
275
0
    }
276
277
0
    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
278
0
        && qemu_log_in_addr_range(db->pc_first)) {
279
0
        FILE *logfile = qemu_log_trylock();
280
0
        if (logfile) {
281
0
            fprintf(logfile, "----------------\n");
282
283
0
            if (!ops->disas_log ||
284
0
                !ops->disas_log(db, cpu, logfile)) {
285
0
                fprintf(logfile, "IN: %s\n", lookup_symbol(db->pc_first));
286
0
                target_disas(logfile, cpu, db);
287
0
            }
288
0
            fprintf(logfile, "\n");
289
0
            qemu_log_unlock(logfile);
290
0
        }
291
0
    }
292
0
}
293
294
static bool translator_ld(CPUArchState *env, DisasContextBase *db,
295
                          void *dest, vaddr pc, size_t len)
296
0
{
297
0
    TranslationBlock *tb = db->tb;
298
0
    vaddr last = pc + len - 1;
299
0
    void *host;
300
0
    vaddr base;
301
302
    /* Use slow path if first page is MMIO. */
303
0
    if (unlikely(tb_page_addr0(tb) == -1)) {
304
        /* We capped translation with first page MMIO in tb_gen_code. */
305
0
        tcg_debug_assert(db->max_insns == 1);
306
0
        return false;
307
0
    }
308
309
0
    host = db->host_addr[0];
310
0
    base = db->pc_first;
311
312
0
    if (likely(((base ^ last) & TARGET_PAGE_MASK) == 0)) {
313
        /* Entire read is from the first page. */
314
0
        goto do_read;
315
0
    }
316
317
0
    if (unlikely(((base ^ pc) & TARGET_PAGE_MASK) == 0)) {
318
        /*
319
         * Read begins on the first page and extends to the second.
320
         * The unaligned read is never atomic.
321
         */
322
0
        size_t len0 = -(pc | TARGET_PAGE_MASK);
323
0
        memcpy(dest, host + (pc - base), len0);
324
0
        pc += len0;
325
0
        dest += len0;
326
0
        len -= len0;
327
0
    }
328
329
    /*
330
     * The read must conclude on the second page and not extend to a third.
331
     *
332
     * TODO: This doesn't handle address space wraparound properly for
333
     * multi-byte reads, as we don't know the size of the address space here.
334
     * But if the target translator wraps pc to 0 itself, and issues aligned
335
     * reads, then this can work.
336
     */
337
0
    if (db->pc_second_page == -1) {
338
0
        db->pc_second_page = pc & TARGET_PAGE_MASK;
339
0
    } else {
340
0
        assert((pc & TARGET_PAGE_MASK) == db->pc_second_page);
341
0
    }
342
0
    assert((last & TARGET_PAGE_MASK) == db->pc_second_page);
343
0
    base = db->pc_second_page;
344
0
    host = db->host_addr[1];
345
346
0
    if (host == NULL) {
347
0
        tb_page_addr_t page0, old_page1, new_page1;
348
349
0
        new_page1 = get_page_addr_code_hostp(env, base, &db->host_addr[1]);
350
351
        /*
352
         * If the second page is MMIO, treat as if the first page
353
         * was MMIO as well, so that we do not cache the TB.
354
         */
355
0
        if (unlikely(new_page1 == -1)) {
356
0
            tb_unlock_pages(tb);
357
0
            tb_set_page_addr0(tb, -1);
358
            /* Require that this be the final insn. */
359
0
            db->max_insns = db->num_insns;
360
0
            return false;
361
0
        }
362
363
        /*
364
         * If this is not the first time around, and page1 matches,
365
         * then we already have the page locked.  Alternately, we're
366
         * not doing anything to prevent the PTE from changing, so
367
         * we might wind up with a different page, requiring us to
368
         * re-do the locking.
369
         */
370
0
        old_page1 = tb_page_addr1(tb);
371
0
        if (likely(new_page1 != old_page1)) {
372
0
            page0 = tb_page_addr0(tb);
373
0
            if (unlikely(old_page1 != -1)) {
374
0
                tb_unlock_page1(page0, old_page1);
375
0
            }
376
0
            tb_set_page_addr1(tb, new_page1);
377
0
            tb_lock_page1(page0, new_page1);
378
0
        }
379
0
        host = db->host_addr[1];
380
0
    }
381
382
0
 do_read:
383
    /*
384
     * Assume aligned reads should be atomic, if possible.
385
     * We're not in a position to jump out with EXCP_ATOMIC.
386
     */
387
0
    host += pc - base;
388
0
    switch (len) {
389
0
    case 2:
390
0
        if (QEMU_IS_ALIGNED(pc, 2)) {
391
0
            uint16_t t = qatomic_read((uint16_t *)host);
392
0
            stw_he_p(dest, t);
393
0
            return true;
394
0
        }
395
0
        break;
396
0
    case 4:
397
0
        if (QEMU_IS_ALIGNED(pc, 4)) {
398
0
            uint32_t t = qatomic_read((uint32_t *)host);
399
0
            stl_he_p(dest, t);
400
0
            return true;
401
0
        }
402
0
        break;
403
0
    case 8:
404
0
        if (QEMU_IS_ALIGNED(pc, 8)) {
405
0
            uint64_t t = qatomic_read((uint64_t *)host);
406
0
            stq_he_p(dest, t);
407
0
            return true;
408
0
        }
409
0
        break;
410
0
    }
411
    /* Unaligned or partial read from the second page is not atomic. */
412
0
    memcpy(dest, host, len);
413
0
    return true;
414
0
}
415
416
static void record_save(DisasContextBase *db, vaddr pc,
417
                        const void *from, int size)
418
0
{
419
0
    int offset;
420
421
    /*
422
     * In translator_ld, we verified that we touched no more than 2 pages,
423
     * but we did not verify that they were virtually contiguous.
424
     * Here, reimagine the two pages as virtually contiguous.
425
     */
426
0
    if (likely(((db->pc_first ^ pc) & TARGET_PAGE_MASK) == 0)) {
427
        /* first page */
428
        /* Do not record probes before the start of TB. */
429
0
        if (pc < db->pc_first) {
430
0
            return;
431
0
        }
432
0
        offset = pc - db->pc_first;
433
0
    } else {
434
0
        int first_page_end_offset = -(db->pc_first | TARGET_PAGE_MASK);
435
0
        assert(db->pc_second_page != -1);
436
0
        assert((pc & TARGET_PAGE_MASK) == db->pc_second_page);
437
0
        offset = pc - db->pc_second_page + first_page_end_offset;
438
0
    }
439
440
    /*
441
     * Either the first or second page may be I/O.  If it is the second,
442
     * then the first byte we need to record will be at a non-zero offset.
443
     * In either case, we should not need to record but a single insn.
444
     *
445
     * A read may re-read bytes that are already recorded: a target may
446
     * fetch a whole aligned word to decode an insn (e.g. riscv Ziccif),
447
     * then probe the following insn, which lies within that same word.
448
     * Such a read extends the record only by the bytes past its end.
449
     */
450
0
    if (db->record_len == 0) {
451
0
        db->record_start = offset;
452
0
        db->record_len = size;
453
0
    } else {
454
0
        int end = offset - db->record_start + size;
455
456
0
        assert(offset >= db->record_start);
457
0
        assert(offset <= db->record_start + db->record_len);
458
0
        assert(end <= sizeof(db->record));
459
0
        db->record_len = MAX(db->record_len, end);
460
0
    }
461
462
0
    memcpy(db->record + (offset - db->record_start), from, size);
463
0
}
464
465
size_t translator_st_len(const DisasContextBase *db)
466
0
{
467
0
    return db->fake_insn ? db->record_len : db->tb->size;
468
0
}
469
470
bool translator_st(const DisasContextBase *db, void *dest,
471
                   vaddr addr, size_t len)
472
0
{
473
0
    size_t offset, offset_end;
474
475
0
    if (addr < db->pc_first) {
476
0
        return false;
477
0
    }
478
0
    offset = addr - db->pc_first;
479
0
    offset_end = offset + len;
480
0
    if (offset_end > translator_st_len(db)) {
481
0
        return false;
482
0
    }
483
484
0
    if (!db->fake_insn) {
485
0
        size_t offset_page1 = -(db->pc_first | TARGET_PAGE_MASK);
486
487
        /* Get all the bytes from the first page. */
488
0
        if (db->host_addr[0]) {
489
0
            if (offset_end <= offset_page1) {
490
0
                memcpy(dest, db->host_addr[0] + offset, len);
491
0
                return true;
492
0
            }
493
0
            if (offset < offset_page1) {
494
0
                size_t len0 = offset_page1 - offset;
495
0
                memcpy(dest, db->host_addr[0] + offset, len0);
496
0
                offset += len0;
497
0
                dest += len0;
498
0
            }
499
0
        }
500
501
        /* Get any bytes from the second page. */
502
0
        if (db->host_addr[1] && offset >= offset_page1) {
503
0
            memcpy(dest, db->host_addr[1] + (offset - offset_page1),
504
0
                   offset_end - offset);
505
0
            return true;
506
0
        }
507
0
    }
508
509
    /* Else get recorded bytes. */
510
0
    if (db->record_len != 0 &&
511
0
        offset >= db->record_start &&
512
0
        offset_end <= db->record_start + db->record_len) {
513
0
        memcpy(dest, db->record + (offset - db->record_start),
514
0
               offset_end - offset);
515
0
        return true;
516
0
    }
517
0
    return false;
518
0
}
519
520
uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, vaddr pc)
521
0
{
522
0
    uint8_t val;
523
524
0
    if (!translator_ld(env, db, &val, pc, sizeof(val))) {
525
0
        MemOpIdx oi = make_memop_idx(MO_UB, db->code_mmuidx);
526
0
        val = cpu_ldb_code_mmu(env, pc, oi, 0);
527
0
        record_save(db, pc, &val, sizeof(val));
528
0
    }
529
0
    return val;
530
0
}
531
532
uint16_t translator_lduw_end(CPUArchState *env, DisasContextBase *db,
533
                             vaddr pc, MemOp endian)
534
0
{
535
0
    uint16_t val;
536
537
0
    if (!translator_ld(env, db, &val, pc, sizeof(val))) {
538
0
        MemOpIdx oi = make_memop_idx(MO_UW, db->code_mmuidx);
539
0
        val = cpu_ldw_code_mmu(env, pc, oi, 0);
540
0
        record_save(db, pc, &val, sizeof(val));
541
0
    }
542
0
    if (endian & MO_BSWAP) {
543
0
        val = bswap16(val);
544
0
    }
545
0
    return val;
546
0
}
547
548
uint32_t translator_ldl_end(CPUArchState *env, DisasContextBase *db,
549
                            vaddr pc, MemOp endian)
550
0
{
551
0
    uint32_t val;
552
553
0
    if (!translator_ld(env, db, &val, pc, sizeof(val))) {
554
0
        MemOpIdx oi = make_memop_idx(MO_UL, db->code_mmuidx);
555
0
        val = cpu_ldl_code_mmu(env, pc, oi, 0);
556
0
        record_save(db, pc, &val, sizeof(val));
557
0
    }
558
0
    if (endian & MO_BSWAP) {
559
0
        val = bswap32(val);
560
0
    }
561
0
    return val;
562
0
}
563
564
uint64_t translator_ldq_end(CPUArchState *env, DisasContextBase *db,
565
                            vaddr pc, MemOp endian)
566
0
{
567
0
    uint64_t val;
568
569
0
    if (!translator_ld(env, db, &val, pc, sizeof(val))) {
570
0
        MemOpIdx oi = make_memop_idx(MO_UQ, db->code_mmuidx);
571
0
        val = cpu_ldq_code_mmu(env, pc, oi, 0);
572
0
        record_save(db, pc, &val, sizeof(val));
573
0
    }
574
0
    if (endian & MO_BSWAP) {
575
0
        val = bswap64(val);
576
0
    }
577
0
    return val;
578
0
}
579
580
void translator_fake_ld(DisasContextBase *db, const void *data, size_t len)
581
0
{
582
    db->fake_insn = true;
583
0
    record_save(db, db->pc_first, data, len);
584
0
}