Coverage Report

Created: 2026-06-07 07:42

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/wasmtime/cranelift/codegen/src/legalizer/mod.rs
Line
Count
Source
1
//! Legalize instructions.
2
//!
3
//! A legal instruction is one that can be mapped directly to a machine code instruction for the
4
//! target ISA. The `legalize_function()` function takes as input any function and transforms it
5
//! into an equivalent function using only legal instructions.
6
//!
7
//! The characteristics of legal instructions depend on the target ISA, so any given instruction
8
//! can be legal for one ISA and illegal for another.
9
//!
10
//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
11
//! which provides a legal encoding recipe for every instruction.
12
//!
13
//! The legalizer does not deal with register allocation constraints. These constraints are derived
14
//! from the encoding recipes, and solved later by the register allocator.
15
16
use crate::cursor::{Cursor, FuncCursor};
17
use crate::ir::{self, InstBuilder, InstructionData, MemFlagsData, Value};
18
use crate::isa::TargetIsa;
19
use crate::trace;
20
use cranelift_entity::EntitySet;
21
use smallvec::SmallVec;
22
23
mod branch_to_trap;
24
mod globalvalue;
25
26
use self::branch_to_trap::BranchToTrap;
27
use self::globalvalue::expand_global_value;
28
29
/// A command describing how the walk over instructions should proceed.
30
enum WalkCommand {
31
    /// Continue walking to the next instruction, if any.
32
    Continue,
33
    /// Revisit the current instruction (presumably because it was legalized
34
    /// into a new instruction that may also require further legalization).
35
    Revisit,
36
}
37
38
/// A simple, naive backwards walk over every instruction in every block in the
39
/// function's layout.
40
///
41
/// This does not guarantee any kind of reverse post-order visitation or
42
/// anything like that, it is just iterating over blocks in reverse layout
43
/// order, not any kind of control-flow graph visitation order.
44
///
45
/// The `f` visitor closure controls how the walk proceeds via its `WalkCommand`
46
/// result.
47
2.80M
fn backward_walk(
48
2.80M
    func: &mut ir::Function,
49
2.80M
    mut f: impl FnMut(&mut ir::Function, ir::Block, ir::Inst) -> WalkCommand,
50
2.80M
) {
51
2.80M
    let mut pos = FuncCursor::new(func);
52
36.5M
    while let Some(block) = pos.prev_block() {
53
        let mut prev_pos;
54
341M
        while let Some(inst) = {
55
341M
            prev_pos = pos.position();
56
341M
            pos.prev_inst()
57
341M
        } {
58
308M
            match f(pos.func, block, inst) {
59
265M
                WalkCommand::Continue => continue,
60
42.7M
                WalkCommand::Revisit => pos.set_position(prev_pos),
61
            }
62
        }
63
    }
64
2.80M
}
65
66
/// Perform a simple legalization by expansion of the function, without
67
/// platform-specific transforms.
68
2.80M
pub fn simple_legalize(func: &mut ir::Function, isa: &dyn TargetIsa) {
69
2.80M
    trace!("Pre-legalization function:\n{}", func.display());
70
71
2.80M
    let mut branch_to_trap = BranchToTrap::default();
72
73
    // We walk the IR backwards because in practice, given the way that
74
    // frontends tend to produce CLIF, this means we will visit in roughly
75
    // reverse post order, which is helpful for getting the most optimizations
76
    // out of the `branch-to-trap` pass that we can (it must analyze trapping
77
    // blocks before it can rewrite branches to them) but the order does not
78
    // actually affect correctness.
79
308M
    backward_walk(func, |func, block, inst| match func.dfg.insts[inst] {
80
        InstructionData::Trap {
81
            opcode: ir::Opcode::Trap,
82
            code: _,
83
        } => {
84
4.93M
            branch_to_trap.analyze_trapping_block(func, block);
85
4.93M
            WalkCommand::Continue
86
        }
87
        InstructionData::Brif {
88
            opcode: ir::Opcode::Brif,
89
11.5M
            arg,
90
11.5M
            blocks,
91
        } => {
92
11.5M
            branch_to_trap.process_brif(func, inst, arg, blocks);
93
11.5M
            WalkCommand::Continue
94
        }
95
96
        InstructionData::UnaryGlobalValue {
97
            opcode: ir::Opcode::GlobalValue,
98
42.7M
            global_value,
99
42.7M
        } => expand_global_value(inst, func, isa, global_value),
100
101
        InstructionData::StackLoad {
102
            opcode: ir::Opcode::StackLoad,
103
2.77M
            stack_slot,
104
2.77M
            offset,
105
2.77M
        } => expand_stack_load(isa, func, inst, stack_slot, offset),
106
107
        InstructionData::StackStore {
108
            opcode: ir::Opcode::StackStore,
109
232k
            arg,
110
232k
            stack_slot,
111
232k
            offset,
112
232k
        } => expand_stack_store(isa, func, inst, arg, stack_slot, offset),
113
114
        InstructionData::DynamicStackLoad {
115
            opcode: ir::Opcode::DynamicStackLoad,
116
0
            dynamic_stack_slot,
117
0
        } => expand_dynamic_stack_load(isa, func, inst, dynamic_stack_slot),
118
119
        InstructionData::DynamicStackStore {
120
            opcode: ir::Opcode::DynamicStackStore,
121
0
            arg,
122
0
            dynamic_stack_slot,
123
0
        } => expand_dynamic_stack_store(isa, func, inst, arg, dynamic_stack_slot),
124
125
31.1M
        InstructionData::Binary { opcode, args } => expand_binary(func, inst, opcode, args),
126
127
214M
        _ => WalkCommand::Continue,
128
308M
    });
129
130
2.80M
    trace!("Post-legalization function:\n{}", func.display());
131
2.80M
}
132
133
31.1M
fn expand_binary(
134
31.1M
    func: &mut ir::Function,
135
31.1M
    inst: ir::Inst,
136
31.1M
    opcode: ir::Opcode,
137
31.1M
    args: [ir::Value; 2],
138
31.1M
) -> WalkCommand {
139
31.1M
    let mut pos = FuncCursor::new(func);
140
31.1M
    pos.goto_inst(inst);
141
142
    // Legalize the fused bitwise-plus-not instructions into simpler
143
    // instructions to assist with optimizations. Lowering will pattern match
144
    // this sequence regardless when architectures support the instruction
145
    // natively.
146
31.1M
    match opcode {
147
16.7k
        ir::Opcode::BandNot => {
148
16.7k
            let neg = pos.ins().bnot(args[1]);
149
16.7k
            pos.func.replace(inst).band(args[0], neg);
150
16.7k
        }
151
12.7k
        ir::Opcode::BorNot => {
152
12.7k
            let neg = pos.ins().bnot(args[1]);
153
12.7k
            pos.func.replace(inst).bor(args[0], neg);
154
12.7k
        }
155
18.5k
        ir::Opcode::BxorNot => {
156
18.5k
            let neg = pos.ins().bnot(args[1]);
157
18.5k
            pos.func.replace(inst).bxor(args[0], neg);
158
18.5k
        }
159
31.0M
        _ => {}
160
    }
161
162
31.1M
    WalkCommand::Continue
163
31.1M
}
164
165
0
fn expand_dynamic_stack_store(
166
0
    isa: &dyn TargetIsa,
167
0
    func: &mut ir::Function,
168
0
    inst: ir::Inst,
169
0
    arg: Value,
170
0
    dynamic_stack_slot: ir::DynamicStackSlot,
171
0
) -> WalkCommand {
172
0
    let mut pos = FuncCursor::new(func);
173
0
    pos.goto_inst(inst);
174
0
    pos.use_srcloc(inst);
175
176
0
    let vector_ty = pos.func.dfg.value_type(arg);
177
0
    assert!(vector_ty.is_dynamic_vector());
178
179
0
    let addr_ty = isa.pointer_type();
180
0
    let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot);
181
182
0
    let mut mflags = MemFlagsData::new();
183
    // Stack slots are required to be accessible and aligned.
184
0
    mflags.set_notrap();
185
0
    mflags.set_aligned();
186
187
0
    pos.func.replace(inst).store(mflags, arg, addr, 0);
188
189
0
    WalkCommand::Continue
190
0
}
191
192
0
fn expand_dynamic_stack_load(
193
0
    isa: &dyn TargetIsa,
194
0
    func: &mut ir::Function,
195
0
    inst: ir::Inst,
196
0
    dynamic_stack_slot: ir::DynamicStackSlot,
197
0
) -> WalkCommand {
198
0
    let mut pos = FuncCursor::new(func).at_inst(inst);
199
0
    pos.use_srcloc(inst);
200
201
0
    let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst));
202
0
    assert!(ty.is_dynamic_vector());
203
204
0
    let addr_ty = isa.pointer_type();
205
0
    let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot);
206
207
    // Stack slots are required to be accessible and aligned.
208
0
    let mflags = MemFlagsData::trusted();
209
210
0
    pos.func.replace(inst).load(ty, mflags, addr, 0);
211
212
0
    WalkCommand::Continue
213
0
}
214
215
232k
fn expand_stack_store(
216
232k
    isa: &dyn TargetIsa,
217
232k
    func: &mut ir::Function,
218
232k
    inst: ir::Inst,
219
232k
    arg: ir::Value,
220
232k
    stack_slot: ir::StackSlot,
221
232k
    offset: ir::immediates::Offset32,
222
232k
) -> WalkCommand {
223
232k
    let mut pos = FuncCursor::new(func).at_inst(inst);
224
232k
    pos.use_srcloc(inst);
225
226
232k
    let addr_ty = isa.pointer_type();
227
232k
    let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
228
229
    // Stack slots are required to be accessible.
230
    // We can't currently ensure that they are aligned.
231
232k
    let mut mflags = MemFlagsData::new();
232
232k
    mflags.set_notrap();
233
234
232k
    pos.func.replace(inst).store(mflags, arg, addr, 0);
235
236
232k
    WalkCommand::Continue
237
232k
}
238
239
2.77M
fn expand_stack_load(
240
2.77M
    isa: &dyn TargetIsa,
241
2.77M
    func: &mut ir::Function,
242
2.77M
    inst: ir::Inst,
243
2.77M
    stack_slot: ir::StackSlot,
244
2.77M
    offset: ir::immediates::Offset32,
245
2.77M
) -> WalkCommand {
246
2.77M
    let mut pos = FuncCursor::new(func).at_inst(inst);
247
2.77M
    pos.use_srcloc(inst);
248
249
2.77M
    let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst));
250
2.77M
    let addr_ty = isa.pointer_type();
251
252
2.77M
    let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
253
254
    // Stack slots are required to be accessible.
255
    // We can't currently ensure that they are aligned.
256
2.77M
    let mut mflags = MemFlagsData::new();
257
2.77M
    mflags.set_notrap();
258
259
2.77M
    pos.func.replace(inst).load(ty, mflags, addr, 0);
260
261
2.77M
    WalkCommand::Continue
262
2.77M
}