/src/wasmtime/cranelift/codegen/src/legalizer/mod.rs
Line | Count | Source |
1 | | //! Legalize instructions. |
2 | | //! |
3 | | //! A legal instruction is one that can be mapped directly to a machine code instruction for the |
4 | | //! target ISA. The `legalize_function()` function takes as input any function and transforms it |
5 | | //! into an equivalent function using only legal instructions. |
6 | | //! |
7 | | //! The characteristics of legal instructions depend on the target ISA, so any given instruction |
8 | | //! can be legal for one ISA and illegal for another. |
9 | | //! |
10 | | //! Besides transforming instructions, the legalizer also fills out the `function.encodings` map |
11 | | //! which provides a legal encoding recipe for every instruction. |
12 | | //! |
13 | | //! The legalizer does not deal with register allocation constraints. These constraints are derived |
14 | | //! from the encoding recipes, and solved later by the register allocator. |
15 | | |
16 | | use crate::cursor::{Cursor, FuncCursor}; |
17 | | use crate::ir::{self, InstBuilder, InstructionData, MemFlagsData, Value}; |
18 | | use crate::isa::TargetIsa; |
19 | | use crate::trace; |
20 | | use cranelift_entity::EntitySet; |
21 | | use smallvec::SmallVec; |
22 | | |
23 | | mod branch_to_trap; |
24 | | mod globalvalue; |
25 | | |
26 | | use self::branch_to_trap::BranchToTrap; |
27 | | use self::globalvalue::expand_global_value; |
28 | | |
29 | | /// A command describing how the walk over instructions should proceed. |
30 | | enum WalkCommand { |
31 | | /// Continue walking to the next instruction, if any. |
32 | | Continue, |
33 | | /// Revisit the current instruction (presumably because it was legalized |
34 | | /// into a new instruction that may also require further legalization). |
35 | | Revisit, |
36 | | } |
37 | | |
38 | | /// A simple, naive backwards walk over every instruction in every block in the |
39 | | /// function's layout. |
40 | | /// |
41 | | /// This does not guarantee any kind of reverse post-order visitation or |
42 | | /// anything like that, it is just iterating over blocks in reverse layout |
43 | | /// order, not any kind of control-flow graph visitation order. |
44 | | /// |
45 | | /// The `f` visitor closure controls how the walk proceeds via its `WalkCommand` |
46 | | /// result. |
47 | 2.80M | fn backward_walk( |
48 | 2.80M | func: &mut ir::Function, |
49 | 2.80M | mut f: impl FnMut(&mut ir::Function, ir::Block, ir::Inst) -> WalkCommand, |
50 | 2.80M | ) { |
51 | 2.80M | let mut pos = FuncCursor::new(func); |
52 | 36.5M | while let Some(block) = pos.prev_block() { |
53 | | let mut prev_pos; |
54 | 341M | while let Some(inst) = { |
55 | 341M | prev_pos = pos.position(); |
56 | 341M | pos.prev_inst() |
57 | 341M | } { |
58 | 308M | match f(pos.func, block, inst) { |
59 | 265M | WalkCommand::Continue => continue, |
60 | 42.7M | WalkCommand::Revisit => pos.set_position(prev_pos), |
61 | | } |
62 | | } |
63 | | } |
64 | 2.80M | } |
65 | | |
66 | | /// Perform a simple legalization by expansion of the function, without |
67 | | /// platform-specific transforms. |
68 | 2.80M | pub fn simple_legalize(func: &mut ir::Function, isa: &dyn TargetIsa) { |
69 | 2.80M | trace!("Pre-legalization function:\n{}", func.display()); |
70 | | |
71 | 2.80M | let mut branch_to_trap = BranchToTrap::default(); |
72 | | |
73 | | // We walk the IR backwards because in practice, given the way that |
74 | | // frontends tend to produce CLIF, this means we will visit in roughly |
75 | | // reverse post order, which is helpful for getting the most optimizations |
76 | | // out of the `branch-to-trap` pass that we can (it must analyze trapping |
77 | | // blocks before it can rewrite branches to them) but the order does not |
78 | | // actually affect correctness. |
79 | 308M | backward_walk(func, |func, block, inst| match func.dfg.insts[inst] { |
80 | | InstructionData::Trap { |
81 | | opcode: ir::Opcode::Trap, |
82 | | code: _, |
83 | | } => { |
84 | 4.93M | branch_to_trap.analyze_trapping_block(func, block); |
85 | 4.93M | WalkCommand::Continue |
86 | | } |
87 | | InstructionData::Brif { |
88 | | opcode: ir::Opcode::Brif, |
89 | 11.5M | arg, |
90 | 11.5M | blocks, |
91 | | } => { |
92 | 11.5M | branch_to_trap.process_brif(func, inst, arg, blocks); |
93 | 11.5M | WalkCommand::Continue |
94 | | } |
95 | | |
96 | | InstructionData::UnaryGlobalValue { |
97 | | opcode: ir::Opcode::GlobalValue, |
98 | 42.7M | global_value, |
99 | 42.7M | } => expand_global_value(inst, func, isa, global_value), |
100 | | |
101 | | InstructionData::StackLoad { |
102 | | opcode: ir::Opcode::StackLoad, |
103 | 2.77M | stack_slot, |
104 | 2.77M | offset, |
105 | 2.77M | } => expand_stack_load(isa, func, inst, stack_slot, offset), |
106 | | |
107 | | InstructionData::StackStore { |
108 | | opcode: ir::Opcode::StackStore, |
109 | 232k | arg, |
110 | 232k | stack_slot, |
111 | 232k | offset, |
112 | 232k | } => expand_stack_store(isa, func, inst, arg, stack_slot, offset), |
113 | | |
114 | | InstructionData::DynamicStackLoad { |
115 | | opcode: ir::Opcode::DynamicStackLoad, |
116 | 0 | dynamic_stack_slot, |
117 | 0 | } => expand_dynamic_stack_load(isa, func, inst, dynamic_stack_slot), |
118 | | |
119 | | InstructionData::DynamicStackStore { |
120 | | opcode: ir::Opcode::DynamicStackStore, |
121 | 0 | arg, |
122 | 0 | dynamic_stack_slot, |
123 | 0 | } => expand_dynamic_stack_store(isa, func, inst, arg, dynamic_stack_slot), |
124 | | |
125 | 31.1M | InstructionData::Binary { opcode, args } => expand_binary(func, inst, opcode, args), |
126 | | |
127 | 214M | _ => WalkCommand::Continue, |
128 | 308M | }); |
129 | | |
130 | 2.80M | trace!("Post-legalization function:\n{}", func.display()); |
131 | 2.80M | } |
132 | | |
133 | 31.1M | fn expand_binary( |
134 | 31.1M | func: &mut ir::Function, |
135 | 31.1M | inst: ir::Inst, |
136 | 31.1M | opcode: ir::Opcode, |
137 | 31.1M | args: [ir::Value; 2], |
138 | 31.1M | ) -> WalkCommand { |
139 | 31.1M | let mut pos = FuncCursor::new(func); |
140 | 31.1M | pos.goto_inst(inst); |
141 | | |
142 | | // Legalize the fused bitwise-plus-not instructions into simpler |
143 | | // instructions to assist with optimizations. Lowering will pattern match |
144 | | // this sequence regardless when architectures support the instruction |
145 | | // natively. |
146 | 31.1M | match opcode { |
147 | 16.7k | ir::Opcode::BandNot => { |
148 | 16.7k | let neg = pos.ins().bnot(args[1]); |
149 | 16.7k | pos.func.replace(inst).band(args[0], neg); |
150 | 16.7k | } |
151 | 12.7k | ir::Opcode::BorNot => { |
152 | 12.7k | let neg = pos.ins().bnot(args[1]); |
153 | 12.7k | pos.func.replace(inst).bor(args[0], neg); |
154 | 12.7k | } |
155 | 18.5k | ir::Opcode::BxorNot => { |
156 | 18.5k | let neg = pos.ins().bnot(args[1]); |
157 | 18.5k | pos.func.replace(inst).bxor(args[0], neg); |
158 | 18.5k | } |
159 | 31.0M | _ => {} |
160 | | } |
161 | | |
162 | 31.1M | WalkCommand::Continue |
163 | 31.1M | } |
164 | | |
165 | 0 | fn expand_dynamic_stack_store( |
166 | 0 | isa: &dyn TargetIsa, |
167 | 0 | func: &mut ir::Function, |
168 | 0 | inst: ir::Inst, |
169 | 0 | arg: Value, |
170 | 0 | dynamic_stack_slot: ir::DynamicStackSlot, |
171 | 0 | ) -> WalkCommand { |
172 | 0 | let mut pos = FuncCursor::new(func); |
173 | 0 | pos.goto_inst(inst); |
174 | 0 | pos.use_srcloc(inst); |
175 | | |
176 | 0 | let vector_ty = pos.func.dfg.value_type(arg); |
177 | 0 | assert!(vector_ty.is_dynamic_vector()); |
178 | | |
179 | 0 | let addr_ty = isa.pointer_type(); |
180 | 0 | let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot); |
181 | | |
182 | 0 | let mut mflags = MemFlagsData::new(); |
183 | | // Stack slots are required to be accessible and aligned. |
184 | 0 | mflags.set_notrap(); |
185 | 0 | mflags.set_aligned(); |
186 | | |
187 | 0 | pos.func.replace(inst).store(mflags, arg, addr, 0); |
188 | | |
189 | 0 | WalkCommand::Continue |
190 | 0 | } |
191 | | |
192 | 0 | fn expand_dynamic_stack_load( |
193 | 0 | isa: &dyn TargetIsa, |
194 | 0 | func: &mut ir::Function, |
195 | 0 | inst: ir::Inst, |
196 | 0 | dynamic_stack_slot: ir::DynamicStackSlot, |
197 | 0 | ) -> WalkCommand { |
198 | 0 | let mut pos = FuncCursor::new(func).at_inst(inst); |
199 | 0 | pos.use_srcloc(inst); |
200 | | |
201 | 0 | let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst)); |
202 | 0 | assert!(ty.is_dynamic_vector()); |
203 | | |
204 | 0 | let addr_ty = isa.pointer_type(); |
205 | 0 | let addr = pos.ins().dynamic_stack_addr(addr_ty, dynamic_stack_slot); |
206 | | |
207 | | // Stack slots are required to be accessible and aligned. |
208 | 0 | let mflags = MemFlagsData::trusted(); |
209 | | |
210 | 0 | pos.func.replace(inst).load(ty, mflags, addr, 0); |
211 | | |
212 | 0 | WalkCommand::Continue |
213 | 0 | } |
214 | | |
215 | 232k | fn expand_stack_store( |
216 | 232k | isa: &dyn TargetIsa, |
217 | 232k | func: &mut ir::Function, |
218 | 232k | inst: ir::Inst, |
219 | 232k | arg: ir::Value, |
220 | 232k | stack_slot: ir::StackSlot, |
221 | 232k | offset: ir::immediates::Offset32, |
222 | 232k | ) -> WalkCommand { |
223 | 232k | let mut pos = FuncCursor::new(func).at_inst(inst); |
224 | 232k | pos.use_srcloc(inst); |
225 | | |
226 | 232k | let addr_ty = isa.pointer_type(); |
227 | 232k | let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset); |
228 | | |
229 | | // Stack slots are required to be accessible. |
230 | | // We can't currently ensure that they are aligned. |
231 | 232k | let mut mflags = MemFlagsData::new(); |
232 | 232k | mflags.set_notrap(); |
233 | | |
234 | 232k | pos.func.replace(inst).store(mflags, arg, addr, 0); |
235 | | |
236 | 232k | WalkCommand::Continue |
237 | 232k | } |
238 | | |
239 | 2.77M | fn expand_stack_load( |
240 | 2.77M | isa: &dyn TargetIsa, |
241 | 2.77M | func: &mut ir::Function, |
242 | 2.77M | inst: ir::Inst, |
243 | 2.77M | stack_slot: ir::StackSlot, |
244 | 2.77M | offset: ir::immediates::Offset32, |
245 | 2.77M | ) -> WalkCommand { |
246 | 2.77M | let mut pos = FuncCursor::new(func).at_inst(inst); |
247 | 2.77M | pos.use_srcloc(inst); |
248 | | |
249 | 2.77M | let ty = pos.func.dfg.value_type(pos.func.dfg.first_result(inst)); |
250 | 2.77M | let addr_ty = isa.pointer_type(); |
251 | | |
252 | 2.77M | let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset); |
253 | | |
254 | | // Stack slots are required to be accessible. |
255 | | // We can't currently ensure that they are aligned. |
256 | 2.77M | let mut mflags = MemFlagsData::new(); |
257 | 2.77M | mflags.set_notrap(); |
258 | | |
259 | 2.77M | pos.func.replace(inst).load(ty, mflags, addr, 0); |
260 | | |
261 | 2.77M | WalkCommand::Continue |
262 | 2.77M | } |