/src/wasmtime/cranelift/codegen/src/legalizer/split.rs
Line | Count | Source (jump to first uncovered line) |
1 | | //! Value splitting. |
2 | | //! |
3 | | //! Some value types are too large to fit in registers, so they need to be split into smaller parts |
4 | | //! that the ISA can operate on. There's two dimensions of splitting, represented by two |
5 | | //! complementary instruction pairs: |
6 | | //! |
7 | | //! - `isplit` and `iconcat` for splitting integer types into smaller integers. |
8 | | //! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same |
9 | | //! lane types. |
10 | | //! |
11 | | //! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably |
12 | | //! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers. |
13 | | //! This breakdown is handled by the ABI lowering. |
14 | | //! |
15 | | //! When legalizing a single instruction, it is wrapped in splits and concatenations: |
16 | | //! |
17 | | //! ```clif |
18 | | //! v1 = bxor.i64 v2, v3 |
19 | | //! ``` |
20 | | //! |
21 | | //! becomes: |
22 | | //! |
23 | | //! ```clif |
24 | | //! v20, v21 = isplit v2 |
25 | | //! v30, v31 = isplit v3 |
26 | | //! v10 = bxor.i32 v20, v30 |
27 | | //! v11 = bxor.i32 v21, v31 |
28 | | //! v1 = iconcat v10, v11 |
29 | | //! ``` |
30 | | //! |
31 | | //! This local expansion approach still leaves the original `i64` values in the code as operands on |
32 | | //! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as |
33 | | //! values are constantly split and concatenated. |
34 | | //! |
35 | | //! # Optimized splitting |
36 | | //! |
37 | | //! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value, |
38 | | //! first check if the value is defined by the corresponding concatenation. If so, then just use |
39 | | //! the two concatenation inputs directly: |
40 | | //! |
41 | | //! ```clif |
42 | | //! v4 = iadd_imm.i64 v1, 1 |
43 | | //! ``` |
44 | | //! |
45 | | //! becomes, using the expanded code from above: |
46 | | //! |
47 | | //! ```clif |
48 | | //! v40, v5 = iadd_imm_cout.i32 v10, 1 |
49 | | //! v6 = bint.i32 |
50 | | //! v41 = iadd.i32 v11, v6 |
51 | | //! v4 = iconcat v40, v41 |
52 | | //! ``` |
53 | | //! |
54 | | //! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they |
55 | | //! can be trivially deleted by a dead code elimination pass. |
56 | | //! |
57 | | //! # block arguments |
58 | | //! |
59 | | //! If all instructions that produce an `i64` value are legalized as above, we will eventually end |
60 | | //! up with no `i64` values anywhere, except for block arguments. We can work around this by |
61 | | //! iteratively splitting block arguments too. That should leave us with no illegal value types |
62 | | //! anywhere. |
63 | | //! |
64 | | //! It is possible to have circular dependencies of block arguments that are never used by any real |
65 | | //! instructions. These loops will remain in the program. |
66 | | |
67 | | use crate::cursor::{Cursor, CursorPosition, FuncCursor}; |
68 | | use crate::flowgraph::{BlockPredecessor, ControlFlowGraph}; |
69 | | use crate::ir::{self, Block, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef}; |
70 | | use alloc::vec::Vec; |
71 | | use core::iter; |
72 | | use smallvec::SmallVec; |
73 | | |
74 | | /// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values |
75 | | /// if possible. |
76 | | pub fn isplit( |
77 | | func: &mut ir::Function, |
78 | | cfg: &ControlFlowGraph, |
79 | | pos: CursorPosition, |
80 | | srcloc: ir::SourceLoc, |
81 | | value: Value, |
82 | | ) -> (Value, Value) { |
83 | | split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat) |
84 | | } |
85 | | |
86 | | /// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if |
87 | | /// possible. |
88 | | pub fn vsplit( |
89 | | func: &mut ir::Function, |
90 | | cfg: &ControlFlowGraph, |
91 | | pos: CursorPosition, |
92 | | srcloc: ir::SourceLoc, |
93 | | value: Value, |
94 | | ) -> (Value, Value) { |
95 | | split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat) |
96 | | } |
97 | | |
98 | | /// After splitting a block argument, we need to go back and fix up all of the predecessor |
99 | | /// instructions. This is potentially a recursive operation, but we don't implement it recursively |
100 | | /// since that could use up too muck stack. |
101 | | /// |
102 | | /// Instead, the repairs are deferred and placed on a work list in stack form. |
103 | | struct Repair { |
104 | | concat: Opcode, |
105 | | // The argument type after splitting. |
106 | | split_type: Type, |
107 | | // The destination block whose arguments have been split. |
108 | | block: Block, |
109 | | // Number of the original block argument which has been replaced by the low part. |
110 | | num: usize, |
111 | | // Number of the new block argument which represents the high part after the split. |
112 | | hi_num: usize, |
113 | | } |
114 | | |
115 | | /// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode. |
116 | | fn split_any( |
117 | | func: &mut ir::Function, |
118 | | cfg: &ControlFlowGraph, |
119 | | pos: CursorPosition, |
120 | | srcloc: ir::SourceLoc, |
121 | | value: Value, |
122 | | concat: Opcode, |
123 | | ) -> (Value, Value) { |
124 | | let mut repairs = Vec::new(); |
125 | | let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc); |
126 | | let result = split_value(pos, value, concat, &mut repairs); |
127 | | |
128 | | perform_repairs(pos, cfg, repairs); |
129 | | |
130 | | result |
131 | | } |
132 | | |
133 | 4.78M | pub fn split_block_params(func: &mut ir::Function, cfg: &ControlFlowGraph, block: Block) { |
134 | 4.78M | let pos = &mut FuncCursor::new(func).at_top(block); |
135 | 4.78M | let block_params = pos.func.dfg.block_params(block); |
136 | 4.78M | |
137 | 4.78M | // Add further splittable types here. |
138 | 9.69M | fn type_requires_splitting(ty: Type) -> bool { |
139 | 9.69M | ty == ir::types::I128 |
140 | 9.69M | } |
141 | 4.78M | |
142 | 4.78M | // A shortcut. If none of the param types require splitting, exit now. This helps because |
143 | 4.78M | // the loop below necessarily has to copy the block params into a new vector, so it's better to |
144 | 4.78M | // avoid doing so when possible. |
145 | 4.78M | if !block_params |
146 | 4.78M | .iter() |
147 | 9.69M | .any(|block_param| type_requires_splitting(pos.func.dfg.value_type(*block_param))) |
148 | | { |
149 | 4.78M | return; |
150 | 0 | } |
151 | 0 |
|
152 | 0 | let mut repairs = Vec::new(); |
153 | 0 | for (num, block_param) in block_params.to_vec().into_iter().enumerate() { |
154 | 0 | if !type_requires_splitting(pos.func.dfg.value_type(block_param)) { |
155 | | continue; |
156 | 0 | } |
157 | 0 |
|
158 | 0 | split_block_param(pos, block, num, block_param, Opcode::Iconcat, &mut repairs); |
159 | | } |
160 | | |
161 | 0 | perform_repairs(pos, cfg, repairs); |
162 | 4.78M | } |
163 | | |
164 | 0 | fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec<Repair>) { |
165 | | // We have split the value requested, and now we may need to fix some block predecessors. |
166 | 0 | while let Some(repair) = repairs.pop() { |
167 | 0 | for BlockPredecessor { inst, .. } in cfg.pred_iter(repair.block) { |
168 | 0 | let branch_opc = pos.func.dfg[inst].opcode(); |
169 | | debug_assert!( |
170 | | branch_opc.is_branch(), |
171 | | "Predecessor not a branch: {}", |
172 | | pos.func.dfg.display_inst(inst, None) |
173 | | ); |
174 | 0 | let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments(); |
175 | 0 | let mut args = pos.func.dfg[inst] |
176 | 0 | .take_value_list() |
177 | 0 | .expect("Branches must have value lists."); |
178 | 0 | let num_args = args.len(&pos.func.dfg.value_lists); |
179 | 0 | // Get the old value passed to the block argument we're repairing. |
180 | 0 | let old_arg = args |
181 | 0 | .get(num_fixed_args + repair.num, &pos.func.dfg.value_lists) |
182 | 0 | .expect("Too few branch arguments"); |
183 | 0 |
|
184 | 0 | // It's possible that the CFG's predecessor list has duplicates. Detect them here. |
185 | 0 | if pos.func.dfg.value_type(old_arg) == repair.split_type { |
186 | 0 | pos.func.dfg[inst].put_value_list(args); |
187 | | continue; |
188 | 0 | } |
189 | 0 |
|
190 | 0 | // Split the old argument, possibly causing more repairs to be scheduled. |
191 | 0 | pos.goto_inst(inst); |
192 | 0 |
|
193 | 0 | let inst_block = pos.func.layout.inst_block(inst).expect("inst in block"); |
194 | 0 |
|
195 | 0 | // Insert split values prior to the terminal branch group. |
196 | 0 | let canonical = pos |
197 | 0 | .func |
198 | 0 | .layout |
199 | 0 | .canonical_branch_inst(&pos.func.dfg, inst_block); |
200 | 0 | if let Some(first_branch) = canonical { |
201 | 0 | pos.goto_inst(first_branch); |
202 | 0 | } |
203 | | |
204 | 0 | let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs); |
205 | 0 |
|
206 | 0 | // The `lo` part replaces the original argument. |
207 | 0 | *args |
208 | 0 | .get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists) |
209 | 0 | .unwrap() = lo; |
210 | 0 |
|
211 | 0 | // The `hi` part goes at the end. Since multiple repairs may have been scheduled to the |
212 | 0 | // same block, there could be multiple arguments missing. |
213 | 0 | if num_args > num_fixed_args + repair.hi_num { |
214 | 0 | *args |
215 | 0 | .get_mut( |
216 | 0 | num_fixed_args + repair.hi_num, |
217 | 0 | &mut pos.func.dfg.value_lists, |
218 | 0 | ) |
219 | 0 | .unwrap() = hi; |
220 | 0 | } else { |
221 | 0 | // We need to append one or more arguments. If we're adding more than one argument, |
222 | 0 | // there must be pending repairs on the stack that will fill in the correct values |
223 | 0 | // instead of `hi`. |
224 | 0 | args.extend( |
225 | 0 | iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args), |
226 | 0 | &mut pos.func.dfg.value_lists, |
227 | 0 | ); |
228 | 0 | } |
229 | | |
230 | | // Put the value list back after manipulating it. |
231 | 0 | pos.func.dfg[inst].put_value_list(args); |
232 | | } |
233 | | } |
234 | 0 | } |
235 | | |
236 | | /// Split a single value using the integer or vector semantics given by the `concat` opcode. |
237 | | /// |
238 | | /// If the value is defined by a `concat` instruction, just reuse the operand values of that |
239 | | /// instruction. |
240 | | /// |
241 | | /// Return the two new values representing the parts of `value`. |
242 | 0 | fn split_value( |
243 | 0 | pos: &mut FuncCursor, |
244 | 0 | value: Value, |
245 | 0 | concat: Opcode, |
246 | 0 | repairs: &mut Vec<Repair>, |
247 | 0 | ) -> (Value, Value) { |
248 | 0 | let value = pos.func.dfg.resolve_aliases(value); |
249 | 0 | let mut reuse = None; |
250 | 0 |
|
251 | 0 | match pos.func.dfg.value_def(value) { |
252 | 0 | ValueDef::Result(inst, num) => { |
253 | | // This is an instruction result. See if the value was created by a `concat` |
254 | | // instruction. |
255 | 0 | if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] { |
256 | 0 | debug_assert_eq!(num, 0); |
257 | 0 | if opcode == concat { |
258 | 0 | reuse = Some((args[0], args[1])); |
259 | 0 | } |
260 | 0 | } |
261 | | } |
262 | 0 | ValueDef::Param(block, num) => { |
263 | 0 | // This is a block parameter. |
264 | 0 | // We can split the parameter value unless this is the entry block. |
265 | 0 | if pos.func.layout.entry_block() != Some(block) { |
266 | 0 | reuse = Some(split_block_param(pos, block, num, value, concat, repairs)); |
267 | 0 | } |
268 | | } |
269 | | } |
270 | | |
271 | | // Did the code above succeed in finding values we can reuse? |
272 | 0 | if let Some(pair) = reuse { |
273 | 0 | pair |
274 | | } else { |
275 | | // No, we'll just have to insert the requested split instruction at `pos`. Note that `pos` |
276 | | // has not been moved by the block argument code above when `reuse` is `None`. |
277 | 0 | match concat { |
278 | 0 | Opcode::Iconcat => pos.ins().isplit(value), |
279 | 0 | Opcode::Vconcat => pos.ins().vsplit(value), |
280 | 0 | _ => panic!("Unhandled concat opcode: {}", concat), |
281 | | } |
282 | | } |
283 | 0 | } |
284 | | |
285 | 0 | fn split_block_param( |
286 | 0 | pos: &mut FuncCursor, |
287 | 0 | block: Block, |
288 | 0 | param_num: usize, |
289 | 0 | value: Value, |
290 | 0 | concat: Opcode, |
291 | 0 | repairs: &mut Vec<Repair>, |
292 | 0 | ) -> (Value, Value) { |
293 | | // We are going to replace the parameter at `num` with two new arguments. |
294 | | // Determine the new value types. |
295 | 0 | let ty = pos.func.dfg.value_type(value); |
296 | 0 | let split_type = match concat { |
297 | 0 | Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"), |
298 | 0 | Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"), |
299 | 0 | _ => panic!("Unhandled concat opcode: {}", concat), |
300 | | }; |
301 | | |
302 | | // Since the `repairs` stack potentially contains other parameter numbers for |
303 | | // `block`, avoid shifting and renumbering block parameters. It could invalidate other |
304 | | // `repairs` entries. |
305 | | // |
306 | | // Replace the original `value` with the low part, and append the high part at the |
307 | | // end of the argument list. |
308 | 0 | let lo = pos.func.dfg.replace_block_param(value, split_type); |
309 | 0 | let hi_num = pos.func.dfg.num_block_params(block); |
310 | 0 | let hi = pos.func.dfg.append_block_param(block, split_type); |
311 | 0 |
|
312 | 0 | // Now the original value is dangling. Insert a concatenation instruction that can |
313 | 0 | // compute it from the two new parameters. This also serves as a record of what we |
314 | 0 | // did so a future call to this function doesn't have to redo the work. |
315 | 0 | // |
316 | 0 | // Note that it is safe to move `pos` here since `reuse` was set above, so we don't |
317 | 0 | // need to insert a split instruction before returning. |
318 | 0 | pos.goto_first_inst(block); |
319 | 0 | pos.ins() |
320 | 0 | .with_result(value) |
321 | 0 | .Binary(concat, split_type, lo, hi); |
322 | 0 |
|
323 | 0 | // Finally, splitting the block parameter is not enough. We also have to repair all |
324 | 0 | // of the predecessor instructions that branch here. |
325 | 0 | add_repair(concat, split_type, block, param_num, hi_num, repairs); |
326 | 0 |
|
327 | 0 | (lo, hi) |
328 | 0 | } |
329 | | |
330 | | // Add a repair entry to the work list. |
331 | | fn add_repair( |
332 | | concat: Opcode, |
333 | | split_type: Type, |
334 | | block: Block, |
335 | | num: usize, |
336 | | hi_num: usize, |
337 | | repairs: &mut Vec<Repair>, |
338 | | ) { |
339 | | repairs.push(Repair { |
340 | | concat, |
341 | | split_type, |
342 | | block, |
343 | | num, |
344 | | hi_num, |
345 | | }); |
346 | | } |
347 | | |
348 | | /// Strip concat-split chains. Return a simpler way of computing the same value. |
349 | | /// |
350 | | /// Given this input: |
351 | | /// |
352 | | /// ```clif |
353 | | /// v10 = iconcat v1, v2 |
354 | | /// v11, v12 = isplit v10 |
355 | | /// ``` |
356 | | /// |
357 | | /// This function resolves `v11` to `v1` and `v12` to `v2`. |
358 | 10.1M | fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value { |
359 | 10.1M | let value = dfg.resolve_aliases(value); |
360 | | |
361 | | // Deconstruct a split instruction. |
362 | | let split_res; |
363 | | let concat_opc; |
364 | | let split_arg; |
365 | 10.1M | if let ValueDef::Result(inst, num) = dfg.value_def(value) { |
366 | 9.39M | split_res = num; |
367 | 9.39M | concat_opc = match dfg[inst].opcode() { |
368 | 0 | Opcode::Isplit => Opcode::Iconcat, |
369 | 0 | Opcode::Vsplit => Opcode::Vconcat, |
370 | 9.39M | _ => return value, |
371 | | }; |
372 | 0 | split_arg = dfg.inst_args(inst)[0]; |
373 | | } else { |
374 | 755k | return value; |
375 | | } |
376 | | |
377 | | // See if split_arg is defined by a concatenation instruction. |
378 | 0 | if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) { |
379 | 0 | if dfg[inst].opcode() == concat_opc { |
380 | 0 | return dfg.inst_args(inst)[split_res]; |
381 | 0 | } |
382 | 0 | } |
383 | | |
384 | 0 | value |
385 | 10.1M | } |
386 | | |
387 | | /// Simplify the arguments to a branch *after* the instructions leading up to the branch have been |
388 | | /// legalized. |
389 | | /// |
390 | | /// The branch argument repairs performed by `split_any()` above may be performed on branches that |
391 | | /// have not yet been legalized. The repaired arguments can be defined by actual split |
392 | | /// instructions in that case. |
393 | | /// |
394 | | /// After legalizing the instructions computing the value that was split, it is likely that we can |
395 | | /// avoid depending on the split instruction. Its input probably comes from a concatenation. |
396 | 11.3M | pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) { |
397 | 11.3M | let mut new_args = SmallVec::<[Value; 32]>::new(); |
398 | | |
399 | 11.3M | for &arg in dfg.inst_args(branch) { |
400 | 10.1M | let new_arg = resolve_splits(dfg, arg); |
401 | 10.1M | new_args.push(new_arg); |
402 | 10.1M | } |
403 | | |
404 | 11.3M | dfg.inst_args_mut(branch).copy_from_slice(&new_args); |
405 | 11.3M | } |