Coverage Report

Created: 2021-03-22 08:29

/src/wasmtime/cranelift/codegen/src/legalizer/split.rs
Line
Count
Source (jump to first uncovered line)
1
//! Value splitting.
2
//!
3
//! Some value types are too large to fit in registers, so they need to be split into smaller parts
4
//! that the ISA can operate on. There's two dimensions of splitting, represented by two
5
//! complementary instruction pairs:
6
//!
7
//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
8
//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
9
//!   lane types.
10
//!
11
//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
12
//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
13
//! This breakdown is handled by the ABI lowering.
14
//!
15
//! When legalizing a single instruction, it is wrapped in splits and concatenations:
16
//!
17
//! ```clif
18
//!     v1 = bxor.i64 v2, v3
19
//! ```
20
//!
21
//! becomes:
22
//!
23
//! ```clif
24
//!     v20, v21 = isplit v2
25
//!     v30, v31 = isplit v3
26
//!     v10 = bxor.i32 v20, v30
27
//!     v11 = bxor.i32 v21, v31
28
//!     v1 = iconcat v10, v11
29
//! ```
30
//!
31
//! This local expansion approach still leaves the original `i64` values in the code as operands on
32
//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
33
//! values are constantly split and concatenated.
34
//!
35
//! # Optimized splitting
36
//!
37
//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
38
//! first check if the value is defined by the corresponding concatenation. If so, then just use
39
//! the two concatenation inputs directly:
40
//!
41
//! ```clif
42
//!     v4 = iadd_imm.i64 v1, 1
43
//! ```
44
//!
45
//! becomes, using the expanded code from above:
46
//!
47
//! ```clif
48
//!     v40, v5 = iadd_imm_cout.i32 v10, 1
49
//!     v6 = bint.i32
50
//!     v41 = iadd.i32 v11, v6
51
//!     v4 = iconcat v40, v41
52
//! ```
53
//!
54
//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
55
//! can be trivially deleted by a dead code elimination pass.
56
//!
57
//! # block arguments
58
//!
59
//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
60
//! up with no `i64` values anywhere, except for block arguments. We can work around this by
61
//! iteratively splitting block arguments too. That should leave us with no illegal value types
62
//! anywhere.
63
//!
64
//! It is possible to have circular dependencies of block arguments that are never used by any real
65
//! instructions. These loops will remain in the program.
66
67
use crate::cursor::{Cursor, CursorPosition, FuncCursor};
68
use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
69
use crate::ir::{self, Block, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
70
use alloc::vec::Vec;
71
use core::iter;
72
use smallvec::SmallVec;
73
74
/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
75
/// if possible.
76
pub fn isplit(
77
    func: &mut ir::Function,
78
    cfg: &ControlFlowGraph,
79
    pos: CursorPosition,
80
    srcloc: ir::SourceLoc,
81
    value: Value,
82
) -> (Value, Value) {
83
    split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
84
}
85
86
/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
87
/// possible.
88
pub fn vsplit(
89
    func: &mut ir::Function,
90
    cfg: &ControlFlowGraph,
91
    pos: CursorPosition,
92
    srcloc: ir::SourceLoc,
93
    value: Value,
94
) -> (Value, Value) {
95
    split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
96
}
97
98
/// After splitting a block argument, we need to go back and fix up all of the predecessor
99
/// instructions. This is potentially a recursive operation, but we don't implement it recursively
100
/// since that could use up too muck stack.
101
///
102
/// Instead, the repairs are deferred and placed on a work list in stack form.
103
struct Repair {
104
    concat: Opcode,
105
    // The argument type after splitting.
106
    split_type: Type,
107
    // The destination block whose arguments have been split.
108
    block: Block,
109
    // Number of the original block argument which has been replaced by the low part.
110
    num: usize,
111
    // Number of the new block argument which represents the high part after the split.
112
    hi_num: usize,
113
}
114
115
/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
116
fn split_any(
117
    func: &mut ir::Function,
118
    cfg: &ControlFlowGraph,
119
    pos: CursorPosition,
120
    srcloc: ir::SourceLoc,
121
    value: Value,
122
    concat: Opcode,
123
) -> (Value, Value) {
124
    let mut repairs = Vec::new();
125
    let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
126
    let result = split_value(pos, value, concat, &mut repairs);
127
128
    perform_repairs(pos, cfg, repairs);
129
130
    result
131
}
132
133
4.78M
pub fn split_block_params(func: &mut ir::Function, cfg: &ControlFlowGraph, block: Block) {
134
4.78M
    let pos = &mut FuncCursor::new(func).at_top(block);
135
4.78M
    let block_params = pos.func.dfg.block_params(block);
136
4.78M
137
4.78M
    // Add further splittable types here.
138
9.69M
    fn type_requires_splitting(ty: Type) -> bool {
139
9.69M
        ty == ir::types::I128
140
9.69M
    }
141
4.78M
142
4.78M
    // A shortcut.  If none of the param types require splitting, exit now.  This helps because
143
4.78M
    // the loop below necessarily has to copy the block params into a new vector, so it's better to
144
4.78M
    // avoid doing so when possible.
145
4.78M
    if !block_params
146
4.78M
        .iter()
147
9.69M
        .any(|block_param| type_requires_splitting(pos.func.dfg.value_type(*block_param)))
148
    {
149
4.78M
        return;
150
0
    }
151
0
152
0
    let mut repairs = Vec::new();
153
0
    for (num, block_param) in block_params.to_vec().into_iter().enumerate() {
154
0
        if !type_requires_splitting(pos.func.dfg.value_type(block_param)) {
155
            continue;
156
0
        }
157
0
158
0
        split_block_param(pos, block, num, block_param, Opcode::Iconcat, &mut repairs);
159
    }
160
161
0
    perform_repairs(pos, cfg, repairs);
162
4.78M
}
163
164
0
fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec<Repair>) {
165
    // We have split the value requested, and now we may need to fix some block predecessors.
166
0
    while let Some(repair) = repairs.pop() {
167
0
        for BlockPredecessor { inst, .. } in cfg.pred_iter(repair.block) {
168
0
            let branch_opc = pos.func.dfg[inst].opcode();
169
            debug_assert!(
170
                branch_opc.is_branch(),
171
                "Predecessor not a branch: {}",
172
                pos.func.dfg.display_inst(inst, None)
173
            );
174
0
            let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments();
175
0
            let mut args = pos.func.dfg[inst]
176
0
                .take_value_list()
177
0
                .expect("Branches must have value lists.");
178
0
            let num_args = args.len(&pos.func.dfg.value_lists);
179
0
            // Get the old value passed to the block argument we're repairing.
180
0
            let old_arg = args
181
0
                .get(num_fixed_args + repair.num, &pos.func.dfg.value_lists)
182
0
                .expect("Too few branch arguments");
183
0
184
0
            // It's possible that the CFG's predecessor list has duplicates. Detect them here.
185
0
            if pos.func.dfg.value_type(old_arg) == repair.split_type {
186
0
                pos.func.dfg[inst].put_value_list(args);
187
                continue;
188
0
            }
189
0
190
0
            // Split the old argument, possibly causing more repairs to be scheduled.
191
0
            pos.goto_inst(inst);
192
0
193
0
            let inst_block = pos.func.layout.inst_block(inst).expect("inst in block");
194
0
195
0
            // Insert split values prior to the terminal branch group.
196
0
            let canonical = pos
197
0
                .func
198
0
                .layout
199
0
                .canonical_branch_inst(&pos.func.dfg, inst_block);
200
0
            if let Some(first_branch) = canonical {
201
0
                pos.goto_inst(first_branch);
202
0
            }
203
204
0
            let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);
205
0
206
0
            // The `lo` part replaces the original argument.
207
0
            *args
208
0
                .get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists)
209
0
                .unwrap() = lo;
210
0
211
0
            // The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
212
0
            // same block, there could be multiple arguments missing.
213
0
            if num_args > num_fixed_args + repair.hi_num {
214
0
                *args
215
0
                    .get_mut(
216
0
                        num_fixed_args + repair.hi_num,
217
0
                        &mut pos.func.dfg.value_lists,
218
0
                    )
219
0
                    .unwrap() = hi;
220
0
            } else {
221
0
                // We need to append one or more arguments. If we're adding more than one argument,
222
0
                // there must be pending repairs on the stack that will fill in the correct values
223
0
                // instead of `hi`.
224
0
                args.extend(
225
0
                    iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args),
226
0
                    &mut pos.func.dfg.value_lists,
227
0
                );
228
0
            }
229
230
            // Put the value list back after manipulating it.
231
0
            pos.func.dfg[inst].put_value_list(args);
232
        }
233
    }
234
0
}
235
236
/// Split a single value using the integer or vector semantics given by the `concat` opcode.
237
///
238
/// If the value is defined by a `concat` instruction, just reuse the operand values of that
239
/// instruction.
240
///
241
/// Return the two new values representing the parts of `value`.
242
0
fn split_value(
243
0
    pos: &mut FuncCursor,
244
0
    value: Value,
245
0
    concat: Opcode,
246
0
    repairs: &mut Vec<Repair>,
247
0
) -> (Value, Value) {
248
0
    let value = pos.func.dfg.resolve_aliases(value);
249
0
    let mut reuse = None;
250
0
251
0
    match pos.func.dfg.value_def(value) {
252
0
        ValueDef::Result(inst, num) => {
253
            // This is an instruction result. See if the value was created by a `concat`
254
            // instruction.
255
0
            if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
256
0
                debug_assert_eq!(num, 0);
257
0
                if opcode == concat {
258
0
                    reuse = Some((args[0], args[1]));
259
0
                }
260
0
            }
261
        }
262
0
        ValueDef::Param(block, num) => {
263
0
            // This is a block parameter.
264
0
            // We can split the parameter value unless this is the entry block.
265
0
            if pos.func.layout.entry_block() != Some(block) {
266
0
                reuse = Some(split_block_param(pos, block, num, value, concat, repairs));
267
0
            }
268
        }
269
    }
270
271
    // Did the code above succeed in finding values we can reuse?
272
0
    if let Some(pair) = reuse {
273
0
        pair
274
    } else {
275
        // No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
276
        // has not been moved by the block argument code above when `reuse` is `None`.
277
0
        match concat {
278
0
            Opcode::Iconcat => pos.ins().isplit(value),
279
0
            Opcode::Vconcat => pos.ins().vsplit(value),
280
0
            _ => panic!("Unhandled concat opcode: {}", concat),
281
        }
282
    }
283
0
}
284
285
0
fn split_block_param(
286
0
    pos: &mut FuncCursor,
287
0
    block: Block,
288
0
    param_num: usize,
289
0
    value: Value,
290
0
    concat: Opcode,
291
0
    repairs: &mut Vec<Repair>,
292
0
) -> (Value, Value) {
293
    // We are going to replace the parameter at `num` with two new arguments.
294
    // Determine the new value types.
295
0
    let ty = pos.func.dfg.value_type(value);
296
0
    let split_type = match concat {
297
0
        Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
298
0
        Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
299
0
        _ => panic!("Unhandled concat opcode: {}", concat),
300
    };
301
302
    // Since the `repairs` stack potentially contains other parameter numbers for
303
    // `block`, avoid shifting and renumbering block parameters. It could invalidate other
304
    // `repairs` entries.
305
    //
306
    // Replace the original `value` with the low part, and append the high part at the
307
    // end of the argument list.
308
0
    let lo = pos.func.dfg.replace_block_param(value, split_type);
309
0
    let hi_num = pos.func.dfg.num_block_params(block);
310
0
    let hi = pos.func.dfg.append_block_param(block, split_type);
311
0
312
0
    // Now the original value is dangling. Insert a concatenation instruction that can
313
0
    // compute it from the two new parameters. This also serves as a record of what we
314
0
    // did so a future call to this function doesn't have to redo the work.
315
0
    //
316
0
    // Note that it is safe to move `pos` here since `reuse` was set above, so we don't
317
0
    // need to insert a split instruction before returning.
318
0
    pos.goto_first_inst(block);
319
0
    pos.ins()
320
0
        .with_result(value)
321
0
        .Binary(concat, split_type, lo, hi);
322
0
323
0
    // Finally, splitting the block parameter is not enough. We also have to repair all
324
0
    // of the predecessor instructions that branch here.
325
0
    add_repair(concat, split_type, block, param_num, hi_num, repairs);
326
0
327
0
    (lo, hi)
328
0
}
329
330
// Add a repair entry to the work list.
331
fn add_repair(
332
    concat: Opcode,
333
    split_type: Type,
334
    block: Block,
335
    num: usize,
336
    hi_num: usize,
337
    repairs: &mut Vec<Repair>,
338
) {
339
    repairs.push(Repair {
340
        concat,
341
        split_type,
342
        block,
343
        num,
344
        hi_num,
345
    });
346
}
347
348
/// Strip concat-split chains. Return a simpler way of computing the same value.
349
///
350
/// Given this input:
351
///
352
/// ```clif
353
///     v10 = iconcat v1, v2
354
///     v11, v12 = isplit v10
355
/// ```
356
///
357
/// This function resolves `v11` to `v1` and `v12` to `v2`.
358
10.1M
fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
359
10.1M
    let value = dfg.resolve_aliases(value);
360
361
    // Deconstruct a split instruction.
362
    let split_res;
363
    let concat_opc;
364
    let split_arg;
365
10.1M
    if let ValueDef::Result(inst, num) = dfg.value_def(value) {
366
9.39M
        split_res = num;
367
9.39M
        concat_opc = match dfg[inst].opcode() {
368
0
            Opcode::Isplit => Opcode::Iconcat,
369
0
            Opcode::Vsplit => Opcode::Vconcat,
370
9.39M
            _ => return value,
371
        };
372
0
        split_arg = dfg.inst_args(inst)[0];
373
    } else {
374
755k
        return value;
375
    }
376
377
    // See if split_arg is defined by a concatenation instruction.
378
0
    if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
379
0
        if dfg[inst].opcode() == concat_opc {
380
0
            return dfg.inst_args(inst)[split_res];
381
0
        }
382
0
    }
383
384
0
    value
385
10.1M
}
386
387
/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
388
/// legalized.
389
///
390
/// The branch argument repairs performed by `split_any()` above may be performed on branches that
391
/// have not yet been legalized. The repaired arguments can be defined by actual split
392
/// instructions in that case.
393
///
394
/// After legalizing the instructions computing the value that was split, it is likely that we can
395
/// avoid depending on the split instruction. Its input probably comes from a concatenation.
396
11.3M
pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
397
11.3M
    let mut new_args = SmallVec::<[Value; 32]>::new();
398
399
11.3M
    for &arg in dfg.inst_args(branch) {
400
10.1M
        let new_arg = resolve_splits(dfg, arg);
401
10.1M
        new_args.push(new_arg);
402
10.1M
    }
403
404
11.3M
    dfg.inst_args_mut(branch).copy_from_slice(&new_args);
405
11.3M
}